Drop the MAX_CONNS macro and define TCP_CONNECTION_LIMIT, TCP_POLL_CAPACITY, TCP_EVENT_CAPACITY instead

inc/ToastyFS.h

@@ -188,8 +188,9 @@ void toasty_free_result(ToastyResult *result);
 // OTHER
 //////////////////////////////////////////////////////////////////////////////////
 
-// This is a hook for the simulation testing framework.
+#define TOASTY_POLL_CAPACITY 514
-// You shouldn't need to use this.
+
+// TODO: comment
 int toasty_process_events(ToastyFS *toasty, void **contexts,
     struct pollfd *polled, int num_polled);
 

src/chunk_server.c

@@ -919,7 +919,7 @@ int chunk_server_free(ChunkServer *state)
 
 int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
 {
-    Event events[MAX_CONNS+1];
+    Event events[TCP_EVENT_CAPACITY];
     int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
 
     Time current_time = get_current_time();

src/client.c

@@ -2061,13 +2061,15 @@ unlock_and_exit:
     return ret;
 }
 
+_Static_assert(TCP_POLL_CAPACITY == TOASTY_POLL_CAPACITY);
+
 int toasty_process_events(ToastyFS *toasty, void **contexts, struct pollfd *polled, int num_polled)
 {
     if (mutex_lock(&toasty->mutex) < 0)
         return -1;
 
     int num_events;
-    Event events[MAX_CONNS+1];
+    Event events[TCP_EVENT_CAPACITY];
 
     num_events = tcp_translate_events(&toasty->tcp, events, contexts, polled, num_polled);
     for (int i = 0; i < num_events; i++) {
@@ -2199,8 +2201,8 @@ int toasty_wait_result(ToastyFS *toasty, ToastyHandle handle, ToastyResult *resu
             return -1;
     }
 
-    void *contexts[MAX_CONNS+1];
+    void *contexts[TCP_POLL_CAPACITY];
-    struct pollfd polled[MAX_CONNS+1];
+    struct pollfd polled[TCP_POLL_CAPACITY];
     int num_polled;
 
     num_polled = toasty_process_events(toasty, contexts, polled, 0);

src/main_server.c

@@ -15,8 +15,8 @@
 
 int metadata_server_main(int argc, char **argv)
 {
-    void *contexts[MAX_CONNS+1];
+    void *contexts[TCP_POLL_CAPACITY];
-    struct pollfd polled[MAX_CONNS+1];
+    struct pollfd polled[TCP_POLL_CAPACITY];
     int num_polled;
     int timeout = -1;
     MetadataServer state;
@@ -37,8 +37,8 @@ int metadata_server_main(int argc, char **argv)
 
 int chunk_server_main(int argc, char **argv)
 {
-    void *contexts[MAX_CONNS+1];
+    void *contexts[TCP_POLL_CAPACITY];
-    struct pollfd polled[MAX_CONNS+1];
+    struct pollfd polled[TCP_POLL_CAPACITY];
     int num_polled;
     int timeout = -1;
     ChunkServer state;

src/metadata_server.c

@@ -1040,7 +1040,7 @@ int metadata_server_free(MetadataServer *state)
 
 int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
 {
-    Event events[MAX_CONNS+1];
+    Event events[TCP_EVENT_CAPACITY];
     int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
 
     Time current_time = get_current_time();

src/system.c

@@ -343,8 +343,8 @@ int spawn_simulated_process(char *args)
         process->desc[i].generation = 0;
     }
 
-    void *contexts[MAX_CONNS+1];
+    void *contexts[TCP_POLL_CAPACITY];
-    struct pollfd polled[MAX_CONNS+1];
+    struct pollfd polled[TCP_POLL_CAPACITY];
     int num_polled;
     int timeout = -1;
 
@@ -711,8 +711,8 @@ void update_simulation(void)
 
             current_process = processes[i];
 
-            void *contexts[MAX_CONNS+1];
+            void *contexts[TCP_POLL_CAPACITY];
-            struct pollfd polled[MAX_CONNS+1];
+            struct pollfd polled[TCP_POLL_CAPACITY];
             int num_polled = setup_poll_array(contexts, polled);
 
             if (num_polled > 0) {
@@ -787,8 +787,8 @@ void update_simulation(void)
 
     current_process = next_process;
 
-    void *contexts[MAX_CONNS+1];
+    void *contexts[TCP_POLL_CAPACITY];
-    struct pollfd polled[MAX_CONNS+1];
+    struct pollfd polled[TCP_POLL_CAPACITY];
     int num_polled = setup_poll_array(contexts, polled);
 
     int timeout = -1;

src/tcp.c

@@ -271,16 +271,13 @@ int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled)
 {
     int num_polled = 0;
 
-    // TODO: Check that there is enough capacity in the polled array.
-    //       At the moment only MAX_CONNS plus 1 for the listener are
-    //       allowed.
     polled[num_polled].fd = tcp->wait_fd;
     polled[num_polled].events = POLLIN;
     polled[num_polled].revents = 0;
     contexts[num_polled] = NULL;
     num_polled++;
 
-    if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < MAX_CONNS) {
+    if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < TCP_CONNECTION_LIMIT) {
         polled[num_polled].fd = tcp->listen_fd;
         polled[num_polled].events = POLLIN;
         polled[num_polled].revents = 0;
@@ -302,11 +299,11 @@ int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled)
     return num_polled;
 }
 
-// The "events" array must be an array of capacity MAX_CONNS+1
+// The "events" array must be an array of capacity TCP_EVENT_CAPACITY,
-// TODO: Actually, it should have capacity MAX_CONNS+2
+// while "contexts" and "polled" must have capacity TCP_POLL_CAPACITY.
 int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled)
 {
-    bool removed[MAX_CONNS+1];
+    bool removed[TCP_CONNECTION_LIMIT+1];
 
     int num_events = 0;
     for (int i = 1; i < num_polled; i++) {
@@ -431,7 +428,7 @@ ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx)
 
 int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
 {
-    if (tcp->num_conns == MAX_CONNS)
+    if (tcp->num_conns == TCP_CONNECTION_LIMIT)
         return -1;
     int conn_idx = tcp->num_conns;
 

src/tcp.h

@@ -12,7 +12,23 @@
 #define CLOSE_SOCKET sys_close
 #endif
 
-#define MAX_CONNS 512
+#ifndef TCP_CONNECTION_LIMIT
+// Maximum number of connections that can be managed
+// simultaneously.
+#define TCP_CONNECTION_LIMIT 512
+#endif
+
+// This is the maximum number of descriptors that the
+// TCP system will want to wait at any given time.
+// One descriptor per connection plus a listener socket
+// and a self-pipe handle for wakeup.
+#define TCP_POLL_CAPACITY (TCP_CONNECTION_LIMIT+2)
+
+// Number of TCP events that can be returned at a given
+// time by "tcp_translate_events". There may be a single
+// event per connection (MESSAGE, CONNECT, DISCONNECT)
+// plus a general WAKEUP event.
+#define TCP_EVENT_CAPACITY (TCP_CONNECTION_LIMIT+1)
 
 typedef enum {
     EVENT_WAKEUP,
@@ -42,7 +58,7 @@ typedef struct {
     SOCKET wait_fd;
     SOCKET signal_fd;
     int    num_conns;
-    Connection conns[MAX_CONNS];
+    Connection conns[TCP_CONNECTION_LIMIT];
 } TCP;
 
 bool addr_eql(Address a, Address b);

web/chttp.h

@@ -526,8 +526,8 @@ int socket_manager_wakeup(SocketManager *sm);
 typedef struct {
     void **ptrs;
     struct pollfd *polled;
-    int max_polled;
     int num_polled;
+    int max_polled;
 } EventRegister;
 
 // Resets the event register with the list of descriptors
@@ -764,6 +764,11 @@ int http_create_test_certificate(HTTP_String C, HTTP_String O, HTTP_String CN,
 #define HTTP_CLIENT_CAPACITY (1<<7)
 #endif
 
+// Maximum number of descriptors the client will want
+// to wait on. It's one per connection plus the wakeup
+// self-pipe.
+#define HTTP_CLIENT_POLL_CAPACITY (HTTP_CLIENT_CAPACITY+1)
+
 typedef enum {
     HTTP_CLIENT_CONN_FREE,
     HTTP_CLIENT_CONN_WAIT_LINE,
@@ -922,6 +927,11 @@ void http_free_response(HTTP_Response *response);
 #define HTTP_SERVER_CAPACITY (1<<9)
 #endif
 
+// Maximum number of descriptors the server will want
+// to wait on. It's one per connection plus two for the
+// TCP and TLS listener, plus one for the wakeup self-pipe.
+#define HTTP_SERVER_POLL_CAPACITY (HTTP_SERVER_CAPACITY+3)
+
 typedef enum {
 
     // This struct is unused

web/main.c

@@ -84,9 +84,6 @@ int main(void)
 
     for (;;) {
 
-        #define HTTP_SERVER_POLL_CAPACITY (HTTP_SERVER_CAPACITY+3)
-        #define TOASTY_POLL_CAPACITY (MAX_CONNS+1)
-
         #define POLL_CAPACITY (HTTP_SERVER_POLL_CAPACITY + TOASTY_POLL_CAPACITY)
 
         EventRegister reg;