Improve simulation coverage by adding multiple chunk servers

Two changes to improve code coverage in deterministic simulation testing:

1. Add 3 chunk servers (was 1) to match the replication factor
   - This enables testing of chunk replication and chunk-to-chunk
     server communication code paths in chunk_server.c
   - Covers: process_chunk_server_download_*, start_download,
     download_targets_* functions

2. Fix Quakey bug where events targeting a closed descriptor caused
   assertion failures
   - Added remove_events_targeting_desc() to clean up DISCONNECT and
     DATA events when a descriptor is freed
   - Modified desc_free() to accept Sim pointer and call cleanup
   - This was exposed by running multiple chunk servers which create
     more connection activity

quakey/src/quakey.c

@@ -369,6 +369,7 @@ static void time_event_wakeup(Sim *sim, Nanos time, Host *host);
 static void time_event_connect(Sim *sim, Nanos time, Desc *desc);
 static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst);
 static void time_event_send_data(Sim *sim, Nanos time, Desc *desc);
+static void remove_events_targeting_desc(Sim *sim, Desc *desc);
 
 /////////////////////////////////////////////////////////////////
 /////////////////////////////////////////////////////////////////
@@ -776,7 +777,7 @@ static bool accept_queue_empty(AcceptQueue *queue)
 // If the descriptor is a connection socket and rst=true,
 // the peer connection will be marked as "reset" instead
 // of simply closed.
-static void desc_free(Desc *desc, bool rst)
+static void desc_free(Sim *sim, Desc *desc, bool rst)
 {
     switch (desc->type) {
     case DESC_EMPTY:
@@ -793,6 +794,9 @@ static void desc_free(Desc *desc, bool rst)
         accept_queue_free(&desc->accept_queue);
         break;
     case DESC_SOCKET_C:
+        // Remove any pending events targeting this descriptor before freeing it
+        if (sim)
+            remove_events_targeting_desc(sim, desc);
         if (desc->peer) {
             // A connection was previously established.
             // We need to update the other end of the connection.
@@ -989,7 +993,7 @@ static void host_free(Host *host)
 
     for (int i = 0; i < HOST_DESC_LIMIT; i++) {
         if (host->desc[i].type != DESC_EMPTY)
-            desc_free(&host->desc[i], true);
+            desc_free(host->sim, &host->desc[i], true);
     }
 
     free(host->state);
@@ -1285,7 +1289,7 @@ static int host_close(Host *host, int desc_idx, bool expect_socket)
     if (host->desc[desc_idx].type == DESC_SOCKET_C)
         time_event_disconnect(host->sim, host->sim->current_time + 10000000, &host->desc[desc_idx], false);
 
-    desc_free(&host->desc[desc_idx], false);
+    desc_free(host->sim, &host->desc[desc_idx], false);
     host->num_desc--;
     return 0;
 }
@@ -1934,6 +1938,26 @@ static b32 remove_connect_event(Sim *sim, Desc *desc)
     return true;
 }
 
+// Remove all events that target a specific descriptor (DISCONNECT and DATA events)
+static void remove_events_targeting_desc(Sim *sim, Desc *desc)
+{
+    int i = 0;
+    while (i < sim->num_events) {
+        TimeEvent *event = &sim->events[i];
+        if (event->dst_desc == desc) {
+            // Free any resources associated with the event
+            if (event->type == EVENT_TYPE_DATA && event->data_queue) {
+                socket_queue_unref(event->data_queue);
+            }
+            // Remove by swapping with last element
+            sim->events[i] = sim->events[--sim->num_events];
+            // Don't increment i - need to check the swapped element
+        } else {
+            i++;
+        }
+    }
+}
+
 static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst)
 {
     if (remove_connect_event(sim, desc))

src/main.c

@@ -160,10 +160,10 @@ int main(void)
         quakey_spawn(quakey, config, "ms --addr 127.0.0.3 --port 8080");
     }
 
-    // Chunk Server
+    // Chunk Server 1
     {
         QuakeySpawn config = {
-            .name       = "cs",
+            .name       = "cs1",
             .state_size = sizeof(ChunkServer),
             .init_func  = chunk_server_init,
             .tick_func  = chunk_server_tick,
@@ -176,6 +176,38 @@ int main(void)
         quakey_spawn(quakey, config, "cs --addr 127.0.0.4 --port 8081 --remote-addr 127.0.0.3 --remote-port 8080");
     }
 
+    // Chunk Server 2
+    {
+        QuakeySpawn config = {
+            .name       = "cs2",
+            .state_size = sizeof(ChunkServer),
+            .init_func  = chunk_server_init,
+            .tick_func  = chunk_server_tick,
+            .free_func  = chunk_server_free,
+            .addrs      = (char*[]) { "127.0.0.5" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "cs --addr 127.0.0.5 --port 8082 --remote-addr 127.0.0.3 --remote-port 8080");
+    }
+
+    // Chunk Server 3
+    {
+        QuakeySpawn config = {
+            .name       = "cs3",
+            .state_size = sizeof(ChunkServer),
+            .init_func  = chunk_server_init,
+            .tick_func  = chunk_server_tick,
+            .free_func  = chunk_server_free,
+            .addrs      = (char*[]) { "127.0.0.6" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "cs --addr 127.0.0.6 --port 8083 --remote-addr 127.0.0.3 --remote-port 8080");
+    }
+
     while (simulation_running)
         quakey_schedule_one(quakey);