Handle allocation and I/O failures gracefully in simulation

Replace assert(0) with proper error handling for:
- malloc failures in client write operations
- binary_read failures during message parsing
- WAL append failures in metadata server
- Chunk server timeout (now drops unresponsive servers)

This fixes crashes exposed by fault injection testing.

src/client.c

@@ -1538,8 +1538,9 @@ static void process_event_for_write(ToastyFS *toasty,
         toasty->operations[opidx].num_chunks = num_all_hasehs;
         toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow
         toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256));
-        if (toasty->operations[opidx].hashes == NULL) {
-            assert(0); // TODO
+        if (toasty->operations[opidx].hashes == NULL && num_hashes > 0) {
+            toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
+            return;
         }
 
         toasty->operations[opidx].uploads = NULL;
@@ -1880,19 +1881,19 @@ static void process_event_for_write(ToastyFS *toasty,
 
         // version
         if (!binary_read(&reader, NULL, sizeof(uint16_t))) {
-            assert(0); // TODO
+            toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
             return;
         }
 
         uint16_t type;
         if (!binary_read(&reader, &type, sizeof(uint16_t))) {
-            assert(0); // TODO
+            toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
             return;
         }
 
         // length
         if (!binary_read(&reader, NULL, sizeof(uint32_t))) {
-            assert(0); // TODO
+            toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
             return;
         }
 
@@ -1909,13 +1910,13 @@ static void process_event_for_write(ToastyFS *toasty,
 
             SHA256 hash;
             if (!binary_read(&reader, &hash, sizeof(hash))) {
-                assert(0); // TODO
+                toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
                 return;
             }
 
             // Check that there is nothing else to read
             if (binary_read(&reader, NULL, 1)) {
-                assert(0); // TODO
+                toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
                 return;
             }
 
@@ -1961,8 +1962,9 @@ static void process_event_for_write(ToastyFS *toasty,
 
             int num_upload_results = toasty->operations[opidx].num_chunks;
             ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult));
-            if (upload_results == NULL) {
-                assert(0); // TODO
+            if (upload_results == NULL && num_upload_results > 0) {
+                toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
+                return;
             }
 
             for (int i = 0; i < num_upload_results; i++) {
@@ -2008,7 +2010,9 @@ static void process_event_for_write(ToastyFS *toasty,
             uint32_t     flags  = toasty->operations[opidx].flags;
 
             if (path.len > UINT16_MAX) {
-                assert(0); // TODO
+                toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
+                free(upload_results);
+                return;
             }
             uint16_t path_len = path.len;
             uint64_t expect_gen = toasty->operations[opidx].expect_gen;
@@ -2046,7 +2050,8 @@ static void process_event_for_write(ToastyFS *toasty,
             free(upload_results);
 
             if (metadata_server_request_end(toasty, &writer, opidx, TAG_COMMIT_WRITE) < 0) {
-                assert(0); // TODO
+                toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
+                return;
             }
         }
 

src/metadata_server.c

@@ -185,7 +185,7 @@ process_client_create(MetadataServer *state, int conn_idx, ByteView msg)
         return -1;
 
     if (wal_append_create(&state->wal, path, is_dir, chunk_size) < 0) {
-        assert(0); // TODO
+        return -1;
     }
 
     uint64_t gen;
@@ -253,7 +253,7 @@ process_client_delete(MetadataServer *state, int conn_idx, ByteView msg)
         return -1;
 
     if (wal_append_delete(&state->wal, path, expect_gen) < 0) {
-        assert(0); // TODO
+        return -1;
     }
 
     // TODO: return unused hashes and add them to the ms_rem_list of holder chunk servers
@@ -659,7 +659,7 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
         new_hashes[i] = results[i].hash;
 
     if (wal_append_write(&state->wal, path, offset, length, num_chunks, expect_gen, new_hashes) < 0) {
-        assert(0); // TODO
+        return -1;
     }
 
     // Extract flag values
@@ -682,7 +682,7 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
 
         // Log the creation in the WAL
         if (wal_append_create(&state->wal, path, false, chunk_size) < 0) {
-            assert(0); // TODO
+            return -1;
         }
 
         uint64_t create_gen;
@@ -859,7 +859,7 @@ static int process_chunk_server_auth(MetadataServer *state,
     //       of this chunk server. If we do, tell it.
 
     if (!message_writer_free(&writer)) {
-        assert(0); // TODO
+        return -1;
     }
     return 0;
 }
@@ -905,7 +905,7 @@ static int process_chunk_server_sync(MetadataServer *state,
 
         if (num_holders <= state->replication_factor) {
             if (hash_set_insert(&chunk_server->ms_add_list, hash) < 0) {
-                assert(0); // TODO
+                return -1;
             }
             continue;
         }
@@ -1115,7 +1115,8 @@ int metadata_server_init(void *state_, int argc, char **argv,
 
     if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) {
         fprintf(stderr, "metadata server :: Couldn't setup WAL\n");
-        assert(0); // TODO
+        tcp_context_free(&state->tcp);
+        return -1;
     }
 
     printf("Metadata server set up (local=%.*s:%d)\n",
@@ -1251,7 +1252,9 @@ int metadata_server_tick(void *state_, void **ctxs,
 
         Time response_timeout = chunk_server->last_response_time + (Time) RESPONSE_TIME_LIMIT * 1000000000;
         if (current_time > response_timeout) {
-            assert(0); // TODO: drop the chunk server
+            // Drop unresponsive chunk server
+            chunk_server->used = false;
+            state->num_chunk_servers--;
             continue;
         }
         nearest_deadline(&next_wakeup, response_timeout);