Fix client timeout propagation and add operation retry logic

The client library had several issues preventing operations from completing
in the simulation:

- toastyfs_register_events discarded the computed timeout instead of
  returning it to callers, so clients never woke up on schedule
- random_client_tick ignored the timeout parameter entirely
- deadline_to_timeout truncated sub-millisecond values to 0 instead of
  rounding up, causing busy-wait loops
- The server didn't reset the heartbeat timer after completing a view
  change, so followers would immediately timeout again
- The client's DELETE handler didn't handle META_RESULT_NOT_FOUND,
  causing an assertion failure when deleting non-existent keys
- Most critically, messages sent before TCP connections were established
  were silently dropped with no retry mechanism — added timeout-based
  retry logic in toastyfs_process_events

https://claude.ai/code/session_0184gHjra7fsmSPZ4kppaGhC

src/basic.c

@@ -71,7 +71,10 @@ int deadline_to_timeout(Time deadline, Time current_time)
 {
     if (deadline == INVALID_TIME)
         return -1;
-    return (deadline - current_time) / 1000000;
+    if (deadline <= current_time)
+        return 0;
+    // Round up so sub-millisecond deadlines become 1ms, not 0
+    return (deadline - current_time + 999999) / 1000000;
 }
 
 bool getargb(int argc, char **argv, char *name)

src/client.c

@@ -570,6 +570,13 @@ static int process_message(ToastyFS *tfs,
                     break;
                 }
 
+                if (reply.result.type == META_RESULT_NOT_FOUND) {
+                    client_log_simple(tfs, "RECV REPLY NOT_FOUND");
+                    tfs->error = TOASTYFS_ERROR_NOT_FOUND;
+                    tfs->step = STEP_DELETE_DONE;
+                    break;
+                }
+
                 assert(reply.result.type == META_RESULT_OK);
                 client_log_simple(tfs, "RECV REPLY OK");
                 tfs->error = TOASTYFS_ERROR_VOID;
@@ -688,12 +695,44 @@ void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i
             tcp_consume_message(&tfs->tcp, conn_idx);
         }
     }
+
+    // Check for operation timeout — retry the current operation if the
+    // deadline has passed (handles initial sends that were dropped because
+    // the TCP connection wasn't established yet, and unresponsive servers).
+    if (tfs->step != STEP_IDLE
+        && tfs->step != STEP_PUT_DONE
+        && tfs->step != STEP_GET_DONE
+        && tfs->step != STEP_DELETE_DONE) {
+        Time now = get_current_time();
+        Time deadline = tfs->step_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
+        if (now >= deadline) {
+            client_log_simple(tfs, "TIMEOUT RETRY");
+            switch (tfs->step) {
+            case STEP_STORE_CHUNK:
+            case STEP_FETCH_CHUNK:
+                for (int i = 0; i < tfs->num_transfers; i++) {
+                    if (tfs->transfers[i].state == TRANSFER_STARTED)
+                        tfs->transfers[i].state = TRANSFER_PENDING;
+                }
+                tfs->step_time = now;
+                begin_transfers(tfs);
+                break;
+            case STEP_COMMIT:
+            case STEP_DELETE:
+            case STEP_GET:
+                replay_request(tfs);
+                break;
+            default:
+                break;
+            }
+        }
+    }
 }
 
 // TODO: The toastyfs client needs to determine a timeout based on the
 //       pending operation status, not just use PRIMARY_DEATH_TIMEOUT_SEC
 //       for everything.
-int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap)
+int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap, int *timeout)
 {
     Time now = get_current_time(); // TODO: Handle INVALID_TIME error
     Time deadline = INVALID_TIME;
@@ -702,7 +741,7 @@ int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i
         nearest_deadline(&deadline, tfs->step_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
     }
 
-    (void) deadline_to_timeout(deadline, now);
+    *timeout = deadline_to_timeout(deadline, now);
     if (pcap < TCP_POLL_CAPACITY)
         return -1;
     return tcp_register_events(&tfs->tcp, ctxs, pdata);
@@ -1019,11 +1058,12 @@ static int wait_until_result(ToastyFS *tfs, ToastyFS_Result *res)
     for (;;) {
         void *ctxs[TCP_POLL_CAPACITY];
         struct pollfd arr[TCP_POLL_CAPACITY];
-        int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY);
+        int poll_timeout;
+        int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY, &poll_timeout);
         if (num < 0)
             return num;
 
-        poll(arr, num, -1); // TODO: use computed timeout
+        poll(arr, num, poll_timeout);
 
         toastyfs_process_events(tfs, ctxs, arr, num);
         *res = toastyfs_get_result(tfs);

src/random_client.c

@@ -82,12 +82,12 @@ int random_client_init(void *state_, int argc, char **argv,
         return -1;
     state->started = false;
 
-    *timeout = 0;
     if (pcap < TCP_POLL_CAPACITY) {
         fprintf(stderr, "Random client :: Not enough poll capacity\n");
         return -1;
     }
-    *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap);
+    *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout);
+    *timeout = 0; // Ensure first tick fires immediately to start an operation
     return 0;
 }
 
@@ -142,8 +142,7 @@ int random_client_tick(void *state_, void **ctxs,
         }
     }
 
-    (void) timeout;
-    *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap);
+    *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout);
     return 0;
 }
 

src/server.c

@@ -517,6 +517,7 @@ complete_view_change_and_become_primary(ServerState *state)
     broadcast_to_peers_ex(state, &begin_view_message.base, state->log.entries, state->log.count * sizeof(LogEntry));
 
     clear_view_change_fields(state);
+    state->heartbeat = state->now;
     return HR_OK;
 }