From 82333928718527e66e49e8900038b2ef30327f75 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Feb 2026 14:08:07 +0000 Subject: [PATCH] Fix client timeout propagation and add operation retry logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client library had several issues preventing operations from completing in the simulation: - toastyfs_register_events discarded the computed timeout instead of returning it to callers, so clients never woke up on schedule - random_client_tick ignored the timeout parameter entirely - deadline_to_timeout truncated sub-millisecond values to 0 instead of rounding up, causing busy-wait loops - The server didn't reset the heartbeat timer after completing a view change, so followers would immediately timeout again - The client's DELETE handler didn't handle META_RESULT_NOT_FOUND, causing an assertion failure when deleting non-existent keys - Most critically, messages sent before TCP connections were established were silently dropped with no retry mechanism — added timeout-based retry logic in toastyfs_process_events https://claude.ai/code/session_0184gHjra7fsmSPZ4kppaGhC --- include/toastyfs.h | 2 +- src/basic.c | 5 ++++- src/client.c | 48 +++++++++++++++++++++++++++++++++++++++++---- src/random_client.c | 7 +++---- src/server.c | 1 + 5 files changed, 53 insertions(+), 10 deletions(-) diff --git a/include/toastyfs.h b/include/toastyfs.h index 306a90b..4018b2a 100644 --- a/include/toastyfs.h +++ b/include/toastyfs.h @@ -33,7 +33,7 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs); void toastyfs_free(ToastyFS *tfs); void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pnum); -int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap); +int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap, int *timeout); int toastyfs_async_put(ToastyFS *tfs, char *key, int key_len, char *data, int data_len); diff --git a/src/basic.c b/src/basic.c index c15078b..3974344 100644 --- a/src/basic.c +++ b/src/basic.c @@ -71,7 +71,10 @@ int deadline_to_timeout(Time deadline, Time current_time) { if (deadline == INVALID_TIME) return -1; - return (deadline - current_time) / 1000000; + if (deadline <= current_time) + return 0; + // Round up so sub-millisecond deadlines become 1ms, not 0 + return (deadline - current_time + 999999) / 1000000; } bool getargb(int argc, char **argv, char *name) diff --git a/src/client.c b/src/client.c index 1c58d9c..c12baf8 100644 --- a/src/client.c +++ b/src/client.c @@ -570,6 +570,13 @@ static int process_message(ToastyFS *tfs, break; } + if (reply.result.type == META_RESULT_NOT_FOUND) { + client_log_simple(tfs, "RECV REPLY NOT_FOUND"); + tfs->error = TOASTYFS_ERROR_NOT_FOUND; + tfs->step = STEP_DELETE_DONE; + break; + } + assert(reply.result.type == META_RESULT_OK); client_log_simple(tfs, "RECV REPLY OK"); tfs->error = TOASTYFS_ERROR_VOID; @@ -688,12 +695,44 @@ void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i tcp_consume_message(&tfs->tcp, conn_idx); } } + + // Check for operation timeout — retry the current operation if the + // deadline has passed (handles initial sends that were dropped because + // the TCP connection wasn't established yet, and unresponsive servers). + if (tfs->step != STEP_IDLE + && tfs->step != STEP_PUT_DONE + && tfs->step != STEP_GET_DONE + && tfs->step != STEP_DELETE_DONE) { + Time now = get_current_time(); + Time deadline = tfs->step_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (now >= deadline) { + client_log_simple(tfs, "TIMEOUT RETRY"); + switch (tfs->step) { + case STEP_STORE_CHUNK: + case STEP_FETCH_CHUNK: + for (int i = 0; i < tfs->num_transfers; i++) { + if (tfs->transfers[i].state == TRANSFER_STARTED) + tfs->transfers[i].state = TRANSFER_PENDING; + } + tfs->step_time = now; + begin_transfers(tfs); + break; + case STEP_COMMIT: + case STEP_DELETE: + case STEP_GET: + replay_request(tfs); + break; + default: + break; + } + } + } } // TODO: The toastyfs client needs to determine a timeout based on the // pending operation status, not just use PRIMARY_DEATH_TIMEOUT_SEC // for everything. -int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap) +int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap, int *timeout) { Time now = get_current_time(); // TODO: Handle INVALID_TIME error Time deadline = INVALID_TIME; @@ -702,7 +741,7 @@ int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i nearest_deadline(&deadline, tfs->step_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL); } - (void) deadline_to_timeout(deadline, now); + *timeout = deadline_to_timeout(deadline, now); if (pcap < TCP_POLL_CAPACITY) return -1; return tcp_register_events(&tfs->tcp, ctxs, pdata); @@ -1019,11 +1058,12 @@ static int wait_until_result(ToastyFS *tfs, ToastyFS_Result *res) for (;;) { void *ctxs[TCP_POLL_CAPACITY]; struct pollfd arr[TCP_POLL_CAPACITY]; - int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY); + int poll_timeout; + int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY, &poll_timeout); if (num < 0) return num; - poll(arr, num, -1); // TODO: use computed timeout + poll(arr, num, poll_timeout); toastyfs_process_events(tfs, ctxs, arr, num); *res = toastyfs_get_result(tfs); diff --git a/src/random_client.c b/src/random_client.c index 489e54a..54253f4 100644 --- a/src/random_client.c +++ b/src/random_client.c @@ -82,12 +82,12 @@ int random_client_init(void *state_, int argc, char **argv, return -1; state->started = false; - *timeout = 0; if (pcap < TCP_POLL_CAPACITY) { fprintf(stderr, "Random client :: Not enough poll capacity\n"); return -1; } - *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap); + *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout); + *timeout = 0; // Ensure first tick fires immediately to start an operation return 0; } @@ -142,8 +142,7 @@ int random_client_tick(void *state_, void **ctxs, } } - (void) timeout; - *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap); + *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout); return 0; } diff --git a/src/server.c b/src/server.c index 18de29f..0091bdd 100644 --- a/src/server.c +++ b/src/server.c @@ -517,6 +517,7 @@ complete_view_change_and_become_primary(ServerState *state) broadcast_to_peers_ex(state, &begin_view_message.base, state->log.entries, state->log.count * sizeof(LogEntry)); clear_view_change_fields(state); + state->heartbeat = state->now; return HR_OK; }