Version 0.9

build.sh

@@ -1,6 +1,6 @@
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c vsr/invariant_checker.c state_machine/kvstore.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/kvstore.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/kvstore.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/invariant_checker.c raft/main.c state_machine/kvstore.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION -DDISABLE_SPURIOUS_IO_ERRORS
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/main.c state_machine/kvstore.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
-gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/main.c state_machine/kvstore.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT

lib/file_system.c

@@ -36,7 +36,7 @@ int file_open(string path, Handle *fd)
     memcpy(zt, path.ptr, path.len);
     zt[path.len] = '\0';
 
-    int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
+    int ret = open(zt, O_RDWR | O_CREAT, 0644);
     if (ret < 0)
         return -1;
 

quakey/include/quakey.h

@@ -26,6 +26,8 @@
 #include <pthread.h>
 #endif
 
+#include <stdbool.h>
+
 typedef struct {} Quakey;
 
 // Function pointers to a simulated program's code
@@ -82,6 +84,9 @@ void *quakey_node_state(QuakeyNode node);
 // Schedule and executes one program until it would block, then returns
 int quakey_schedule_one(Quakey *quakey);
 
+// Get the current simulated time in nanoseconds
+QuakeyUInt64 quakey_current_time(Quakey *quakey);
+
 // Generate a random u64
 QuakeyUInt64 quakey_random(void);
 
@@ -93,6 +98,12 @@ typedef struct {
 void quakey_signal(char *name);
 int  quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
 
+// Limit the number of hosts that can be crashed at the same time.
+// Set to 0 for no limit (default).
+void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
+
+void quakey_network_partitioning(Quakey *quakey, bool enable);
+
 // Access spawned host information
 int         quakey_num_hosts(Quakey *quakey);
 void       *quakey_host_state(Quakey *quakey, int idx);  // Returns NULL if host is dead
@@ -135,6 +146,7 @@ int   mock_clock_gettime(clockid_t clockid, struct timespec *tp);
 int   mock_open(char *path, int flags, int mode);
 int   mock_fcntl(int fd, int cmd, int flags);
 int   mock_close(int fd);
+int   mock_access(const char *path, int mode);
 int   mock_ftruncate(int fd, size_t new_size);
 int   mock_fstat(int fd, struct stat *buf);
 int   mock_read(int fd, char *dst, int len);
@@ -183,6 +195,7 @@ void  mock_free(void *ptr);
 #define open             mock_open
 #define fcntl            mock_fcntl
 #define close            mock_close
+#define access           mock_access
 #define ftruncate        mock_ftruncate
 #define CreateFileW      mock_CreateFileW
 #define CloseHandle      mock_CloseHandle

quakey/src/quakey.c

@@ -11,6 +11,10 @@
 
 #define TODO __builtin_trap()
 
+#define SIM_TRACE(sim, fmt, ...) \
+    fprintf(stderr, "SIM: [%.3fs] " fmt "\n", \
+        (double)(sim)->current_time / 1e9, ##__VA_ARGS__)
+
 #ifdef NDEBUG
 #define UNREACHABLE {}
 #define ASSERT(X) {}
@@ -317,6 +321,7 @@ typedef enum {
     EVENT_TYPE_DATA,
     EVENT_TYPE_WAKEUP,
     EVENT_TYPE_RESTART,
+    EVENT_TYPE_PARTITION,
 } TimeEventType;
 
 typedef struct {
@@ -355,6 +360,17 @@ typedef struct {
 
 #define SIM_SIGNAL_LIMIT 8
 
+typedef struct {
+    Host *a;
+    Host *b;
+} HostPair;
+
+typedef struct {
+    int count;
+    int capacity;
+    HostPair *pairs;
+} HostPairList;
+
 struct Sim {
 
     uint64_t seed;
@@ -378,15 +394,30 @@ struct Sim {
     int num_signals;
     int head_signal;
     QuakeySignal signals[SIM_SIGNAL_LIMIT];
+
+    // Network partition simulation. The partition list holds pairs
+    // of hosts whose link is currently broken. The target_partition
+    // holds the desired end state. A periodic timer event gradually
+    // breaks or repairs links to converge toward the target. Once
+    // reached, a new random target is generated.
+    HostPairList partition;
+    HostPairList target_partition;
+
+    // Maximum number of hosts that can be dead at the same time.
+    int max_crashes;
+
+    bool network_partitioning;
 };
 
 static void time_event_wakeup(Sim *sim, Nanos time, Host *host);
 static void time_event_connect(Sim *sim, Nanos time, Desc *desc);
 static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst);
-static void time_event_send_data(Sim *sim, Nanos time, Desc *desc);
+static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count);
 static void time_event_free(TimeEvent *event);
 static void time_event_restart(Sim *sim, Nanos time, Host *host);
+static void time_event_partition(Sim *sim, Nanos time);
 static void remove_events_targeting_desc(Sim *sim, Desc *desc);
+static b32  remove_wakeup_event(Sim *sim, Host *host);
 
 /////////////////////////////////////////////////////////////////
 /////////////////////////////////////////////////////////////////
@@ -714,11 +745,6 @@ static b32 socket_queue_empty(SocketQueue *queue)
     return queue->used == 0;
 }
 
-static b32 socket_queue_used(SocketQueue *queue)
-{
-    return queue->used;
-}
-
 /////////////////////////////////////////////////////////////////
 // Accept Queue Code
 
@@ -1003,9 +1029,10 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg)
         TODO;
     }
 
-    if (host->poll_timeout > 0)
+    if (host->poll_timeout > 0) {
+        remove_wakeup_event(host->sim, host);
         time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
-    else if (host->poll_timeout == 0)
+    } else if (host->poll_timeout == 0)
         host->timedout = true;  // Immediate timeout, don't create event at current_time
 }
 
@@ -1035,8 +1062,11 @@ static void host_free(Host *host)
 }
 
 // Remove all events associated with a host (WAKEUP events and events
-// whose source or destination descriptors belong to this host)
+// whose source or destination descriptors belong to this host).
-static void remove_events_for_host(Sim *sim, Host *host)
+// If skip_restart is true, RESTART events are kept (used during
+// restart failure to avoid removing the event currently being
+// processed by process_events_at_current_time).
+static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
 {
     int i = 0;
     while (i < sim->num_events) {
@@ -1045,7 +1075,7 @@ static void remove_events_for_host(Sim *sim, Host *host)
 
         if (event->type == EVENT_TYPE_WAKEUP && event->host == host)
             remove = true;
-        if (event->type == EVENT_TYPE_RESTART && event->host == host)
+        if (!skip_restart && event->type == EVENT_TYPE_RESTART && event->host == host)
             remove = true;
         if (event->src_desc && event->src_desc->host == host)
             remove = true;
@@ -1061,6 +1091,11 @@ static void remove_events_for_host(Sim *sim, Host *host)
     }
 }
 
+static void remove_events_for_host(Sim *sim, Host *host)
+{
+    remove_events_for_host_ex(sim, host, false);
+}
+
 static int count_dead(Sim *sim)
 {
     int n = 0;
@@ -1088,6 +1123,11 @@ static void host_crash(Host *host)
     // Remove all pending events for this host
     remove_events_for_host(sim, host);
 
+    // Let the application clean up (e.g. log the crash)
+    host___ = host;
+    host->free_func(host->state);
+    host___ = NULL;
+
     // Free application state
     free(host->state);
     host->state = NULL;
@@ -1098,7 +1138,6 @@ static void host_crash(Host *host)
     host->poll_count = 0;
     host->poll_timeout = -1;
 
-    printf("Quakey: Host crash (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
 }
 
 // Restart a crashed host: re-initialize from saved config,
@@ -1163,17 +1202,37 @@ static void host_restart(Host *host)
     );
     host___ = NULL;
     if (ret < 0) {
-        // If the node fails to restart, mark it as dead again
+        // Clean up any descriptors created during the failed init
+        for (int i = 0; i < HOST_DESC_LIMIT; i++) {
+            if (host->desc[i].type != DESC_EMPTY)
+                desc_free(sim, &host->desc[i], true);
+        }
+        host->num_desc = 0;
+        // Skip removing RESTART events: this function is called from
+        // time_event_process (EVENT_TYPE_RESTART), so the current
+        // restart event is still in the array being iterated by
+        // process_events_at_current_time. Removing it here would cause
+        // the outer loop's swap-removal to discard an unrelated event.
+        remove_events_for_host_ex(sim, host, true);
+        free(host->state);
+        host->state = NULL;
         host->dead = true;
+
+        // Schedule another restart attempt. Init can fail due to
+        // transient I/O errors from fault injection; permanently
+        // killing the host would be unrealistic since a real
+        // process supervisor would keep retrying.
+        Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL);
+        time_event_restart(sim, sim->current_time + restart_delay, host);
         return;
     }
 
-    if (host->poll_timeout > 0)
+    if (host->poll_timeout > 0) {
+        remove_wakeup_event(sim, host);
         time_event_wakeup(sim, sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
-    else if (host->poll_timeout == 0)
+    } else if (host->poll_timeout == 0)
         host->timedout = true;
 
-    printf("Quakey: Host restart (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
 }
 
 static b32 host_is_linux(Host *host)
@@ -1343,12 +1402,16 @@ static void host_update(Host *host)
     );
     host___ = NULL;
     if (ret < 0) {
-        TODO;
+        host_crash(host);
+        Nanos restart_delay = 1000000000ULL + (sim_random(host->sim) % 9000000000ULL);
+        time_event_restart(host->sim, host->sim->current_time + restart_delay, host);
+        return;
     }
 
-    if (host->poll_timeout > 0)
+    if (host->poll_timeout > 0) {
+        remove_wakeup_event(host->sim, host);
         time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
-    else if (host->poll_timeout == 0)
+    } else if (host->poll_timeout == 0)
         host->timedout = true;  // Immediate timeout, don't create event at current_time
 }
 
@@ -1445,9 +1508,15 @@ static int host_create_pipe(Host *host, int *desc_idxs)
         return HOST_ERROR_FULL;
     Desc *desc_1 = &host->desc[desc_idx_1];
 
+    // Mark the first slot as in-use before searching for the second,
+    // otherwise find_empty_desc_struct returns the same index twice.
+    desc_1->type = DESC_PIPE;
+
     int desc_idx_2 = find_empty_desc_struct(host);
-    if (desc_idx_2 < 0)
+    if (desc_idx_2 < 0) {
+        desc_1->type = DESC_EMPTY;
         return HOST_ERROR_FULL;
+    }
     Desc *desc_2 = &host->desc[desc_idx_2];
 
     desc_1->type = DESC_PIPE;
@@ -1894,7 +1963,7 @@ static int send_inner(Desc *desc, char *src, int len)
     uint64_t rng = sim_random(sim);
     latency = 1000000 + (rng % 99000000); // between 1ms and 100ms
 #endif
-    time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc);
+    time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc, ret);
 
     return ret;
 }
@@ -1916,7 +1985,7 @@ static int host_read(Host *host, int desc_idx, char *dst, int len)
         desc->type == DESC_PIPE) {
         num = recv_inner(desc, dst, len);
     } else if (desc->type == DESC_FILE) {
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
         uint64_t roll = sim_random(host->sim) % 1000;
         if (roll == 0) return HOST_ERROR_IO;
 #endif
@@ -1960,7 +2029,7 @@ static int host_write(Host *host, int desc_idx, char *src, int len)
         desc->type == DESC_PIPE) {
         num = send_inner(desc, src, len);
     } else if (desc->type == DESC_FILE) {
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
         uint64_t roll = sim_random(host->sim) % 1000;
         if (roll == 0) return HOST_ERROR_IO;
         if (roll == 1) return HOST_ERROR_NOSPC;
@@ -2000,9 +2069,11 @@ static int host_recv(Host *host, int desc_idx, char *dst, int len)
     Desc *desc = &host->desc[desc_idx];
 
 #ifdef FAULT_INJECTION
-    Sim *sim = desc->host->sim;
+    if (len > 0) {
-    uint64_t rng = sim_random(sim);
+        Sim *sim = desc->host->sim;
-    len = 1 + (rng % len);
+        uint64_t rng = sim_random(sim);
+        len = 1 + (rng % len);
+    }
 #endif
 
     int num = 0;
@@ -2024,9 +2095,11 @@ static int host_send(Host *host, int desc_idx, char *src, int len)
     Desc *desc = &host->desc[desc_idx];
 
 #ifdef FAULT_INJECTION
-    Sim *sim = desc->host->sim;
+    if (len > 0) {
-    uint64_t rng = sim_random(sim);
+        Sim *sim = desc->host->sim;
-    len = 1 + (rng % len);
+        uint64_t rng = sim_random(sim);
+        len = 1 + (rng % len);
+    }
 #endif
 
     int num = 0;
@@ -2132,7 +2205,7 @@ static int host_fsync(Host *host, int desc_idx)
     if (desc->type != DESC_FILE)
         return HOST_ERROR_BADIDX;
 
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
     uint64_t roll = sim_random(host->sim) % 100;
     if (roll == 0)
         return HOST_ERROR_IO;
@@ -2251,6 +2324,19 @@ static b32 remove_connect_event(Sim *sim, Desc *desc)
     return true;
 }
 
+static b32 remove_wakeup_event(Sim *sim, Host *host)
+{
+    int i = 0;
+    while (i < sim->num_events && (sim->events[i].type != EVENT_TYPE_WAKEUP || sim->events[i].host != host))
+        i++;
+
+    if (i == sim->num_events)
+        return false;
+
+    sim->events[i] = sim->events[--sim->num_events];
+    return true;
+}
+
 // Remove all events that target a specific descriptor (DISCONNECT and DATA events)
 static void remove_events_targeting_desc(Sim *sim, Desc *desc)
 {
@@ -2292,7 +2378,7 @@ static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst)
     append_event(sim, event);
 }
 
-static void time_event_send_data(Sim *sim, Nanos time, Desc *desc)
+static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count)
 {
     TimeEvent event = {
         .type = EVENT_TYPE_DATA,
@@ -2300,7 +2386,7 @@ static void time_event_send_data(Sim *sim, Nanos time, Desc *desc)
         .host = NULL,
         .src_desc = NULL,
         .dst_desc = desc->peer,
-        .data_count = socket_queue_used(desc->output),
+        .data_count = count,
         .data_queue = socket_queue_ref(desc->output),
         .rst = false,
     };
@@ -2328,12 +2414,52 @@ static void time_event_restart(Sim *sim, Nanos time, Host *host)
     append_event(sim, event);
 }
 
-static int sim_find_host(Sim *sim, Addr addr);
+static void time_event_partition(Sim *sim, Nanos time)
+{
+    TimeEvent event = {
+        .type = EVENT_TYPE_PARTITION,
+        .time = time,
+    };
+    append_event(sim, event);
+}
+
+static int  sim_find_host(Sim *sim, Addr addr);
+static int  change_target_partition(Sim *sim);
+static bool break_or_repair_link(Sim *sim);
+static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b);
+
+static Nanos pick_partition_delay(Sim *sim, bool longer)
+{
+    Nanos min_delay = 1000000000; // 1s
+    Nanos max_delay = 10000000000; // 10s
+    if (longer) {
+        min_delay = 10000000000;
+        max_delay = 20000000000;
+    }
+    return min_delay + sim_random(sim) % (max_delay - min_delay);
+}
 
 static b32 time_event_process(TimeEvent *event, Sim *sim)
 {
     b32 consumed = true;
     switch (event->type) {
+    case EVENT_TYPE_PARTITION:
+        {
+            // Try to move one step toward the target partition.
+            // If current already matches target, pick a new target.
+            if (!break_or_repair_link(sim)) {
+                if (sim->network_partitioning) {
+                    if (change_target_partition(sim) < 0) {
+                        TODO;
+                    }
+                }
+            }
+
+            // Reschedule for the next partition step
+            event->time = sim->current_time + pick_partition_delay(sim, true);
+            consumed = false;
+        }
+        break;
     case EVENT_TYPE_CONNECT:
         {
             Desc *src_desc = event->src_desc;
@@ -2346,6 +2472,13 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
             }
             Host *peer_host = sim->hosts[idx];
 
+            if (hosts_are_partitioned(sim, src_desc->host, peer_host)) {
+                //SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
+                //    src_desc->host->name, peer_host->name);
+                src_desc->connect_status = CONNECT_STATUS_NOHOST;
+                break;
+            }
+
             Desc *peer = host_find_desc_bound_to(peer_host,
                 src_desc->connect_addr, src_desc->connect_port);
             if (peer == NULL) {
@@ -2410,6 +2543,242 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
 /////////////////////////////////////////////////////////////////
 // Sim Code
 
+static void
+host_pair_list_init(HostPairList *list)
+{
+    list->count = 0;
+    list->capacity = 0;
+    list->pairs = NULL;
+}
+
+static void
+host_pair_list_free(HostPairList *list)
+{
+    free(list->pairs);
+}
+
+static bool
+host_pair_list_exists(HostPairList *list, Host *a, Host *b)
+{
+    for (int i = 0; i < list->count; i++)
+        if ((list->pairs[i].a == a && list->pairs[i].b == b) ||
+            (list->pairs[i].a == b && list->pairs[i].b == a))
+            return true;
+    return false;
+}
+
+static int
+host_pair_list_add(HostPairList *list, Host *a, Host *b)
+{
+    if (!host_pair_list_exists(list, a, b)) {
+        if (list->count == list->capacity) {
+            int n = 2 * list->count;
+            if (n < 8) n = 8;
+            void *p = realloc(list->pairs, n * sizeof(HostPair));
+            if (p == NULL)
+                return -1;
+            list->capacity = n;
+            list->pairs = p;
+        }
+        list->pairs[list->count++] = (HostPair) { a, b };
+    }
+    return 0;
+}
+
+static void
+host_pair_list_remove_at(HostPairList *list, int idx)
+{
+    list->pairs[idx] = list->pairs[--list->count];
+}
+
+// Generate a new random target partition by assigning hosts to
+// random buckets. Hosts in the same bucket can communicate; hosts
+// in different buckets have their link marked as broken.
+static int change_target_partition(Sim *sim)
+{
+    HostPairList list;
+    host_pair_list_init(&list);
+
+    #define MAX_BUCKETS 3
+    #define MAX_NODES_PER_BUCKET 8
+
+    int   num_buckets = 1 + sim_random(sim) % MAX_BUCKETS;
+
+    Host *buckets[MAX_BUCKETS][MAX_NODES_PER_BUCKET];
+    int   bucket_sizes[MAX_BUCKETS];
+
+    for (int i = 0; i < num_buckets; i++)
+        bucket_sizes[i] = 0;
+
+    // Randomly assign each host to a bucket
+    for (int i = 0; i < sim->num_hosts; i++) {
+        int bucket_index = sim_random(sim) % num_buckets;
+        buckets[bucket_index][bucket_sizes[bucket_index]++] = sim->hosts[i];
+    }
+
+    // Every cross-bucket pair is a broken link in the target
+    for (int i = 0; i < num_buckets; i++) {
+        for (int j = 0; j < bucket_sizes[i]; j++) {
+            for (int k = 0; k < num_buckets; k++) {
+                if (k == i) continue;
+                for (int g = 0; g < bucket_sizes[k]; g++) {
+                    Host *a = buckets[i][j];
+                    Host *b = buckets[k][g];
+                    if (host_pair_list_add(&list, a, b) < 0) {
+                        host_pair_list_free(&list);
+                        return -1;
+                    }
+                }
+            }
+        }
+    }
+
+    host_pair_list_free(&sim->target_partition);
+    sim->target_partition = list;
+    //SIM_TRACE(sim, "PARTITION: new target partition with %d broken links (%d buckets)",
+    //    list.count, num_buckets);
+    return 0;
+}
+
+// Pick a random pair that exists in 'left' but not in 'right'.
+// Returns the index into 'left', or -1 if no such pair exists.
+static int pick_random_from_left_not_in_right(Sim *sim,
+    HostPairList *left, HostPairList *right)
+{
+    int *arr = malloc(left->count * sizeof(int));
+    if (arr == NULL) {
+        TODO; // Allowing this to return error would make execution not deterministic
+    }
+    int num = 0;
+
+    for (int i = 0; i < left->count; i++) {
+        HostPair pair = left->pairs[i];
+        if (!host_pair_list_exists(right, pair.a, pair.b)) {
+            arr[num++] = i;
+        }
+    }
+
+    int idx;
+    if (num == 0) {
+        idx = -1;
+    } else {
+        idx = arr[sim_random(sim) % num];
+    }
+
+    free(arr);
+    return idx;
+}
+
+// Forcefully reset a connection socket and notify its peer.
+// Removes any pending time events that reference either end.
+static void reset_conn(Sim *sim, Desc *desc)
+{
+    assert(desc->type == DESC_SOCKET_C);
+
+    if (desc->connect_status == CONNECT_STATUS_WAIT ||
+        desc->connect_status == CONNECT_STATUS_DONE) {
+
+        remove_events_targeting_desc(sim, desc);
+
+        if (desc->peer) {
+            assert(desc->peer->type == DESC_SOCKET_C
+                || desc->peer->type == DESC_SOCKET_L);
+
+            if (desc->peer->type == DESC_SOCKET_C) {
+                remove_events_targeting_desc(sim, desc->peer);
+                desc->peer->connect_status = CONNECT_STATUS_RESET;
+                desc->peer->peer = NULL;
+            } else {
+                accept_queue_remove(&desc->peer->accept_queue, desc);
+            }
+            desc->peer = NULL;
+        }
+
+        desc->connect_status = CONNECT_STATUS_RESET;
+    }
+}
+
+// Reset all active connections between two hosts (both directions)
+static void drop_conns_between_hosts(Sim *sim, Host *host, Host *peer)
+{
+    for (int i = 0, j = 0; j < host->num_desc; i++) {
+
+        Desc *desc = &host->desc[i];
+        if (desc->type == DESC_EMPTY)
+            continue;
+        j++;
+
+        if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == peer)
+            reset_conn(sim, desc);
+    }
+
+    for (int i = 0, j = 0; j < peer->num_desc; i++) {
+
+        Desc *desc = &peer->desc[i];
+        if (desc->type == DESC_EMPTY)
+            continue;
+        j++;
+
+        if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == host)
+            reset_conn(sim, desc);
+    }
+}
+
+// Break a link that is in the target but not yet in current
+static bool break_link(Sim *sim)
+{
+    int idx = pick_random_from_left_not_in_right(sim,
+        &sim->target_partition, &sim->partition);
+    if (idx < 0)
+        return false;
+    HostPair pair = sim->target_partition.pairs[idx];
+
+    host_pair_list_add(&sim->partition, pair.a, pair.b);
+    drop_conns_between_hosts(sim, pair.a, pair.b);
+    //SIM_TRACE(sim, "PARTITION: break link %s <-> %s (%d broken links)",
+    //    pair.a->name, pair.b->name, sim->partition.count);
+    return true;
+}
+
+// Repair a link that is broken in current but not in the target
+static bool repair_link(Sim *sim)
+{
+    int idx = pick_random_from_left_not_in_right(sim,
+        &sim->partition, &sim->target_partition);
+    if (idx < 0)
+        return false;
+
+    //HostPair pair = sim->partition.pairs[idx];
+    host_pair_list_remove_at(&sim->partition, idx);
+    //SIM_TRACE(sim, "PARTITION: repair link %s <-> %s (%d broken links)",
+    //    pair.a->name, pair.b->name, sim->partition.count);
+    return true;
+}
+
+// Try to move one step toward the target: randomly attempt a
+// break or repair first, falling back to the other on failure.
+// Returns false when current already matches the target.
+static bool break_or_repair_link(Sim *sim)
+{
+    if (sim_random(sim) & 1) {
+        if (break_link(sim))
+            return true;
+        if (repair_link(sim))
+            return true;
+    } else {
+        if (repair_link(sim))
+            return true;
+        if (break_link(sim))
+            return true;
+    }
+    return false;
+}
+
+static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b)
+{
+    return host_pair_list_exists(&sim->partition, a, b);
+}
+
 static void sim_init(Sim *sim, uint64_t seed)
 {
     sim->seed = seed;
@@ -2422,10 +2791,27 @@ static void sim_init(Sim *sim, uint64_t seed)
     sim->events = NULL;
     sim->num_signals = 0;
     sim->head_signal = 0;
+    sim->max_crashes = 0;
+    sim->network_partitioning = true;
+    host_pair_list_init(&sim->partition);
+    host_pair_list_init(&sim->target_partition);
+    time_event_partition(sim, pick_partition_delay(sim, false));
+}
+
+static void sim_network_partitioning(Sim *sim, bool enable)
+{
+    sim->network_partitioning = enable;
+    if (!enable) {
+        host_pair_list_free(&sim->target_partition);
+        host_pair_list_init(&sim->target_partition);
+    }
 }
 
 static void sim_free(Sim *sim)
 {
+    host_pair_list_free(&sim->target_partition);
+    host_pair_list_free(&sim->partition);
+
     for (int i = 0; i < sim->num_hosts; i++)
         host_free(sim->hosts[i]);
     free(sim->hosts);
@@ -2503,6 +2889,9 @@ static void process_events_at_current_time(Sim *sim)
                 }
         }
     }
+
+    //if (sim->num_events > 10000)
+    //    SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
 }
 
 static int find_first_ready_host(Sim *sim)
@@ -2565,14 +2954,14 @@ static b32 sim_update(Sim *sim)
     if (sim->num_hosts > 1) {
         uint64_t rng = sim_random(sim);
         if ((rng % 10000) == 0) {
-            // Pick a random live host to crash
+
             int live_count = 0;
             for (int i = 0; i < sim->num_hosts; i++)
                 if (!sim->hosts[i]->dead)
                     live_count++;
+            int dead_count = sim->num_hosts - live_count;
 
-            // Only crash if at least 2 hosts are alive (keep a majority)
+            if (dead_count < sim->max_crashes) {
-            if (live_count >= 2) {
                 // Pick one of the live hosts at random
                 int target = sim_random(sim) % live_count;
                 int j = 0;
@@ -2664,6 +3053,18 @@ void quakey_free(Quakey *quakey)
     }
 }
 
+void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
+{
+    Sim *sim = (Sim*) quakey;
+    sim->max_crashes = max_crashes;
+}
+
+void quakey_network_partitioning(Quakey *quakey, bool enable)
+{
+    Sim *sim = (Sim*) quakey;
+    sim_network_partitioning(sim, enable);
+}
+
 QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg)
 {
     return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg);
@@ -2680,6 +3081,12 @@ int quakey_schedule_one(Quakey *quakey)
     return sim_update((Sim*) quakey);
 }
 
+QuakeyUInt64 quakey_current_time(Quakey *quakey)
+{
+    Sim *sim = (Sim*) quakey;
+    return sim->current_time;
+}
+
 QuakeyUInt64 quakey_random(void)
 {
     Host *host = host___;
@@ -2823,6 +3230,11 @@ int mock_close(int fd)
     return 0;
 }
 
+int mock_access(const char *path, int mode)
+{
+    assert(0); // TODO
+}
+
 static int convert_addr(void *addr, size_t addr_len,
     Addr *converted_addr, uint16_t *converted_port)
 {

raft/client.c

@@ -9,13 +9,64 @@
 #include "node.h"
 #include "client.h"
 
-#define CLIENT_TRACE(fmt, ...) {}
+//#define CLIENT_TRACE(fmt, ...) {}
-//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
+#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
 
 #define CLIENT_REQUEST_TIMEOUT_SEC 3
+#define KEY_POOL_SIZE 128
 
 static uint64_t next_client_id = 1;
 
+static uint64_t client_random(void)
+{
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+    return quakey_random();
+#else
+    return (uint64_t)rand();
+#endif
+}
+
+static KVStoreOper random_oper(void)
+{
+    KVStoreOper oper = {0};
+    snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
+
+    switch (client_random() % 3) {
+    case 0:
+        oper.type = KVSTORE_OPER_SET;
+        oper.val  = client_random();
+        break;
+    case 1:
+        oper.type = KVSTORE_OPER_GET;
+        break;
+    case 2:
+        oper.type = KVSTORE_OPER_DEL;
+        break;
+    }
+    return oper;
+}
+
+static const char *oper_type_name(KVStoreOperType t)
+{
+    switch (t) {
+    case KVSTORE_OPER_NOOP: return "NOOP";
+    case KVSTORE_OPER_SET:  return "SET";
+    case KVSTORE_OPER_GET:  return "GET";
+    case KVSTORE_OPER_DEL:  return "DEL";
+    }
+    return "???";
+}
+
+static const char *result_type_name(KVStoreResultType t)
+{
+    switch (t) {
+    case KVSTORE_RESULT_OK:      return "OK";
+    case KVSTORE_RESULT_FULL:    return "FULL";
+    case KVSTORE_RESULT_MISSING: return "MISSING";
+    }
+    return "???";
+}
+
 static int
 process_message(ClientState *state,
     int conn_idx, uint8_t type, ByteView msg)
@@ -31,7 +82,12 @@ process_message(ClientState *state,
         if (redirect_message.leader_idx >= 0 && redirect_message.leader_idx < state->num_servers) {
             CLIENT_TRACE("Redirected to leader %d", redirect_message.leader_idx);
             state->current_leader = redirect_message.leader_idx;
-            // Retry immediately with the correct leader
+            // Retry immediately with the correct leader.
+            // A redirect means the server did not process the request,
+            // so mark as rejected (not timeout) for the linearizability
+            // checker: the outcome is unambiguous (no effect).
+            state->last_was_rejected = true;
+            state->last_was_timeout = false;
             state->pending = false;
         }
         return 0;
@@ -48,8 +104,24 @@ process_message(ClientState *state,
         return -1;
     memcpy(&reply_message, msg.ptr, sizeof(reply_message));
 
-    CLIENT_TRACE("Received reply for request %lu", (unsigned long)state->request_id);
+    // Ignore stale replies from previous requests. After a timeout
+    // the client moves to a new leader and sends a new request, but
+    // the old leader may still deliver a reply for the old request
+    // on the previous connection. Without this check the client
+    // would accept the stale result for the wrong operation.
+    if (reply_message.request_id != state->request_id)
+        return 0;
 
+    CLIENT_TRACE("REPLY: %s key=\"%.16s\" -> %s val=%lu (req_id=%lu)",
+        oper_type_name(state->last_oper.type),
+        state->last_oper.key,
+        result_type_name(reply_message.result.type),
+        (unsigned long)reply_message.result.val,
+        (unsigned long)state->request_id);
+
+    state->last_result = reply_message.result;
+    state->last_was_timeout = false;
+    state->last_was_rejected = false;
     state->pending = false;
     return 0;
 }
@@ -167,6 +239,8 @@ int client_tick(void *state_, void **ctxs,
         if (now >= request_deadline) {
             CLIENT_TRACE("Request %lu timed out, trying next server",
                 (unsigned long)state->request_id);
+            state->last_was_timeout = true;
+            state->last_was_rejected = false;
             state->pending = false;
             state->current_leader = (state->current_leader + 1) % state->num_servers;
         }
@@ -181,6 +255,7 @@ int client_tick(void *state_, void **ctxs,
             tcp_connect(&state->tcp, state->server_addrs[leader], leader, NULL);
         } else {
             state->request_id++;
+            state->last_oper = random_oper();
 
             RequestMessage request_message = {
                 .base = {
@@ -188,11 +263,18 @@ int client_tick(void *state_, void **ctxs,
                     .type    = MESSAGE_TYPE_REQUEST,
                     .length  = sizeof(RequestMessage),
                 },
-                .oper = OPERATION_A,
+                .oper = state->last_oper,
                 .client_id = state->client_id,
                 .request_id = state->request_id,
             };
 
+            CLIENT_TRACE("REQUEST: %s key=\"%.16s\" val=%lu (req_id=%lu, leader=%d)",
+                oper_type_name(state->last_oper.type),
+                state->last_oper.key,
+                (unsigned long)state->last_oper.val,
+                (unsigned long)state->request_id,
+                leader);
+
             ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
             if (output)
                 byte_queue_write(output, &request_message, sizeof(request_message));

raft/client.h

@@ -4,6 +4,8 @@
 #include <lib/tcp.h>
 #include <lib/basic.h>
 
+#include <state_machine/kvstore.h>
+
 #include "config.h"
 
 typedef struct {
@@ -13,6 +15,14 @@ typedef struct {
     // True if we are waiting for a response
     bool pending;
 
+    // The operation sent in the current pending request (for logging)
+    KVStoreOper last_oper;
+
+    // Linearizability checker support
+    KVStoreResult last_result;
+    bool          last_was_timeout;
+    bool          last_was_rejected;
+
     Address server_addrs[NODE_LIMIT];
     int num_servers;
 

raft/client_table.c

@@ -0,0 +1,45 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+
+#include "client_table.h"
+
+void client_table_init(ClientTable *ct)
+{
+    ct->count = 0;
+    ct->capacity = 0;
+    ct->entries = NULL;
+}
+
+void client_table_free(ClientTable *ct)
+{
+    free(ct->entries);
+}
+
+ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id)
+{
+    for (int i = 0; i < ct->count; i++)
+        if (ct->entries[i].client_id == client_id)
+            return &ct->entries[i];
+    return NULL;
+}
+
+int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag)
+{
+    if (ct->count == ct->capacity) {
+        int n = ct->capacity ? 2 * ct->capacity : 8;
+        void *p = realloc(ct->entries, n * sizeof(ClientTableEntry));
+        if (p == NULL) return -1;
+        ct->capacity = n;
+        ct->entries = p;
+    }
+    ct->entries[ct->count++] = (ClientTableEntry) {
+        .client_id = client_id,
+        .last_request_id = request_id,
+        .pending = true,
+        .conn_tag = conn_tag,
+    };
+    return 0;
+}

raft/client_table.h

@@ -0,0 +1,28 @@
+#ifndef CLIENT_TABLE_INCLUDED
+#define CLIENT_TABLE_INCLUDED
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <state_machine/kvstore.h>
+
+typedef struct {
+    uint64_t        client_id;
+    uint64_t        last_request_id;
+    KVStoreResult   last_result;
+    bool            pending;
+    int             conn_tag;
+} ClientTableEntry;
+
+typedef struct {
+    int count;
+    int capacity;
+    ClientTableEntry *entries;
+} ClientTable;
+
+void client_table_init(ClientTable *ct);
+void client_table_free(ClientTable *ct);
+ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id);
+int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag);
+
+#endif // CLIENT_TABLE_INCLUDED

raft/invariant_checker.c

@@ -0,0 +1,351 @@
+// Raft invariant checker with external shadow log tracking.
+//
+// This file runs in the main simulation loop outside Quakey-scheduled
+// processes. It includes node.h for struct definitions, then restores
+// real allocators since mock_malloc/realloc/free abort outside process
+// context.
+
+#include "invariant_checker.h"
+
+#include <assert.h>
+#include <string.h>
+
+// Restore real allocators (see checker/linearizability.c for precedent).
+// The header chain (invariant_checker.h -> node.h -> wal.h ->
+// lib/file_system.h) defines QUAKEY_ENABLE_MOCKS and replaces
+// malloc/realloc/free with mock versions. We need the real ones
+// because this code runs outside any Quakey-scheduled process.
+#undef malloc
+#undef realloc
+#undef free
+#include <stdio.h>
+#include <stdlib.h>
+
+static int self_idx(NodeState *state)
+{
+    for (int i = 0; i < state->num_nodes; i++)
+        if (addr_eql(state->node_addrs[i], state->self_addr))
+            return i;
+    UNREACHABLE;
+}
+
+static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
+{
+    if (ic->shadow_count == ic->shadow_capacity) {
+        int n = 2 * ic->shadow_capacity;
+        if (n < 8)
+            n = 8;
+        KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
+        if (p == NULL)
+            return -1;
+        ic->shadow_log = p;
+        ic->shadow_capacity = n;
+    }
+    ic->shadow_log[ic->shadow_count++] = oper;
+    return 0;
+}
+
+void invariant_checker_init(InvariantChecker *ic)
+{
+    ic->shadow_log = NULL;
+    ic->shadow_count = 0;
+    ic->shadow_capacity = 0;
+}
+
+void invariant_checker_free(InvariantChecker *ic)
+{
+    fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
+        ic->shadow_count);
+    free(ic->shadow_log);
+}
+
+void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
+{
+    for (int i = 0; i < num_nodes; i++) {
+        NodeState *s = nodes[i];
+        if (s == NULL)
+            continue;
+
+        int log_count = wal_entry_count(&s->wal);
+
+        // The commit index starts at -1 (nothing committed) and only
+        // increases. It must never go below -1.
+        if (s->commit_index < -1) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < -1\n",
+                i, s->commit_index);
+            __builtin_trap();
+        }
+
+        // A node cannot have committed an entry beyond what exists
+        // in its log. With an empty log (count=0), commit_index
+        // must be -1.
+        if (s->commit_index >= log_count) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) >= log_count (%d)\n",
+                i, s->commit_index, log_count);
+            __builtin_trap();
+        }
+
+        // Entries are applied to the state machine in order, up to
+        // the commit index. The last applied index must never exceed
+        // the commit index.
+        if (s->last_applied > s->commit_index) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) > commit_index (%d)\n",
+                i, s->last_applied, s->commit_index);
+            __builtin_trap();
+        }
+
+        if (s->last_applied < -1) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) < -1\n",
+                i, s->last_applied);
+            __builtin_trap();
+        }
+
+        // voted_for is either -1 (no vote) or a valid node index.
+        if (s->term_and_vote.voted_for != -1 && (s->term_and_vote.voted_for < 0 || s->term_and_vote.voted_for >= s->num_nodes)) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: voted_for (%d) is not -1 "
+                "and not in [0, %d)\n",
+                i, s->term_and_vote.voted_for, s->num_nodes);
+            __builtin_trap();
+        }
+
+        // Leaders append entries with their current term, and terms
+        // only increase. Truncation preserves this property because
+        // replacement entries come from a leader whose log already
+        // satisfies non-decreasing terms.
+        for (int k = 1; k < log_count; k++) {
+            if (wal_peek_entry(&s->wal, k)->term < wal_peek_entry(&s->wal, k-1)->term) {
+                fprintf(stderr, "INVARIANT VIOLATED: node %d: wal[%d].term (%lu) "
+                    "< wal[%d].term (%lu) (non-monotonic)\n",
+                    i, k, (unsigned long)wal_peek_entry(&s->wal, k)->term,
+                    k-1, (unsigned long)wal_peek_entry(&s->wal, k-1)->term);
+                __builtin_trap();
+            }
+        }
+
+        // A leader must have voted for itself in the current term.
+        // A node becomes leader by winning an election, which requires
+        // voting for itself. The voted_for value is only reset when
+        // stepping down (which changes the role to FOLLOWER).
+        if (s->role == ROLE_LEADER) {
+            if (s->term_and_vote.voted_for != self_idx(s)) {
+                fprintf(stderr, "INVARIANT VIOLATED: node %d: role is LEADER but "
+                    "voted_for (%d) != self_idx (%d)\n",
+                    i, s->term_and_vote.voted_for, self_idx(s));
+                __builtin_trap();
+            }
+        }
+
+        // A candidate must have voted for itself in the current term.
+        // A node enters candidate state only through start_election(),
+        // which increments the term and sets voted_for to self_idx.
+        if (s->role == ROLE_CANDIDATE) {
+            if (s->term_and_vote.voted_for != self_idx(s)) {
+                fprintf(stderr, "INVARIANT VIOLATED: node %d: role is CANDIDATE but "
+                    "voted_for (%d) != self_idx (%d)\n",
+                    i, s->term_and_vote.voted_for, self_idx(s));
+                __builtin_trap();
+            }
+        }
+
+        // For a leader, match_indices[self] must equal the last log
+        // index. The leader always has its own entries matched.
+        if (s->role == ROLE_LEADER) {
+            int expected_match = log_count - 1;
+            if (s->match_indices[self_idx(s)] != expected_match) {
+                fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
+                    "match_indices[self] (%d) != last log index (%d)\n",
+                    i, s->match_indices[self_idx(s)], expected_match);
+                __builtin_trap();
+            }
+        }
+
+        // For a leader, next_indices[k] must be >= match_indices[k] + 1
+        // for all followers (k != self). The next index to send is
+        // always at least one past the highest known replicated index.
+        // The leader's own next_indices[self] is not maintained since
+        // the leader never sends entries to itself.
+        if (s->role == ROLE_LEADER) {
+            int si = self_idx(s);
+            for (int k = 0; k < s->num_nodes; k++) {
+                if (k == si)
+                    continue;
+                if (s->next_indices[k] < s->match_indices[k] + 1) {
+                    fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
+                        "next_indices[%d] (%d) < match_indices[%d] + 1 (%d)\n",
+                        i, k, s->next_indices[k], k, s->match_indices[k] + 1);
+                    __builtin_trap();
+                }
+            }
+        }
+    }
+
+    // Election Safety: at most one leader per term.
+    // Each term has at most one leader because a candidate needs
+    // a majority of votes, and each node votes at most once per term.
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL || nodes[i]->role != ROLE_LEADER)
+            continue;
+        for (int j = i + 1; j < num_nodes; j++) {
+            if (nodes[j] == NULL || nodes[j]->role != ROLE_LEADER)
+                continue;
+            if (nodes[i]->term_and_vote.term == nodes[j]->term_and_vote.term) {
+                fprintf(stderr, "INVARIANT VIOLATED: two leaders in term %lu: "
+                    "node %d and node %d\n",
+                    (unsigned long)nodes[i]->term_and_vote.term, i, j);
+                __builtin_trap();
+            }
+        }
+    }
+
+    // State Machine Safety (committed prefix agreement).
+    // For any two nodes, their logs must agree on all entries up to
+    // min(commit_index_i, commit_index_j). This is the core safety
+    // property: all committed operations are identical across replicas.
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL)
+            continue;
+        for (int j = i + 1; j < num_nodes; j++) {
+            if (nodes[j] == NULL)
+                continue;
+
+            int min_commit = nodes[i]->commit_index;
+            if (nodes[j]->commit_index < min_commit)
+                min_commit = nodes[j]->commit_index;
+
+            for (int k = 0; k <= min_commit; k++) {
+                WALEntry *ei = wal_peek_entry(&nodes[i]->wal, k);
+                WALEntry *ej = wal_peek_entry(&nodes[j]->wal, k);
+
+                if (memcmp(&ei->oper, &ej->oper, sizeof(KVStoreOper)) != 0) {
+                    fprintf(stderr, "INVARIANT VIOLATED: committed log operation "
+                        "mismatch at index %d between node %d and node %d\n",
+                        k, i, j);
+                    __builtin_trap();
+                }
+
+                if (ei->term != ej->term) {
+                    fprintf(stderr, "INVARIANT VIOLATED: committed log term "
+                        "mismatch at index %d between node %d (term %lu) "
+                        "and node %d (term %lu)\n",
+                        k, i, (unsigned long)ei->term,
+                        j, (unsigned long)ej->term);
+                    __builtin_trap();
+                }
+            }
+        }
+    }
+
+    ////////////////////////////////////////////////////////////////////
+    // Shadow log: external commit tracking
+    ////////////////////////////////////////////////////////////////////
+
+    // Phase 1: Find the observed max commit index and a source node.
+    //
+    // In Raft, commit_index == -1 means nothing committed, 0 means the
+    // first entry is committed, etc. We track the number of committed
+    // entries as (commit_index + 1) so it aligns with shadow_count.
+    int observed_committed = 0;  // number of committed entries observed
+    int source_node_idx = -1;
+
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL)
+            continue;
+        int node_committed = nodes[i]->commit_index + 1;
+        if (node_committed > observed_committed) {
+            observed_committed = node_committed;
+            source_node_idx = i;
+        }
+    }
+
+    // Phase 2: Append newly committed entries to the shadow log.
+    if (source_node_idx >= 0 && observed_committed > ic->shadow_count) {
+
+        NodeState *source = nodes[source_node_idx];
+        assert(wal_entry_count(&source->wal) >= observed_committed);
+
+        for (int k = ic->shadow_count; k < observed_committed; k++) {
+
+            KVStoreOper *source_oper = &wal_peek_entry(&source->wal, k)->oper;
+
+            // Cross-validate against other live nodes that have also
+            // committed this entry.
+            for (int j = 0; j < num_nodes; j++) {
+                if (j == source_node_idx)
+                    continue;
+                if (nodes[j] == NULL)
+                    continue;
+                if (nodes[j]->commit_index < k)
+                    continue;
+                if (wal_entry_count(&nodes[j]->wal) <= k)
+                    continue;
+
+                if (memcmp(&wal_peek_entry(&nodes[j]->wal, k)->oper, source_oper, sizeof(KVStoreOper)) != 0) {
+                    fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
+                        "between source node %d and node %d during shadow log append\n",
+                        k, source_node_idx, j);
+                    __builtin_trap();
+                }
+            }
+
+            if (shadow_log_append(ic, *source_oper) < 0) {
+                fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
+                __builtin_trap();
+            }
+        }
+    }
+
+    // Phase 3: Verify shadow log against the cluster.
+
+    // Sub-check A: Committed entries must match the shadow log.
+    for (int k = 0; k < ic->shadow_count; k++) {
+        for (int i = 0; i < num_nodes; i++) {
+            if (nodes[i] == NULL)
+                continue;
+            if (wal_entry_count(&nodes[i]->wal) <= k)
+                continue;
+            if (nodes[i]->commit_index < k)
+                continue;
+
+            if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
+                char shadow_buf[128], node_buf[128];
+                kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
+                kvstore_snprint_oper(node_buf, sizeof(node_buf), wal_peek_entry(&nodes[i]->wal, k)->oper);
+                fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
+                    "  shadow: %s\n"
+                    "  node:   %s\n",
+                    k, i, shadow_buf, node_buf);
+                __builtin_trap();
+            }
+        }
+    }
+
+    // Sub-check B: When commit regresses, previously committed entries
+    // must still be held by a majority of the cluster.
+    if (observed_committed < ic->shadow_count) {
+        for (int k = observed_committed; k < ic->shadow_count; k++) {
+            int holders = 0;
+            int num_dead = 0;
+
+            for (int i = 0; i < num_nodes; i++) {
+                if (nodes[i] == NULL) {
+                    num_dead++;
+                    continue;
+                }
+                if (wal_entry_count(&nodes[i]->wal) <= k)
+                    continue;
+                if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
+                    holders++;
+            }
+
+            if (holders + num_dead <= num_nodes / 2) {
+                char oper_buf[128];
+                kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
+                fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
+                    "no longer held by majority (holders=%d, dead=%d, total=%d)\n"
+                    "  entry: %s\n",
+                    k, holders, num_dead, num_nodes, oper_buf);
+                __builtin_trap();
+            }
+        }
+    }
+}

raft/invariant_checker.h

@@ -0,0 +1,17 @@
+#ifndef INVARIANT_CHECKER_INCLUDED
+#define INVARIANT_CHECKER_INCLUDED
+
+#include "node.h"
+
+typedef struct {
+    // External shadow log of committed operations (unbounded, dynamically allocated)
+    KVStoreOper *shadow_log;
+    int shadow_count;
+    int shadow_capacity;
+} InvariantChecker;
+
+void invariant_checker_init(InvariantChecker *ic);
+void invariant_checker_free(InvariantChecker *ic);
+void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
+
+#endif // INVARIANT_CHECKER_INCLUDED

raft/main.c

@@ -8,6 +8,7 @@
 
 #include "node.h"
 #include "client.h"
+#include "invariant_checker.h"
 
 static volatile int simulation_running = 1;
 
@@ -17,15 +18,32 @@ static void sigint_handler(int sig)
     simulation_running = 0;
 }
 
-int main(void)
+int main(int argc, char **argv)
 {
     signal(SIGINT, sigint_handler);
 
+    QuakeyUInt64 seed = 2;
+    QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
+
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
+            seed = strtoull(argv[++i], NULL, 10);
+        } else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
+            time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
+        }
+    }
+
     Quakey *quakey;
-    int ret = quakey_init(&quakey, 2);
+    int ret = quakey_init(&quakey, seed);
     if (ret < 0)
         return -1;
 
+    QuakeyNode cli_1;
+    QuakeyNode cli_2;
+    QuakeyNode node_1;
+    QuakeyNode node_2;
+    QuakeyNode node_3;
+
     // Client 1
     {
         QuakeySpawn config = {
@@ -39,7 +57,7 @@ int main(void)
             .disk_size  = 10<<20,
             .platform   = QUAKEY_LINUX,
         };
-        quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+        cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
     }
 
     // Client 2
@@ -55,7 +73,7 @@ int main(void)
             .disk_size  = 10<<20,
             .platform   = QUAKEY_LINUX,
         };
-        quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+        cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
     }
 
     // Node 1
@@ -71,7 +89,7 @@ int main(void)
             .disk_size  = 10<<20,
             .platform   = QUAKEY_LINUX,
         };
-        quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
+        node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
     }
 
     // Node 2
@@ -87,7 +105,7 @@ int main(void)
             .disk_size  = 10<<20,
             .platform   = QUAKEY_LINUX,
         };
-        quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
+        node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
     }
 
     // Node 3
@@ -103,12 +121,30 @@ int main(void)
             .disk_size  = 10<<20,
             .platform   = QUAKEY_LINUX,
         };
-        quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
+        node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
     }
 
-    while (simulation_running)
+    // Limit crashes to 1 node at a time (within fault tolerance of f=1).
+    quakey_set_max_crashes(quakey, 1);
+
+    quakey_network_partitioning(quakey, true);
+
+    InvariantChecker invariant_checker;
+    invariant_checker_init(&invariant_checker);
+
+    while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
+
         quakey_schedule_one(quakey);
 
+        NodeState *arr[] = {
+            quakey_node_state(node_1),
+            quakey_node_state(node_2),
+            quakey_node_state(node_3),
+        };
+        invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
+    }
+
+    invariant_checker_free(&invariant_checker);
     quakey_free(quakey);
     return 0;
 }

raft/node.h

@@ -7,6 +7,7 @@
 
 #include "wal.h"
 #include "config.h"
+#include "client_table.h"
 
 enum {
     MESSAGE_TYPE_REQUEST_VOTE,
@@ -28,6 +29,7 @@ typedef struct {
 
 typedef struct {
     MessageHeader base;
+    int      sender_idx;
     uint64_t term;
     uint8_t  value;
 } VotedMessage;
@@ -53,14 +55,15 @@ typedef struct {
 
 typedef struct {
     MessageHeader base;
-    Operation oper;
+    KVStoreOper oper;
     uint64_t client_id;
     uint64_t request_id;
 } RequestMessage;
 
 typedef struct {
     MessageHeader base;
-    OperationResult result;
+    KVStoreResult result;
+    uint64_t request_id;
 } ReplyMessage;
 
 typedef struct {
@@ -68,20 +71,6 @@ typedef struct {
     int leader_idx;
 } RedirectMessage;
 
-typedef struct {
-    uint64_t        client_id;
-    uint64_t        last_request_id;
-    OperationResult last_result;
-    bool            pending;
-    int             conn_tag;
-} ClientTableEntry;
-
-typedef struct {
-    int count;
-    int capacity;
-    ClientTableEntry *entries;
-} ClientTable;
-
 typedef enum {
     ROLE_FOLLOWER,
     ROLE_CANDIDATE,
@@ -90,46 +79,74 @@ typedef enum {
 
 typedef struct {
 
-    TCP tcp;
+    // In-memory copy of the term and vote values.
-    WAL wal;
+    // These must always be updated after the version
+    // stored on disk.
+    uint64_t term;
+    int      voted_for;
 
+    // Handle to the file backing the term and vote
+    // values.
+    Handle handle;
+} TermAndVote;
+
+typedef struct {
+
+    // Networking subsystem
+    TCP tcp;
+
+    // Static list of cluster nodes
     Address self_addr;
     Address node_addrs[NODE_LIMIT];
     int     num_nodes;
 
+    // Current loder of the node and the
+    // current leader. If no leader, -1.
     Role role;
+    int  leader_idx;
 
-    uint64_t term;
+    // Votes received when candidate.
-    int      voted_for;
+    // Each bit is associated to a node. The
-    Handle   term_and_vote_handle;
+    // number of votes is obtained by counting
+    // set bits. This ensures nodes won't vote
+    // twice.
+    uint32_t votes;
 
-    // Index of the current leader
+    // Durable log subsystem
-    int leader_idx;
+    WAL wal;
+
+    // Durable fixed-size data subsystem
+    TermAndVote term_and_vote;
+
+    // Current election timeout. This value is relative
+    // and must be summed to the heartbeat in order to
+    // get the absolute election deadline.
+    uint64_t election_timeout;
+
+    // This is the current value of the node.
+    // It is set at the start of each event
+    // handler call.
+    Time now;
+
+    // If leader, this is the time when the last
+    // heartbeat was sent. If follower, this is
+    // the last time a heartbeat was received.
+    Time heartbeat;
+
+    // The state machine to be replicated.
+    KVStore kvstore;
+
+    // Table of client requests. This ensures request
+    // are applied in order.
+    ClientTable client_table;
+    int next_client_tag;
 
     int commit_index;
     int last_applied;
 
-    // When CANDIDATE
-    int votes_received;
-
-    // When LEADER
     int next_indices[NODE_LIMIT];
     int match_indices[NODE_LIMIT];
 
-    // Relative timeout in nanoseconds
-    uint64_t election_timeout;
-
-    // Last heartbeat time
-    uint64_t watchdog;
-
-    // Keep track of client request order to enforce
-    // linearizability
-    ClientTable client_table;
-    int next_client_tag;
-
-    // The state machine to be replicated
-    StateMachine state_machine;
-
 } NodeState;
 
 struct pollfd;

raft/wal.c

@@ -4,10 +4,26 @@
 
 #include <quakey.h>
 #include <assert.h>
+#include <stddef.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "wal.h"
 
+// FNV-1a checksum over all WALEntry fields except the checksum itself.
+uint32_t wal_entry_checksum(WALEntry *entry)
+{
+    uint32_t h = 2166136261u;
+    const unsigned char *p = (const unsigned char *)entry;
+    // Hash all bytes up to (but not including) the checksum field
+    size_t len = offsetof(WALEntry, checksum);
+    for (size_t i = 0; i < len; i++) {
+        h ^= p[i];
+        h *= 16777619u;
+    }
+    return h;
+}
+
 int wal_init(WAL *wal, string file)
 {
     Handle handle;
@@ -21,25 +37,46 @@ int wal_init(WAL *wal, string file)
         return -1;
     }
 
-    if (size % sizeof(WALEntry) != 0) {
+    // Discard any partial trailing entry (crash during write).
+    int raw_count = size / sizeof(WALEntry);
+    size_t valid_size = raw_count * sizeof(WALEntry);
+    if (valid_size < size) {
+        file_truncate(handle, valid_size);
+    }
+
+    WALEntry *entries = malloc(raw_count * sizeof(WALEntry));
+    if (entries == NULL && raw_count > 0) {
         file_close(handle);
         return -1;
     }
-    int count = size / sizeof(WALEntry);
+
+    if (file_set_offset(handle, 0) < 0) {
+        file_close(handle);
+        free(entries);
+        return -1;
+    }
+
+    if (raw_count > 0 && file_read_exact(handle, (char*) entries, raw_count * sizeof(WALEntry)) < 0) {
+        file_close(handle);
+        free(entries);
+        return -1;
+    }
+
+    // Verify checksums: truncate at the first corrupted entry.
+    // All entries from a corrupted one onward are discarded.
+    int count = raw_count;
+    for (int i = 0; i < raw_count; i++) {
+        if (entries[i].checksum != wal_entry_checksum(&entries[i])) {
+            count = i;
+            file_truncate(handle, count * sizeof(WALEntry));
+            break;
+        }
+    }
 
     wal->count = count;
-    wal->capacity = count;
+    wal->capacity = raw_count; // capacity >= count
-    wal->entries = malloc(count * sizeof(WALEntry));
+    wal->entries = entries;
-    if (wal->entries == NULL) {
+    wal->handle = handle;
-        file_close(handle);
-        return -1;
-    }
-
-    if (file_read_exact(handle, (char*) wal->entries, count * sizeof(WALEntry)) < 0) {
-        file_close(handle);
-        return -1;
-    }
-
     return 0;
 }
 
@@ -61,12 +98,26 @@ int wal_append(WAL *wal, WALEntry *entry)
         wal->entries = p;
     }
 
-    // TODO: Should truncate file on partial writes
+    // Compute checksum before writing to disk
-    if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0)
+    entry->checksum = wal_entry_checksum(entry);
-        return -1;
 
-    if (file_sync(wal->handle) < 0)
+    if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0) {
+        // A partial write may have advanced the file offset. Truncate
+        // and rewind so the file stays consistent with the in-memory
+        // entry count.
+        file_truncate(wal->handle, wal->count * sizeof(WALEntry));
+        file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
         return -1;
+    }
+
+    if (file_sync(wal->handle) < 0) {
+        // The write succeeded but sync failed. Rewind the offset and
+        // truncate the phantom entry so the next append writes to the
+        // correct position.
+        file_truncate(wal->handle, wal->count * sizeof(WALEntry));
+        file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
+        return -1;
+    }
 
     wal->entries[wal->count++] = *entry;
     return 0;

raft/wal.h

@@ -3,14 +3,18 @@
 
 #include <lib/file_system.h>
 
-#include <state_machine/state_machine.h>
+#include <state_machine/kvstore.h>
 
 typedef struct {
-    uint64_t  term;
+    uint64_t    term;
-    uint64_t  client_id;
+    uint64_t    client_id;
-    Operation oper;
+    uint64_t    request_id;
+    KVStoreOper oper;
+    uint32_t    checksum;
 } WALEntry;
 
+uint32_t wal_entry_checksum(WALEntry *entry);
+
 typedef struct {
     int       count;
     int       capacity;

state_machine/kvstore.c

@@ -0,0 +1,118 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "kvstore.h"
+
+void kvstore_init(KVStore *kvs)
+{
+    memset(kvs, 0, sizeof(*kvs));
+}
+
+void kvstore_free(KVStore *kvs)
+{
+    (void) kvs;
+}
+
+static int find_entry(KVStore *kvs, char *key)
+{
+    for (int i = 0; i < kvs->count; i++)
+        if (!memcmp(kvs->entries[i].key, key, KVSTORE_KEY_SIZE))
+            return i;
+    return -1;
+}
+
+KVStoreResult kvstore_update(KVStore *kvs, KVStoreOper oper)
+{
+    KVStoreResult result;
+    switch (oper.type) {
+    case KVSTORE_OPER_NOOP:
+        result.type = KVSTORE_RESULT_OK;
+        result.val = 0;
+        break;
+    case KVSTORE_OPER_SET:
+        {
+            int i = find_entry(kvs, oper.key);
+            if (i < 0) {
+                if (kvs->count >= KVSTORE_ENTRY_LIMIT) {
+                    result.type = KVSTORE_RESULT_FULL;
+                    result.val = 0;
+                    break;
+                }
+                i = kvs->count++;
+                memcpy(kvs->entries[i].key, oper.key, KVSTORE_KEY_SIZE);
+            }
+            kvs->entries[i].val = oper.val;
+
+            result.type = KVSTORE_RESULT_OK;
+            result.val = 0;
+        }
+        break;
+    case KVSTORE_OPER_GET:
+        {
+            int i = find_entry(kvs, oper.key);
+            if (i < 0) {
+                result.type = KVSTORE_RESULT_MISSING;
+                result.val = 0;
+            } else {
+                result.type = KVSTORE_RESULT_OK;
+                result.val  = kvs->entries[i].val;
+            }
+        }
+        break;
+    case KVSTORE_OPER_DEL:
+        {
+            int i = find_entry(kvs, oper.key);
+            if (i < 0) {
+                result.type = KVSTORE_RESULT_MISSING;
+                result.val = 0;
+            } else {
+                result.type = KVSTORE_RESULT_OK;
+                result.val  = kvs->entries[i].val;
+                kvs->entries[i] = kvs->entries[--kvs->count];
+            }
+        }
+        break;
+    default:
+        assert(0);
+        break;
+    }
+
+    return result;
+}
+
+void kvstore_dump_oper(KVStoreOper oper)
+{
+    switch (oper.type) {
+        case KVSTORE_OPER_NOOP: printf("NOOP"); break;
+        case KVSTORE_OPER_SET: printf("SET(%.16s, %lu)", oper.key, oper.val); break;
+        case KVSTORE_OPER_GET: printf("GET(%.16s)", oper.key); break;
+        case KVSTORE_OPER_DEL: printf("DEL(%.16s)", oper.key); break;
+        default: printf("???\n");
+    }
+}
+
+int kvstore_snprint_oper(char *buf, int size, KVStoreOper oper)
+{
+    switch (oper.type) {
+    case KVSTORE_OPER_NOOP: return snprintf(buf, size, "NOOP");
+    case KVSTORE_OPER_SET:  return snprintf(buf, size, "SET(%.16s, %lu)", oper.key, oper.val);
+    case KVSTORE_OPER_GET:  return snprintf(buf, size, "GET(%.16s)", oper.key);
+    case KVSTORE_OPER_DEL:  return snprintf(buf, size, "DEL(%.16s)", oper.key);
+    default:                return snprintf(buf, size, "???");
+    }
+}
+
+int kvstore_snprint_result(char *buf, int size, KVStoreResult result)
+{
+    switch (result.type) {
+    case KVSTORE_RESULT_OK:      return snprintf(buf, size, "OK(%lu)", result.val);
+    case KVSTORE_RESULT_FULL:    return snprintf(buf, size, "FULL");
+    case KVSTORE_RESULT_MISSING: return snprintf(buf, size, "MISSING");
+    default:                     return snprintf(buf, size, "???");
+    }
+}

state_machine/kvstore.h

@@ -0,0 +1,51 @@
+#ifndef KVSTORE_INCLUDED
+#define KVSTORE_INCLUDED
+
+#include <stdint.h>
+
+#define KVSTORE_KEY_SIZE 16
+#define KVSTORE_ENTRY_LIMIT 64
+
+typedef enum {
+    KVSTORE_OPER_NOOP,
+    KVSTORE_OPER_SET,
+    KVSTORE_OPER_GET,
+    KVSTORE_OPER_DEL,
+} KVStoreOperType;
+
+typedef enum {
+    KVSTORE_RESULT_OK,
+    KVSTORE_RESULT_FULL,
+    KVSTORE_RESULT_MISSING,
+} KVStoreResultType;
+
+typedef struct {
+    KVStoreOperType type;
+    char key[KVSTORE_KEY_SIZE];
+    uint64_t val;
+} KVStoreOper;
+
+typedef struct {
+    KVStoreResultType type;
+    uint64_t val;
+} KVStoreResult;
+
+typedef struct {
+    char     key[KVSTORE_KEY_SIZE];
+    uint64_t val;
+} KVEntry;
+
+typedef struct {
+    int count;
+    KVEntry entries[KVSTORE_ENTRY_LIMIT];
+} KVStore;
+
+void          kvstore_init(KVStore *kvs);
+void          kvstore_free(KVStore *kvs);
+KVStoreResult kvstore_update(KVStore *kvs, KVStoreOper oper);
+
+void kvstore_dump_oper(KVStoreOper oper);
+int  kvstore_snprint_oper(char *buf, int size, KVStoreOper oper);
+int  kvstore_snprint_result(char *buf, int size, KVStoreResult result);
+
+#endif // KVSTORE_INCLUDED

state_machine/state_machine.c

@@ -1,25 +0,0 @@
-#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
-#define QUAKEY_ENABLE_MOCKS
-#endif
-
-#include <quakey.h>
-#include <assert.h>
-
-#include "state_machine.h"
-
-void state_machine_init(StateMachine *sm)
-{
-    (void)sm;
-}
-
-void state_machine_free(StateMachine *sm)
-{
-    (void)sm;
-}
-
-OperationResult state_machine_update(StateMachine *sm, Operation op)
-{
-    (void)sm;
-    (void)op;
-    return OPERATION_RESULT_X;
-}

state_machine/state_machine.h

@@ -1,26 +0,0 @@
-#ifndef STATE_MACHINE_INCLUDED
-#define STATE_MACHINE_INCLUDED
-
-typedef enum {
-    OPERATION_NOOP,
-    OPERATION_A,
-    OPERATION_B,
-    OPERATION_C,
-} Operation;
-
-typedef enum {
-    OPERATION_RESULT_X,
-    OPERATION_RESULT_Y,
-    OPERATION_RESULT_Z,
-} OperationResult;
-
-typedef struct {
-
-} StateMachine;
-
-void state_machine_init(StateMachine *sm);
-void state_machine_free(StateMachine *sm);
-
-OperationResult state_machine_update(StateMachine *sm, Operation op);
-
-#endif // STATE_MACHINE_INCLUDED

table.txt

@@ -0,0 +1,247 @@
+Consensus Protocol Simulation Results
+Simulated time: 1200s per run, seeds 1-100
+3 nodes, 2 clients, fault injection enabled, network partitioning enabled
+
+Column descriptions:
+  Seed        - RNG seed used for the simulation run
+  Proto       - Consensus protocol (VSR or Raft)
+  Invocations - Number of client requests initiated
+  OK          - Requests that completed successfully with a response
+  Timeouts    - Requests that timed out without a response
+  Rejected    - Requests explicitly rejected (e.g. sent to a non-leader)
+  Checks      - Linearizability checks performed by the checker
+  Steps       - Total simulation scheduling steps (ticks across all hosts)
+
+Seed  Proto  Invocations       OK  Timeouts  Rejected   Checks     Steps
+-----------------------------------------------------------------------
+   1    VSR         2244      2206        33         5     2244    105648
+   1   Raft         3276      1700       368      1206     3274    363190
+   2    VSR         2883      2839        39         5     2883    126828
+   2   Raft         3394      2761       285       346     3392    171163
+   3    VSR          592       497        94         1      592     42548
+   3   Raft         2789      2358       330        99     2787    150699
+   4    VSR         1274      1231        37         6     1274     78222
+   4   Raft         2068      1362       464       240     2066     88436
+   5    VSR         1405      1372        31         1     1404     75712
+   5   Raft         2304      1630       416       256     2302    131192
+   6    VSR         2025      1975        49         0     2024    169653
+   6   Raft         2182      1244       447       489     2180    378921
+   7    VSR          955       856        98         1      955     56039
+   7   Raft         3433      1609       311      1511     3431    297830
+   8    VSR          720       636        79         4      719     46945
+   8   Raft         2498      1759       436       301     2496    123015
+   9    VSR          859       780        77         2      859     56003
+   9   Raft         2778      1816       362       599     2777    225590
+  10    VSR         1486      1401        84         1     1486     95773
+  10   Raft         3114      2093       321       698     3112    341220
+  11    VSR          456       302       153         1      456     33396
+  11   Raft         3441      1209       375      1855     3439    345485
+  12    VSR         1220      1128        88         2     1218     66620
+  12   Raft         2555      1064       458      1031     2553    370909
+  13    VSR          863       753       108         1      862     52065
+  13   Raft         1965      1210       494       259     1963    162897
+  14    VSR          645       524       115         5      644     44976
+  14   Raft         2498      1918       399       179     2496    138682
+  15    VSR          690       638        50         1      689     48727
+  15   Raft         2541      1871       381       287     2539    200612
+  16    VSR          944       835       106         1      942     52517
+  16   Raft         2369      1451       466       450     2367    465559
+  17    VSR          647       564        80         2      646     40272
+  17   Raft         2446      1846       399       199     2444    124084
+  18    VSR          392       310        76         4      390     35414
+  18   Raft         1955       923       527       503     1953    218171
+  19    VSR         1165      1068        97         0     1165     60283
+  19   Raft         2987      1849       363       773     2985    230511
+  20    VSR         2229      2164        62         2     2228    108081
+  20   Raft         2996      1742       381       871     2994    450695
+  21    VSR         1390      1341        44         5     1390     78704
+  21   Raft         2441      1485       412       542     2439    120767
+  22    VSR         1844      1758        81         5     1844     96250
+  22   Raft         3561      1897       306      1356     3559    212082
+  23    VSR         1393      1335        55         3     1393     79177
+  23   Raft         2524      1496       434       593     2523    222489
+  24    VSR          971       951        18         0      969     56647
+  24   Raft         2177      1606       412       157     2175    177816
+  25    VSR          458       376        77         5      458     35323
+  25   Raft         2507      1898       374       233     2505    139076
+  26    VSR         1177      1117        60         0     1177     70344
+  26   Raft         2583      1978       365       238     2581    186689
+  27    VSR         1042       937       104         1     1042     70667
+  27   Raft         2794      2237       355       200     2792    171007
+  28    VSR         1118      1067        43         6     1116     67368
+  28   Raft         2790      1870       357       561     2788    257457
+  29    VSR          683       610        73         0      683     45166
+  29   Raft         2557      1439       408       708     2555    110388
+  30    VSR         1202      1134        63         3     1200     64864
+  30   Raft         3312      2297       316       697     3310    406253
+  31    VSR         1466      1399        65         2     1466     83025
+  31   Raft         2818      1612       377       827     2816    311506
+  32    VSR         1207      1161        41         5     1207     68520
+  32   Raft         2676      1141       416      1117     2674    255671
+  33    VSR         1219      1134        83         1     1218     65923
+  33   Raft         3260      2844       274       140     3258    191445
+  34    VSR         1255      1229        24         1     1254     71120
+  34   Raft         3311      1372       405      1532     3309    428477
+  35    VSR         1197      1142        50         4     1196     73779
+  35   Raft         2435      1793       418       222     2433    147736
+  36    VSR         1560      1500        55         4     1559     89849
+  36   Raft         2110      1186       515       407     2108    310505
+  37    VSR         1487      1424        58         4     1486    148383
+  37   Raft         3223      1407       335      1479     3221    242814
+  38    VSR         1397      1349        42         5     1396     76827
+  38   Raft         2466      1260       441       763     2464    352797
+  39    VSR         1304      1249        53         1     1303     78321
+  39   Raft         3027      2062       343       620     3025    334793
+  40    VSR         1299      1270        25         2     1297     72474
+  40   Raft         2183      1213       479       489     2181    229529
+  41    VSR         1091      1040        41         8     1089     65193
+  41   Raft         2856      2333       339       182     2854    162726
+  42    VSR          496       410        84         1      495     36493
+  42   Raft         2421      1797       400       222     2419    163015
+  43    VSR          658       562        92         2      656     46387
+  43   Raft         2834      2134       362       336     2832    161248
+  44    VSR         1372      1318        49         4     1371     75563
+  44   Raft         2654      2076       370       206     2652    262333
+  45    VSR          257       127       129         0      256     33593
+  45   Raft         3146      1319       366      1459     3144    395077
+  46    VSR         1515      1458        53         2     1513    105709
+  46   Raft         3320      1456       382      1480     3318    444523
+  47    VSR         1057      1017        39         0     1056     61117
+  47   Raft         2243      1351       467       423     2241    210836
+  48    VSR         1195      1140        51         3     1194     69487
+  48   Raft         2378      1708       422       246     2376    148188
+  49    VSR          950       876        71         2      949     55394
+  49   Raft         2984      2444       309       229     2982    153760
+  50    VSR          964       875        88         1      964     57668
+  50   Raft         2889      2295       344       248     2887    182648
+  51    VSR         1302      1250        49         1     1300     74028
+  51   Raft         2363      1734       415       212     2361    127568
+  52    VSR         1205      1111        91         2     1204     64727
+  52   Raft         2905      2380       303       221     2904    170346
+  53    VSR          869       744       124         0      868     95545
+  53   Raft         2203       964       444       794     2202    306208
+  54    VSR          408       333        72         3      408     36481
+  54   Raft         2113      1409       434       268     2111    103004
+  55    VSR          494       443        50         1      494     40161
+  55   Raft         2618      1937       375       305     2617    239299
+  56    VSR         1300      1231        66         2     1299     70686
+  56   Raft         3401      2723       282       394     3399    194017
+  57    VSR          238       145        93         0      238     34783
+  57   Raft         1941      1255       451       233     1939     90352
+  58    VSR          922       854        63         5      922     54772
+  58   Raft         2173      1303       448       420     2171    225395
+  59    VSR         1941      1853        84         3     1940    103258
+  59   Raft         2501      1237       420       842     2499    214547
+  60    VSR         1766      1645       114         5     1764     86384
+  60   Raft         2909      1062       437      1408     2907    446530
+  61    VSR         1630      1557        68         4     1629     76775
+  61   Raft         2956      2258       333       363     2954    199725
+  62    VSR         1077      1023        51         2     1076    107542
+  62   Raft         3418      1993       303      1120     3416    178317
+  63    VSR         2158      2107        47         3     2157    107086
+  63   Raft         2245      1039       473       731     2243    331394
+  64    VSR         1804      1743        53         7     1803    101345
+  64   Raft         2804      2152       325       325     2802    206594
+  65    VSR          934       825       106         1      932     77299
+  65   Raft         2279      1090       486       701     2277    266669
+  66    VSR          798       726        69         2      797     59313
+  66   Raft         2105      1135       444       524     2103    246688
+  67    VSR          965       874        91         0      965     71117
+  67   Raft         2268      1598       439       229     2266    103555
+  68    VSR          438       327       109         1      437     54528
+  68   Raft         2787      1755       395       635     2785    347905
+  69    VSR          922       869        52         1      922     61234
+  69   Raft         2588      1752       425       409     2586    218784
+  70    VSR          720       607       110         3      720     47228
+  70   Raft         2424      1495       414       513     2422    255104
+  71    VSR         1067       982        81         4     1067     62847
+  71   Raft         2207      1583       398       224     2205    165271
+  72    VSR          673       561       110         0      671     48927
+  72   Raft         2927      2052       348       525     2925    145377
+  73    VSR         1361      1270        88         2     1360     70464
+  73   Raft         2798      1976       353       467     2796    131273
+  74    VSR         1891      1842        43         5     1890    104058
+  74   Raft         2176      1640       405       129     2174    123942
+  75    VSR          970       879        91         0      970     86026
+  75   Raft         2456       879       491      1084     2454    380910
+  76    VSR          851       737       111         2      850     55131
+  76   Raft         3226      1945       343       936     3224    158287
+  77    VSR          610       474       133         2      609     40458
+  77   Raft         2492      1882       387       221     2490    134293
+  78    VSR         1269      1232        31         6     1269     69782
+  78   Raft         3411      1575       329      1506     3410    238137
+  79    VSR          921       859        62         0      921     55960
+  79   Raft         3008      2516       335       155     3006    176544
+  80    VSR         1271      1230        36         5     1271     77391
+  80   Raft         2605      1033       426      1144     2603    287940
+  81    VSR          835       708       124         2      834     52299
+  81   Raft         2582      1845       361       374     2580    140619
+  82    VSR         1024       913       109         1     1023     58016
+  82   Raft         2921      2264       330       325     2919    143004
+  83    VSR          922       802       120         0      922     51388
+  83   Raft         2600      1885       412       301     2598    129191
+  84    VSR         1533      1480        52         1     1533     79724
+  84   Raft         2344      1753       440       149     2342    129605
+  85    VSR         1650      1588        58         4     1650     88562
+  85   Raft         3366      1419       355      1590     3364    379641
+  86    VSR          884       763       118         1      882     49372
+  86   Raft         2736      1258       457      1019     2734    552128
+  87    VSR         1072      1003        68         0     1071     59323
+  87   Raft         1875       968       506       399     1873    253771
+  88    VSR         1493      1437        52         4     1493     92016
+  88   Raft         1902      1285       445       170     1900    109923
+  89    VSR         1294      1239        53         0     1292     69379
+  89   Raft         1922      1204       471       245     1920     91616
+  90    VSR         1215      1119        93         2     1214     67086
+  90   Raft         3440      2057       296      1085     3438    284840
+  91    VSR         2176      2142        28         6     2176    108769
+  91   Raft         2606      2025       358       221     2604    136452
+  92    VSR          594       550        42         0      592     43714
+  92   Raft         2563      1504       432       625     2561    370793
+  93    VSR          775       699        76         0      775     67757
+  93   Raft         1990       826       509       653     1988    320932
+  94    VSR          546       455        88         3      546     35972
+  94   Raft         2449      1686       399       362     2447    216855
+  95    VSR          420       326        94         0      420     33058
+  95   Raft         2506      1891       379       234     2504    141706
+  96    VSR          583       509        70         4      583     40050
+  96   Raft         2307      1263       473       569     2305    325134
+  97    VSR         1991      1965        21         3     1989     95473
+  97   Raft         2679      1360       397       920     2677    281323
+  98    VSR         1198      1123        73         2     1198     78181
+  98   Raft         2550      1467       467       614     2548    416874
+  99    VSR          954       884        67         3      954     72020
+  99   Raft         2051      1356       481       212     2049    102556
+ 100    VSR         1426      1319       105         1     1425     76369
+ 100   Raft         2817      1402       440       973     2815    474134
+-----------------------------------------------------------------------
+
+Summary Statistics
+
+VSR (100 completed runs, 0 violations):
+  Avg invocations:     1132.8
+  Avg ok:              1057.4
+  Avg timeouts:          72.3
+  Avg rejected:           2.3
+  Avg steps:          69089.2
+  Total invocations:   113278
+  Total ok:            105742
+  Total timeouts:        7226
+  Total rejected:         234
+  OK rate:              93.3%
+  Timeout rate:          6.4%
+  Rejection rate:        0.2%
+
+Raft (100 completed runs, 0 violations):
+  Avg invocations:     2645.8
+  Avg ok:              1664.9
+  Avg timeouts:         397.6
+  Avg rejected:         581.4
+  Avg steps:         234875.8
+  Total invocations:   264582
+  Total ok:            166491
+  Total timeouts:       39755
+  Total rejected:       58142
+  OK rate:              62.9%
+  Timeout rate:         15.0%
+  Rejection rate:       22.0%

vsr/client.c

@@ -6,18 +6,68 @@
 #include <stdint.h>
 #include <assert.h>
 
-#include <lib/file_system.h>
-
 #include "node.h"
 #include "client.h"
 
 //#define CLIENT_TRACE(fmt, ...) {}
 #define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
 
+#define KEY_POOL_SIZE 128
+
+static uint64_t next_client_id = 1;
+
+static uint64_t client_random(void)
+{
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+    return quakey_random();
+#else
+    return (uint64_t)rand();
+#endif
+}
+
+static KVStoreOper random_oper(void)
+{
+    KVStoreOper oper = {0};
+    snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
+
+    switch (client_random() % 3) {
+    case 0:
+        oper.type = KVSTORE_OPER_SET;
+        oper.val  = client_random();
+        break;
+    case 1:
+        oper.type = KVSTORE_OPER_GET;
+        break;
+    case 2:
+        oper.type = KVSTORE_OPER_DEL;
+        break;
+    }
+    return oper;
+}
+
 // Format time as seconds with 3 decimal places for trace output
-#define TIME_FMT "%.3fs"
+#define TIME_FMT "%7.3fs"
 #define TIME_VAL(t) ((double)(t) / 1000000000.0)
 
+static void client_log_impl(ClientState *state, Time now, const char *event, const char *detail)
+{
+    printf("[" TIME_FMT "] CLIENT %lu | V%-3lu | %-20s %s\n",
+        TIME_VAL(now),
+        state->client_id,
+        state->view_number,
+        event,
+        detail ? detail : "");
+}
+
+#define client_log(state, now, event, fmt, ...) do {                \
+    char _detail[256];                                              \
+    snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__);         \
+    client_log_impl(state, now, event, _detail);                    \
+} while (0)
+
+#define client_log_simple(state, now, event) \
+    client_log_impl(state, now, event, NULL)
+
 static int leader_idx(ClientState *state)
 {
     return state->view_number % state->num_servers;
@@ -29,24 +79,61 @@ process_message(ClientState *state,
 {
     (void) conn_idx;
 
+    if (type == MESSAGE_TYPE_REDIRECT) {
+        RedirectMessage redirect_message;
+        if (msg.len != sizeof(RedirectMessage))
+            return -1;
+        memcpy(&redirect_message, msg.ptr, sizeof(redirect_message));
+
+        if (redirect_message.view_number > state->view_number) {
+            Time now = get_current_time();
+            client_log(state, now, "RECV REDIRECT", "view=%lu -> %lu leader=%d",
+                (unsigned long)state->view_number,
+                (unsigned long)redirect_message.view_number,
+                (int)(redirect_message.view_number % state->num_servers));
+            state->view_number = redirect_message.view_number;
+            state->last_was_rejected = true;
+            state->last_was_timeout = false;
+            state->pending = false;
+        }
+        return 0;
+    }
+
     if (!state->pending)
         return -1;
 
     if (type != MESSAGE_TYPE_REPLY)
         return -1;
 
-    ReplyMessage reply_message;
+    ReplyMessage message;
     if (msg.len != sizeof(ReplyMessage))
         return -1;
-    memcpy(&reply_message, msg.ptr, sizeof(reply_message));
+    memcpy(&message, msg.ptr, sizeof(message));
+
+    // Ignore stale replies from previous requests. After a timeout
+    // the client moves to a new view and sends a new request, but
+    // the old leader may still deliver a reply for the old request
+    // on the previous connection. Without this check the client
+    // would accept the stale result for the wrong operation.
+    if (message.request_id != state->request_id)
+        return 0;
 
     {
         Time now = get_current_time();
-        CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)",
+        char oper_buf[64];
-            TIME_VAL(now),
+        kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
-            reply_message.rejected ? "true" : "false");
+        if (message.rejected) {
+            client_log(state, now, "RECV REPLY", "key=%.16s %s -> REJECTED", state->last_oper.key, oper_buf);
+        } else {
+            char result_buf[64];
+            kvstore_snprint_result(result_buf, sizeof(result_buf), message.result);
+            client_log(state, now, "RECV REPLY", "key=%.16s %s -> %s", state->last_oper.key, oper_buf, result_buf);
+        }
     }
 
+    state->last_result = message.result;
+    state->last_was_timeout = false;
+    state->last_was_rejected = message.rejected;
     state->pending = false;
     return 0;
 }
@@ -92,35 +179,9 @@ int client_init(void *state_, int argc, char **argv,
 
     state->view_number = 0;
     state->request_id = 0;
+    state->reconnect_time = 0;
 
-    // Load or generate a persistent client_id from disk.
+    state->client_id = next_client_id++;
-    // This ensures the client_id survives process restarts.
-    {
-        string path = S("client_id");
-        Handle fd;
-        bool loaded = false;
-        if (file_exists(path)) {
-            if (file_open(path, &fd) == 0) {
-                file_set_offset(fd, 0);
-                uint64_t saved_id;
-                if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) {
-                    state->client_id = saved_id;
-                    loaded = true;
-                }
-                file_close(fd);
-            }
-        }
-        if (!loaded) {
-            Time now = get_current_time();
-            state->client_id = (uint64_t)now;
-            if (file_open(path, &fd) == 0) {
-                file_set_offset(fd, 0);
-                file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id));
-                file_sync(fd);
-                file_close(fd);
-            }
-        }
-    }
 
     // Connect to all known servers
     for (int i = 0; i < state->num_servers; i++) {
@@ -133,8 +194,7 @@ int client_init(void *state_, int argc, char **argv,
 
     {
         Time now = get_current_time();
-        CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d",
+        client_log(state, now, "INIT", "servers=%d leader=%d", state->num_servers, leader_idx(state));
-            TIME_VAL(now), state->num_servers, leader_idx(state));
     }
 
     *timeout = 0;
@@ -155,20 +215,24 @@ int client_tick(void *state_, void **ctxs,
     int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
 
     for (int i = 0; i < num_events; i++) {
+
         if (events[i].type == EVENT_DISCONNECT) {
             int conn_idx = events[i].conn_idx;
             int tag = tcp_get_tag(&state->tcp, conn_idx);
             if (tag == leader_idx(state) && state->pending) {
                 Time now = get_current_time();
-                CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request",
+                client_log(state, now, "DISCONNECT", "key=%.16s lost leader (node %d)", state->last_oper.key, leader_idx(state));
-                    TIME_VAL(now), leader_idx(state));
+                state->last_was_timeout = true;
+                state->last_was_rejected = false;
                 state->pending = false;
             }
             tcp_close(&state->tcp, conn_idx);
             continue;
         }
+
         if (events[i].type != EVENT_MESSAGE)
             continue;
+
         int conn_idx = events[i].conn_idx;
 
         for (;;) {
@@ -201,10 +265,16 @@ int client_tick(void *state_, void **ctxs,
     if (state->pending) {
         Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
         if (request_deadline <= now) {
-            CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server",
+
-                TIME_VAL(now), leader_idx(state),
+            {
-                (unsigned long)state->view_number);
+                char oper_buf[64];
+                kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
+                client_log(state, now, "TIMEOUT", "key=%.16s %s", state->last_oper.key, oper_buf);
+            }
+
             state->view_number++;
+            state->last_was_timeout = true;
+            state->last_was_rejected = false;
             state->pending = false;
         }
     }
@@ -213,15 +283,15 @@ int client_tick(void *state_, void **ctxs,
 
         int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
         if (conn_idx < 0) {
-            // Leader connection not available, try reconnecting
+            if (state->reconnect_time <= now) {
-            {
+                tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
-                CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting",
+                state->reconnect_time = now + HEARTBEAT_INTERVAL_SEC * 1000000000ULL;
-                    TIME_VAL(now), leader_idx(state));
             }
-            tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
         } else {
+
             // Now start a new operation
             state->request_id++;
+            state->last_oper = random_oper();
 
             RequestMessage request_message = {
                 .base = {
@@ -229,7 +299,7 @@ int client_tick(void *state_, void **ctxs,
                     .type    = MESSAGE_TYPE_REQUEST,
                     .length  = sizeof(RequestMessage),
                 },
-                .oper = OPERATION_A,
+                .oper = state->last_oper,
                 .client_id = state->client_id,
                 .request_id = state->request_id,
             };
@@ -240,11 +310,9 @@ int client_tick(void *state_, void **ctxs,
             byte_queue_write(output, &request_message, request_message.base.length);
 
             {
-                CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)",
+                char oper_buf[64];
-                    TIME_VAL(now), leader_idx(state),
+                kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
-                    (unsigned long)state->view_number,
+                client_log(state, now, "SEND REQUEST", "key=%.16s %s", state->last_oper.key, oper_buf);
-                    (unsigned long)state->client_id,
-                    (unsigned long)state->request_id);
             }
 
             state->pending = true;
@@ -252,10 +320,15 @@ int client_tick(void *state_, void **ctxs,
         }
     }
 
-    // Set timeout based on pending request deadline
+    // Set timeout based on pending request deadline or reconnection delay
     Time deadline = INVALID_TIME;
     if (state->pending) {
         nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
+    } else {
+        int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
+        if (conn_idx < 0 && state->reconnect_time > now) {
+            nearest_deadline(&deadline, state->reconnect_time);
+        }
     }
     *timeout = deadline_to_timeout(deadline, now);
     if (pcap < TCP_POLL_CAPACITY)
@@ -268,6 +341,11 @@ int client_free(void *state_)
 {
     ClientState *state = state_;
 
+    {
+        Time now = get_current_time();
+        client_log_simple(state, now, "CRASHED");
+    }
+
     tcp_context_free(&state->tcp);
     return 0;
 }

vsr/client.h

@@ -4,6 +4,8 @@
 #include <lib/tcp.h>
 #include <lib/basic.h>
 
+#include <state_machine/kvstore.h>
+
 #include "config.h"
 
 typedef struct {
@@ -14,6 +16,14 @@ typedef struct {
     bool pending;
     Time request_time; // When the current request was sent
 
+    // The operation sent in the current pending request (for logging)
+    KVStoreOper last_oper;
+
+    // Linearizability checker support
+    KVStoreResult last_result;
+    bool          last_was_timeout;
+    bool          last_was_rejected;
+
     Address server_addrs[NODE_LIMIT];
     int num_servers;
 
@@ -22,6 +32,8 @@ typedef struct {
     uint64_t client_id;
     uint64_t request_id;
 
+    Time reconnect_time; // Earliest time to retry connecting to leader
+
 } ClientState;
 
 struct pollfd;

vsr/client_table.c

@@ -28,7 +28,7 @@ ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_i
     return NULL;
 }
 
-int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx)
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
 {
     if (client_table->count == client_table->capacity) {
         int n = 2 * client_table->capacity;
@@ -44,7 +44,7 @@ int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t req
         .client_id = client_id,
         .last_request_id = request_id,
         .pending = true,
-        .conn_idx = conn_idx,
+        .conn_tag = conn_tag,
     };
     return 0;
 }

vsr/client_table.h

@@ -3,14 +3,17 @@
 
 #include <stdint.h>
 
-#include <state_machine/state_machine.h>
+#include <state_machine/kvstore.h>
 
 typedef struct {
     uint64_t        client_id;
     uint64_t        last_request_id;
-    OperationResult last_result;
+    KVStoreResult   last_result;
-    bool            pending;
+    bool            pending; // Only meaningful on the leader that received
-    int             conn_idx;
+                             // the REQUEST. After a view change, a new leader
+                             // may find stale entries with pending=false from
+                             // a previous view when it was leader before.
+    int             conn_tag;
 } ClientTableEntry;
 
 typedef struct {
@@ -22,7 +25,7 @@ typedef struct {
 void client_table_init(ClientTable *client_table);
 void client_table_free(ClientTable *client_table);
 ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
-int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx);
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
 int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
 
 #endif // CLIENT_TABLE_INCLUDED

vsr/config.h

@@ -5,7 +5,9 @@
 #define FUTURE_LIMIT 32
 #define HEARTBEAT_INTERVAL_SEC 1
 #define PRIMARY_DEATH_TIMEOUT_SEC 5
+#define VIEW_CHANGE_TIMEOUT_SEC 10
 #define RECOVERY_TIMEOUT_SEC 3
 #define RECOVERY_ATTEMPT_LIMIT 10
+#define STATE_TRANSFER_TIMEOUT_SEC 2
 
 #endif // CONFIG_INCLUDED

vsr/invariant_checker.c

@@ -0,0 +1,395 @@
+// VSR invariant checker with external shadow log tracking.
+//
+// This file runs in the main simulation loop outside Quakey-scheduled
+// processes. It includes node.h for struct definitions, then restores
+// real allocators since mock_malloc/realloc/free abort outside process
+// context.
+
+#include "node.h"
+
+#include <assert.h>
+
+// Restore real allocators (see checker/linearizability.c for precedent).
+#undef malloc
+#undef realloc
+#undef free
+#include <stdlib.h>
+
+// These helpers are static in node.c; duplicated here for the checker.
+
+static int self_idx(NodeState *state)
+{
+    for (int i = 0; i < state->num_nodes; i++)
+        if (addr_eql(state->node_addrs[i], state->self_addr))
+            return i;
+    UNREACHABLE;
+}
+
+static int leader_idx(NodeState *state)
+{
+    return state->view_number % state->num_nodes;
+}
+
+static bool is_leader(NodeState *state)
+{
+    if (state->status == STATUS_RECOVERY)
+        return false;
+    return self_idx(state) == leader_idx(state);
+}
+
+static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
+{
+    if (ic->shadow_count == ic->shadow_capacity) {
+        int n = 2 * ic->shadow_capacity;
+        if (n < 8)
+            n = 8;
+        KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
+        if (p == NULL)
+            return -1;
+        ic->shadow_log = p;
+        ic->shadow_capacity = n;
+    }
+    ic->shadow_log[ic->shadow_count++] = oper;
+    return 0;
+}
+
+void invariant_checker_init(InvariantChecker *ic)
+{
+    ic->last_min_commit = -1;
+    ic->last_max_commit = -1;
+    for (int i = 0; i < NODE_LIMIT; i++)
+        ic->prev_status[i] = STATUS_NORMAL;
+    ic->shadow_log = NULL;
+    ic->shadow_count = 0;
+    ic->shadow_capacity = 0;
+}
+
+void invariant_checker_free(InvariantChecker *ic)
+{
+    fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
+        ic->shadow_count);
+    free(ic->shadow_log);
+}
+
+void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
+{
+    int min_commit = -1;
+    int max_commit = -1;
+
+    bool primary = false;
+    uint64_t primary_view_number = 0;
+
+    bool min_commit_just_recovered = false;
+
+    uint64_t max_view_number = 0;
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i])
+            max_view_number = MAX(max_view_number, nodes[i]->view_number);
+    }
+
+    for (int i = 0; i < num_nodes; i++) {
+        NodeState *n = nodes[i];
+        if (n == NULL || n->status == STATUS_RECOVERY)
+            continue;
+
+        if (min_commit < 0 || min_commit > n->commit_index) {
+            min_commit = n->commit_index;
+            min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY);
+        }
+
+        if (max_commit < 0 || max_commit < n->commit_index) {
+            max_commit = n->commit_index;
+        }
+
+        if (is_leader(n) && n->view_number > primary_view_number) {
+            primary = true;
+            primary_view_number = n->view_number;
+        }
+    }
+
+    // If the primary isn't up to date, it's not the
+    // real primary.
+    if (primary_view_number < max_view_number)
+        primary = false;
+
+    if (min_commit < 0) {
+        assert(ic->last_min_commit == -1);
+    } else {
+        // The minimum number of committed entries should
+        // only increase, but there are some corner-cases
+        // when this is not true.
+        // When a node completes the recovery state, its
+        // log is technically outdated as it was sent some
+        // point in the past. If operations are committed
+        // while the recovery response is in transit over
+        // the network, the minimum number of committed
+        // entries will decrease.
+        if (!min_commit_just_recovered) {
+            assert(ic->last_min_commit <= min_commit);
+        }
+    }
+
+    if (max_commit < 0) {
+        assert(ic->last_max_commit == -1);
+    } else {
+        // The maximum number of committed entries
+        // should only increase. The primary generally
+        // has more committed entries than replicas. If
+        // the primary dies, the maximum commit number
+        // of the live nodes may decrease. This is still
+        // okay since the new primary will commit up to
+        // that point as the view change completes. This
+        // implies that the maximum commit number should
+        // always increase unless there is no primary.
+        if (!primary) {
+            max_commit = ic->last_max_commit;
+        } else {
+            //assert(ic->last_max_commit <= max_commit);
+        }
+    }
+
+    ic->last_min_commit = min_commit;
+    ic->last_max_commit = max_commit;
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i])
+            ic->prev_status[i] = nodes[i]->status;
+    }
+
+    for (int i = 0; i < num_nodes; i++) {
+        NodeState *s = nodes[i];
+        if (s == NULL)
+            continue;
+
+        // 1. commit_index <= log.count
+        //    A node cannot have committed more entries than it has in its log.
+        if (s->commit_index > s->log.count) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n",
+                i, s->commit_index, s->log.count);
+            __builtin_trap();
+        }
+
+        // 2. commit_index >= 0
+        if (s->commit_index < 0) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n",
+                i, s->commit_index);
+            __builtin_trap();
+        }
+
+        // 4. Future buffer count is in valid range.
+        if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n",
+                i, s->num_future, FUTURE_LIMIT);
+            __builtin_trap();
+        }
+
+        // 7. last_normal_view <= view_number
+        //    The most recent view in which this node was in NORMAL status
+        //    cannot exceed its current view number.
+        if (s->last_normal_view > s->view_number) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: last_normal_view (%lu) > view_number (%lu)\n",
+                i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
+            __builtin_trap();
+        }
+
+        // 8. When status is NORMAL, last_normal_view must equal view_number.
+        //    Every transition to NORMAL status sets last_normal_view = view_number.
+        //    If they diverge while in NORMAL, a transition forgot to update it.
+        if (s->status == STATUS_NORMAL && s->last_normal_view != s->view_number) {
+            fprintf(stderr, "INVARIANT VIOLATED: node %d: status is NORMAL but "
+                "last_normal_view (%lu) != view_number (%lu)\n",
+                i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
+            __builtin_trap();
+        }
+
+        // 9. Log entry view numbers must not exceed the node's current view.
+        //    Entries are created in the view they were proposed. No entry
+        //    should carry a view number from the future.
+        for (int k = 0; k < s->log.count; k++) {
+            if ((uint64_t)s->log.entries[k].view_number > s->view_number) {
+                fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) "
+                    "> view_number (%lu)\n",
+                    i, k, s->log.entries[k].view_number,
+                    (unsigned long)s->view_number);
+                __builtin_trap();
+            }
+        }
+
+        // 10. For a leader in NORMAL status, every uncommitted log entry
+        //     must have the leader's own vote bit set. The leader always
+        //     votes for its own entries.
+        if (s->status == STATUS_NORMAL && is_leader(s)) {
+            int idx = self_idx(s);
+            for (int k = s->commit_index; k < s->log.count; k++) {
+                if (!(s->log.entries[k].votes & (1 << idx))) {
+                    fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): "
+                        "uncommitted log[%d] missing leader's own vote bit\n",
+                        i, k);
+                    __builtin_trap();
+                }
+            }
+        }
+    }
+
+    // Cross-node invariants
+
+    // 5. At most one leader in normal status per view.
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i]))
+            continue;
+        for (int j = i + 1; j < num_nodes; j++) {
+            if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j]))
+                continue;
+            if (nodes[i]->view_number == nodes[j]->view_number) {
+                fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n",
+                    (unsigned long)nodes[i]->view_number, i, j);
+                __builtin_trap();
+            }
+        }
+    }
+
+    // 6. Committed prefix agreement (State Machine Safety).
+    //    For any two nodes, their logs must agree on all entries up to
+    //    min(commit_index_i, commit_index_j). This is the core safety
+    //    property of VSR: all committed operations are identical across
+    //    replicas.
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL)
+            continue;
+        for (int j = i + 1; j < num_nodes; j++) {
+            if (nodes[j] == NULL)
+                continue;
+
+            int mc = nodes[i]->commit_index;
+            if (nodes[j]->commit_index < mc)
+                mc = nodes[j]->commit_index;
+
+            for (int k = 0; k < mc; k++) {
+                if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(KVStoreOper)) != 0) {
+                    fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d "
+                        "between node %d and node %d\n", k, i, j);
+                    __builtin_trap();
+                }
+            }
+        }
+    }
+
+    ////////////////////////////////////////////////////////////////////
+    // Shadow log: external commit tracking
+    ////////////////////////////////////////////////////////////////////
+
+    // Phase 1: Find the observed max commit index and a source node.
+    int observed_max_commit = 0;
+    int source_node_idx = -1;
+
+    for (int i = 0; i < num_nodes; i++) {
+        if (nodes[i] == NULL)
+            continue;
+        if (nodes[i]->status == STATUS_RECOVERY)
+            continue;
+        if (nodes[i]->commit_index > observed_max_commit) {
+            observed_max_commit = nodes[i]->commit_index;
+            source_node_idx = i;
+        }
+    }
+
+    // Phase 2: Append newly committed entries to the shadow log.
+    if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
+
+        NodeState *source = nodes[source_node_idx];
+        assert(source->log.count >= observed_max_commit);
+
+        for (int k = ic->shadow_count; k < observed_max_commit; k++) {
+
+            KVStoreOper *source_oper = &source->log.entries[k].oper;
+
+            // Cross-validate against other live non-recovering nodes
+            // that have also committed this entry.
+            for (int j = 0; j < num_nodes; j++) {
+                if (j == source_node_idx)
+                    continue;
+                if (nodes[j] == NULL)
+                    continue;
+                if (nodes[j]->status == STATUS_RECOVERY)
+                    continue;
+                if (nodes[j]->commit_index <= k)
+                    continue;
+                if (nodes[j]->log.count <= k)
+                    continue;
+
+                if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(KVStoreOper)) != 0) {
+                    fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
+                        "between source node %d and node %d during shadow log append\n",
+                        k, source_node_idx, j);
+                    __builtin_trap();
+                }
+            }
+
+            if (shadow_log_append(ic, *source_oper) < 0) {
+                fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
+                __builtin_trap();
+            }
+        }
+    }
+
+    // Phase 3: Verify shadow log against the cluster.
+
+    // Sub-check A: Committed entries must match the shadow log.
+    for (int k = 0; k < ic->shadow_count; k++) {
+        for (int i = 0; i < num_nodes; i++) {
+            if (nodes[i] == NULL)
+                continue;
+            if (nodes[i]->log.count <= k)
+                continue;
+            if (nodes[i]->commit_index <= k)
+                continue;
+
+            if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
+                char shadow_buf[128], node_buf[128];
+                kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
+                kvstore_snprint_oper(node_buf, sizeof(node_buf), nodes[i]->log.entries[k].oper);
+                fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
+                    "  shadow: %s\n"
+                    "  node:   %s\n",
+                    k, i, shadow_buf, node_buf);
+                __builtin_trap();
+            }
+        }
+    }
+
+    // Sub-check B: When commit regresses, previously committed entries
+    // must still be held by a majority of the cluster.
+    // Recovering nodes are treated like dead nodes: they haven't
+    // restored their log yet and may still recover the entry through
+    // the recovery protocol.
+    if (observed_max_commit < ic->shadow_count) {
+        for (int k = observed_max_commit; k < ic->shadow_count; k++) {
+            int holders = 0;
+            int num_dead = 0;
+
+            for (int i = 0; i < num_nodes; i++) {
+                if (nodes[i] == NULL) {
+                    num_dead++;
+                    continue;
+                }
+                if (nodes[i]->status == STATUS_RECOVERY) {
+                    num_dead++;
+                    continue;
+                }
+                if (nodes[i]->log.count <= k)
+                    continue;
+                if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
+                    holders++;
+            }
+
+            if (holders + num_dead <= num_nodes / 2) {
+                char oper_buf[128];
+                kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
+                fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
+                    "no longer held by majority (holders=%d, dead=%d, total=%d)\n"
+                    "  entry: %s\n",
+                    k, holders, num_dead, num_nodes, oper_buf);
+                __builtin_trap();
+            }
+        }
+    }
+}

vsr/log.c

@@ -13,11 +13,29 @@ void log_init(Log *log)
     log->entries = NULL;
 }
 
+int log_init_from_network(Log *log, void *src, int num)
+{
+    log->count = num;
+    log->capacity = num;
+    log->entries = malloc(num * sizeof(LogEntry));
+    if (log->entries == NULL)
+        return -1;
+    memcpy(log->entries, src, num * sizeof(LogEntry));
+    return 0;
+}
+
 void log_free(Log *log)
 {
     free(log->entries);
 }
 
+void log_move(Log *dst, Log *src)
+{
+    log_free(dst);
+    *dst = *src;
+    log_init(src);
+}
+
 int log_append(Log *log, LogEntry entry)
 {
     if (log->count == log->capacity) {

vsr/log.h

@@ -1,15 +1,16 @@
 #ifndef LOG_INCLUDED
 #define LOG_INCLUDED
 
-#include <state_machine/state_machine.h>
+#include <state_machine/kvstore.h>
 
 #include "config.h"
 
 typedef struct {
-    Operation oper;
+    KVStoreOper oper;
     uint32_t votes;
     int view_number;
     uint64_t client_id;
+    uint64_t request_id;
 } LogEntry;
 
 _Static_assert(NODE_LIMIT <= 32, "");
@@ -21,7 +22,9 @@ typedef struct {
 } Log;
 
 void log_init(Log *log);
+int  log_init_from_network(Log *log, void *src, int num);
 void log_free(Log *log);
+void log_move(Log *dst, Log *src);
 int  log_append(Log *log, LogEntry entry);
 
 #endif // LOG_INCLUDED

vsr/main.c

@@ -4,7 +4,7 @@
 #include <signal.h>
 #include <stdint.h>
 #include <quakey.h>
-
+#include <assert.h>
 #include "node.h"
 #include "client.h"
 
@@ -16,12 +16,23 @@ static void sigint_handler(int sig)
     simulation_running = 0;
 }
 
-int main(void)
+int main(int argc, char **argv)
 {
     signal(SIGINT, sigint_handler);
 
+    QuakeyUInt64 seed = 1;
+    QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
+
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
+            seed = strtoull(argv[++i], NULL, 10);
+        } else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
+            time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
+        }
+    }
+
     Quakey *quakey;
-    int ret = quakey_init(&quakey, 1);
+    int ret = quakey_init(&quakey, seed);
     if (ret < 0)
         return -1;
 
@@ -111,7 +122,18 @@ int main(void)
         node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
     }
 
-    while (simulation_running) {
+    // Limit crashes to 1 node at a time (within fault tolerance of f=1).
+    // This is dynamically adjusted below: crashes are disabled while
+    // any node is still recovering.
+    quakey_set_max_crashes(quakey, 1);
+
+    quakey_network_partitioning(quakey, true);
+
+    InvariantChecker invariant_checker;
+    invariant_checker_init(&invariant_checker);
+
+    while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
+
         quakey_schedule_one(quakey);
 
         NodeState *arr[] = {
@@ -119,9 +141,21 @@ int main(void)
             quakey_node_state(node_2),
             quakey_node_state(node_3),
         };
-        check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0]));
+        invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
+
+        // VR-Revisited Section 8.2: "a replica is considered failed
+        // until it has recovered its state." Disable crashes while
+        // any node is recovering to avoid exceeding f simultaneous
+        // failures (dead + recovering).
+        bool any_recovering = false;
+        for (int i = 0; i < 3; i++) {
+            if (arr[i] && arr[i]->status == STATUS_RECOVERY)
+                any_recovering = true;
+        }
+        quakey_set_max_crashes(quakey, any_recovering ? 0 : 1);
     }
 
+    invariant_checker_free(&invariant_checker);
     quakey_free(quakey);
     return 0;
 }

vsr/node.h

@@ -5,7 +5,7 @@
 #include <lib/basic.h>
 #include <lib/message.h>
 
-#include <state_machine/state_machine.h>
+#include <state_machine/kvstore.h>
 
 #include "log.h"
 #include "config.h"
@@ -28,22 +28,31 @@ enum {
     // Recovery Protocol
     MESSAGE_TYPE_RECOVERY,
     MESSAGE_TYPE_RECOVERY_RESPONSE,
+
+    // State Transfer Protocol
+    MESSAGE_TYPE_GET_STATE,
+    MESSAGE_TYPE_NEW_STATE,
+
+    // Client Redirect
+    MESSAGE_TYPE_REDIRECT,
 };
 
 typedef struct {
     MessageHeader base;
-    Operation oper;
+    KVStoreOper oper;
     uint64_t client_id;
     uint64_t request_id;
 } RequestMessage;
 
 typedef struct {
     MessageHeader base;
-    Operation oper;
+    KVStoreOper oper;
     int sender_idx;
     int log_index;
     int commit_index;
     uint64_t view_number;
+    uint64_t client_id;
+    uint64_t request_id;
 } PrepareMessage;
 
 typedef struct {
@@ -61,7 +70,8 @@ typedef struct {
 typedef struct {
     MessageHeader base;
     bool rejected;
-    OperationResult result;
+    KVStoreResult result;
+    uint64_t request_id;
 } ReplyMessage;
 
 typedef struct {
@@ -103,6 +113,26 @@ typedef struct {
     int sender_idx;
 } RecoveryResponseMessage;
 
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+    int op_number;      // Requester's current log count
+    int sender_idx;
+} GetStateMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+    int op_number;      // Number of log entries that follow
+    int commit_index;
+    // Followed by: LogEntry log[op_number]
+} NewStateMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+} RedirectMessage;
+
 typedef enum {
     STATUS_NORMAL,
     STATUS_CHANGE_VIEW,
@@ -120,27 +150,28 @@ typedef struct {
     Status status;
 
     ClientTable client_table;
+    int next_client_tag;
 
     uint64_t view_number;
+    uint64_t last_normal_view;  // Latest view where status was NORMAL
 
     // These fields are used in recovery mode
     uint32_t recovery_votes;
     uint64_t recovery_nonce;
-    uint64_t max_view_seen;
+    uint64_t recovery_view;
-    int      latest_primary_idx;
+    Log      recovery_log;
-    bool     received_primary_state;
+    uint64_t recovery_log_view;
-    Log      potential_primary_log;
+    Time     recovery_time;
-    Time     recovery_start_time;
+    int      recovery_commit;
-    int      recovery_attempt_count;
 
     ///////////////////////////////////////////////////////////
     // VIEW CHANGE
 
-    uint32_t begin_view_change_votes;  // Bitmask of received BeginViewChange messages
+    uint32_t view_change_begin_votes;
-    uint32_t do_view_change_votes;     // Bitmask of received DoViewChange messages
+    uint32_t view_change_apply_votes;
-    uint64_t do_view_change_best_old_view;  // Best old_view_number seen in DoViewChange
+    Log      view_change_log; // Best log seen
-    int      do_view_change_best_commit;    // Best commit_index seen
+    uint64_t view_change_old_view;  // Best old_view_number seen in DoViewChange
-    Log      do_view_change_best_log;       // Best log seen
+    int      view_change_commit;    // Best commit_index seen
 
     ///////////////////////////////////////////////////////////
 
@@ -152,11 +183,17 @@ typedef struct {
     PrepareMessage future[FUTURE_LIMIT];
     int num_future;
 
+    bool state_transfer_pending;
+    Time state_transfer_time;
+
     Log log;
 
-    Time last_heartbeat_time;
+    Time heartbeat;
 
-    StateMachine state_machine;
+    KVStore kvstore;
+
+    // Set at each wakeup
+    Time now;
 
 } NodeState;
 
@@ -171,6 +208,19 @@ int node_tick(void *state, void **ctxs,
 
 int node_free(void *state);
 
-void check_vsr_invariants(NodeState **nodes, int num_nodes);
+typedef struct {
+    int last_min_commit;
+    int last_max_commit;
+    Status prev_status[NODE_LIMIT];
+
+    // External shadow log of committed operations (unbounded, dynamically allocated)
+    KVStoreOper *shadow_log;
+    int shadow_count;
+    int shadow_capacity;
+} InvariantChecker;
+
+void invariant_checker_init(InvariantChecker *ic);
+void invariant_checker_free(InvariantChecker *ic);
+void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
 
 #endif // NODE_INCLUDED