cozis/Thesis_Code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 0.9

Browse Source
This commit is contained in:
cozis
2026-02-19 15:28:06 +01:00
parent 868312edf8
commit 238e35b581
31 changed files with 4256 additions and 2253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.sh
+6 -6
View File
@@ -1,6 +1,6 @@
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c vsr/invariant_checker.c state_machine/kvstore.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/kvstore.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/kvstore.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/invariant_checker.c raft/main.c state_machine/kvstore.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION -DDISABLE_SPURIOUS_IO_ERRORS
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/main.c state_machine/kvstore.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/client_table.c raft/main.c state_machine/kvstore.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
lib/file_system.c
+1 -1
View File
@@ -36,7 +36,7 @@ int file_open(string path, Handle *fd)
memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0';
int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
int ret = open(zt, O_RDWR | O_CREAT, 0644);
if (ret < 0)
return -1;
quakey/include/quakey.h
+13
View File
@@ -26,6 +26,8 @@
#include <pthread.h>
#endif
#include <stdbool.h>
typedef struct {} Quakey;
// Function pointers to a simulated program's code
@@ -82,6 +84,9 @@ void *quakey_node_state(QuakeyNode node);
// Schedule and executes one program until it would block, then returns
int quakey_schedule_one(Quakey *quakey);
// Get the current simulated time in nanoseconds
QuakeyUInt64 quakey_current_time(Quakey *quakey);
// Generate a random u64
QuakeyUInt64 quakey_random(void);
@@ -93,6 +98,12 @@ typedef struct {
void quakey_signal(char *name);
int quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
// Limit the number of hosts that can be crashed at the same time.
// Set to 0 for no limit (default).
void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
void quakey_network_partitioning(Quakey *quakey, bool enable);
// Access spawned host information
int quakey_num_hosts(Quakey *quakey);
void *quakey_host_state(Quakey *quakey, int idx); // Returns NULL if host is dead
@@ -135,6 +146,7 @@ int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_fcntl(int fd, int cmd, int flags);
int mock_close(int fd);
int mock_access(const char *path, int mode);
int mock_ftruncate(int fd, size_t new_size);
int mock_fstat(int fd, struct stat *buf);
int mock_read(int fd, char *dst, int len);
@@ -183,6 +195,7 @@ void mock_free(void *ptr);
#define open mock_open
#define fcntl mock_fcntl
#define close mock_close
#define access mock_access
#define ftruncate mock_ftruncate
#define CreateFileW mock_CreateFileW
#define CloseHandle mock_CloseHandle
quakey/src/quakey.c
+441 -29
View File
@@ -11,6 +11,10 @@
#define TODO __builtin_trap()
#define SIM_TRACE(sim, fmt, ...) \
fprintf(stderr, "SIM: [%.3fs] " fmt "\n", \
(double)(sim)->current_time / 1e9, ##__VA_ARGS__)
#ifdef NDEBUG
#define UNREACHABLE {}
#define ASSERT(X) {}
@@ -317,6 +321,7 @@ typedef enum {
EVENT_TYPE_DATA,
EVENT_TYPE_WAKEUP,
EVENT_TYPE_RESTART,
EVENT_TYPE_PARTITION,
} TimeEventType;
typedef struct {
@@ -355,6 +360,17 @@ typedef struct {
#define SIM_SIGNAL_LIMIT 8
typedef struct {
Host *a;
Host *b;
} HostPair;
typedef struct {
int count;
int capacity;
HostPair *pairs;
} HostPairList;
struct Sim {
uint64_t seed;
@@ -378,15 +394,30 @@ struct Sim {
int num_signals;
int head_signal;
QuakeySignal signals[SIM_SIGNAL_LIMIT];
// Network partition simulation. The partition list holds pairs
// of hosts whose link is currently broken. The target_partition
// holds the desired end state. A periodic timer event gradually
// breaks or repairs links to converge toward the target. Once
// reached, a new random target is generated.
HostPairList partition;
HostPairList target_partition;
// Maximum number of hosts that can be dead at the same time.
int max_crashes;
bool network_partitioning;
};
static void time_event_wakeup(Sim *sim, Nanos time, Host *host);
static void time_event_connect(Sim *sim, Nanos time, Desc *desc);
static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst);
static void time_event_send_data(Sim *sim, Nanos time, Desc *desc);
static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count);
static void time_event_free(TimeEvent *event);
static void time_event_restart(Sim *sim, Nanos time, Host *host);
static void time_event_partition(Sim *sim, Nanos time);
static void remove_events_targeting_desc(Sim *sim, Desc *desc);
static b32 remove_wakeup_event(Sim *sim, Host *host);
/////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////
@@ -714,11 +745,6 @@ static b32 socket_queue_empty(SocketQueue *queue)
return queue->used == 0;
}
static b32 socket_queue_used(SocketQueue *queue)
{
return queue->used;
}
/////////////////////////////////////////////////////////////////
// Accept Queue Code
@@ -1003,9 +1029,10 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg)
TODO;
}
if (host->poll_timeout > 0)
if (host->poll_timeout > 0) {
remove_wakeup_event(host->sim, host);
time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
else if (host->poll_timeout == 0)
} else if (host->poll_timeout == 0)
host->timedout = true; // Immediate timeout, don't create event at current_time
}
@@ -1035,8 +1062,11 @@ static void host_free(Host *host)
}
// Remove all events associated with a host (WAKEUP events and events
// whose source or destination descriptors belong to this host)
static void remove_events_for_host(Sim *sim, Host *host)
// whose source or destination descriptors belong to this host).
// If skip_restart is true, RESTART events are kept (used during
// restart failure to avoid removing the event currently being
// processed by process_events_at_current_time).
static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
{
int i = 0;
while (i < sim->num_events) {
@@ -1045,7 +1075,7 @@ static void remove_events_for_host(Sim *sim, Host *host)
if (event->type == EVENT_TYPE_WAKEUP && event->host == host)
remove = true;
if (event->type == EVENT_TYPE_RESTART && event->host == host)
if (!skip_restart && event->type == EVENT_TYPE_RESTART && event->host == host)
remove = true;
if (event->src_desc && event->src_desc->host == host)
remove = true;
@@ -1061,6 +1091,11 @@ static void remove_events_for_host(Sim *sim, Host *host)
}
}
static void remove_events_for_host(Sim *sim, Host *host)
{
remove_events_for_host_ex(sim, host, false);
}
static int count_dead(Sim *sim)
{
int n = 0;
@@ -1088,6 +1123,11 @@ static void host_crash(Host *host)
// Remove all pending events for this host
remove_events_for_host(sim, host);
// Let the application clean up (e.g. log the crash)
host___ = host;
host->free_func(host->state);
host___ = NULL;
// Free application state
free(host->state);
host->state = NULL;
@@ -1098,7 +1138,6 @@ static void host_crash(Host *host)
host->poll_count = 0;
host->poll_timeout = -1;
printf("Quakey: Host crash (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
}
// Restart a crashed host: re-initialize from saved config,
@@ -1163,17 +1202,37 @@ static void host_restart(Host *host)
);
host___ = NULL;
if (ret < 0) {
// If the node fails to restart, mark it as dead again
// Clean up any descriptors created during the failed init
for (int i = 0; i < HOST_DESC_LIMIT; i++) {
if (host->desc[i].type != DESC_EMPTY)
desc_free(sim, &host->desc[i], true);
}
host->num_desc = 0;
// Skip removing RESTART events: this function is called from
// time_event_process (EVENT_TYPE_RESTART), so the current
// restart event is still in the array being iterated by
// process_events_at_current_time. Removing it here would cause
// the outer loop's swap-removal to discard an unrelated event.
remove_events_for_host_ex(sim, host, true);
free(host->state);
host->state = NULL;
host->dead = true;
// Schedule another restart attempt. Init can fail due to
// transient I/O errors from fault injection; permanently
// killing the host would be unrealistic since a real
// process supervisor would keep retrying.
Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL);
time_event_restart(sim, sim->current_time + restart_delay, host);
return;
}
if (host->poll_timeout > 0)
if (host->poll_timeout > 0) {
remove_wakeup_event(sim, host);
time_event_wakeup(sim, sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
else if (host->poll_timeout == 0)
} else if (host->poll_timeout == 0)
host->timedout = true;
printf("Quakey: Host restart (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
}
static b32 host_is_linux(Host *host)
@@ -1343,12 +1402,16 @@ static void host_update(Host *host)
);
host___ = NULL;
if (ret < 0) {
TODO;
host_crash(host);
Nanos restart_delay = 1000000000ULL + (sim_random(host->sim) % 9000000000ULL);
time_event_restart(host->sim, host->sim->current_time + restart_delay, host);
return;
}
if (host->poll_timeout > 0)
if (host->poll_timeout > 0) {
remove_wakeup_event(host->sim, host);
time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host);
else if (host->poll_timeout == 0)
} else if (host->poll_timeout == 0)
host->timedout = true; // Immediate timeout, don't create event at current_time
}
@@ -1445,9 +1508,15 @@ static int host_create_pipe(Host *host, int *desc_idxs)
return HOST_ERROR_FULL;
Desc *desc_1 = &host->desc[desc_idx_1];
// Mark the first slot as in-use before searching for the second,
// otherwise find_empty_desc_struct returns the same index twice.
desc_1->type = DESC_PIPE;
int desc_idx_2 = find_empty_desc_struct(host);
if (desc_idx_2 < 0)
if (desc_idx_2 < 0) {
desc_1->type = DESC_EMPTY;
return HOST_ERROR_FULL;
}
Desc *desc_2 = &host->desc[desc_idx_2];
desc_1->type = DESC_PIPE;
@@ -1894,7 +1963,7 @@ static int send_inner(Desc *desc, char *src, int len)
uint64_t rng = sim_random(sim);
latency = 1000000 + (rng % 99000000); // between 1ms and 100ms
#endif
time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc);
time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc, ret);
return ret;
}
@@ -1916,7 +1985,7 @@ static int host_read(Host *host, int desc_idx, char *dst, int len)
desc->type == DESC_PIPE) {
num = recv_inner(desc, dst, len);
} else if (desc->type == DESC_FILE) {
#ifdef FAULT_INJECTION
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
uint64_t roll = sim_random(host->sim) % 1000;
if (roll == 0) return HOST_ERROR_IO;
#endif
@@ -1960,7 +2029,7 @@ static int host_write(Host *host, int desc_idx, char *src, int len)
desc->type == DESC_PIPE) {
num = send_inner(desc, src, len);
} else if (desc->type == DESC_FILE) {
#ifdef FAULT_INJECTION
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
uint64_t roll = sim_random(host->sim) % 1000;
if (roll == 0) return HOST_ERROR_IO;
if (roll == 1) return HOST_ERROR_NOSPC;
@@ -2000,9 +2069,11 @@ static int host_recv(Host *host, int desc_idx, char *dst, int len)
Desc *desc = &host->desc[desc_idx];
#ifdef FAULT_INJECTION
if (len > 0) {
Sim *sim = desc->host->sim;
uint64_t rng = sim_random(sim);
len = 1 + (rng % len);
}
#endif
int num = 0;
@@ -2024,9 +2095,11 @@ static int host_send(Host *host, int desc_idx, char *src, int len)
Desc *desc = &host->desc[desc_idx];
#ifdef FAULT_INJECTION
if (len > 0) {
Sim *sim = desc->host->sim;
uint64_t rng = sim_random(sim);
len = 1 + (rng % len);
}
#endif
int num = 0;
@@ -2132,7 +2205,7 @@ static int host_fsync(Host *host, int desc_idx)
if (desc->type != DESC_FILE)
return HOST_ERROR_BADIDX;
#ifdef FAULT_INJECTION
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
uint64_t roll = sim_random(host->sim) % 100;
if (roll == 0)
return HOST_ERROR_IO;
@@ -2251,6 +2324,19 @@ static b32 remove_connect_event(Sim *sim, Desc *desc)
return true;
}
static b32 remove_wakeup_event(Sim *sim, Host *host)
{
int i = 0;
while (i < sim->num_events && (sim->events[i].type != EVENT_TYPE_WAKEUP || sim->events[i].host != host))
i++;
if (i == sim->num_events)
return false;
sim->events[i] = sim->events[--sim->num_events];
return true;
}
// Remove all events that target a specific descriptor (DISCONNECT and DATA events)
static void remove_events_targeting_desc(Sim *sim, Desc *desc)
{
@@ -2292,7 +2378,7 @@ static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst)
append_event(sim, event);
}
static void time_event_send_data(Sim *sim, Nanos time, Desc *desc)
static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count)
{
TimeEvent event = {
.type = EVENT_TYPE_DATA,
@@ -2300,7 +2386,7 @@ static void time_event_send_data(Sim *sim, Nanos time, Desc *desc)
.host = NULL,
.src_desc = NULL,
.dst_desc = desc->peer,
.data_count = socket_queue_used(desc->output),
.data_count = count,
.data_queue = socket_queue_ref(desc->output),
.rst = false,
};
@@ -2328,12 +2414,52 @@ static void time_event_restart(Sim *sim, Nanos time, Host *host)
append_event(sim, event);
}
static void time_event_partition(Sim *sim, Nanos time)
{
TimeEvent event = {
.type = EVENT_TYPE_PARTITION,
.time = time,
};
append_event(sim, event);
}
static int sim_find_host(Sim *sim, Addr addr);
static int change_target_partition(Sim *sim);
static bool break_or_repair_link(Sim *sim);
static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b);
static Nanos pick_partition_delay(Sim *sim, bool longer)
{
Nanos min_delay = 1000000000; // 1s
Nanos max_delay = 10000000000; // 10s
if (longer) {
min_delay = 10000000000;
max_delay = 20000000000;
}
return min_delay + sim_random(sim) % (max_delay - min_delay);
}
static b32 time_event_process(TimeEvent *event, Sim *sim)
{
b32 consumed = true;
switch (event->type) {
case EVENT_TYPE_PARTITION:
{
// Try to move one step toward the target partition.
// If current already matches target, pick a new target.
if (!break_or_repair_link(sim)) {
if (sim->network_partitioning) {
if (change_target_partition(sim) < 0) {
TODO;
}
}
}
// Reschedule for the next partition step
event->time = sim->current_time + pick_partition_delay(sim, true);
consumed = false;
}
break;
case EVENT_TYPE_CONNECT:
{
Desc *src_desc = event->src_desc;
@@ -2346,6 +2472,13 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
}
Host *peer_host = sim->hosts[idx];
if (hosts_are_partitioned(sim, src_desc->host, peer_host)) {
//SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
// src_desc->host->name, peer_host->name);
src_desc->connect_status = CONNECT_STATUS_NOHOST;
break;
}
Desc *peer = host_find_desc_bound_to(peer_host,
src_desc->connect_addr, src_desc->connect_port);
if (peer == NULL) {
@@ -2410,6 +2543,242 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
/////////////////////////////////////////////////////////////////
// Sim Code
static void
host_pair_list_init(HostPairList *list)
{
list->count = 0;
list->capacity = 0;
list->pairs = NULL;
}
static void
host_pair_list_free(HostPairList *list)
{
free(list->pairs);
}
static bool
host_pair_list_exists(HostPairList *list, Host *a, Host *b)
{
for (int i = 0; i < list->count; i++)
if ((list->pairs[i].a == a && list->pairs[i].b == b) ||
(list->pairs[i].a == b && list->pairs[i].b == a))
return true;
return false;
}
static int
host_pair_list_add(HostPairList *list, Host *a, Host *b)
{
if (!host_pair_list_exists(list, a, b)) {
if (list->count == list->capacity) {
int n = 2 * list->count;
if (n < 8) n = 8;
void *p = realloc(list->pairs, n * sizeof(HostPair));
if (p == NULL)
return -1;
list->capacity = n;
list->pairs = p;
}
list->pairs[list->count++] = (HostPair) { a, b };
}
return 0;
}
static void
host_pair_list_remove_at(HostPairList *list, int idx)
{
list->pairs[idx] = list->pairs[--list->count];
}
// Generate a new random target partition by assigning hosts to
// random buckets. Hosts in the same bucket can communicate; hosts
// in different buckets have their link marked as broken.
static int change_target_partition(Sim *sim)
{
HostPairList list;
host_pair_list_init(&list);
#define MAX_BUCKETS 3
#define MAX_NODES_PER_BUCKET 8
int num_buckets = 1 + sim_random(sim) % MAX_BUCKETS;
Host *buckets[MAX_BUCKETS][MAX_NODES_PER_BUCKET];
int bucket_sizes[MAX_BUCKETS];
for (int i = 0; i < num_buckets; i++)
bucket_sizes[i] = 0;
// Randomly assign each host to a bucket
for (int i = 0; i < sim->num_hosts; i++) {
int bucket_index = sim_random(sim) % num_buckets;
buckets[bucket_index][bucket_sizes[bucket_index]++] = sim->hosts[i];
}
// Every cross-bucket pair is a broken link in the target
for (int i = 0; i < num_buckets; i++) {
for (int j = 0; j < bucket_sizes[i]; j++) {
for (int k = 0; k < num_buckets; k++) {
if (k == i) continue;
for (int g = 0; g < bucket_sizes[k]; g++) {
Host *a = buckets[i][j];
Host *b = buckets[k][g];
if (host_pair_list_add(&list, a, b) < 0) {
host_pair_list_free(&list);
return -1;
}
}
}
}
}
host_pair_list_free(&sim->target_partition);
sim->target_partition = list;
//SIM_TRACE(sim, "PARTITION: new target partition with %d broken links (%d buckets)",
// list.count, num_buckets);
return 0;
}
// Pick a random pair that exists in 'left' but not in 'right'.
// Returns the index into 'left', or -1 if no such pair exists.
static int pick_random_from_left_not_in_right(Sim *sim,
HostPairList *left, HostPairList *right)
{
int *arr = malloc(left->count * sizeof(int));
if (arr == NULL) {
TODO; // Allowing this to return error would make execution not deterministic
}
int num = 0;
for (int i = 0; i < left->count; i++) {
HostPair pair = left->pairs[i];
if (!host_pair_list_exists(right, pair.a, pair.b)) {
arr[num++] = i;
}
}
int idx;
if (num == 0) {
idx = -1;
} else {
idx = arr[sim_random(sim) % num];
}
free(arr);
return idx;
}
// Forcefully reset a connection socket and notify its peer.
// Removes any pending time events that reference either end.
static void reset_conn(Sim *sim, Desc *desc)
{
assert(desc->type == DESC_SOCKET_C);
if (desc->connect_status == CONNECT_STATUS_WAIT ||
desc->connect_status == CONNECT_STATUS_DONE) {
remove_events_targeting_desc(sim, desc);
if (desc->peer) {
assert(desc->peer->type == DESC_SOCKET_C
|| desc->peer->type == DESC_SOCKET_L);
if (desc->peer->type == DESC_SOCKET_C) {
remove_events_targeting_desc(sim, desc->peer);
desc->peer->connect_status = CONNECT_STATUS_RESET;
desc->peer->peer = NULL;
} else {
accept_queue_remove(&desc->peer->accept_queue, desc);
}
desc->peer = NULL;
}
desc->connect_status = CONNECT_STATUS_RESET;
}
}
// Reset all active connections between two hosts (both directions)
static void drop_conns_between_hosts(Sim *sim, Host *host, Host *peer)
{
for (int i = 0, j = 0; j < host->num_desc; i++) {
Desc *desc = &host->desc[i];
if (desc->type == DESC_EMPTY)
continue;
j++;
if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == peer)
reset_conn(sim, desc);
}
for (int i = 0, j = 0; j < peer->num_desc; i++) {
Desc *desc = &peer->desc[i];
if (desc->type == DESC_EMPTY)
continue;
j++;
if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == host)
reset_conn(sim, desc);
}
}
// Break a link that is in the target but not yet in current
static bool break_link(Sim *sim)
{
int idx = pick_random_from_left_not_in_right(sim,
&sim->target_partition, &sim->partition);
if (idx < 0)
return false;
HostPair pair = sim->target_partition.pairs[idx];
host_pair_list_add(&sim->partition, pair.a, pair.b);
drop_conns_between_hosts(sim, pair.a, pair.b);
//SIM_TRACE(sim, "PARTITION: break link %s <-> %s (%d broken links)",
// pair.a->name, pair.b->name, sim->partition.count);
return true;
}
// Repair a link that is broken in current but not in the target
static bool repair_link(Sim *sim)
{
int idx = pick_random_from_left_not_in_right(sim,
&sim->partition, &sim->target_partition);
if (idx < 0)
return false;
//HostPair pair = sim->partition.pairs[idx];
host_pair_list_remove_at(&sim->partition, idx);
//SIM_TRACE(sim, "PARTITION: repair link %s <-> %s (%d broken links)",
// pair.a->name, pair.b->name, sim->partition.count);
return true;
}
// Try to move one step toward the target: randomly attempt a
// break or repair first, falling back to the other on failure.
// Returns false when current already matches the target.
static bool break_or_repair_link(Sim *sim)
{
if (sim_random(sim) & 1) {
if (break_link(sim))
return true;
if (repair_link(sim))
return true;
} else {
if (repair_link(sim))
return true;
if (break_link(sim))
return true;
}
return false;
}
static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b)
{
return host_pair_list_exists(&sim->partition, a, b);
}
static void sim_init(Sim *sim, uint64_t seed)
{
sim->seed = seed;
@@ -2422,10 +2791,27 @@ static void sim_init(Sim *sim, uint64_t seed)
sim->events = NULL;
sim->num_signals = 0;
sim->head_signal = 0;
sim->max_crashes = 0;
sim->network_partitioning = true;
host_pair_list_init(&sim->partition);
host_pair_list_init(&sim->target_partition);
time_event_partition(sim, pick_partition_delay(sim, false));
}
static void sim_network_partitioning(Sim *sim, bool enable)
{
sim->network_partitioning = enable;
if (!enable) {
host_pair_list_free(&sim->target_partition);
host_pair_list_init(&sim->target_partition);
}
}
static void sim_free(Sim *sim)
{
host_pair_list_free(&sim->target_partition);
host_pair_list_free(&sim->partition);
for (int i = 0; i < sim->num_hosts; i++)
host_free(sim->hosts[i]);
free(sim->hosts);
@@ -2503,6 +2889,9 @@ static void process_events_at_current_time(Sim *sim)
}
}
}
//if (sim->num_events > 10000)
// SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
}
static int find_first_ready_host(Sim *sim)
@@ -2565,14 +2954,14 @@ static b32 sim_update(Sim *sim)
if (sim->num_hosts > 1) {
uint64_t rng = sim_random(sim);
if ((rng % 10000) == 0) {
// Pick a random live host to crash
int live_count = 0;
for (int i = 0; i < sim->num_hosts; i++)
if (!sim->hosts[i]->dead)
live_count++;
int dead_count = sim->num_hosts - live_count;
// Only crash if at least 2 hosts are alive (keep a majority)
if (live_count >= 2) {
if (dead_count < sim->max_crashes) {
// Pick one of the live hosts at random
int target = sim_random(sim) % live_count;
int j = 0;
@@ -2664,6 +3053,18 @@ void quakey_free(Quakey *quakey)
}
}
void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
{
Sim *sim = (Sim*) quakey;
sim->max_crashes = max_crashes;
}
void quakey_network_partitioning(Quakey *quakey, bool enable)
{
Sim *sim = (Sim*) quakey;
sim_network_partitioning(sim, enable);
}
QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg)
{
return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg);
@@ -2680,6 +3081,12 @@ int quakey_schedule_one(Quakey *quakey)
return sim_update((Sim*) quakey);
}
QuakeyUInt64 quakey_current_time(Quakey *quakey)
{
Sim *sim = (Sim*) quakey;
return sim->current_time;
}
QuakeyUInt64 quakey_random(void)
{
Host *host = host___;
@@ -2823,6 +3230,11 @@ int mock_close(int fd)
return 0;
}
int mock_access(const char *path, int mode)
{
assert(0); // TODO
}
static int convert_addr(void *addr, size_t addr_len,
Addr *converted_addr, uint16_t *converted_port)
{
raft/client.c
+87 -5
View File
@@ -9,13 +9,64 @@
#include "node.h"
#include "client.h"
#define CLIENT_TRACE(fmt, ...) {}
//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
//#define CLIENT_TRACE(fmt, ...) {}
#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
#define CLIENT_REQUEST_TIMEOUT_SEC 3
#define KEY_POOL_SIZE 128
static uint64_t next_client_id = 1;
static uint64_t client_random(void)
{
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
return quakey_random();
#else
return (uint64_t)rand();
#endif
}
static KVStoreOper random_oper(void)
{
KVStoreOper oper = {0};
snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
switch (client_random() % 3) {
case 0:
oper.type = KVSTORE_OPER_SET;
oper.val = client_random();
break;
case 1:
oper.type = KVSTORE_OPER_GET;
break;
case 2:
oper.type = KVSTORE_OPER_DEL;
break;
}
return oper;
}
static const char *oper_type_name(KVStoreOperType t)
{
switch (t) {
case KVSTORE_OPER_NOOP: return "NOOP";
case KVSTORE_OPER_SET: return "SET";
case KVSTORE_OPER_GET: return "GET";
case KVSTORE_OPER_DEL: return "DEL";
}
return "???";
}
static const char *result_type_name(KVStoreResultType t)
{
switch (t) {
case KVSTORE_RESULT_OK: return "OK";
case KVSTORE_RESULT_FULL: return "FULL";
case KVSTORE_RESULT_MISSING: return "MISSING";
}
return "???";
}
static int
process_message(ClientState *state,
int conn_idx, uint8_t type, ByteView msg)
@@ -31,7 +82,12 @@ process_message(ClientState *state,
if (redirect_message.leader_idx >= 0 && redirect_message.leader_idx < state->num_servers) {
CLIENT_TRACE("Redirected to leader %d", redirect_message.leader_idx);
state->current_leader = redirect_message.leader_idx;
// Retry immediately with the correct leader
// Retry immediately with the correct leader.
// A redirect means the server did not process the request,
// so mark as rejected (not timeout) for the linearizability
// checker: the outcome is unambiguous (no effect).
state->last_was_rejected = true;
state->last_was_timeout = false;
state->pending = false;
}
return 0;
@@ -48,8 +104,24 @@ process_message(ClientState *state,
return -1;
memcpy(&reply_message, msg.ptr, sizeof(reply_message));
CLIENT_TRACE("Received reply for request %lu", (unsigned long)state->request_id);
// Ignore stale replies from previous requests. After a timeout
// the client moves to a new leader and sends a new request, but
// the old leader may still deliver a reply for the old request
// on the previous connection. Without this check the client
// would accept the stale result for the wrong operation.
if (reply_message.request_id != state->request_id)
return 0;
CLIENT_TRACE("REPLY: %s key=\"%.16s\" -> %s val=%lu (req_id=%lu)",
oper_type_name(state->last_oper.type),
state->last_oper.key,
result_type_name(reply_message.result.type),
(unsigned long)reply_message.result.val,
(unsigned long)state->request_id);
state->last_result = reply_message.result;
state->last_was_timeout = false;
state->last_was_rejected = false;
state->pending = false;
return 0;
}
@@ -167,6 +239,8 @@ int client_tick(void *state_, void **ctxs,
if (now >= request_deadline) {
CLIENT_TRACE("Request %lu timed out, trying next server",
(unsigned long)state->request_id);
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
state->current_leader = (state->current_leader + 1) % state->num_servers;
}
@@ -181,6 +255,7 @@ int client_tick(void *state_, void **ctxs,
tcp_connect(&state->tcp, state->server_addrs[leader], leader, NULL);
} else {
state->request_id++;
state->last_oper = random_oper();
RequestMessage request_message = {
.base = {
@@ -188,11 +263,18 @@ int client_tick(void *state_, void **ctxs,
.type = MESSAGE_TYPE_REQUEST,
.length = sizeof(RequestMessage),
},
.oper = OPERATION_A,
.oper = state->last_oper,
.client_id = state->client_id,
.request_id = state->request_id,
};
CLIENT_TRACE("REQUEST: %s key=\"%.16s\" val=%lu (req_id=%lu, leader=%d)",
oper_type_name(state->last_oper.type),
state->last_oper.key,
(unsigned long)state->last_oper.val,
(unsigned long)state->request_id,
leader);
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output)
byte_queue_write(output, &request_message, sizeof(request_message));
raft/client.h
+10
View File
@@ -4,6 +4,8 @@
#include <lib/tcp.h>
#include <lib/basic.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
@@ -13,6 +15,14 @@ typedef struct {
// True if we are waiting for a response
bool pending;
// The operation sent in the current pending request (for logging)
KVStoreOper last_oper;
// Linearizability checker support
KVStoreResult last_result;
bool last_was_timeout;
bool last_was_rejected;
Address server_addrs[NODE_LIMIT];
int num_servers;
raft/client_table.c
+45
View File
@@ -0,0 +1,45 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include "client_table.h"
void client_table_init(ClientTable *ct)
{
ct->count = 0;
ct->capacity = 0;
ct->entries = NULL;
}
void client_table_free(ClientTable *ct)
{
free(ct->entries);
}
ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id)
{
for (int i = 0; i < ct->count; i++)
if (ct->entries[i].client_id == client_id)
return &ct->entries[i];
return NULL;
}
int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag)
{
if (ct->count == ct->capacity) {
int n = ct->capacity ? 2 * ct->capacity : 8;
void *p = realloc(ct->entries, n * sizeof(ClientTableEntry));
if (p == NULL) return -1;
ct->capacity = n;
ct->entries = p;
}
ct->entries[ct->count++] = (ClientTableEntry) {
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_tag = conn_tag,
};
return 0;
}
raft/client_table.h
+28
View File
@@ -0,0 +1,28 @@
#ifndef CLIENT_TABLE_INCLUDED
#define CLIENT_TABLE_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
KVStoreResult last_result;
bool pending;
int conn_tag;
} ClientTableEntry;
typedef struct {
int count;
int capacity;
ClientTableEntry *entries;
} ClientTable;
void client_table_init(ClientTable *ct);
void client_table_free(ClientTable *ct);
ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id);
int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag);
#endif // CLIENT_TABLE_INCLUDED
raft/invariant_checker.c
+351
View File
@@ -0,0 +1,351 @@
// Raft invariant checker with external shadow log tracking.
//
// This file runs in the main simulation loop outside Quakey-scheduled
// processes. It includes node.h for struct definitions, then restores
// real allocators since mock_malloc/realloc/free abort outside process
// context.
#include "invariant_checker.h"
#include <assert.h>
#include <string.h>
// Restore real allocators (see checker/linearizability.c for precedent).
// The header chain (invariant_checker.h -> node.h -> wal.h ->
// lib/file_system.h) defines QUAKEY_ENABLE_MOCKS and replaces
// malloc/realloc/free with mock versions. We need the real ones
// because this code runs outside any Quakey-scheduled process.
#undef malloc
#undef realloc
#undef free
#include <stdio.h>
#include <stdlib.h>
static int self_idx(NodeState *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
return i;
UNREACHABLE;
}
static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
{
if (ic->shadow_count == ic->shadow_capacity) {
int n = 2 * ic->shadow_capacity;
if (n < 8)
n = 8;
KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
if (p == NULL)
return -1;
ic->shadow_log = p;
ic->shadow_capacity = n;
}
ic->shadow_log[ic->shadow_count++] = oper;
return 0;
}
void invariant_checker_init(InvariantChecker *ic)
{
ic->shadow_log = NULL;
ic->shadow_count = 0;
ic->shadow_capacity = 0;
}
void invariant_checker_free(InvariantChecker *ic)
{
fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
ic->shadow_count);
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
{
for (int i = 0; i < num_nodes; i++) {
NodeState *s = nodes[i];
if (s == NULL)
continue;
int log_count = wal_entry_count(&s->wal);
// The commit index starts at -1 (nothing committed) and only
// increases. It must never go below -1.
if (s->commit_index < -1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < -1\n",
i, s->commit_index);
__builtin_trap();
}
// A node cannot have committed an entry beyond what exists
// in its log. With an empty log (count=0), commit_index
// must be -1.
if (s->commit_index >= log_count) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) >= log_count (%d)\n",
i, s->commit_index, log_count);
__builtin_trap();
}
// Entries are applied to the state machine in order, up to
// the commit index. The last applied index must never exceed
// the commit index.
if (s->last_applied > s->commit_index) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) > commit_index (%d)\n",
i, s->last_applied, s->commit_index);
__builtin_trap();
}
if (s->last_applied < -1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) < -1\n",
i, s->last_applied);
__builtin_trap();
}
// voted_for is either -1 (no vote) or a valid node index.
if (s->term_and_vote.voted_for != -1 && (s->term_and_vote.voted_for < 0 || s->term_and_vote.voted_for >= s->num_nodes)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: voted_for (%d) is not -1 "
"and not in [0, %d)\n",
i, s->term_and_vote.voted_for, s->num_nodes);
__builtin_trap();
}
// Leaders append entries with their current term, and terms
// only increase. Truncation preserves this property because
// replacement entries come from a leader whose log already
// satisfies non-decreasing terms.
for (int k = 1; k < log_count; k++) {
if (wal_peek_entry(&s->wal, k)->term < wal_peek_entry(&s->wal, k-1)->term) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: wal[%d].term (%lu) "
"< wal[%d].term (%lu) (non-monotonic)\n",
i, k, (unsigned long)wal_peek_entry(&s->wal, k)->term,
k-1, (unsigned long)wal_peek_entry(&s->wal, k-1)->term);
__builtin_trap();
}
}
// A leader must have voted for itself in the current term.
// A node becomes leader by winning an election, which requires
// voting for itself. The voted_for value is only reset when
// stepping down (which changes the role to FOLLOWER).
if (s->role == ROLE_LEADER) {
if (s->term_and_vote.voted_for != self_idx(s)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: role is LEADER but "
"voted_for (%d) != self_idx (%d)\n",
i, s->term_and_vote.voted_for, self_idx(s));
__builtin_trap();
}
}
// A candidate must have voted for itself in the current term.
// A node enters candidate state only through start_election(),
// which increments the term and sets voted_for to self_idx.
if (s->role == ROLE_CANDIDATE) {
if (s->term_and_vote.voted_for != self_idx(s)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: role is CANDIDATE but "
"voted_for (%d) != self_idx (%d)\n",
i, s->term_and_vote.voted_for, self_idx(s));
__builtin_trap();
}
}
// For a leader, match_indices[self] must equal the last log
// index. The leader always has its own entries matched.
if (s->role == ROLE_LEADER) {
int expected_match = log_count - 1;
if (s->match_indices[self_idx(s)] != expected_match) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
"match_indices[self] (%d) != last log index (%d)\n",
i, s->match_indices[self_idx(s)], expected_match);
__builtin_trap();
}
}
// For a leader, next_indices[k] must be >= match_indices[k] + 1
// for all followers (k != self). The next index to send is
// always at least one past the highest known replicated index.
// The leader's own next_indices[self] is not maintained since
// the leader never sends entries to itself.
if (s->role == ROLE_LEADER) {
int si = self_idx(s);
for (int k = 0; k < s->num_nodes; k++) {
if (k == si)
continue;
if (s->next_indices[k] < s->match_indices[k] + 1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
"next_indices[%d] (%d) < match_indices[%d] + 1 (%d)\n",
i, k, s->next_indices[k], k, s->match_indices[k] + 1);
__builtin_trap();
}
}
}
}
// Election Safety: at most one leader per term.
// Each term has at most one leader because a candidate needs
// a majority of votes, and each node votes at most once per term.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL || nodes[i]->role != ROLE_LEADER)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL || nodes[j]->role != ROLE_LEADER)
continue;
if (nodes[i]->term_and_vote.term == nodes[j]->term_and_vote.term) {
fprintf(stderr, "INVARIANT VIOLATED: two leaders in term %lu: "
"node %d and node %d\n",
(unsigned long)nodes[i]->term_and_vote.term, i, j);
__builtin_trap();
}
}
}
// State Machine Safety (committed prefix agreement).
// For any two nodes, their logs must agree on all entries up to
// min(commit_index_i, commit_index_j). This is the core safety
// property: all committed operations are identical across replicas.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL)
continue;
int min_commit = nodes[i]->commit_index;
if (nodes[j]->commit_index < min_commit)
min_commit = nodes[j]->commit_index;
for (int k = 0; k <= min_commit; k++) {
WALEntry *ei = wal_peek_entry(&nodes[i]->wal, k);
WALEntry *ej = wal_peek_entry(&nodes[j]->wal, k);
if (memcmp(&ei->oper, &ej->oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed log operation "
"mismatch at index %d between node %d and node %d\n",
k, i, j);
__builtin_trap();
}
if (ei->term != ej->term) {
fprintf(stderr, "INVARIANT VIOLATED: committed log term "
"mismatch at index %d between node %d (term %lu) "
"and node %d (term %lu)\n",
k, i, (unsigned long)ei->term,
j, (unsigned long)ej->term);
__builtin_trap();
}
}
}
}
////////////////////////////////////////////////////////////////////
// Shadow log: external commit tracking
////////////////////////////////////////////////////////////////////
// Phase 1: Find the observed max commit index and a source node.
//
// In Raft, commit_index == -1 means nothing committed, 0 means the
// first entry is committed, etc. We track the number of committed
// entries as (commit_index + 1) so it aligns with shadow_count.
int observed_committed = 0; // number of committed entries observed
int source_node_idx = -1;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
int node_committed = nodes[i]->commit_index + 1;
if (node_committed > observed_committed) {
observed_committed = node_committed;
source_node_idx = i;
}
}
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_committed > ic->shadow_count) {
NodeState *source = nodes[source_node_idx];
assert(wal_entry_count(&source->wal) >= observed_committed);
for (int k = ic->shadow_count; k < observed_committed; k++) {
KVStoreOper *source_oper = &wal_peek_entry(&source->wal, k)->oper;
// Cross-validate against other live nodes that have also
// committed this entry.
for (int j = 0; j < num_nodes; j++) {
if (j == source_node_idx)
continue;
if (nodes[j] == NULL)
continue;
if (nodes[j]->commit_index < k)
continue;
if (wal_entry_count(&nodes[j]->wal) <= k)
continue;
if (memcmp(&wal_peek_entry(&nodes[j]->wal, k)->oper, source_oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
"between source node %d and node %d during shadow log append\n",
k, source_node_idx, j);
__builtin_trap();
}
}
if (shadow_log_append(ic, *source_oper) < 0) {
fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
__builtin_trap();
}
}
}
// Phase 3: Verify shadow log against the cluster.
// Sub-check A: Committed entries must match the shadow log.
for (int k = 0; k < ic->shadow_count; k++) {
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (wal_entry_count(&nodes[i]->wal) <= k)
continue;
if (nodes[i]->commit_index < k)
continue;
if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
char shadow_buf[128], node_buf[128];
kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
kvstore_snprint_oper(node_buf, sizeof(node_buf), wal_peek_entry(&nodes[i]->wal, k)->oper);
fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
" shadow: %s\n"
" node: %s\n",
k, i, shadow_buf, node_buf);
__builtin_trap();
}
}
}
// Sub-check B: When commit regresses, previously committed entries
// must still be held by a majority of the cluster.
if (observed_committed < ic->shadow_count) {
for (int k = observed_committed; k < ic->shadow_count; k++) {
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
if (wal_entry_count(&nodes[i]->wal) <= k)
continue;
if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
holders++;
}
if (holders + num_dead <= num_nodes / 2) {
char oper_buf[128];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
"no longer held by majority (holders=%d, dead=%d, total=%d)\n"
" entry: %s\n",
k, holders, num_dead, num_nodes, oper_buf);
__builtin_trap();
}
}
}
}
raft/invariant_checker.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef INVARIANT_CHECKER_INCLUDED
#define INVARIANT_CHECKER_INCLUDED
#include "node.h"
typedef struct {
// External shadow log of committed operations (unbounded, dynamically allocated)
KVStoreOper *shadow_log;
int shadow_count;
int shadow_capacity;
} InvariantChecker;
void invariant_checker_init(InvariantChecker *ic);
void invariant_checker_free(InvariantChecker *ic);
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
#endif // INVARIANT_CHECKER_INCLUDED
raft/main.c
+44 -8
View File
@@ -8,6 +8,7 @@
#include "node.h"
#include "client.h"
#include "invariant_checker.h"
static volatile int simulation_running = 1;
@@ -17,15 +18,32 @@ static void sigint_handler(int sig)
simulation_running = 0;
}
int main(void)
int main(int argc, char **argv)
{
signal(SIGINT, sigint_handler);
QuakeyUInt64 seed = 2;
QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
seed = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
}
}
Quakey *quakey;
int ret = quakey_init(&quakey, 2);
int ret = quakey_init(&quakey, seed);
if (ret < 0)
return -1;
QuakeyNode cli_1;
QuakeyNode cli_2;
QuakeyNode node_1;
QuakeyNode node_2;
QuakeyNode node_3;
// Client 1
{
QuakeySpawn config = {
@@ -39,7 +57,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Client 2
@@ -55,7 +73,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Node 1
@@ -71,7 +89,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Node 2
@@ -87,7 +105,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Node 3
@@ -103,12 +121,30 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
while (simulation_running)
// Limit crashes to 1 node at a time (within fault tolerance of f=1).
quakey_set_max_crashes(quakey, 1);
quakey_network_partitioning(quakey, true);
InvariantChecker invariant_checker;
invariant_checker_init(&invariant_checker);
while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
quakey_schedule_one(quakey);
NodeState *arr[] = {
quakey_node_state(node_1),
quakey_node_state(node_2),
quakey_node_state(node_3),
};
invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
}
invariant_checker_free(&invariant_checker);
quakey_free(quakey);
return 0;
}
raft/node.c
+558 -530
View File
File diff suppressed because it is too large Load Diff
raft/node.h
+59 -42
View File
@@ -7,6 +7,7 @@
#include "wal.h"
#include "config.h"
#include "client_table.h"
enum {
MESSAGE_TYPE_REQUEST_VOTE,
@@ -28,6 +29,7 @@ typedef struct {
typedef struct {
MessageHeader base;
int sender_idx;
uint64_t term;
uint8_t value;
} VotedMessage;
@@ -53,14 +55,15 @@ typedef struct {
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
OperationResult result;
KVStoreResult result;
uint64_t request_id;
} ReplyMessage;
typedef struct {
@@ -68,20 +71,6 @@ typedef struct {
int leader_idx;
} RedirectMessage;
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
OperationResult last_result;
bool pending;
int conn_tag;
} ClientTableEntry;
typedef struct {
int count;
int capacity;
ClientTableEntry *entries;
} ClientTable;
typedef enum {
ROLE_FOLLOWER,
ROLE_CANDIDATE,
@@ -90,46 +79,74 @@ typedef enum {
typedef struct {
TCP tcp;
WAL wal;
// In-memory copy of the term and vote values.
// These must always be updated after the version
// stored on disk.
uint64_t term;
int voted_for;
// Handle to the file backing the term and vote
// values.
Handle handle;
} TermAndVote;
typedef struct {
// Networking subsystem
TCP tcp;
// Static list of cluster nodes
Address self_addr;
Address node_addrs[NODE_LIMIT];
int num_nodes;
// Current loder of the node and the
// current leader. If no leader, -1.
Role role;
uint64_t term;
int voted_for;
Handle term_and_vote_handle;
// Index of the current leader
int leader_idx;
// Votes received when candidate.
// Each bit is associated to a node. The
// number of votes is obtained by counting
// set bits. This ensures nodes won't vote
// twice.
uint32_t votes;
// Durable log subsystem
WAL wal;
// Durable fixed-size data subsystem
TermAndVote term_and_vote;
// Current election timeout. This value is relative
// and must be summed to the heartbeat in order to
// get the absolute election deadline.
uint64_t election_timeout;
// This is the current value of the node.
// It is set at the start of each event
// handler call.
Time now;
// If leader, this is the time when the last
// heartbeat was sent. If follower, this is
// the last time a heartbeat was received.
Time heartbeat;
// The state machine to be replicated.
KVStore kvstore;
// Table of client requests. This ensures request
// are applied in order.
ClientTable client_table;
int next_client_tag;
int commit_index;
int last_applied;
// When CANDIDATE
int votes_received;
// When LEADER
int next_indices[NODE_LIMIT];
int match_indices[NODE_LIMIT];
// Relative timeout in nanoseconds
uint64_t election_timeout;
// Last heartbeat time
uint64_t watchdog;
// Keep track of client request order to enforce
// linearizability
ClientTable client_table;
int next_client_tag;
// The state machine to be replicated
StateMachine state_machine;
} NodeState;
struct pollfd;
raft/wal.c
+69 -18
View File
@@ -4,10 +4,26 @@
#include <quakey.h>
#include <assert.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include "wal.h"
// FNV-1a checksum over all WALEntry fields except the checksum itself.
uint32_t wal_entry_checksum(WALEntry *entry)
{
uint32_t h = 2166136261u;
const unsigned char *p = (const unsigned char *)entry;
// Hash all bytes up to (but not including) the checksum field
size_t len = offsetof(WALEntry, checksum);
for (size_t i = 0; i < len; i++) {
h ^= p[i];
h *= 16777619u;
}
return h;
}
int wal_init(WAL *wal, string file)
{
Handle handle;
@@ -21,25 +37,46 @@ int wal_init(WAL *wal, string file)
return -1;
}
if (size % sizeof(WALEntry) != 0) {
// Discard any partial trailing entry (crash during write).
int raw_count = size / sizeof(WALEntry);
size_t valid_size = raw_count * sizeof(WALEntry);
if (valid_size < size) {
file_truncate(handle, valid_size);
}
WALEntry *entries = malloc(raw_count * sizeof(WALEntry));
if (entries == NULL && raw_count > 0) {
file_close(handle);
return -1;
}
int count = size / sizeof(WALEntry);
if (file_set_offset(handle, 0) < 0) {
file_close(handle);
free(entries);
return -1;
}
if (raw_count > 0 && file_read_exact(handle, (char*) entries, raw_count * sizeof(WALEntry)) < 0) {
file_close(handle);
free(entries);
return -1;
}
// Verify checksums: truncate at the first corrupted entry.
// All entries from a corrupted one onward are discarded.
int count = raw_count;
for (int i = 0; i < raw_count; i++) {
if (entries[i].checksum != wal_entry_checksum(&entries[i])) {
count = i;
file_truncate(handle, count * sizeof(WALEntry));
break;
}
}
wal->count = count;
wal->capacity = count;
wal->entries = malloc(count * sizeof(WALEntry));
if (wal->entries == NULL) {
file_close(handle);
return -1;
}
if (file_read_exact(handle, (char*) wal->entries, count * sizeof(WALEntry)) < 0) {
file_close(handle);
return -1;
}
wal->capacity = raw_count; // capacity >= count
wal->entries = entries;
wal->handle = handle;
return 0;
}
@@ -61,12 +98,26 @@ int wal_append(WAL *wal, WALEntry *entry)
wal->entries = p;
}
// TODO: Should truncate file on partial writes
if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0)
return -1;
// Compute checksum before writing to disk
entry->checksum = wal_entry_checksum(entry);
if (file_sync(wal->handle) < 0)
if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0) {
// A partial write may have advanced the file offset. Truncate
// and rewind so the file stays consistent with the in-memory
// entry count.
file_truncate(wal->handle, wal->count * sizeof(WALEntry));
file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
return -1;
}
if (file_sync(wal->handle) < 0) {
// The write succeeded but sync failed. Rewind the offset and
// truncate the phantom entry so the next append writes to the
// correct position.
file_truncate(wal->handle, wal->count * sizeof(WALEntry));
file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
return -1;
}
wal->entries[wal->count++] = *entry;
return 0;
raft/wal.h
+6 -2
View File
@@ -3,14 +3,18 @@
#include <lib/file_system.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t term;
uint64_t client_id;
Operation oper;
uint64_t request_id;
KVStoreOper oper;
uint32_t checksum;
} WALEntry;
uint32_t wal_entry_checksum(WALEntry *entry);
typedef struct {
int count;
int capacity;
state_machine/kvstore.c
+118
View File
@@ -0,0 +1,118 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include <stdio.h>
#include "kvstore.h"
void kvstore_init(KVStore *kvs)
{
memset(kvs, 0, sizeof(*kvs));
}
void kvstore_free(KVStore *kvs)
{
(void) kvs;
}
static int find_entry(KVStore *kvs, char *key)
{
for (int i = 0; i < kvs->count; i++)
if (!memcmp(kvs->entries[i].key, key, KVSTORE_KEY_SIZE))
return i;
return -1;
}
KVStoreResult kvstore_update(KVStore *kvs, KVStoreOper oper)
{
KVStoreResult result;
switch (oper.type) {
case KVSTORE_OPER_NOOP:
result.type = KVSTORE_RESULT_OK;
result.val = 0;
break;
case KVSTORE_OPER_SET:
{
int i = find_entry(kvs, oper.key);
if (i < 0) {
if (kvs->count >= KVSTORE_ENTRY_LIMIT) {
result.type = KVSTORE_RESULT_FULL;
result.val = 0;
break;
}
i = kvs->count++;
memcpy(kvs->entries[i].key, oper.key, KVSTORE_KEY_SIZE);
}
kvs->entries[i].val = oper.val;
result.type = KVSTORE_RESULT_OK;
result.val = 0;
}
break;
case KVSTORE_OPER_GET:
{
int i = find_entry(kvs, oper.key);
if (i < 0) {
result.type = KVSTORE_RESULT_MISSING;
result.val = 0;
} else {
result.type = KVSTORE_RESULT_OK;
result.val = kvs->entries[i].val;
}
}
break;
case KVSTORE_OPER_DEL:
{
int i = find_entry(kvs, oper.key);
if (i < 0) {
result.type = KVSTORE_RESULT_MISSING;
result.val = 0;
} else {
result.type = KVSTORE_RESULT_OK;
result.val = kvs->entries[i].val;
kvs->entries[i] = kvs->entries[--kvs->count];
}
}
break;
default:
assert(0);
break;
}
return result;
}
void kvstore_dump_oper(KVStoreOper oper)
{
switch (oper.type) {
case KVSTORE_OPER_NOOP: printf("NOOP"); break;
case KVSTORE_OPER_SET: printf("SET(%.16s, %lu)", oper.key, oper.val); break;
case KVSTORE_OPER_GET: printf("GET(%.16s)", oper.key); break;
case KVSTORE_OPER_DEL: printf("DEL(%.16s)", oper.key); break;
default: printf("???\n");
}
}
int kvstore_snprint_oper(char *buf, int size, KVStoreOper oper)
{
switch (oper.type) {
case KVSTORE_OPER_NOOP: return snprintf(buf, size, "NOOP");
case KVSTORE_OPER_SET: return snprintf(buf, size, "SET(%.16s, %lu)", oper.key, oper.val);
case KVSTORE_OPER_GET: return snprintf(buf, size, "GET(%.16s)", oper.key);
case KVSTORE_OPER_DEL: return snprintf(buf, size, "DEL(%.16s)", oper.key);
default: return snprintf(buf, size, "???");
}
}
int kvstore_snprint_result(char *buf, int size, KVStoreResult result)
{
switch (result.type) {
case KVSTORE_RESULT_OK: return snprintf(buf, size, "OK(%lu)", result.val);
case KVSTORE_RESULT_FULL: return snprintf(buf, size, "FULL");
case KVSTORE_RESULT_MISSING: return snprintf(buf, size, "MISSING");
default: return snprintf(buf, size, "???");
}
}
state_machine/kvstore.h
+51
View File
@@ -0,0 +1,51 @@
#ifndef KVSTORE_INCLUDED
#define KVSTORE_INCLUDED
#include <stdint.h>
#define KVSTORE_KEY_SIZE 16
#define KVSTORE_ENTRY_LIMIT 64
typedef enum {
KVSTORE_OPER_NOOP,
KVSTORE_OPER_SET,
KVSTORE_OPER_GET,
KVSTORE_OPER_DEL,
} KVStoreOperType;
typedef enum {
KVSTORE_RESULT_OK,
KVSTORE_RESULT_FULL,
KVSTORE_RESULT_MISSING,
} KVStoreResultType;
typedef struct {
KVStoreOperType type;
char key[KVSTORE_KEY_SIZE];
uint64_t val;
} KVStoreOper;
typedef struct {
KVStoreResultType type;
uint64_t val;
} KVStoreResult;
typedef struct {
char key[KVSTORE_KEY_SIZE];
uint64_t val;
} KVEntry;
typedef struct {
int count;
KVEntry entries[KVSTORE_ENTRY_LIMIT];
} KVStore;
void kvstore_init(KVStore *kvs);
void kvstore_free(KVStore *kvs);
KVStoreResult kvstore_update(KVStore *kvs, KVStoreOper oper);
void kvstore_dump_oper(KVStoreOper oper);
int kvstore_snprint_oper(char *buf, int size, KVStoreOper oper);
int kvstore_snprint_result(char *buf, int size, KVStoreResult result);
#endif // KVSTORE_INCLUDED
state_machine/state_machine.c
-25
View File
@@ -1,25 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include "state_machine.h"
void state_machine_init(StateMachine *sm)
{
(void)sm;
}
void state_machine_free(StateMachine *sm)
{
(void)sm;
}
OperationResult state_machine_update(StateMachine *sm, Operation op)
{
(void)sm;
(void)op;
return OPERATION_RESULT_X;
}
state_machine/state_machine.h
-26
View File
@@ -1,26 +0,0 @@
#ifndef STATE_MACHINE_INCLUDED
#define STATE_MACHINE_INCLUDED
typedef enum {
OPERATION_NOOP,
OPERATION_A,
OPERATION_B,
OPERATION_C,
} Operation;
typedef enum {
OPERATION_RESULT_X,
OPERATION_RESULT_Y,
OPERATION_RESULT_Z,
} OperationResult;
typedef struct {
} StateMachine;
void state_machine_init(StateMachine *sm);
void state_machine_free(StateMachine *sm);
OperationResult state_machine_update(StateMachine *sm, Operation op);
#endif // STATE_MACHINE_INCLUDED
table.txt
+247
View File
@@ -0,0 +1,247 @@
Consensus Protocol Simulation Results
Simulated time: 1200s per run, seeds 1-100
3 nodes, 2 clients, fault injection enabled, network partitioning enabled
Column descriptions:
Seed - RNG seed used for the simulation run
Proto - Consensus protocol (VSR or Raft)
Invocations - Number of client requests initiated
OK - Requests that completed successfully with a response
Timeouts - Requests that timed out without a response
Rejected - Requests explicitly rejected (e.g. sent to a non-leader)
Checks - Linearizability checks performed by the checker
Steps - Total simulation scheduling steps (ticks across all hosts)
Seed Proto Invocations OK Timeouts Rejected Checks Steps
-----------------------------------------------------------------------
1 VSR 2244 2206 33 5 2244 105648
1 Raft 3276 1700 368 1206 3274 363190
2 VSR 2883 2839 39 5 2883 126828
2 Raft 3394 2761 285 346 3392 171163
3 VSR 592 497 94 1 592 42548
3 Raft 2789 2358 330 99 2787 150699
4 VSR 1274 1231 37 6 1274 78222
4 Raft 2068 1362 464 240 2066 88436
5 VSR 1405 1372 31 1 1404 75712
5 Raft 2304 1630 416 256 2302 131192
6 VSR 2025 1975 49 0 2024 169653
6 Raft 2182 1244 447 489 2180 378921
7 VSR 955 856 98 1 955 56039
7 Raft 3433 1609 311 1511 3431 297830
8 VSR 720 636 79 4 719 46945
8 Raft 2498 1759 436 301 2496 123015
9 VSR 859 780 77 2 859 56003
9 Raft 2778 1816 362 599 2777 225590
10 VSR 1486 1401 84 1 1486 95773
10 Raft 3114 2093 321 698 3112 341220
11 VSR 456 302 153 1 456 33396
11 Raft 3441 1209 375 1855 3439 345485
12 VSR 1220 1128 88 2 1218 66620
12 Raft 2555 1064 458 1031 2553 370909
13 VSR 863 753 108 1 862 52065
13 Raft 1965 1210 494 259 1963 162897
14 VSR 645 524 115 5 644 44976
14 Raft 2498 1918 399 179 2496 138682
15 VSR 690 638 50 1 689 48727
15 Raft 2541 1871 381 287 2539 200612
16 VSR 944 835 106 1 942 52517
16 Raft 2369 1451 466 450 2367 465559
17 VSR 647 564 80 2 646 40272
17 Raft 2446 1846 399 199 2444 124084
18 VSR 392 310 76 4 390 35414
18 Raft 1955 923 527 503 1953 218171
19 VSR 1165 1068 97 0 1165 60283
19 Raft 2987 1849 363 773 2985 230511
20 VSR 2229 2164 62 2 2228 108081
20 Raft 2996 1742 381 871 2994 450695
21 VSR 1390 1341 44 5 1390 78704
21 Raft 2441 1485 412 542 2439 120767
22 VSR 1844 1758 81 5 1844 96250
22 Raft 3561 1897 306 1356 3559 212082
23 VSR 1393 1335 55 3 1393 79177
23 Raft 2524 1496 434 593 2523 222489
24 VSR 971 951 18 0 969 56647
24 Raft 2177 1606 412 157 2175 177816
25 VSR 458 376 77 5 458 35323
25 Raft 2507 1898 374 233 2505 139076
26 VSR 1177 1117 60 0 1177 70344
26 Raft 2583 1978 365 238 2581 186689
27 VSR 1042 937 104 1 1042 70667
27 Raft 2794 2237 355 200 2792 171007
28 VSR 1118 1067 43 6 1116 67368
28 Raft 2790 1870 357 561 2788 257457
29 VSR 683 610 73 0 683 45166
29 Raft 2557 1439 408 708 2555 110388
30 VSR 1202 1134 63 3 1200 64864
30 Raft 3312 2297 316 697 3310 406253
31 VSR 1466 1399 65 2 1466 83025
31 Raft 2818 1612 377 827 2816 311506
32 VSR 1207 1161 41 5 1207 68520
32 Raft 2676 1141 416 1117 2674 255671
33 VSR 1219 1134 83 1 1218 65923
33 Raft 3260 2844 274 140 3258 191445
34 VSR 1255 1229 24 1 1254 71120
34 Raft 3311 1372 405 1532 3309 428477
35 VSR 1197 1142 50 4 1196 73779
35 Raft 2435 1793 418 222 2433 147736
36 VSR 1560 1500 55 4 1559 89849
36 Raft 2110 1186 515 407 2108 310505
37 VSR 1487 1424 58 4 1486 148383
37 Raft 3223 1407 335 1479 3221 242814
38 VSR 1397 1349 42 5 1396 76827
38 Raft 2466 1260 441 763 2464 352797
39 VSR 1304 1249 53 1 1303 78321
39 Raft 3027 2062 343 620 3025 334793
40 VSR 1299 1270 25 2 1297 72474
40 Raft 2183 1213 479 489 2181 229529
41 VSR 1091 1040 41 8 1089 65193
41 Raft 2856 2333 339 182 2854 162726
42 VSR 496 410 84 1 495 36493
42 Raft 2421 1797 400 222 2419 163015
43 VSR 658 562 92 2 656 46387
43 Raft 2834 2134 362 336 2832 161248
44 VSR 1372 1318 49 4 1371 75563
44 Raft 2654 2076 370 206 2652 262333
45 VSR 257 127 129 0 256 33593
45 Raft 3146 1319 366 1459 3144 395077
46 VSR 1515 1458 53 2 1513 105709
46 Raft 3320 1456 382 1480 3318 444523
47 VSR 1057 1017 39 0 1056 61117
47 Raft 2243 1351 467 423 2241 210836
48 VSR 1195 1140 51 3 1194 69487
48 Raft 2378 1708 422 246 2376 148188
49 VSR 950 876 71 2 949 55394
49 Raft 2984 2444 309 229 2982 153760
50 VSR 964 875 88 1 964 57668
50 Raft 2889 2295 344 248 2887 182648
51 VSR 1302 1250 49 1 1300 74028
51 Raft 2363 1734 415 212 2361 127568
52 VSR 1205 1111 91 2 1204 64727
52 Raft 2905 2380 303 221 2904 170346
53 VSR 869 744 124 0 868 95545
53 Raft 2203 964 444 794 2202 306208
54 VSR 408 333 72 3 408 36481
54 Raft 2113 1409 434 268 2111 103004
55 VSR 494 443 50 1 494 40161
55 Raft 2618 1937 375 305 2617 239299
56 VSR 1300 1231 66 2 1299 70686
56 Raft 3401 2723 282 394 3399 194017
57 VSR 238 145 93 0 238 34783
57 Raft 1941 1255 451 233 1939 90352
58 VSR 922 854 63 5 922 54772
58 Raft 2173 1303 448 420 2171 225395
59 VSR 1941 1853 84 3 1940 103258
59 Raft 2501 1237 420 842 2499 214547
60 VSR 1766 1645 114 5 1764 86384
60 Raft 2909 1062 437 1408 2907 446530
61 VSR 1630 1557 68 4 1629 76775
61 Raft 2956 2258 333 363 2954 199725
62 VSR 1077 1023 51 2 1076 107542
62 Raft 3418 1993 303 1120 3416 178317
63 VSR 2158 2107 47 3 2157 107086
63 Raft 2245 1039 473 731 2243 331394
64 VSR 1804 1743 53 7 1803 101345
64 Raft 2804 2152 325 325 2802 206594
65 VSR 934 825 106 1 932 77299
65 Raft 2279 1090 486 701 2277 266669
66 VSR 798 726 69 2 797 59313
66 Raft 2105 1135 444 524 2103 246688
67 VSR 965 874 91 0 965 71117
67 Raft 2268 1598 439 229 2266 103555
68 VSR 438 327 109 1 437 54528
68 Raft 2787 1755 395 635 2785 347905
69 VSR 922 869 52 1 922 61234
69 Raft 2588 1752 425 409 2586 218784
70 VSR 720 607 110 3 720 47228
70 Raft 2424 1495 414 513 2422 255104
71 VSR 1067 982 81 4 1067 62847
71 Raft 2207 1583 398 224 2205 165271
72 VSR 673 561 110 0 671 48927
72 Raft 2927 2052 348 525 2925 145377
73 VSR 1361 1270 88 2 1360 70464
73 Raft 2798 1976 353 467 2796 131273
74 VSR 1891 1842 43 5 1890 104058
74 Raft 2176 1640 405 129 2174 123942
75 VSR 970 879 91 0 970 86026
75 Raft 2456 879 491 1084 2454 380910
76 VSR 851 737 111 2 850 55131
76 Raft 3226 1945 343 936 3224 158287
77 VSR 610 474 133 2 609 40458
77 Raft 2492 1882 387 221 2490 134293
78 VSR 1269 1232 31 6 1269 69782
78 Raft 3411 1575 329 1506 3410 238137
79 VSR 921 859 62 0 921 55960
79 Raft 3008 2516 335 155 3006 176544
80 VSR 1271 1230 36 5 1271 77391
80 Raft 2605 1033 426 1144 2603 287940
81 VSR 835 708 124 2 834 52299
81 Raft 2582 1845 361 374 2580 140619
82 VSR 1024 913 109 1 1023 58016
82 Raft 2921 2264 330 325 2919 143004
83 VSR 922 802 120 0 922 51388
83 Raft 2600 1885 412 301 2598 129191
84 VSR 1533 1480 52 1 1533 79724
84 Raft 2344 1753 440 149 2342 129605
85 VSR 1650 1588 58 4 1650 88562
85 Raft 3366 1419 355 1590 3364 379641
86 VSR 884 763 118 1 882 49372
86 Raft 2736 1258 457 1019 2734 552128
87 VSR 1072 1003 68 0 1071 59323
87 Raft 1875 968 506 399 1873 253771
88 VSR 1493 1437 52 4 1493 92016
88 Raft 1902 1285 445 170 1900 109923
89 VSR 1294 1239 53 0 1292 69379
89 Raft 1922 1204 471 245 1920 91616
90 VSR 1215 1119 93 2 1214 67086
90 Raft 3440 2057 296 1085 3438 284840
91 VSR 2176 2142 28 6 2176 108769
91 Raft 2606 2025 358 221 2604 136452
92 VSR 594 550 42 0 592 43714
92 Raft 2563 1504 432 625 2561 370793
93 VSR 775 699 76 0 775 67757
93 Raft 1990 826 509 653 1988 320932
94 VSR 546 455 88 3 546 35972
94 Raft 2449 1686 399 362 2447 216855
95 VSR 420 326 94 0 420 33058
95 Raft 2506 1891 379 234 2504 141706
96 VSR 583 509 70 4 583 40050
96 Raft 2307 1263 473 569 2305 325134
97 VSR 1991 1965 21 3 1989 95473
97 Raft 2679 1360 397 920 2677 281323
98 VSR 1198 1123 73 2 1198 78181
98 Raft 2550 1467 467 614 2548 416874
99 VSR 954 884 67 3 954 72020
99 Raft 2051 1356 481 212 2049 102556
100 VSR 1426 1319 105 1 1425 76369
100 Raft 2817 1402 440 973 2815 474134
-----------------------------------------------------------------------
Summary Statistics
VSR (100 completed runs, 0 violations):
Avg invocations: 1132.8
Avg ok: 1057.4
Avg timeouts: 72.3
Avg rejected: 2.3
Avg steps: 69089.2
Total invocations: 113278
Total ok: 105742
Total timeouts: 7226
Total rejected: 234
OK rate: 93.3%
Timeout rate: 6.4%
Rejection rate: 0.2%
Raft (100 completed runs, 0 violations):
Avg invocations: 2645.8
Avg ok: 1664.9
Avg timeouts: 397.6
Avg rejected: 581.4
Avg steps: 234875.8
Total invocations: 264582
Total ok: 166491
Total timeouts: 39755
Total rejected: 58142
OK rate: 62.9%
Timeout rate: 15.0%
Rejection rate: 22.0%
vsr/client.c
+133 -55
View File
@@ -6,18 +6,68 @@
#include <stdint.h>
#include <assert.h>
#include <lib/file_system.h>
#include "node.h"
#include "client.h"
//#define CLIENT_TRACE(fmt, ...) {}
#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
#define KEY_POOL_SIZE 128
static uint64_t next_client_id = 1;
static uint64_t client_random(void)
{
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
return quakey_random();
#else
return (uint64_t)rand();
#endif
}
static KVStoreOper random_oper(void)
{
KVStoreOper oper = {0};
snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
switch (client_random() % 3) {
case 0:
oper.type = KVSTORE_OPER_SET;
oper.val = client_random();
break;
case 1:
oper.type = KVSTORE_OPER_GET;
break;
case 2:
oper.type = KVSTORE_OPER_DEL;
break;
}
return oper;
}
// Format time as seconds with 3 decimal places for trace output
#define TIME_FMT "%.3fs"
#define TIME_FMT "%7.3fs"
#define TIME_VAL(t) ((double)(t) / 1000000000.0)
static void client_log_impl(ClientState *state, Time now, const char *event, const char *detail)
{
printf("[" TIME_FMT "] CLIENT %lu | V%-3lu | %-20s %s\n",
TIME_VAL(now),
state->client_id,
state->view_number,
event,
detail ? detail : "");
}
#define client_log(state, now, event, fmt, ...) do { \
char _detail[256]; \
snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \
client_log_impl(state, now, event, _detail); \
} while (0)
#define client_log_simple(state, now, event) \
client_log_impl(state, now, event, NULL)
static int leader_idx(ClientState *state)
{
return state->view_number % state->num_servers;
@@ -29,24 +79,61 @@ process_message(ClientState *state,
{
(void) conn_idx;
if (type == MESSAGE_TYPE_REDIRECT) {
RedirectMessage redirect_message;
if (msg.len != sizeof(RedirectMessage))
return -1;
memcpy(&redirect_message, msg.ptr, sizeof(redirect_message));
if (redirect_message.view_number > state->view_number) {
Time now = get_current_time();
client_log(state, now, "RECV REDIRECT", "view=%lu -> %lu leader=%d",
(unsigned long)state->view_number,
(unsigned long)redirect_message.view_number,
(int)(redirect_message.view_number % state->num_servers));
state->view_number = redirect_message.view_number;
state->last_was_rejected = true;
state->last_was_timeout = false;
state->pending = false;
}
return 0;
}
if (!state->pending)
return -1;
if (type != MESSAGE_TYPE_REPLY)
return -1;
ReplyMessage reply_message;
ReplyMessage message;
if (msg.len != sizeof(ReplyMessage))
return -1;
memcpy(&reply_message, msg.ptr, sizeof(reply_message));
memcpy(&message, msg.ptr, sizeof(message));
// Ignore stale replies from previous requests. After a timeout
// the client moves to a new view and sends a new request, but
// the old leader may still deliver a reply for the old request
// on the previous connection. Without this check the client
// would accept the stale result for the wrong operation.
if (message.request_id != state->request_id)
return 0;
{
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)",
TIME_VAL(now),
reply_message.rejected ? "true" : "false");
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
if (message.rejected) {
client_log(state, now, "RECV REPLY", "key=%.16s %s -> REJECTED", state->last_oper.key, oper_buf);
} else {
char result_buf[64];
kvstore_snprint_result(result_buf, sizeof(result_buf), message.result);
client_log(state, now, "RECV REPLY", "key=%.16s %s -> %s", state->last_oper.key, oper_buf, result_buf);
}
}
state->last_result = message.result;
state->last_was_timeout = false;
state->last_was_rejected = message.rejected;
state->pending = false;
return 0;
}
@@ -92,35 +179,9 @@ int client_init(void *state_, int argc, char **argv,
state->view_number = 0;
state->request_id = 0;
state->reconnect_time = 0;
// Load or generate a persistent client_id from disk.
// This ensures the client_id survives process restarts.
{
string path = S("client_id");
Handle fd;
bool loaded = false;
if (file_exists(path)) {
if (file_open(path, &fd) == 0) {
file_set_offset(fd, 0);
uint64_t saved_id;
if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) {
state->client_id = saved_id;
loaded = true;
}
file_close(fd);
}
}
if (!loaded) {
Time now = get_current_time();
state->client_id = (uint64_t)now;
if (file_open(path, &fd) == 0) {
file_set_offset(fd, 0);
file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id));
file_sync(fd);
file_close(fd);
}
}
}
state->client_id = next_client_id++;
// Connect to all known servers
for (int i = 0; i < state->num_servers; i++) {
@@ -133,8 +194,7 @@ int client_init(void *state_, int argc, char **argv,
{
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d",
TIME_VAL(now), state->num_servers, leader_idx(state));
client_log(state, now, "INIT", "servers=%d leader=%d", state->num_servers, leader_idx(state));
}
*timeout = 0;
@@ -155,20 +215,24 @@ int client_tick(void *state_, void **ctxs,
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
for (int i = 0; i < num_events; i++) {
if (events[i].type == EVENT_DISCONNECT) {
int conn_idx = events[i].conn_idx;
int tag = tcp_get_tag(&state->tcp, conn_idx);
if (tag == leader_idx(state) && state->pending) {
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request",
TIME_VAL(now), leader_idx(state));
client_log(state, now, "DISCONNECT", "key=%.16s lost leader (node %d)", state->last_oper.key, leader_idx(state));
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
}
tcp_close(&state->tcp, conn_idx);
continue;
}
if (events[i].type != EVENT_MESSAGE)
continue;
int conn_idx = events[i].conn_idx;
for (;;) {
@@ -201,10 +265,16 @@ int client_tick(void *state_, void **ctxs,
if (state->pending) {
Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
if (request_deadline <= now) {
CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server",
TIME_VAL(now), leader_idx(state),
(unsigned long)state->view_number);
{
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
client_log(state, now, "TIMEOUT", "key=%.16s %s", state->last_oper.key, oper_buf);
}
state->view_number++;
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
}
}
@@ -213,15 +283,15 @@ int client_tick(void *state_, void **ctxs,
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
if (conn_idx < 0) {
// Leader connection not available, try reconnecting
{
CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting",
TIME_VAL(now), leader_idx(state));
}
if (state->reconnect_time <= now) {
tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
state->reconnect_time = now + HEARTBEAT_INTERVAL_SEC * 1000000000ULL;
}
} else {
// Now start a new operation
state->request_id++;
state->last_oper = random_oper();
RequestMessage request_message = {
.base = {
@@ -229,7 +299,7 @@ int client_tick(void *state_, void **ctxs,
.type = MESSAGE_TYPE_REQUEST,
.length = sizeof(RequestMessage),
},
.oper = OPERATION_A,
.oper = state->last_oper,
.client_id = state->client_id,
.request_id = state->request_id,
};
@@ -240,11 +310,9 @@ int client_tick(void *state_, void **ctxs,
byte_queue_write(output, &request_message, request_message.base.length);
{
CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)",
TIME_VAL(now), leader_idx(state),
(unsigned long)state->view_number,
(unsigned long)state->client_id,
(unsigned long)state->request_id);
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
client_log(state, now, "SEND REQUEST", "key=%.16s %s", state->last_oper.key, oper_buf);
}
state->pending = true;
@@ -252,10 +320,15 @@ int client_tick(void *state_, void **ctxs,
}
}
// Set timeout based on pending request deadline
// Set timeout based on pending request deadline or reconnection delay
Time deadline = INVALID_TIME;
if (state->pending) {
nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
} else {
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
if (conn_idx < 0 && state->reconnect_time > now) {
nearest_deadline(&deadline, state->reconnect_time);
}
}
*timeout = deadline_to_timeout(deadline, now);
if (pcap < TCP_POLL_CAPACITY)
@@ -268,6 +341,11 @@ int client_free(void *state_)
{
ClientState *state = state_;
{
Time now = get_current_time();
client_log_simple(state, now, "CRASHED");
}
tcp_context_free(&state->tcp);
return 0;
}
vsr/client.h
+12
View File
@@ -4,6 +4,8 @@
#include <lib/tcp.h>
#include <lib/basic.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
@@ -14,6 +16,14 @@ typedef struct {
bool pending;
Time request_time; // When the current request was sent
// The operation sent in the current pending request (for logging)
KVStoreOper last_oper;
// Linearizability checker support
KVStoreResult last_result;
bool last_was_timeout;
bool last_was_rejected;
Address server_addrs[NODE_LIMIT];
int num_servers;
@@ -22,6 +32,8 @@ typedef struct {
uint64_t client_id;
uint64_t request_id;
Time reconnect_time; // Earliest time to retry connecting to leader
} ClientState;
struct pollfd;
vsr/client_table.c
+2 -2
View File
@@ -28,7 +28,7 @@ ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_i
return NULL;
}
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx)
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
{
if (client_table->count == client_table->capacity) {
int n = 2 * client_table->capacity;
@@ -44,7 +44,7 @@ int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t req
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_idx = conn_idx,
.conn_tag = conn_tag,
};
return 0;
}
vsr/client_table.h
+8 -5
View File
@@ -3,14 +3,17 @@
#include <stdint.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
OperationResult last_result;
bool pending;
int conn_idx;
KVStoreResult last_result;
bool pending; // Only meaningful on the leader that received
// the REQUEST. After a view change, a new leader
// may find stale entries with pending=false from
// a previous view when it was leader before.
int conn_tag;
} ClientTableEntry;
typedef struct {
@@ -22,7 +25,7 @@ typedef struct {
void client_table_init(ClientTable *client_table);
void client_table_free(ClientTable *client_table);
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
#endif // CLIENT_TABLE_INCLUDED
vsr/config.h
+2
View File
@@ -5,7 +5,9 @@
#define FUTURE_LIMIT 32
#define HEARTBEAT_INTERVAL_SEC 1
#define PRIMARY_DEATH_TIMEOUT_SEC 5
#define VIEW_CHANGE_TIMEOUT_SEC 10
#define RECOVERY_TIMEOUT_SEC 3
#define RECOVERY_ATTEMPT_LIMIT 10
#define STATE_TRANSFER_TIMEOUT_SEC 2
#endif // CONFIG_INCLUDED
vsr/invariant_checker.c
+395
View File
@@ -0,0 +1,395 @@
// VSR invariant checker with external shadow log tracking.
//
// This file runs in the main simulation loop outside Quakey-scheduled
// processes. It includes node.h for struct definitions, then restores
// real allocators since mock_malloc/realloc/free abort outside process
// context.
#include "node.h"
#include <assert.h>
// Restore real allocators (see checker/linearizability.c for precedent).
#undef malloc
#undef realloc
#undef free
#include <stdlib.h>
// These helpers are static in node.c; duplicated here for the checker.
static int self_idx(NodeState *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
return i;
UNREACHABLE;
}
static int leader_idx(NodeState *state)
{
return state->view_number % state->num_nodes;
}
static bool is_leader(NodeState *state)
{
if (state->status == STATUS_RECOVERY)
return false;
return self_idx(state) == leader_idx(state);
}
static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
{
if (ic->shadow_count == ic->shadow_capacity) {
int n = 2 * ic->shadow_capacity;
if (n < 8)
n = 8;
KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
if (p == NULL)
return -1;
ic->shadow_log = p;
ic->shadow_capacity = n;
}
ic->shadow_log[ic->shadow_count++] = oper;
return 0;
}
void invariant_checker_init(InvariantChecker *ic)
{
ic->last_min_commit = -1;
ic->last_max_commit = -1;
for (int i = 0; i < NODE_LIMIT; i++)
ic->prev_status[i] = STATUS_NORMAL;
ic->shadow_log = NULL;
ic->shadow_count = 0;
ic->shadow_capacity = 0;
}
void invariant_checker_free(InvariantChecker *ic)
{
fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
ic->shadow_count);
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
{
int min_commit = -1;
int max_commit = -1;
bool primary = false;
uint64_t primary_view_number = 0;
bool min_commit_just_recovered = false;
uint64_t max_view_number = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
max_view_number = MAX(max_view_number, nodes[i]->view_number);
}
for (int i = 0; i < num_nodes; i++) {
NodeState *n = nodes[i];
if (n == NULL || n->status == STATUS_RECOVERY)
continue;
if (min_commit < 0 || min_commit > n->commit_index) {
min_commit = n->commit_index;
min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY);
}
if (max_commit < 0 || max_commit < n->commit_index) {
max_commit = n->commit_index;
}
if (is_leader(n) && n->view_number > primary_view_number) {
primary = true;
primary_view_number = n->view_number;
}
}
// If the primary isn't up to date, it's not the
// real primary.
if (primary_view_number < max_view_number)
primary = false;
if (min_commit < 0) {
assert(ic->last_min_commit == -1);
} else {
// The minimum number of committed entries should
// only increase, but there are some corner-cases
// when this is not true.
// When a node completes the recovery state, its
// log is technically outdated as it was sent some
// point in the past. If operations are committed
// while the recovery response is in transit over
// the network, the minimum number of committed
// entries will decrease.
if (!min_commit_just_recovered) {
assert(ic->last_min_commit <= min_commit);
}
}
if (max_commit < 0) {
assert(ic->last_max_commit == -1);
} else {
// The maximum number of committed entries
// should only increase. The primary generally
// has more committed entries than replicas. If
// the primary dies, the maximum commit number
// of the live nodes may decrease. This is still
// okay since the new primary will commit up to
// that point as the view change completes. This
// implies that the maximum commit number should
// always increase unless there is no primary.
if (!primary) {
max_commit = ic->last_max_commit;
} else {
//assert(ic->last_max_commit <= max_commit);
}
}
ic->last_min_commit = min_commit;
ic->last_max_commit = max_commit;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
ic->prev_status[i] = nodes[i]->status;
}
for (int i = 0; i < num_nodes; i++) {
NodeState *s = nodes[i];
if (s == NULL)
continue;
// 1. commit_index <= log.count
// A node cannot have committed more entries than it has in its log.
if (s->commit_index > s->log.count) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n",
i, s->commit_index, s->log.count);
__builtin_trap();
}
// 2. commit_index >= 0
if (s->commit_index < 0) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n",
i, s->commit_index);
__builtin_trap();
}
// 4. Future buffer count is in valid range.
if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n",
i, s->num_future, FUTURE_LIMIT);
__builtin_trap();
}
// 7. last_normal_view <= view_number
// The most recent view in which this node was in NORMAL status
// cannot exceed its current view number.
if (s->last_normal_view > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_normal_view (%lu) > view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 8. When status is NORMAL, last_normal_view must equal view_number.
// Every transition to NORMAL status sets last_normal_view = view_number.
// If they diverge while in NORMAL, a transition forgot to update it.
if (s->status == STATUS_NORMAL && s->last_normal_view != s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: status is NORMAL but "
"last_normal_view (%lu) != view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 9. Log entry view numbers must not exceed the node's current view.
// Entries are created in the view they were proposed. No entry
// should carry a view number from the future.
for (int k = 0; k < s->log.count; k++) {
if ((uint64_t)s->log.entries[k].view_number > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) "
"> view_number (%lu)\n",
i, k, s->log.entries[k].view_number,
(unsigned long)s->view_number);
__builtin_trap();
}
}
// 10. For a leader in NORMAL status, every uncommitted log entry
// must have the leader's own vote bit set. The leader always
// votes for its own entries.
if (s->status == STATUS_NORMAL && is_leader(s)) {
int idx = self_idx(s);
for (int k = s->commit_index; k < s->log.count; k++) {
if (!(s->log.entries[k].votes & (1 << idx))) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): "
"uncommitted log[%d] missing leader's own vote bit\n",
i, k);
__builtin_trap();
}
}
}
}
// Cross-node invariants
// 5. At most one leader in normal status per view.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i]))
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j]))
continue;
if (nodes[i]->view_number == nodes[j]->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n",
(unsigned long)nodes[i]->view_number, i, j);
__builtin_trap();
}
}
}
// 6. Committed prefix agreement (State Machine Safety).
// For any two nodes, their logs must agree on all entries up to
// min(commit_index_i, commit_index_j). This is the core safety
// property of VSR: all committed operations are identical across
// replicas.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL)
continue;
int mc = nodes[i]->commit_index;
if (nodes[j]->commit_index < mc)
mc = nodes[j]->commit_index;
for (int k = 0; k < mc; k++) {
if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d "
"between node %d and node %d\n", k, i, j);
__builtin_trap();
}
}
}
}
////////////////////////////////////////////////////////////////////
// Shadow log: external commit tracking
////////////////////////////////////////////////////////////////////
// Phase 1: Find the observed max commit index and a source node.
int observed_max_commit = 0;
int source_node_idx = -1;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->status == STATUS_RECOVERY)
continue;
if (nodes[i]->commit_index > observed_max_commit) {
observed_max_commit = nodes[i]->commit_index;
source_node_idx = i;
}
}
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
NodeState *source = nodes[source_node_idx];
assert(source->log.count >= observed_max_commit);
for (int k = ic->shadow_count; k < observed_max_commit; k++) {
KVStoreOper *source_oper = &source->log.entries[k].oper;
// Cross-validate against other live non-recovering nodes
// that have also committed this entry.
for (int j = 0; j < num_nodes; j++) {
if (j == source_node_idx)
continue;
if (nodes[j] == NULL)
continue;
if (nodes[j]->status == STATUS_RECOVERY)
continue;
if (nodes[j]->commit_index <= k)
continue;
if (nodes[j]->log.count <= k)
continue;
if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
"between source node %d and node %d during shadow log append\n",
k, source_node_idx, j);
__builtin_trap();
}
}
if (shadow_log_append(ic, *source_oper) < 0) {
fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
__builtin_trap();
}
}
}
// Phase 3: Verify shadow log against the cluster.
// Sub-check A: Committed entries must match the shadow log.
for (int k = 0; k < ic->shadow_count; k++) {
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->log.count <= k)
continue;
if (nodes[i]->commit_index <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
char shadow_buf[128], node_buf[128];
kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
kvstore_snprint_oper(node_buf, sizeof(node_buf), nodes[i]->log.entries[k].oper);
fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
" shadow: %s\n"
" node: %s\n",
k, i, shadow_buf, node_buf);
__builtin_trap();
}
}
}
// Sub-check B: When commit regresses, previously committed entries
// must still be held by a majority of the cluster.
// Recovering nodes are treated like dead nodes: they haven't
// restored their log yet and may still recover the entry through
// the recovery protocol.
if (observed_max_commit < ic->shadow_count) {
for (int k = observed_max_commit; k < ic->shadow_count; k++) {
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
if (nodes[i]->status == STATUS_RECOVERY) {
num_dead++;
continue;
}
if (nodes[i]->log.count <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
holders++;
}
if (holders + num_dead <= num_nodes / 2) {
char oper_buf[128];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
"no longer held by majority (holders=%d, dead=%d, total=%d)\n"
" entry: %s\n",
k, holders, num_dead, num_nodes, oper_buf);
__builtin_trap();
}
}
}
}
vsr/log.c
+18
View File
@@ -13,11 +13,29 @@ void log_init(Log *log)
log->entries = NULL;
}
int log_init_from_network(Log *log, void *src, int num)
{
log->count = num;
log->capacity = num;
log->entries = malloc(num * sizeof(LogEntry));
if (log->entries == NULL)
return -1;
memcpy(log->entries, src, num * sizeof(LogEntry));
return 0;
}
void log_free(Log *log)
{
free(log->entries);
}
void log_move(Log *dst, Log *src)
{
log_free(dst);
*dst = *src;
log_init(src);
}
int log_append(Log *log, LogEntry entry)
{
if (log->count == log->capacity) {
vsr/log.h
+5 -2
View File
@@ -1,15 +1,16 @@
#ifndef LOG_INCLUDED
#define LOG_INCLUDED
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
Operation oper;
KVStoreOper oper;
uint32_t votes;
int view_number;
uint64_t client_id;
uint64_t request_id;
} LogEntry;
_Static_assert(NODE_LIMIT <= 32, "");
@@ -21,7 +22,9 @@ typedef struct {
} Log;
void log_init(Log *log);
int log_init_from_network(Log *log, void *src, int num);
void log_free(Log *log);
void log_move(Log *dst, Log *src);
int log_append(Log *log, LogEntry entry);
#endif // LOG_INCLUDED
vsr/main.c
+39 -5
View File
@@ -4,7 +4,7 @@
#include <signal.h>
#include <stdint.h>
#include <quakey.h>
#include <assert.h>
#include "node.h"
#include "client.h"
@@ -16,12 +16,23 @@ static void sigint_handler(int sig)
simulation_running = 0;
}
int main(void)
int main(int argc, char **argv)
{
signal(SIGINT, sigint_handler);
QuakeyUInt64 seed = 1;
QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
seed = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
}
}
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
int ret = quakey_init(&quakey, seed);
if (ret < 0)
return -1;
@@ -111,7 +122,18 @@ int main(void)
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
while (simulation_running) {
// Limit crashes to 1 node at a time (within fault tolerance of f=1).
// This is dynamically adjusted below: crashes are disabled while
// any node is still recovering.
quakey_set_max_crashes(quakey, 1);
quakey_network_partitioning(quakey, true);
InvariantChecker invariant_checker;
invariant_checker_init(&invariant_checker);
while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
quakey_schedule_one(quakey);
NodeState *arr[] = {
@@ -119,9 +141,21 @@ int main(void)
quakey_node_state(node_2),
quakey_node_state(node_3),
};
check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0]));
invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
// VR-Revisited Section 8.2: "a replica is considered failed
// until it has recovered its state." Disable crashes while
// any node is recovering to avoid exceeding f simultaneous
// failures (dead + recovering).
bool any_recovering = false;
for (int i = 0; i < 3; i++) {
if (arr[i] && arr[i]->status == STATUS_RECOVERY)
any_recovering = true;
}
quakey_set_max_crashes(quakey, any_recovering ? 0 : 1);
}
invariant_checker_free(&invariant_checker);
quakey_free(quakey);
return 0;
}
vsr/node.c
+1129 -1180
View File
File diff suppressed because it is too large Load Diff
vsr/node.h
+68 -18
View File
@@ -5,7 +5,7 @@
#include <lib/basic.h>
#include <lib/message.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
#include "log.h"
#include "config.h"
@@ -28,22 +28,31 @@ enum {
// Recovery Protocol
MESSAGE_TYPE_RECOVERY,
MESSAGE_TYPE_RECOVERY_RESPONSE,
// State Transfer Protocol
MESSAGE_TYPE_GET_STATE,
MESSAGE_TYPE_NEW_STATE,
// Client Redirect
MESSAGE_TYPE_REDIRECT,
};
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
int sender_idx;
int log_index;
int commit_index;
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
} PrepareMessage;
typedef struct {
@@ -61,7 +70,8 @@ typedef struct {
typedef struct {
MessageHeader base;
bool rejected;
OperationResult result;
KVStoreResult result;
uint64_t request_id;
} ReplyMessage;
typedef struct {
@@ -103,6 +113,26 @@ typedef struct {
int sender_idx;
} RecoveryResponseMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Requester's current log count
int sender_idx;
} GetStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Number of log entries that follow
int commit_index;
// Followed by: LogEntry log[op_number]
} NewStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
} RedirectMessage;
typedef enum {
STATUS_NORMAL,
STATUS_CHANGE_VIEW,
@@ -120,27 +150,28 @@ typedef struct {
Status status;
ClientTable client_table;
int next_client_tag;
uint64_t view_number;
uint64_t last_normal_view; // Latest view where status was NORMAL
// These fields are used in recovery mode
uint32_t recovery_votes;
uint64_t recovery_nonce;
uint64_t max_view_seen;
int latest_primary_idx;
bool received_primary_state;
Log potential_primary_log;
Time recovery_start_time;
int recovery_attempt_count;
uint64_t recovery_view;
Log recovery_log;
uint64_t recovery_log_view;
Time recovery_time;
int recovery_commit;
///////////////////////////////////////////////////////////
// VIEW CHANGE
uint32_t begin_view_change_votes; // Bitmask of received BeginViewChange messages
uint32_t do_view_change_votes; // Bitmask of received DoViewChange messages
uint64_t do_view_change_best_old_view; // Best old_view_number seen in DoViewChange
int do_view_change_best_commit; // Best commit_index seen
Log do_view_change_best_log; // Best log seen
uint32_t view_change_begin_votes;
uint32_t view_change_apply_votes;
Log view_change_log; // Best log seen
uint64_t view_change_old_view; // Best old_view_number seen in DoViewChange
int view_change_commit; // Best commit_index seen
///////////////////////////////////////////////////////////
@@ -152,11 +183,17 @@ typedef struct {
PrepareMessage future[FUTURE_LIMIT];
int num_future;
bool state_transfer_pending;
Time state_transfer_time;
Log log;
Time last_heartbeat_time;
Time heartbeat;
StateMachine state_machine;
KVStore kvstore;
// Set at each wakeup
Time now;
} NodeState;
@@ -171,6 +208,19 @@ int node_tick(void *state, void **ctxs,
int node_free(void *state);
void check_vsr_invariants(NodeState **nodes, int num_nodes);
typedef struct {
int last_min_commit;
int last_max_commit;
Status prev_status[NODE_LIMIT];
// External shadow log of committed operations (unbounded, dynamically allocated)
KVStoreOper *shadow_log;
int shadow_count;
int shadow_capacity;
} InvariantChecker;
void invariant_checker_init(InvariantChecker *ic);
void invariant_checker_free(InvariantChecker *ic);
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
#endif // NODE_INCLUDED