cozis/Thesis_Code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 0.9

Browse Source
This commit is contained in:
cozis
2026-02-19 15:28:06 +01:00
parent 868312edf8
commit 238e35b581
31 changed files with 4256 additions and 2253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vsr/client.c
+133 -55
View File
@@ -6,18 +6,68 @@
#include <stdint.h>
#include <assert.h>
#include <lib/file_system.h>
#include "node.h"
#include "client.h"
//#define CLIENT_TRACE(fmt, ...) {}
#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
#define KEY_POOL_SIZE 128
static uint64_t next_client_id = 1;
static uint64_t client_random(void)
{
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
return quakey_random();
#else
return (uint64_t)rand();
#endif
}
static KVStoreOper random_oper(void)
{
KVStoreOper oper = {0};
snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
switch (client_random() % 3) {
case 0:
oper.type = KVSTORE_OPER_SET;
oper.val = client_random();
break;
case 1:
oper.type = KVSTORE_OPER_GET;
break;
case 2:
oper.type = KVSTORE_OPER_DEL;
break;
}
return oper;
}
// Format time as seconds with 3 decimal places for trace output
#define TIME_FMT "%.3fs"
#define TIME_FMT "%7.3fs"
#define TIME_VAL(t) ((double)(t) / 1000000000.0)
static void client_log_impl(ClientState *state, Time now, const char *event, const char *detail)
{
printf("[" TIME_FMT "] CLIENT %lu | V%-3lu | %-20s %s\n",
TIME_VAL(now),
state->client_id,
state->view_number,
event,
detail ? detail : "");
}
#define client_log(state, now, event, fmt, ...) do { \
char _detail[256]; \
snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \
client_log_impl(state, now, event, _detail); \
} while (0)
#define client_log_simple(state, now, event) \
client_log_impl(state, now, event, NULL)
static int leader_idx(ClientState *state)
{
return state->view_number % state->num_servers;
@@ -29,24 +79,61 @@ process_message(ClientState *state,
{
(void) conn_idx;
if (type == MESSAGE_TYPE_REDIRECT) {
RedirectMessage redirect_message;
if (msg.len != sizeof(RedirectMessage))
return -1;
memcpy(&redirect_message, msg.ptr, sizeof(redirect_message));
if (redirect_message.view_number > state->view_number) {
Time now = get_current_time();
client_log(state, now, "RECV REDIRECT", "view=%lu -> %lu leader=%d",
(unsigned long)state->view_number,
(unsigned long)redirect_message.view_number,
(int)(redirect_message.view_number % state->num_servers));
state->view_number = redirect_message.view_number;
state->last_was_rejected = true;
state->last_was_timeout = false;
state->pending = false;
}
return 0;
}
if (!state->pending)
return -1;
if (type != MESSAGE_TYPE_REPLY)
return -1;
ReplyMessage reply_message;
ReplyMessage message;
if (msg.len != sizeof(ReplyMessage))
return -1;
memcpy(&reply_message, msg.ptr, sizeof(reply_message));
memcpy(&message, msg.ptr, sizeof(message));
// Ignore stale replies from previous requests. After a timeout
// the client moves to a new view and sends a new request, but
// the old leader may still deliver a reply for the old request
// on the previous connection. Without this check the client
// would accept the stale result for the wrong operation.
if (message.request_id != state->request_id)
return 0;
{
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)",
TIME_VAL(now),
reply_message.rejected ? "true" : "false");
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
if (message.rejected) {
client_log(state, now, "RECV REPLY", "key=%.16s %s -> REJECTED", state->last_oper.key, oper_buf);
} else {
char result_buf[64];
kvstore_snprint_result(result_buf, sizeof(result_buf), message.result);
client_log(state, now, "RECV REPLY", "key=%.16s %s -> %s", state->last_oper.key, oper_buf, result_buf);
}
}
state->last_result = message.result;
state->last_was_timeout = false;
state->last_was_rejected = message.rejected;
state->pending = false;
return 0;
}
@@ -92,35 +179,9 @@ int client_init(void *state_, int argc, char **argv,
state->view_number = 0;
state->request_id = 0;
state->reconnect_time = 0;
// Load or generate a persistent client_id from disk.
// This ensures the client_id survives process restarts.
{
string path = S("client_id");
Handle fd;
bool loaded = false;
if (file_exists(path)) {
if (file_open(path, &fd) == 0) {
file_set_offset(fd, 0);
uint64_t saved_id;
if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) {
state->client_id = saved_id;
loaded = true;
}
file_close(fd);
}
}
if (!loaded) {
Time now = get_current_time();
state->client_id = (uint64_t)now;
if (file_open(path, &fd) == 0) {
file_set_offset(fd, 0);
file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id));
file_sync(fd);
file_close(fd);
}
}
}
state->client_id = next_client_id++;
// Connect to all known servers
for (int i = 0; i < state->num_servers; i++) {
@@ -133,8 +194,7 @@ int client_init(void *state_, int argc, char **argv,
{
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d",
TIME_VAL(now), state->num_servers, leader_idx(state));
client_log(state, now, "INIT", "servers=%d leader=%d", state->num_servers, leader_idx(state));
}
*timeout = 0;
@@ -155,20 +215,24 @@ int client_tick(void *state_, void **ctxs,
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
for (int i = 0; i < num_events; i++) {
if (events[i].type == EVENT_DISCONNECT) {
int conn_idx = events[i].conn_idx;
int tag = tcp_get_tag(&state->tcp, conn_idx);
if (tag == leader_idx(state) && state->pending) {
Time now = get_current_time();
CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request",
TIME_VAL(now), leader_idx(state));
client_log(state, now, "DISCONNECT", "key=%.16s lost leader (node %d)", state->last_oper.key, leader_idx(state));
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
}
tcp_close(&state->tcp, conn_idx);
continue;
}
if (events[i].type != EVENT_MESSAGE)
continue;
int conn_idx = events[i].conn_idx;
for (;;) {
@@ -201,10 +265,16 @@ int client_tick(void *state_, void **ctxs,
if (state->pending) {
Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
if (request_deadline <= now) {
CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server",
TIME_VAL(now), leader_idx(state),
(unsigned long)state->view_number);
{
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
client_log(state, now, "TIMEOUT", "key=%.16s %s", state->last_oper.key, oper_buf);
}
state->view_number++;
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
}
}
@@ -213,15 +283,15 @@ int client_tick(void *state_, void **ctxs,
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
if (conn_idx < 0) {
// Leader connection not available, try reconnecting
{
CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting",
TIME_VAL(now), leader_idx(state));
if (state->reconnect_time <= now) {
tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
state->reconnect_time = now + HEARTBEAT_INTERVAL_SEC * 1000000000ULL;
}
tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
} else {
// Now start a new operation
state->request_id++;
state->last_oper = random_oper();
RequestMessage request_message = {
.base = {
@@ -229,7 +299,7 @@ int client_tick(void *state_, void **ctxs,
.type = MESSAGE_TYPE_REQUEST,
.length = sizeof(RequestMessage),
},
.oper = OPERATION_A,
.oper = state->last_oper,
.client_id = state->client_id,
.request_id = state->request_id,
};
@@ -240,11 +310,9 @@ int client_tick(void *state_, void **ctxs,
byte_queue_write(output, &request_message, request_message.base.length);
{
CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)",
TIME_VAL(now), leader_idx(state),
(unsigned long)state->view_number,
(unsigned long)state->client_id,
(unsigned long)state->request_id);
char oper_buf[64];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), state->last_oper);
client_log(state, now, "SEND REQUEST", "key=%.16s %s", state->last_oper.key, oper_buf);
}
state->pending = true;
@@ -252,10 +320,15 @@ int client_tick(void *state_, void **ctxs,
}
}
// Set timeout based on pending request deadline
// Set timeout based on pending request deadline or reconnection delay
Time deadline = INVALID_TIME;
if (state->pending) {
nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
} else {
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
if (conn_idx < 0 && state->reconnect_time > now) {
nearest_deadline(&deadline, state->reconnect_time);
}
}
*timeout = deadline_to_timeout(deadline, now);
if (pcap < TCP_POLL_CAPACITY)
@@ -268,6 +341,11 @@ int client_free(void *state_)
{
ClientState *state = state_;
{
Time now = get_current_time();
client_log_simple(state, now, "CRASHED");
}
tcp_context_free(&state->tcp);
return 0;
}
vsr/client.h
+12
View File
@@ -4,6 +4,8 @@
#include <lib/tcp.h>
#include <lib/basic.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
@@ -14,6 +16,14 @@ typedef struct {
bool pending;
Time request_time; // When the current request was sent
// The operation sent in the current pending request (for logging)
KVStoreOper last_oper;
// Linearizability checker support
KVStoreResult last_result;
bool last_was_timeout;
bool last_was_rejected;
Address server_addrs[NODE_LIMIT];
int num_servers;
@@ -22,6 +32,8 @@ typedef struct {
uint64_t client_id;
uint64_t request_id;
Time reconnect_time; // Earliest time to retry connecting to leader
} ClientState;
struct pollfd;
vsr/client_table.c
+2 -2
View File
@@ -28,7 +28,7 @@ ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_i
return NULL;
}
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx)
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
{
if (client_table->count == client_table->capacity) {
int n = 2 * client_table->capacity;
@@ -44,7 +44,7 @@ int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t req
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_idx = conn_idx,
.conn_tag = conn_tag,
};
return 0;
}
vsr/client_table.h
+8 -5
View File
@@ -3,14 +3,17 @@
#include <stdint.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
OperationResult last_result;
bool pending;
int conn_idx;
KVStoreResult last_result;
bool pending; // Only meaningful on the leader that received
// the REQUEST. After a view change, a new leader
// may find stale entries with pending=false from
// a previous view when it was leader before.
int conn_tag;
} ClientTableEntry;
typedef struct {
@@ -22,7 +25,7 @@ typedef struct {
void client_table_init(ClientTable *client_table);
void client_table_free(ClientTable *client_table);
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
#endif // CLIENT_TABLE_INCLUDED
vsr/config.h
+2
View File
@@ -5,7 +5,9 @@
#define FUTURE_LIMIT 32
#define HEARTBEAT_INTERVAL_SEC 1
#define PRIMARY_DEATH_TIMEOUT_SEC 5
#define VIEW_CHANGE_TIMEOUT_SEC 10
#define RECOVERY_TIMEOUT_SEC 3
#define RECOVERY_ATTEMPT_LIMIT 10
#define STATE_TRANSFER_TIMEOUT_SEC 2
#endif // CONFIG_INCLUDED
vsr/invariant_checker.c
+395
View File
@@ -0,0 +1,395 @@
// VSR invariant checker with external shadow log tracking.
//
// This file runs in the main simulation loop outside Quakey-scheduled
// processes. It includes node.h for struct definitions, then restores
// real allocators since mock_malloc/realloc/free abort outside process
// context.
#include "node.h"
#include <assert.h>
// Restore real allocators (see checker/linearizability.c for precedent).
#undef malloc
#undef realloc
#undef free
#include <stdlib.h>
// These helpers are static in node.c; duplicated here for the checker.
static int self_idx(NodeState *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
return i;
UNREACHABLE;
}
static int leader_idx(NodeState *state)
{
return state->view_number % state->num_nodes;
}
static bool is_leader(NodeState *state)
{
if (state->status == STATUS_RECOVERY)
return false;
return self_idx(state) == leader_idx(state);
}
static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
{
if (ic->shadow_count == ic->shadow_capacity) {
int n = 2 * ic->shadow_capacity;
if (n < 8)
n = 8;
KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
if (p == NULL)
return -1;
ic->shadow_log = p;
ic->shadow_capacity = n;
}
ic->shadow_log[ic->shadow_count++] = oper;
return 0;
}
void invariant_checker_init(InvariantChecker *ic)
{
ic->last_min_commit = -1;
ic->last_max_commit = -1;
for (int i = 0; i < NODE_LIMIT; i++)
ic->prev_status[i] = STATUS_NORMAL;
ic->shadow_log = NULL;
ic->shadow_count = 0;
ic->shadow_capacity = 0;
}
void invariant_checker_free(InvariantChecker *ic)
{
fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
ic->shadow_count);
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
{
int min_commit = -1;
int max_commit = -1;
bool primary = false;
uint64_t primary_view_number = 0;
bool min_commit_just_recovered = false;
uint64_t max_view_number = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
max_view_number = MAX(max_view_number, nodes[i]->view_number);
}
for (int i = 0; i < num_nodes; i++) {
NodeState *n = nodes[i];
if (n == NULL || n->status == STATUS_RECOVERY)
continue;
if (min_commit < 0 || min_commit > n->commit_index) {
min_commit = n->commit_index;
min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY);
}
if (max_commit < 0 || max_commit < n->commit_index) {
max_commit = n->commit_index;
}
if (is_leader(n) && n->view_number > primary_view_number) {
primary = true;
primary_view_number = n->view_number;
}
}
// If the primary isn't up to date, it's not the
// real primary.
if (primary_view_number < max_view_number)
primary = false;
if (min_commit < 0) {
assert(ic->last_min_commit == -1);
} else {
// The minimum number of committed entries should
// only increase, but there are some corner-cases
// when this is not true.
// When a node completes the recovery state, its
// log is technically outdated as it was sent some
// point in the past. If operations are committed
// while the recovery response is in transit over
// the network, the minimum number of committed
// entries will decrease.
if (!min_commit_just_recovered) {
assert(ic->last_min_commit <= min_commit);
}
}
if (max_commit < 0) {
assert(ic->last_max_commit == -1);
} else {
// The maximum number of committed entries
// should only increase. The primary generally
// has more committed entries than replicas. If
// the primary dies, the maximum commit number
// of the live nodes may decrease. This is still
// okay since the new primary will commit up to
// that point as the view change completes. This
// implies that the maximum commit number should
// always increase unless there is no primary.
if (!primary) {
max_commit = ic->last_max_commit;
} else {
//assert(ic->last_max_commit <= max_commit);
}
}
ic->last_min_commit = min_commit;
ic->last_max_commit = max_commit;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
ic->prev_status[i] = nodes[i]->status;
}
for (int i = 0; i < num_nodes; i++) {
NodeState *s = nodes[i];
if (s == NULL)
continue;
// 1. commit_index <= log.count
// A node cannot have committed more entries than it has in its log.
if (s->commit_index > s->log.count) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n",
i, s->commit_index, s->log.count);
__builtin_trap();
}
// 2. commit_index >= 0
if (s->commit_index < 0) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n",
i, s->commit_index);
__builtin_trap();
}
// 4. Future buffer count is in valid range.
if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n",
i, s->num_future, FUTURE_LIMIT);
__builtin_trap();
}
// 7. last_normal_view <= view_number
// The most recent view in which this node was in NORMAL status
// cannot exceed its current view number.
if (s->last_normal_view > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_normal_view (%lu) > view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 8. When status is NORMAL, last_normal_view must equal view_number.
// Every transition to NORMAL status sets last_normal_view = view_number.
// If they diverge while in NORMAL, a transition forgot to update it.
if (s->status == STATUS_NORMAL && s->last_normal_view != s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: status is NORMAL but "
"last_normal_view (%lu) != view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 9. Log entry view numbers must not exceed the node's current view.
// Entries are created in the view they were proposed. No entry
// should carry a view number from the future.
for (int k = 0; k < s->log.count; k++) {
if ((uint64_t)s->log.entries[k].view_number > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) "
"> view_number (%lu)\n",
i, k, s->log.entries[k].view_number,
(unsigned long)s->view_number);
__builtin_trap();
}
}
// 10. For a leader in NORMAL status, every uncommitted log entry
// must have the leader's own vote bit set. The leader always
// votes for its own entries.
if (s->status == STATUS_NORMAL && is_leader(s)) {
int idx = self_idx(s);
for (int k = s->commit_index; k < s->log.count; k++) {
if (!(s->log.entries[k].votes & (1 << idx))) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): "
"uncommitted log[%d] missing leader's own vote bit\n",
i, k);
__builtin_trap();
}
}
}
}
// Cross-node invariants
// 5. At most one leader in normal status per view.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i]))
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j]))
continue;
if (nodes[i]->view_number == nodes[j]->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n",
(unsigned long)nodes[i]->view_number, i, j);
__builtin_trap();
}
}
}
// 6. Committed prefix agreement (State Machine Safety).
// For any two nodes, their logs must agree on all entries up to
// min(commit_index_i, commit_index_j). This is the core safety
// property of VSR: all committed operations are identical across
// replicas.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL)
continue;
int mc = nodes[i]->commit_index;
if (nodes[j]->commit_index < mc)
mc = nodes[j]->commit_index;
for (int k = 0; k < mc; k++) {
if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d "
"between node %d and node %d\n", k, i, j);
__builtin_trap();
}
}
}
}
////////////////////////////////////////////////////////////////////
// Shadow log: external commit tracking
////////////////////////////////////////////////////////////////////
// Phase 1: Find the observed max commit index and a source node.
int observed_max_commit = 0;
int source_node_idx = -1;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->status == STATUS_RECOVERY)
continue;
if (nodes[i]->commit_index > observed_max_commit) {
observed_max_commit = nodes[i]->commit_index;
source_node_idx = i;
}
}
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
NodeState *source = nodes[source_node_idx];
assert(source->log.count >= observed_max_commit);
for (int k = ic->shadow_count; k < observed_max_commit; k++) {
KVStoreOper *source_oper = &source->log.entries[k].oper;
// Cross-validate against other live non-recovering nodes
// that have also committed this entry.
for (int j = 0; j < num_nodes; j++) {
if (j == source_node_idx)
continue;
if (nodes[j] == NULL)
continue;
if (nodes[j]->status == STATUS_RECOVERY)
continue;
if (nodes[j]->commit_index <= k)
continue;
if (nodes[j]->log.count <= k)
continue;
if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
"between source node %d and node %d during shadow log append\n",
k, source_node_idx, j);
__builtin_trap();
}
}
if (shadow_log_append(ic, *source_oper) < 0) {
fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
__builtin_trap();
}
}
}
// Phase 3: Verify shadow log against the cluster.
// Sub-check A: Committed entries must match the shadow log.
for (int k = 0; k < ic->shadow_count; k++) {
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->log.count <= k)
continue;
if (nodes[i]->commit_index <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
char shadow_buf[128], node_buf[128];
kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
kvstore_snprint_oper(node_buf, sizeof(node_buf), nodes[i]->log.entries[k].oper);
fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
" shadow: %s\n"
" node: %s\n",
k, i, shadow_buf, node_buf);
__builtin_trap();
}
}
}
// Sub-check B: When commit regresses, previously committed entries
// must still be held by a majority of the cluster.
// Recovering nodes are treated like dead nodes: they haven't
// restored their log yet and may still recover the entry through
// the recovery protocol.
if (observed_max_commit < ic->shadow_count) {
for (int k = observed_max_commit; k < ic->shadow_count; k++) {
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
if (nodes[i]->status == STATUS_RECOVERY) {
num_dead++;
continue;
}
if (nodes[i]->log.count <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
holders++;
}
if (holders + num_dead <= num_nodes / 2) {
char oper_buf[128];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
"no longer held by majority (holders=%d, dead=%d, total=%d)\n"
" entry: %s\n",
k, holders, num_dead, num_nodes, oper_buf);
__builtin_trap();
}
}
}
}
vsr/log.c
+18
View File
@@ -13,11 +13,29 @@ void log_init(Log *log)
log->entries = NULL;
}
int log_init_from_network(Log *log, void *src, int num)
{
log->count = num;
log->capacity = num;
log->entries = malloc(num * sizeof(LogEntry));
if (log->entries == NULL)
return -1;
memcpy(log->entries, src, num * sizeof(LogEntry));
return 0;
}
void log_free(Log *log)
{
free(log->entries);
}
void log_move(Log *dst, Log *src)
{
log_free(dst);
*dst = *src;
log_init(src);
}
int log_append(Log *log, LogEntry entry)
{
if (log->count == log->capacity) {
vsr/log.h
+5 -2
View File
@@ -1,15 +1,16 @@
#ifndef LOG_INCLUDED
#define LOG_INCLUDED
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
Operation oper;
KVStoreOper oper;
uint32_t votes;
int view_number;
uint64_t client_id;
uint64_t request_id;
} LogEntry;
_Static_assert(NODE_LIMIT <= 32, "");
@@ -21,7 +22,9 @@ typedef struct {
} Log;
void log_init(Log *log);
int log_init_from_network(Log *log, void *src, int num);
void log_free(Log *log);
void log_move(Log *dst, Log *src);
int log_append(Log *log, LogEntry entry);
#endif // LOG_INCLUDED
vsr/main.c
+39 -5
View File
@@ -4,7 +4,7 @@
#include <signal.h>
#include <stdint.h>
#include <quakey.h>
#include <assert.h>
#include "node.h"
#include "client.h"
@@ -16,12 +16,23 @@ static void sigint_handler(int sig)
simulation_running = 0;
}
int main(void)
int main(int argc, char **argv)
{
signal(SIGINT, sigint_handler);
QuakeyUInt64 seed = 1;
QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
seed = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
}
}
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
int ret = quakey_init(&quakey, seed);
if (ret < 0)
return -1;
@@ -111,7 +122,18 @@ int main(void)
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
while (simulation_running) {
// Limit crashes to 1 node at a time (within fault tolerance of f=1).
// This is dynamically adjusted below: crashes are disabled while
// any node is still recovering.
quakey_set_max_crashes(quakey, 1);
quakey_network_partitioning(quakey, true);
InvariantChecker invariant_checker;
invariant_checker_init(&invariant_checker);
while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
quakey_schedule_one(quakey);
NodeState *arr[] = {
@@ -119,9 +141,21 @@ int main(void)
quakey_node_state(node_2),
quakey_node_state(node_3),
};
check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0]));
invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
// VR-Revisited Section 8.2: "a replica is considered failed
// until it has recovered its state." Disable crashes while
// any node is recovering to avoid exceeding f simultaneous
// failures (dead + recovering).
bool any_recovering = false;
for (int i = 0; i < 3; i++) {
if (arr[i] && arr[i]->status == STATUS_RECOVERY)
any_recovering = true;
}
quakey_set_max_crashes(quakey, any_recovering ? 0 : 1);
}
invariant_checker_free(&invariant_checker);
quakey_free(quakey);
return 0;
}
vsr/node.c
+1278 -1329
View File
File diff suppressed because it is too large Load Diff
vsr/node.h
+68 -18
View File
@@ -5,7 +5,7 @@
#include <lib/basic.h>
#include <lib/message.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
#include "log.h"
#include "config.h"
@@ -28,22 +28,31 @@ enum {
// Recovery Protocol
MESSAGE_TYPE_RECOVERY,
MESSAGE_TYPE_RECOVERY_RESPONSE,
// State Transfer Protocol
MESSAGE_TYPE_GET_STATE,
MESSAGE_TYPE_NEW_STATE,
// Client Redirect
MESSAGE_TYPE_REDIRECT,
};
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
int sender_idx;
int log_index;
int commit_index;
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
} PrepareMessage;
typedef struct {
@@ -61,7 +70,8 @@ typedef struct {
typedef struct {
MessageHeader base;
bool rejected;
OperationResult result;
KVStoreResult result;
uint64_t request_id;
} ReplyMessage;
typedef struct {
@@ -103,6 +113,26 @@ typedef struct {
int sender_idx;
} RecoveryResponseMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Requester's current log count
int sender_idx;
} GetStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Number of log entries that follow
int commit_index;
// Followed by: LogEntry log[op_number]
} NewStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
} RedirectMessage;
typedef enum {
STATUS_NORMAL,
STATUS_CHANGE_VIEW,
@@ -120,27 +150,28 @@ typedef struct {
Status status;
ClientTable client_table;
int next_client_tag;
uint64_t view_number;
uint64_t last_normal_view; // Latest view where status was NORMAL
// These fields are used in recovery mode
uint32_t recovery_votes;
uint64_t recovery_nonce;
uint64_t max_view_seen;
int latest_primary_idx;
bool received_primary_state;
Log potential_primary_log;
Time recovery_start_time;
int recovery_attempt_count;
uint64_t recovery_view;
Log recovery_log;
uint64_t recovery_log_view;
Time recovery_time;
int recovery_commit;
///////////////////////////////////////////////////////////
// VIEW CHANGE
uint32_t begin_view_change_votes; // Bitmask of received BeginViewChange messages
uint32_t do_view_change_votes; // Bitmask of received DoViewChange messages
uint64_t do_view_change_best_old_view; // Best old_view_number seen in DoViewChange
int do_view_change_best_commit; // Best commit_index seen
Log do_view_change_best_log; // Best log seen
uint32_t view_change_begin_votes;
uint32_t view_change_apply_votes;
Log view_change_log; // Best log seen
uint64_t view_change_old_view; // Best old_view_number seen in DoViewChange
int view_change_commit; // Best commit_index seen
///////////////////////////////////////////////////////////
@@ -152,11 +183,17 @@ typedef struct {
PrepareMessage future[FUTURE_LIMIT];
int num_future;
bool state_transfer_pending;
Time state_transfer_time;
Log log;
Time last_heartbeat_time;
Time heartbeat;
StateMachine state_machine;
KVStore kvstore;
// Set at each wakeup
Time now;
} NodeState;
@@ -171,6 +208,19 @@ int node_tick(void *state, void **ctxs,
int node_free(void *state);
void check_vsr_invariants(NodeState **nodes, int num_nodes);
typedef struct {
int last_min_commit;
int last_max_commit;
Status prev_status[NODE_LIMIT];
// External shadow log of committed operations (unbounded, dynamically allocated)
KVStoreOper *shadow_log;
int shadow_count;
int shadow_capacity;
} InvariantChecker;
void invariant_checker_init(InvariantChecker *ic);
void invariant_checker_free(InvariantChecker *ic);
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
#endif // NODE_INCLUDED