cozis/Thesis_Code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 0.9

Browse Source
This commit is contained in:
cozis
2026-02-19 15:28:06 +01:00
parent 868312edf8
commit 238e35b581
31 changed files with 4256 additions and 2253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
raft/client.c
+87 -5
View File
@@ -9,13 +9,64 @@
#include "node.h"
#include "client.h"
#define CLIENT_TRACE(fmt, ...) {}
//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
//#define CLIENT_TRACE(fmt, ...) {}
#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
#define CLIENT_REQUEST_TIMEOUT_SEC 3
#define KEY_POOL_SIZE 128
static uint64_t next_client_id = 1;
static uint64_t client_random(void)
{
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
return quakey_random();
#else
return (uint64_t)rand();
#endif
}
static KVStoreOper random_oper(void)
{
KVStoreOper oper = {0};
snprintf(oper.key, KVSTORE_KEY_SIZE, "k%d", (int)(client_random() % KEY_POOL_SIZE));
switch (client_random() % 3) {
case 0:
oper.type = KVSTORE_OPER_SET;
oper.val = client_random();
break;
case 1:
oper.type = KVSTORE_OPER_GET;
break;
case 2:
oper.type = KVSTORE_OPER_DEL;
break;
}
return oper;
}
static const char *oper_type_name(KVStoreOperType t)
{
switch (t) {
case KVSTORE_OPER_NOOP: return "NOOP";
case KVSTORE_OPER_SET: return "SET";
case KVSTORE_OPER_GET: return "GET";
case KVSTORE_OPER_DEL: return "DEL";
}
return "???";
}
static const char *result_type_name(KVStoreResultType t)
{
switch (t) {
case KVSTORE_RESULT_OK: return "OK";
case KVSTORE_RESULT_FULL: return "FULL";
case KVSTORE_RESULT_MISSING: return "MISSING";
}
return "???";
}
static int
process_message(ClientState *state,
int conn_idx, uint8_t type, ByteView msg)
@@ -31,7 +82,12 @@ process_message(ClientState *state,
if (redirect_message.leader_idx >= 0 && redirect_message.leader_idx < state->num_servers) {
CLIENT_TRACE("Redirected to leader %d", redirect_message.leader_idx);
state->current_leader = redirect_message.leader_idx;
// Retry immediately with the correct leader
// Retry immediately with the correct leader.
// A redirect means the server did not process the request,
// so mark as rejected (not timeout) for the linearizability
// checker: the outcome is unambiguous (no effect).
state->last_was_rejected = true;
state->last_was_timeout = false;
state->pending = false;
}
return 0;
@@ -48,8 +104,24 @@ process_message(ClientState *state,
return -1;
memcpy(&reply_message, msg.ptr, sizeof(reply_message));
CLIENT_TRACE("Received reply for request %lu", (unsigned long)state->request_id);
// Ignore stale replies from previous requests. After a timeout
// the client moves to a new leader and sends a new request, but
// the old leader may still deliver a reply for the old request
// on the previous connection. Without this check the client
// would accept the stale result for the wrong operation.
if (reply_message.request_id != state->request_id)
return 0;
CLIENT_TRACE("REPLY: %s key=\"%.16s\" -> %s val=%lu (req_id=%lu)",
oper_type_name(state->last_oper.type),
state->last_oper.key,
result_type_name(reply_message.result.type),
(unsigned long)reply_message.result.val,
(unsigned long)state->request_id);
state->last_result = reply_message.result;
state->last_was_timeout = false;
state->last_was_rejected = false;
state->pending = false;
return 0;
}
@@ -167,6 +239,8 @@ int client_tick(void *state_, void **ctxs,
if (now >= request_deadline) {
CLIENT_TRACE("Request %lu timed out, trying next server",
(unsigned long)state->request_id);
state->last_was_timeout = true;
state->last_was_rejected = false;
state->pending = false;
state->current_leader = (state->current_leader + 1) % state->num_servers;
}
@@ -181,6 +255,7 @@ int client_tick(void *state_, void **ctxs,
tcp_connect(&state->tcp, state->server_addrs[leader], leader, NULL);
} else {
state->request_id++;
state->last_oper = random_oper();
RequestMessage request_message = {
.base = {
@@ -188,11 +263,18 @@ int client_tick(void *state_, void **ctxs,
.type = MESSAGE_TYPE_REQUEST,
.length = sizeof(RequestMessage),
},
.oper = OPERATION_A,
.oper = state->last_oper,
.client_id = state->client_id,
.request_id = state->request_id,
};
CLIENT_TRACE("REQUEST: %s key=\"%.16s\" val=%lu (req_id=%lu, leader=%d)",
oper_type_name(state->last_oper.type),
state->last_oper.key,
(unsigned long)state->last_oper.val,
(unsigned long)state->request_id,
leader);
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output)
byte_queue_write(output, &request_message, sizeof(request_message));
raft/client.h
+10
View File
@@ -4,6 +4,8 @@
#include <lib/tcp.h>
#include <lib/basic.h>
#include <state_machine/kvstore.h>
#include "config.h"
typedef struct {
@@ -13,6 +15,14 @@ typedef struct {
// True if we are waiting for a response
bool pending;
// The operation sent in the current pending request (for logging)
KVStoreOper last_oper;
// Linearizability checker support
KVStoreResult last_result;
bool last_was_timeout;
bool last_was_rejected;
Address server_addrs[NODE_LIMIT];
int num_servers;
raft/client_table.c
+45
View File
@@ -0,0 +1,45 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include "client_table.h"
void client_table_init(ClientTable *ct)
{
ct->count = 0;
ct->capacity = 0;
ct->entries = NULL;
}
void client_table_free(ClientTable *ct)
{
free(ct->entries);
}
ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id)
{
for (int i = 0; i < ct->count; i++)
if (ct->entries[i].client_id == client_id)
return &ct->entries[i];
return NULL;
}
int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag)
{
if (ct->count == ct->capacity) {
int n = ct->capacity ? 2 * ct->capacity : 8;
void *p = realloc(ct->entries, n * sizeof(ClientTableEntry));
if (p == NULL) return -1;
ct->capacity = n;
ct->entries = p;
}
ct->entries[ct->count++] = (ClientTableEntry) {
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_tag = conn_tag,
};
return 0;
}
raft/client_table.h
+28
View File
@@ -0,0 +1,28 @@
#ifndef CLIENT_TABLE_INCLUDED
#define CLIENT_TABLE_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
KVStoreResult last_result;
bool pending;
int conn_tag;
} ClientTableEntry;
typedef struct {
int count;
int capacity;
ClientTableEntry *entries;
} ClientTable;
void client_table_init(ClientTable *ct);
void client_table_free(ClientTable *ct);
ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id);
int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag);
#endif // CLIENT_TABLE_INCLUDED
raft/invariant_checker.c
+351
View File
@@ -0,0 +1,351 @@
// Raft invariant checker with external shadow log tracking.
//
// This file runs in the main simulation loop outside Quakey-scheduled
// processes. It includes node.h for struct definitions, then restores
// real allocators since mock_malloc/realloc/free abort outside process
// context.
#include "invariant_checker.h"
#include <assert.h>
#include <string.h>
// Restore real allocators (see checker/linearizability.c for precedent).
// The header chain (invariant_checker.h -> node.h -> wal.h ->
// lib/file_system.h) defines QUAKEY_ENABLE_MOCKS and replaces
// malloc/realloc/free with mock versions. We need the real ones
// because this code runs outside any Quakey-scheduled process.
#undef malloc
#undef realloc
#undef free
#include <stdio.h>
#include <stdlib.h>
static int self_idx(NodeState *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
return i;
UNREACHABLE;
}
static int shadow_log_append(InvariantChecker *ic, KVStoreOper oper)
{
if (ic->shadow_count == ic->shadow_capacity) {
int n = 2 * ic->shadow_capacity;
if (n < 8)
n = 8;
KVStoreOper *p = realloc(ic->shadow_log, n * sizeof(KVStoreOper));
if (p == NULL)
return -1;
ic->shadow_log = p;
ic->shadow_capacity = n;
}
ic->shadow_log[ic->shadow_count++] = oper;
return 0;
}
void invariant_checker_init(InvariantChecker *ic)
{
ic->shadow_log = NULL;
ic->shadow_count = 0;
ic->shadow_capacity = 0;
}
void invariant_checker_free(InvariantChecker *ic)
{
fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
ic->shadow_count);
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes)
{
for (int i = 0; i < num_nodes; i++) {
NodeState *s = nodes[i];
if (s == NULL)
continue;
int log_count = wal_entry_count(&s->wal);
// The commit index starts at -1 (nothing committed) and only
// increases. It must never go below -1.
if (s->commit_index < -1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < -1\n",
i, s->commit_index);
__builtin_trap();
}
// A node cannot have committed an entry beyond what exists
// in its log. With an empty log (count=0), commit_index
// must be -1.
if (s->commit_index >= log_count) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) >= log_count (%d)\n",
i, s->commit_index, log_count);
__builtin_trap();
}
// Entries are applied to the state machine in order, up to
// the commit index. The last applied index must never exceed
// the commit index.
if (s->last_applied > s->commit_index) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) > commit_index (%d)\n",
i, s->last_applied, s->commit_index);
__builtin_trap();
}
if (s->last_applied < -1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_applied (%d) < -1\n",
i, s->last_applied);
__builtin_trap();
}
// voted_for is either -1 (no vote) or a valid node index.
if (s->term_and_vote.voted_for != -1 && (s->term_and_vote.voted_for < 0 || s->term_and_vote.voted_for >= s->num_nodes)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: voted_for (%d) is not -1 "
"and not in [0, %d)\n",
i, s->term_and_vote.voted_for, s->num_nodes);
__builtin_trap();
}
// Leaders append entries with their current term, and terms
// only increase. Truncation preserves this property because
// replacement entries come from a leader whose log already
// satisfies non-decreasing terms.
for (int k = 1; k < log_count; k++) {
if (wal_peek_entry(&s->wal, k)->term < wal_peek_entry(&s->wal, k-1)->term) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: wal[%d].term (%lu) "
"< wal[%d].term (%lu) (non-monotonic)\n",
i, k, (unsigned long)wal_peek_entry(&s->wal, k)->term,
k-1, (unsigned long)wal_peek_entry(&s->wal, k-1)->term);
__builtin_trap();
}
}
// A leader must have voted for itself in the current term.
// A node becomes leader by winning an election, which requires
// voting for itself. The voted_for value is only reset when
// stepping down (which changes the role to FOLLOWER).
if (s->role == ROLE_LEADER) {
if (s->term_and_vote.voted_for != self_idx(s)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: role is LEADER but "
"voted_for (%d) != self_idx (%d)\n",
i, s->term_and_vote.voted_for, self_idx(s));
__builtin_trap();
}
}
// A candidate must have voted for itself in the current term.
// A node enters candidate state only through start_election(),
// which increments the term and sets voted_for to self_idx.
if (s->role == ROLE_CANDIDATE) {
if (s->term_and_vote.voted_for != self_idx(s)) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: role is CANDIDATE but "
"voted_for (%d) != self_idx (%d)\n",
i, s->term_and_vote.voted_for, self_idx(s));
__builtin_trap();
}
}
// For a leader, match_indices[self] must equal the last log
// index. The leader always has its own entries matched.
if (s->role == ROLE_LEADER) {
int expected_match = log_count - 1;
if (s->match_indices[self_idx(s)] != expected_match) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
"match_indices[self] (%d) != last log index (%d)\n",
i, s->match_indices[self_idx(s)], expected_match);
__builtin_trap();
}
}
// For a leader, next_indices[k] must be >= match_indices[k] + 1
// for all followers (k != self). The next index to send is
// always at least one past the highest known replicated index.
// The leader's own next_indices[self] is not maintained since
// the leader never sends entries to itself.
if (s->role == ROLE_LEADER) {
int si = self_idx(s);
for (int k = 0; k < s->num_nodes; k++) {
if (k == si)
continue;
if (s->next_indices[k] < s->match_indices[k] + 1) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (LEADER): "
"next_indices[%d] (%d) < match_indices[%d] + 1 (%d)\n",
i, k, s->next_indices[k], k, s->match_indices[k] + 1);
__builtin_trap();
}
}
}
}
// Election Safety: at most one leader per term.
// Each term has at most one leader because a candidate needs
// a majority of votes, and each node votes at most once per term.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL || nodes[i]->role != ROLE_LEADER)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL || nodes[j]->role != ROLE_LEADER)
continue;
if (nodes[i]->term_and_vote.term == nodes[j]->term_and_vote.term) {
fprintf(stderr, "INVARIANT VIOLATED: two leaders in term %lu: "
"node %d and node %d\n",
(unsigned long)nodes[i]->term_and_vote.term, i, j);
__builtin_trap();
}
}
}
// State Machine Safety (committed prefix agreement).
// For any two nodes, their logs must agree on all entries up to
// min(commit_index_i, commit_index_j). This is the core safety
// property: all committed operations are identical across replicas.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL)
continue;
int min_commit = nodes[i]->commit_index;
if (nodes[j]->commit_index < min_commit)
min_commit = nodes[j]->commit_index;
for (int k = 0; k <= min_commit; k++) {
WALEntry *ei = wal_peek_entry(&nodes[i]->wal, k);
WALEntry *ej = wal_peek_entry(&nodes[j]->wal, k);
if (memcmp(&ei->oper, &ej->oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed log operation "
"mismatch at index %d between node %d and node %d\n",
k, i, j);
__builtin_trap();
}
if (ei->term != ej->term) {
fprintf(stderr, "INVARIANT VIOLATED: committed log term "
"mismatch at index %d between node %d (term %lu) "
"and node %d (term %lu)\n",
k, i, (unsigned long)ei->term,
j, (unsigned long)ej->term);
__builtin_trap();
}
}
}
}
////////////////////////////////////////////////////////////////////
// Shadow log: external commit tracking
////////////////////////////////////////////////////////////////////
// Phase 1: Find the observed max commit index and a source node.
//
// In Raft, commit_index == -1 means nothing committed, 0 means the
// first entry is committed, etc. We track the number of committed
// entries as (commit_index + 1) so it aligns with shadow_count.
int observed_committed = 0; // number of committed entries observed
int source_node_idx = -1;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
int node_committed = nodes[i]->commit_index + 1;
if (node_committed > observed_committed) {
observed_committed = node_committed;
source_node_idx = i;
}
}
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_committed > ic->shadow_count) {
NodeState *source = nodes[source_node_idx];
assert(wal_entry_count(&source->wal) >= observed_committed);
for (int k = ic->shadow_count; k < observed_committed; k++) {
KVStoreOper *source_oper = &wal_peek_entry(&source->wal, k)->oper;
// Cross-validate against other live nodes that have also
// committed this entry.
for (int j = 0; j < num_nodes; j++) {
if (j == source_node_idx)
continue;
if (nodes[j] == NULL)
continue;
if (nodes[j]->commit_index < k)
continue;
if (wal_entry_count(&nodes[j]->wal) <= k)
continue;
if (memcmp(&wal_peek_entry(&nodes[j]->wal, k)->oper, source_oper, sizeof(KVStoreOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
"between source node %d and node %d during shadow log append\n",
k, source_node_idx, j);
__builtin_trap();
}
}
if (shadow_log_append(ic, *source_oper) < 0) {
fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
__builtin_trap();
}
}
}
// Phase 3: Verify shadow log against the cluster.
// Sub-check A: Committed entries must match the shadow log.
for (int k = 0; k < ic->shadow_count; k++) {
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (wal_entry_count(&nodes[i]->wal) <= k)
continue;
if (nodes[i]->commit_index < k)
continue;
if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) != 0) {
char shadow_buf[128], node_buf[128];
kvstore_snprint_oper(shadow_buf, sizeof(shadow_buf), ic->shadow_log[k]);
kvstore_snprint_oper(node_buf, sizeof(node_buf), wal_peek_entry(&nodes[i]->wal, k)->oper);
fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
" shadow: %s\n"
" node: %s\n",
k, i, shadow_buf, node_buf);
__builtin_trap();
}
}
}
// Sub-check B: When commit regresses, previously committed entries
// must still be held by a majority of the cluster.
if (observed_committed < ic->shadow_count) {
for (int k = observed_committed; k < ic->shadow_count; k++) {
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
if (wal_entry_count(&nodes[i]->wal) <= k)
continue;
if (memcmp(&wal_peek_entry(&nodes[i]->wal, k)->oper, &ic->shadow_log[k], sizeof(KVStoreOper)) == 0)
holders++;
}
if (holders + num_dead <= num_nodes / 2) {
char oper_buf[128];
kvstore_snprint_oper(oper_buf, sizeof(oper_buf), ic->shadow_log[k]);
fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
"no longer held by majority (holders=%d, dead=%d, total=%d)\n"
" entry: %s\n",
k, holders, num_dead, num_nodes, oper_buf);
__builtin_trap();
}
}
}
}
raft/invariant_checker.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef INVARIANT_CHECKER_INCLUDED
#define INVARIANT_CHECKER_INCLUDED
#include "node.h"
typedef struct {
// External shadow log of committed operations (unbounded, dynamically allocated)
KVStoreOper *shadow_log;
int shadow_count;
int shadow_capacity;
} InvariantChecker;
void invariant_checker_init(InvariantChecker *ic);
void invariant_checker_free(InvariantChecker *ic);
void invariant_checker_run(InvariantChecker *ic, NodeState **nodes, int num_nodes);
#endif // INVARIANT_CHECKER_INCLUDED
raft/main.c
+44 -8
View File
@@ -8,6 +8,7 @@
#include "node.h"
#include "client.h"
#include "invariant_checker.h"
static volatile int simulation_running = 1;
@@ -17,15 +18,32 @@ static void sigint_handler(int sig)
simulation_running = 0;
}
int main(void)
int main(int argc, char **argv)
{
signal(SIGINT, sigint_handler);
QuakeyUInt64 seed = 2;
QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
seed = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
}
}
Quakey *quakey;
int ret = quakey_init(&quakey, 2);
int ret = quakey_init(&quakey, seed);
if (ret < 0)
return -1;
QuakeyNode cli_1;
QuakeyNode cli_2;
QuakeyNode node_1;
QuakeyNode node_2;
QuakeyNode node_3;
// Client 1
{
QuakeySpawn config = {
@@ -39,7 +57,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Client 2
@@ -55,7 +73,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Node 1
@@ -71,7 +89,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Node 2
@@ -87,7 +105,7 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Node 3
@@ -103,12 +121,30 @@ int main(void)
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
while (simulation_running)
// Limit crashes to 1 node at a time (within fault tolerance of f=1).
quakey_set_max_crashes(quakey, 1);
quakey_network_partitioning(quakey, true);
InvariantChecker invariant_checker;
invariant_checker_init(&invariant_checker);
while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
quakey_schedule_one(quakey);
NodeState *arr[] = {
quakey_node_state(node_1),
quakey_node_state(node_2),
quakey_node_state(node_3),
};
invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]));
}
invariant_checker_free(&invariant_checker);
quakey_free(quakey);
return 0;
}
raft/node.c
+695 -667
View File
File diff suppressed because it is too large Load Diff
raft/node.h
+58 -41
View File
@@ -7,6 +7,7 @@
#include "wal.h"
#include "config.h"
#include "client_table.h"
enum {
MESSAGE_TYPE_REQUEST_VOTE,
@@ -28,6 +29,7 @@ typedef struct {
typedef struct {
MessageHeader base;
int sender_idx;
uint64_t term;
uint8_t value;
} VotedMessage;
@@ -53,14 +55,15 @@ typedef struct {
typedef struct {
MessageHeader base;
Operation oper;
KVStoreOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
OperationResult result;
KVStoreResult result;
uint64_t request_id;
} ReplyMessage;
typedef struct {
@@ -68,20 +71,6 @@ typedef struct {
int leader_idx;
} RedirectMessage;
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
OperationResult last_result;
bool pending;
int conn_tag;
} ClientTableEntry;
typedef struct {
int count;
int capacity;
ClientTableEntry *entries;
} ClientTable;
typedef enum {
ROLE_FOLLOWER,
ROLE_CANDIDATE,
@@ -90,46 +79,74 @@ typedef enum {
typedef struct {
TCP tcp;
WAL wal;
// In-memory copy of the term and vote values.
// These must always be updated after the version
// stored on disk.
uint64_t term;
int voted_for;
// Handle to the file backing the term and vote
// values.
Handle handle;
} TermAndVote;
typedef struct {
// Networking subsystem
TCP tcp;
// Static list of cluster nodes
Address self_addr;
Address node_addrs[NODE_LIMIT];
int num_nodes;
// Current loder of the node and the
// current leader. If no leader, -1.
Role role;
int leader_idx;
uint64_t term;
int voted_for;
Handle term_and_vote_handle;
// Votes received when candidate.
// Each bit is associated to a node. The
// number of votes is obtained by counting
// set bits. This ensures nodes won't vote
// twice.
uint32_t votes;
// Index of the current leader
int leader_idx;
// Durable log subsystem
WAL wal;
// Durable fixed-size data subsystem
TermAndVote term_and_vote;
// Current election timeout. This value is relative
// and must be summed to the heartbeat in order to
// get the absolute election deadline.
uint64_t election_timeout;
// This is the current value of the node.
// It is set at the start of each event
// handler call.
Time now;
// If leader, this is the time when the last
// heartbeat was sent. If follower, this is
// the last time a heartbeat was received.
Time heartbeat;
// The state machine to be replicated.
KVStore kvstore;
// Table of client requests. This ensures request
// are applied in order.
ClientTable client_table;
int next_client_tag;
int commit_index;
int last_applied;
// When CANDIDATE
int votes_received;
// When LEADER
int next_indices[NODE_LIMIT];
int match_indices[NODE_LIMIT];
// Relative timeout in nanoseconds
uint64_t election_timeout;
// Last heartbeat time
uint64_t watchdog;
// Keep track of client request order to enforce
// linearizability
ClientTable client_table;
int next_client_tag;
// The state machine to be replicated
StateMachine state_machine;
} NodeState;
struct pollfd;
raft/wal.c
+69 -18
View File
@@ -4,10 +4,26 @@
#include <quakey.h>
#include <assert.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include "wal.h"
// FNV-1a checksum over all WALEntry fields except the checksum itself.
uint32_t wal_entry_checksum(WALEntry *entry)
{
uint32_t h = 2166136261u;
const unsigned char *p = (const unsigned char *)entry;
// Hash all bytes up to (but not including) the checksum field
size_t len = offsetof(WALEntry, checksum);
for (size_t i = 0; i < len; i++) {
h ^= p[i];
h *= 16777619u;
}
return h;
}
int wal_init(WAL *wal, string file)
{
Handle handle;
@@ -21,25 +37,46 @@ int wal_init(WAL *wal, string file)
return -1;
}
if (size % sizeof(WALEntry) != 0) {
// Discard any partial trailing entry (crash during write).
int raw_count = size / sizeof(WALEntry);
size_t valid_size = raw_count * sizeof(WALEntry);
if (valid_size < size) {
file_truncate(handle, valid_size);
}
WALEntry *entries = malloc(raw_count * sizeof(WALEntry));
if (entries == NULL && raw_count > 0) {
file_close(handle);
return -1;
}
int count = size / sizeof(WALEntry);
if (file_set_offset(handle, 0) < 0) {
file_close(handle);
free(entries);
return -1;
}
if (raw_count > 0 && file_read_exact(handle, (char*) entries, raw_count * sizeof(WALEntry)) < 0) {
file_close(handle);
free(entries);
return -1;
}
// Verify checksums: truncate at the first corrupted entry.
// All entries from a corrupted one onward are discarded.
int count = raw_count;
for (int i = 0; i < raw_count; i++) {
if (entries[i].checksum != wal_entry_checksum(&entries[i])) {
count = i;
file_truncate(handle, count * sizeof(WALEntry));
break;
}
}
wal->count = count;
wal->capacity = count;
wal->entries = malloc(count * sizeof(WALEntry));
if (wal->entries == NULL) {
file_close(handle);
return -1;
}
if (file_read_exact(handle, (char*) wal->entries, count * sizeof(WALEntry)) < 0) {
file_close(handle);
return -1;
}
wal->capacity = raw_count; // capacity >= count
wal->entries = entries;
wal->handle = handle;
return 0;
}
@@ -61,12 +98,26 @@ int wal_append(WAL *wal, WALEntry *entry)
wal->entries = p;
}
// TODO: Should truncate file on partial writes
if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0)
return -1;
// Compute checksum before writing to disk
entry->checksum = wal_entry_checksum(entry);
if (file_sync(wal->handle) < 0)
if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0) {
// A partial write may have advanced the file offset. Truncate
// and rewind so the file stays consistent with the in-memory
// entry count.
file_truncate(wal->handle, wal->count * sizeof(WALEntry));
file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
return -1;
}
if (file_sync(wal->handle) < 0) {
// The write succeeded but sync failed. Rewind the offset and
// truncate the phantom entry so the next append writes to the
// correct position.
file_truncate(wal->handle, wal->count * sizeof(WALEntry));
file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
return -1;
}
wal->entries[wal->count++] = *entry;
return 0;
raft/wal.h
+8 -4
View File
@@ -3,14 +3,18 @@
#include <lib/file_system.h>
#include <state_machine/state_machine.h>
#include <state_machine/kvstore.h>
typedef struct {
uint64_t term;
uint64_t client_id;
Operation oper;
uint64_t term;
uint64_t client_id;
uint64_t request_id;
KVStoreOper oper;
uint32_t checksum;
} WALEntry;
uint32_t wal_entry_checksum(WALEntry *entry);
typedef struct {
int count;
int capacity;