Fix: enter recovery on clean restart to prevent log divergence
A node that crashed and restarted with a clean (non-truncated) WAL was entering STATUS_NORMAL directly, skipping the recovery protocol. This violated VR-Revisited Section 4.3: a crashed node must recover state at least as recent as what it had before the crash. The bug allows committed prefix agreement to break: while the node was down, a view change may have replaced uncommitted log entries at positions the node already holds. State transfer (GET_STATE/NEW_STATE) only appends the suffix the node is missing, so the divergent entries persist and can later be committed, causing replicas to disagree on committed operations. Fix: any restart with a non-empty WAL (clean or corrupt) now enters STATUS_RECOVERY, which atomically replaces the local log with the primary's authoritative copy before resuming normal operation. Also fix wal_init_from_network to handle num=0 (malloc(0) may return NULL on some platforms, causing a spurious failure). Verified with deterministic simulation (seeds 1-25, 30-60s each): all WAL invariants pass (committed prefix agreement, shadow log, min_commit monotonicity). https://claude.ai/code/session_01X2b5VRntA7LM32kdN9tPAM
This commit is contained in:
+17
-11
@@ -1702,10 +1702,19 @@ int server_init(void *state_, int argc, char **argv,
|
|||||||
state->last_normal_view = state->vc.last_normal_view;
|
state->last_normal_view = state->vc.last_normal_view;
|
||||||
state->commit_index = 0; // Will be advanced below after replaying the log
|
state->commit_index = 0; // Will be advanced below after replaying the log
|
||||||
|
|
||||||
// Load the WAL from disk. Three cases:
|
// Load the WAL from disk. Two cases:
|
||||||
// 1. Empty log (first start) -> enter normal mode
|
// 1. Empty log, no truncation (first start) -> enter normal mode
|
||||||
// 2. Corrupt log (truncated) -> enter recovery mode
|
// 2. Any restart (clean or corrupt log) -> enter recovery mode
|
||||||
// 3. Clean log -> replay up to commit_index, enter normal mode
|
//
|
||||||
|
// VR-Revisited Section 4.3: "When a node recovers after a crash it
|
||||||
|
// cannot participate in request processing and view changes until it
|
||||||
|
// has a state at least as recent as what it had before it crashed."
|
||||||
|
//
|
||||||
|
// A clean restart without recovery is unsafe: while the node was
|
||||||
|
// down, a view change may have replaced uncommitted log entries.
|
||||||
|
// Entering NORMAL with the stale log would cause the node to hold
|
||||||
|
// divergent entries that could later be committed via state transfer,
|
||||||
|
// violating committed prefix agreement.
|
||||||
bool was_truncated = false;
|
bool was_truncated = false;
|
||||||
if (wal_init_from_file(&state->wal, S("vsr.log"), &was_truncated) < 0) {
|
if (wal_init_from_file(&state->wal, S("vsr.log"), &was_truncated) < 0) {
|
||||||
fprintf(stderr, "Node :: Couldn't open WAL file\n");
|
fprintf(stderr, "Node :: Couldn't open WAL file\n");
|
||||||
@@ -1717,14 +1726,11 @@ int server_init(void *state_, int argc, char **argv,
|
|||||||
if (state->wal.count == 0 && !was_truncated) {
|
if (state->wal.count == 0 && !was_truncated) {
|
||||||
// First start: no previous log on disk
|
// First start: no previous log on disk
|
||||||
enter_recovery = false;
|
enter_recovery = false;
|
||||||
} else if (was_truncated) {
|
|
||||||
// Log was corrupt — must enter recovery to get a valid log
|
|
||||||
// from peers
|
|
||||||
enter_recovery = true;
|
|
||||||
} else {
|
} else {
|
||||||
// Clean restart with a valid log — replay committed entries
|
// Restart (clean or corrupt log). The node must enter recovery
|
||||||
// and resume normal operation
|
// to learn the current view and obtain an authoritative log
|
||||||
enter_recovery = false;
|
// from the primary.
|
||||||
|
enter_recovery = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (enter_recovery) {
|
if (enter_recovery) {
|
||||||
|
|||||||
@@ -150,10 +150,13 @@ int wal_init_from_network(WAL *wal, void *src, int num)
|
|||||||
{
|
{
|
||||||
wal->count = num;
|
wal->count = num;
|
||||||
wal->capacity = num;
|
wal->capacity = num;
|
||||||
wal->entries = malloc(num * sizeof(WALEntry));
|
wal->entries = NULL;
|
||||||
if (wal->entries == NULL)
|
if (num > 0) {
|
||||||
return -1;
|
wal->entries = malloc(num * sizeof(WALEntry));
|
||||||
memcpy(wal->entries, src, num * sizeof(WALEntry));
|
if (wal->entries == NULL)
|
||||||
|
return -1;
|
||||||
|
memcpy(wal->entries, src, num * sizeof(WALEntry));
|
||||||
|
}
|
||||||
wal->handle = (Handle) { 0 };
|
wal->handle = (Handle) { 0 };
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user