#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) #define QUAKEY_ENABLE_MOCKS #endif #include #include #include #include "node.h" //#define NODE_TRACE(fmt, ...) {} #define NODE_TRACE(fmt, ...) fprintf(stderr, "NODE: " fmt "\n", ##__VA_ARGS__); // Format time as seconds with 3 decimal places for trace output #define TIME_FMT "%.3fs" #define TIME_VAL(t) ((double)(t) / 1000000000.0) static const char *message_type_name(uint8_t type) { switch (type) { case MESSAGE_TYPE_REQUEST: return "REQUEST"; case MESSAGE_TYPE_REPLY: return "REPLY"; case MESSAGE_TYPE_PREPARE: return "PREPARE"; case MESSAGE_TYPE_PREPARE_OK: return "PREPARE_OK"; case MESSAGE_TYPE_COMMIT: return "COMMIT"; case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: return "BEGIN_VIEW_CHANGE"; case MESSAGE_TYPE_DO_VIEW_CHANGE: return "DO_VIEW_CHANGE"; case MESSAGE_TYPE_BEGIN_VIEW: return "BEGIN_VIEW"; case MESSAGE_TYPE_RECOVERY: return "RECOVERY"; case MESSAGE_TYPE_RECOVERY_RESPONSE: return "RECOVERY_RESPONSE"; default: return "UNKNOWN"; } } static const char *status_name(Status status) { switch (status) { case STATUS_NORMAL: return "NORMAL"; case STATUS_CHANGE_VIEW: return "CHANGE_VIEW"; case STATUS_RECOVERY: return "RECOVERY"; } return "UNKNOWN"; } static int self_idx(NodeState *state) { for (int i = 0; i < state->num_nodes; i++) if (addr_eql(state->node_addrs[i], state->self_addr)) return i; UNREACHABLE; } static int leader_idx(NodeState *state) { return state->view_number % state->num_nodes; } static bool is_leader(NodeState *state) { return self_idx(state) == leader_idx(state); } static int send_to_peer_ex(NodeState *state, int peer_idx, MessageHeader *msg, void *extra, int extra_len) { ByteQueue *output; int conn_idx = tcp_index_from_tag(&state->tcp, peer_idx); if (conn_idx < 0) { int ret = tcp_connect(&state->tcp, state->node_addrs[peer_idx], peer_idx, &output); if (ret < 0) return -1; } else { output = tcp_output_buffer(&state->tcp, conn_idx); if (output == NULL) { assert(0); } } byte_queue_write(output, msg, msg->length - extra_len); byte_queue_write(output, extra, extra_len); return 0; } static int send_to_peer(NodeState *state, int peer_idx, MessageHeader *msg) { return send_to_peer_ex(state, peer_idx, msg, NULL, 0); } static int broadcast_to_peers_ex(NodeState *state, MessageHeader *msg, void *extra, int extra_len) { for (int i = 0; i < state->num_nodes; i++) { if (i != self_idx(state)) if (send_to_peer_ex(state, i, msg, extra, extra_len) < 0) return -1; } return 0; } static int broadcast_to_peers(NodeState *state, MessageHeader *msg) { return broadcast_to_peers_ex(state, msg, NULL, 0); } // TODO: test this function static int count_set(uint32_t word) { int n = 0; for (int i = 0; i < sizeof(word) * 8; i++) if (word & (1 << i)) n++; return n; } static int process_message_as_leader(NodeState *state, int conn_idx, uint8_t type, ByteView msg) { switch (type) { case MESSAGE_TYPE_REQUEST: { // Don't accept new requests during a view change. // The leader must complete the view change first. if (state->status != STATUS_NORMAL) break; RequestMessage request_message; if (msg.len != sizeof(request_message)) return -1; memcpy(&request_message, msg.ptr, sizeof(request_message)); // We must first add or update the client table to // invalidate the request ID. This makes it so any // subsequent requests with the same ID are rejected // while the first one is in progress. // // If the request ID is lower than the one stored in // the table, the request is rejected. // // If the request ID is the same as the one in the table // but no result was saved as the original one is still // in progress, the request is rejected. // // If the request ID is the same and a result is available, // it is returned immediately. { ClientTableEntry *entry = client_table_find(&state->client_table, request_message.client_id); if (entry == NULL) { if (client_table_add(&state->client_table, request_message.client_id, request_message.request_id, conn_idx) < 0) { ReplyMessage reply_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_REPLY, .length = sizeof(ReplyMessage), }, .rejected = true, }; ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); assert(output); byte_queue_write(output, &reply_message, sizeof(reply_message)); break; } } else { if (entry->pending) break; // Only one pending operation per client is allowed. Ignore the message. if (entry->last_request_id > request_message.request_id) break; // Request is old. Ignore. if (entry->last_request_id == request_message.request_id) { // This request was already processed and its value was cached. // Respond with the cached value. ReplyMessage reply_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_REPLY, .length = sizeof(ReplyMessage), }, .rejected = false, .result = entry->last_result, }; ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); assert(output); byte_queue_write(output, &reply_message, sizeof(reply_message)); break; } entry->last_request_id = request_message.request_id; entry->pending = true; } } LogEntry log_entry = { .oper = request_message.oper, .votes = 1 << self_idx(state), .view_number = state->view_number, .client_id = request_message.client_id, }; if (log_append(&state->log, log_entry) < 0) { assert(0); // TODO } { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): REQUEST from client %lu (req_id=%lu), appended to log[%d], view=%lu", TIME_VAL(now), self_idx(state), (unsigned long)request_message.client_id, (unsigned long)request_message.request_id, state->log.count-1, (unsigned long)state->view_number); } PrepareMessage prepare_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_PREPARE, .length = sizeof(PrepareMessage), }, .oper = request_message.oper, .sender_idx = self_idx(state), .log_index = state->log.count-1, .commit_index = state->commit_index, .view_number = state->view_number, }; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER) -> ALL: PREPARE log_index=%d commit_index=%d view=%lu", TIME_VAL(now), self_idx(state), state->log.count-1, state->commit_index, (unsigned long)state->view_number); } if (broadcast_to_peers(state, &prepare_message.base) < 0) { assert(0); } // We forwarded the message to all peers. As soon as // we get enough PREPARE_OK responses, we'll commit // and reply to the client. } break; case MESSAGE_TYPE_PREPARE_OK: { PrepareOKMessage prepare_ok_message; if (msg.len != sizeof(prepare_ok_message)) return -1; memcpy(&prepare_ok_message, msg.ptr, sizeof(prepare_ok_message)); if (prepare_ok_message.view_number != state->view_number) { assert(0); } // TODO: check log_index int log_index = prepare_ok_message.log_index; if (log_index < 0 || log_index >= state->log.count) { assert(0); // TODO } // When the primary appends an entry to its log, it sends a // PREPARE message to all backups. Backups add the entry to // their own logs and reply with PREPARE_OK messages. When // the primary receives a quorum of PREPARE_OKs, it commits // the entry. // // In a reliable network and with no node crashes, we would // expect entries to be committed linearly in the log. If // log entries A and B are added to the log in that order, // we expect A to reach PREPARE_OK messages before B. // // Unfortunately, we must assume messages will be lost (*). // If that happens, instead of worrying about resending the // PREPARE_OK for that entry, we rely on the fact that OK // messages for future messages imply OK messages for the // previous ones. The first message for which a quorum of // OK messages is reached can work as an OK for all previous // entries. // // For this reason, we allow "holes" in the log and if an // entry reached quorum we advance the commit index to it. // // If the log index is lower than the log, it means we // received an OK message that was not necessary anymore // so we can ignore it. // // (*) This implementation uses TCP as a transport protocol, // which means messages will be retransmitted if lost on // the network. Nevertheless, if a node crashes while receiving // a node or we crash before sending it, the message will // be lost. if (log_index < state->commit_index) break; // Already processed LogEntry *entry = &state->log.entries[log_index]; entry->votes |= 1 << prepare_ok_message.sender_idx; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): PREPARE_OK from node %d for log[%d], prep_oks=%d/%d view=%lu", TIME_VAL(now), self_idx(state), prepare_ok_message.sender_idx, log_index, count_set(entry->votes), state->num_nodes/2 + 1, (unsigned long)prepare_ok_message.view_number); } // Quorum reached for this log entry? if (count_set(entry->votes) > state->num_nodes/2) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): quorum reached for log[%d], advancing commit_index %d -> %d", TIME_VAL(now), self_idx(state), log_index, state->commit_index, log_index + 1); } // Okay, we can advance the commit index to here while (state->commit_index <= log_index) { uint64_t client_id = state->log.entries[state->commit_index].client_id; Operation oper = state->log.entries[state->commit_index].oper; state->commit_index++; OperationResult result = state_machine_update(&state->state_machine, oper); ClientTableEntry *table_entry = client_table_find(&state->client_table, client_id); if (table_entry) { assert(table_entry->pending); table_entry->pending = false; table_entry->last_result = result; ReplyMessage reply_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_REPLY, .length = sizeof(ReplyMessage), }, .result = result, }; ByteQueue *output = tcp_output_buffer(&state->tcp, table_entry->conn_idx); assert(output); byte_queue_write(output, &reply_message, sizeof(reply_message)); } } } } break; case MESSAGE_TYPE_DO_VIEW_CHANGE: { DoViewChangeMessage do_view_change_message; if (msg.len < sizeof(do_view_change_message)) return -1; memcpy(&do_view_change_message, msg.ptr, sizeof(do_view_change_message)); // Parse the variable-sized log from the message int num_entries = (msg.len - sizeof(DoViewChangeMessage)) / sizeof(LogEntry); if (num_entries != do_view_change_message.op_number) return -1; // Message size mismatch // Only process if the view matches what we're transitioning to if (do_view_change_message.view_number != state->view_number) break; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d: DO_VIEW_CHANGE from node %d, view=%lu old_view=%lu op_number=%d commit_index=%d", TIME_VAL(now), self_idx(state), do_view_change_message.sender_idx, (unsigned long)do_view_change_message.view_number, (unsigned long)do_view_change_message.old_view_number, do_view_change_message.op_number, do_view_change_message.commit_index); } // Track this vote uint32_t vote_mask = 1 << do_view_change_message.sender_idx; if ((state->do_view_change_votes & vote_mask) == 0) { state->do_view_change_votes |= vote_mask; // Check if this log is better than the one we have stored // Best log: highest old_view_number, then highest op_number bool is_better = (do_view_change_message.old_view_number > state->do_view_change_best_old_view) || (do_view_change_message.old_view_number == state->do_view_change_best_old_view && do_view_change_message.op_number > state->do_view_change_best_log.count); if (is_better) { state->do_view_change_best_old_view = do_view_change_message.old_view_number; // Copy the log entries from the message LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); if (entries == NULL) { assert(0); } memcpy(entries, (uint8_t*)msg.ptr + sizeof(DoViewChangeMessage), num_entries * sizeof(LogEntry)); log_free(&state->do_view_change_best_log); state->do_view_change_best_log.count = num_entries; state->do_view_change_best_log.capacity = num_entries; state->do_view_change_best_log.entries = entries; } // Track the maximum commit index if (do_view_change_message.commit_index > state->do_view_change_best_commit) { state->do_view_change_best_commit = do_view_change_message.commit_index; } } // Count votes int vote_count = 0; for (int i = 0; i < state->num_nodes; i++) { if (state->do_view_change_votes & (1 << i)) vote_count++; } // Need f + 1 DoViewChange messages (including own) if (vote_count >= state->num_nodes / 2 + 1) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d: view change complete, becoming LEADER for view %lu (votes=%d/%d)", TIME_VAL(now), self_idx(state), (unsigned long)state->view_number, vote_count, state->num_nodes); } // Use the best log we collected log_free(&state->log); state->log = state->do_view_change_best_log; state->do_view_change_best_log = (Log){0}; // Clear to avoid double-free state->commit_index = state->do_view_change_best_commit; state->status = STATUS_NORMAL; // Reset vote tracking for uncommitted entries. The log // entries inherited from DO_VIEW_CHANGE have stale // votes from the previous view. The new leader starts // with its own vote for each entry. for (int i = state->commit_index; i < state->log.count; i++) state->log.entries[i].votes = 1 << self_idx(state); BeginViewMessage begin_view_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_BEGIN_VIEW, .length = sizeof(BeginViewMessage) + state->log.count * sizeof(LogEntry), }, .view_number = state->view_number, .commit_index = state->commit_index, .op_number = state->log.count, }; if (broadcast_to_peers_ex(state, &begin_view_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { assert(0); } { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER) -> ALL: BEGIN_VIEW view=%lu commit_index=%d log_count=%d", TIME_VAL(now), self_idx(state), (unsigned long)state->view_number, state->commit_index, state->log.count); } // Reset view change state for next time state->do_view_change_votes = 0; state->do_view_change_best_old_view = 0; state->do_view_change_best_commit = 0; } } break; case MESSAGE_TYPE_RECOVERY: { if (state->status != STATUS_NORMAL) break; // Ignore message. RecoveryMessage recovery_message; if (msg.len != sizeof(RecoveryMessage)) return -1; memcpy(&recovery_message, msg.ptr, sizeof(recovery_message)); { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): RECOVERY from node %d (nonce=%lu), sending RECOVERY_RESPONSE with log_count=%d", TIME_VAL(now), self_idx(state), recovery_message.sender_idx, (unsigned long)recovery_message.nonce, state->log.count); } RecoveryResponseMessage recovery_response_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_RECOVERY_RESPONSE, .length = sizeof(RecoveryResponseMessage) + state->log.count * sizeof(LogEntry), }, .view_number = state->view_number, .op_number = state->log.count-1, // TODO: What if the log is empty? .nonce = recovery_message.nonce, .commit_index = state->commit_index, .sender_idx = self_idx(state), }; if (send_to_peer_ex(state, recovery_message.sender_idx, &recovery_response_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { assert(0); } } break; case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: { if (state->status == STATUS_RECOVERY) break; BeginViewChangeMessage begin_view_change_message; if (msg.len != sizeof(BeginViewChangeMessage)) return -1; memcpy(&begin_view_change_message, msg.ptr, sizeof(begin_view_change_message)); if (begin_view_change_message.view_number > state->view_number) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): BEGIN_VIEW_CHANGE from node %d, transitioning view %lu -> %lu", TIME_VAL(now), self_idx(state), begin_view_change_message.sender_idx, (unsigned long)state->view_number, (unsigned long)begin_view_change_message.view_number); } // Reset vote tracking when transitioning to a new view state->begin_view_change_votes = 0; state->view_number = begin_view_change_message.view_number; state->status = STATUS_CHANGE_VIEW; // Send our own BeginViewChange to all peers BeginViewChangeMessage own_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, .length = sizeof(BeginViewChangeMessage), }, .view_number = state->view_number, .sender_idx = self_idx(state), }; if (broadcast_to_peers(state, &own_message.base) < 0) { assert(0); } // Count our own vote state->begin_view_change_votes |= (1 << self_idx(state)); } // Track this sender's vote (only if view matches) if (begin_view_change_message.view_number == state->view_number) { state->begin_view_change_votes |= (1 << begin_view_change_message.sender_idx); } // Count votes int vote_count = 0; for (int i = 0; i < state->num_nodes; i++) { if (state->begin_view_change_votes & (1 << i)) vote_count++; } // Need f + 1 votes to send DoViewChange if (vote_count >= state->num_nodes / 2 + 1) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): f+1 BEGIN_VIEW_CHANGE votes reached (%d/%d), sending DO_VIEW_CHANGE to node %d for view %lu", TIME_VAL(now), self_idx(state), vote_count, state->num_nodes, (int)(state->view_number % state->num_nodes), (unsigned long)state->view_number); } DoViewChangeMessage do_view_change_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_DO_VIEW_CHANGE, .length = sizeof(DoViewChangeMessage) + state->log.count * sizeof(LogEntry), }, .view_number = state->view_number, .old_view_number = state->view_number - 1, // View before we started view change .op_number = state->log.count, .commit_index = state->commit_index, .sender_idx = self_idx(state), }; if (leader_idx(state) == self_idx(state)) { // We are the new leader: count our own vote directly // since send_to_peer_ex skips self-sends. state->do_view_change_votes |= (1 << self_idx(state)); } else { if (send_to_peer_ex(state, leader_idx(state), &do_view_change_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { assert(0); } } // Clear the future array since we're changing views state->num_future = 0; } } break; case MESSAGE_TYPE_BEGIN_VIEW: { BeginViewMessage message; if (msg.len < sizeof(message)) return -1; memcpy(&message, msg.ptr, sizeof(message)); // Only process messages containing a view-number // that matches or advances the one we know. if (message.view_number < state->view_number) break; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (LEADER): BEGIN_VIEW received, stepping down and adopting view %lu (commit_index=%d, log_entries=%d)", TIME_VAL(now), self_idx(state), (unsigned long)message.view_number, message.commit_index, message.op_number); } // Update view and status state->view_number = message.view_number; state->status = STATUS_NORMAL; // Paper state: change status to normal // Replace the local log with the authoritative log from the primary { int num_entries = (msg.len - sizeof(BeginViewMessage)) / sizeof(LogEntry); LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); if (entries == NULL) { assert(0); } memcpy(entries, (uint8_t*)msg.ptr + sizeof(BeginViewMessage), num_entries * sizeof(LogEntry)); log_free(&state->log); state->log.count = num_entries; state->log.capacity = num_entries; state->log.entries = entries; } // Reset view change state state->begin_view_change_votes = 0; // If there are non-committed operations in the log, // send a PREPAREOK to the new primary if (state->log.count > message.commit_index) { PrepareOKMessage ok_msg = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_PREPARE_OK, .length = sizeof(PrepareOKMessage), }, .sender_idx = self_idx(state), .log_index = state->log.count - 1, .view_number = state->view_number, }; if (send_to_peer(state, leader_idx(state), &ok_msg.base) < 0) { assert(0); } } // Execute all operations known to be committed that haven't // been executed previously while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); state->commit_index++; } Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } break; case MESSAGE_TYPE_RECOVERY_RESPONSE: { RecoveryResponseMessage message; if (msg.len < sizeof(message)) return -1; memcpy(&message, msg.ptr, sizeof(message)); // 1. Only process responses if we are actually in the recovering state // 2. Ensure the nonce matches the one we sent to prevent accepting // delayed responses from previous recovery attempts. if (state->status != STATUS_RECOVERY || message.nonce != state->recovery_nonce) { break; } // Track this response state->recovery_votes |= (1 << message.sender_idx); { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (RECOVERY): RECOVERY_RESPONSE from node %d, view=%lu op_number=%d commit_index=%d", TIME_VAL(now), self_idx(state), message.sender_idx, (unsigned long)message.view_number, message.op_number, message.commit_index); } // Track the highest view number we've seen and who the primary is for it if (message.view_number > state->max_view_seen) { state->max_view_seen = message.view_number; state->latest_primary_idx = (message.view_number % state->num_nodes); } // Store the state from the primary if this message is from them if (message.sender_idx == (int) (message.view_number % state->num_nodes)) { // Copy the log from the variable-sized portion of the message int num_entries = (msg.len - sizeof(RecoveryResponseMessage)) / sizeof(LogEntry); LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); if (entries == NULL) { assert(0); } memcpy(entries, (uint8_t*)msg.ptr + sizeof(RecoveryResponseMessage), num_entries * sizeof(LogEntry)); log_free(&state->potential_primary_log); state->potential_primary_log.count = num_entries; state->potential_primary_log.capacity = num_entries; state->potential_primary_log.entries = entries; state->received_primary_state = true; } // Check if we have f + 1 responses int response_count = 0; for (int i = 0; i < state->num_nodes; i++) { if (state->recovery_votes & (1 << i)) response_count++; } // The threshold f is derived from 2f + 1 = num_nodes int f = (state->num_nodes - 1) / 2; if (response_count >= f + 1 && state->received_primary_state) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d: recovery complete (responses=%d, f+1=%d), adopting view %lu", TIME_VAL(now), self_idx(state), response_count, f + 1, (unsigned long)state->max_view_seen); } // Update state using information from the primary state->view_number = state->max_view_seen; // Move the log (no need to copy since we're transferring ownership) log_free(&state->log); state->log = state->potential_primary_log; state->potential_primary_log = (Log){0}; // Clear to avoid double-free state->commit_index = message.commit_index; // Change status to normal; the recovery protocol is complete state->status = STATUS_NORMAL; // Update heartbeat to avoid immediate view change timeout Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } } break; default: break; } return 0; } static int process_message_as_replica(NodeState *state, int conn_idx, uint8_t type, ByteView msg) { (void) conn_idx; switch (type) { case MESSAGE_TYPE_REQUEST: { // Do nothing. Replicas ignore client requests. } break; case MESSAGE_TYPE_PREPARE: { PrepareMessage prepare_message; if (msg.len != sizeof(prepare_message)) return -1; memcpy(&prepare_message, msg.ptr, sizeof(prepare_message)); // Stale peer if (prepare_message.view_number < state->view_number) break; if (prepare_message.view_number > state->view_number) { // The new leader has started a view we haven't seen. // Adopt the new view and process the PREPARE normally. { Time t = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): adopting view %lu from PREPARE (was view %lu, status %s)", TIME_VAL(t), self_idx(state), (unsigned long)prepare_message.view_number, (unsigned long)state->view_number, status_name(state->status)); } state->view_number = prepare_message.view_number; state->status = STATUS_NORMAL; state->begin_view_change_votes = 0; state->num_future = 0; } if (prepare_message.log_index > state->log.count) { // The prepare message for a previous log entry was not received yet. // Buffer this message to process it later if (state->num_future == FUTURE_LIMIT) { assert(0); // TODO } state->future[state->num_future++] = prepare_message; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): PREPARE log_index=%d buffered (expected %d), num_future=%d", TIME_VAL(now), self_idx(state), prepare_message.log_index, state->log.count, state->num_future); } } else if (prepare_message.log_index < state->log.count) { // Message refers to an old entry. Ignore. } else { LogEntry log_entry = { .oper = prepare_message.oper, .votes = 0, .view_number = state->view_number, }; if (log_append(&state->log, log_entry) < 0) { assert(0); // TODO } PrepareOKMessage prepare_ok_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_PREPARE_OK, .length = sizeof(PrepareOKMessage), }, .sender_idx = self_idx(state), .log_index = state->log.count-1, .view_number = state->view_number, }; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA) -> node %d: PREPARE_OK log_index=%d view=%lu", TIME_VAL(now), self_idx(state), prepare_message.sender_idx, state->log.count-1, (unsigned long)state->view_number); } if (send_to_peer(state, prepare_message.sender_idx, &prepare_ok_message.base) < 0) { assert(0); } // Now try to process future log bool processed_at_least_one; do { processed_at_least_one = false; for (int i = 0; i < state->num_future; i++) { if (state->future[i].log_index < state->log.count) { state->future[i--] = state->future[--state->num_future]; continue; } if (state->future[i].log_index == state->log.count) { LogEntry future_log_entry = { .oper = state->future[i].oper, .votes = 0, .view_number = state->view_number, }; if (log_append(&state->log, future_log_entry) < 0) { assert(0); // TODO } PrepareOKMessage prepare_ok_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_PREPARE_OK, .length = sizeof(PrepareOKMessage), }, .sender_idx = self_idx(state), .log_index = state->log.count-1, .view_number = state->view_number, }; if (send_to_peer(state, state->future[i].sender_idx, &prepare_ok_message.base) < 0) { assert(0); } processed_at_least_one = true; break; } } } while (processed_at_least_one); while (state->commit_index < prepare_message.commit_index && state->commit_index < state->log.count) { state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); state->commit_index++; } Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } } break; case MESSAGE_TYPE_COMMIT: { CommitMessage message; if (msg.len != sizeof(CommitMessage)) return -1; memcpy(&message, msg.ptr, sizeof(message)); // Don't process heartbeats from a stale leader during // a view change, as it would reset the retry timer and // prevent the view change from completing. if (state->status == STATUS_CHANGE_VIEW) break; if (message.commit_index > state->commit_index) { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): COMMIT advancing commit_index %d -> %d", TIME_VAL(now), self_idx(state), state->commit_index, message.commit_index); } while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); state->commit_index++; } Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } break; case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: { if (state->status == STATUS_RECOVERY) break; BeginViewChangeMessage begin_view_change_message; if (msg.len != sizeof(BeginViewChangeMessage)) return -1; memcpy(&begin_view_change_message, msg.ptr, sizeof(begin_view_change_message)); if (begin_view_change_message.view_number > state->view_number) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): BEGIN_VIEW_CHANGE from node %d, transitioning view %lu -> %lu", TIME_VAL(now), self_idx(state), begin_view_change_message.sender_idx, (unsigned long)state->view_number, (unsigned long)begin_view_change_message.view_number); } // Reset vote tracking when transitioning to a new view state->begin_view_change_votes = 0; state->view_number = begin_view_change_message.view_number; state->status = STATUS_CHANGE_VIEW; // Send our own BeginViewChange to all peers BeginViewChangeMessage own_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, .length = sizeof(BeginViewChangeMessage), }, .view_number = state->view_number, .sender_idx = self_idx(state), }; if (broadcast_to_peers(state, &own_message.base) < 0) { assert(0); } // Count our own vote state->begin_view_change_votes |= (1 << self_idx(state)); } // Track this sender's vote (only if view matches) if (begin_view_change_message.view_number == state->view_number) { state->begin_view_change_votes |= (1 << begin_view_change_message.sender_idx); } // Count votes int vote_count = 0; for (int i = 0; i < state->num_nodes; i++) { if (state->begin_view_change_votes & (1 << i)) vote_count++; } // Need f + 1 votes to send DoViewChange if (vote_count >= state->num_nodes / 2 + 1) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): f+1 BEGIN_VIEW_CHANGE votes reached (%d/%d), sending DO_VIEW_CHANGE to node %d for view %lu", TIME_VAL(now), self_idx(state), vote_count, state->num_nodes, (int)(state->view_number % state->num_nodes), (unsigned long)state->view_number); } DoViewChangeMessage do_view_change_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_DO_VIEW_CHANGE, .length = sizeof(DoViewChangeMessage) + state->log.count * sizeof(LogEntry), }, .view_number = state->view_number, .old_view_number = state->view_number - 1, // View before we started view change .op_number = state->log.count, .commit_index = state->commit_index, .sender_idx = self_idx(state), }; if (leader_idx(state) == self_idx(state)) { // We are the new leader: count our own vote directly // since send_to_peer_ex skips self-sends. state->do_view_change_votes |= (1 << self_idx(state)); } else { if (send_to_peer_ex(state, leader_idx(state), &do_view_change_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { assert(0); } } // Clear the future array since we're changing views state->num_future = 0; } } break; case MESSAGE_TYPE_BEGIN_VIEW: { BeginViewMessage message; if (msg.len < sizeof(message)) return -1; memcpy(&message, msg.ptr, sizeof(message)); // Replicas only process messages containing a view-number // that matches or advances the one they know[cite: 738, 812]. if (message.view_number < state->view_number) break; { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): BEGIN_VIEW received, adopting view %lu (commit_index=%d, log_entries=%d)", TIME_VAL(now), self_idx(state), (unsigned long)message.view_number, message.commit_index, message.op_number); } // Update view and status state->view_number = message.view_number; state->status = STATUS_NORMAL; // Paper state: change status to normal // Replace the local log with the authoritative log from the primary { int num_entries = (msg.len - sizeof(BeginViewMessage)) / sizeof(LogEntry); LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); if (entries == NULL) { assert(0); } memcpy(entries, (uint8_t*)msg.ptr + sizeof(BeginViewMessage), num_entries * sizeof(LogEntry)); log_free(&state->log); state->log.count = num_entries; state->log.capacity = num_entries; state->log.entries = entries; } // Reset view change state state->begin_view_change_votes = 0; // If there are non-committed operations in the log, // send a PREPAREOK to the new primary if (state->log.count > message.commit_index) { PrepareOKMessage ok_msg = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_PREPARE_OK, .length = sizeof(PrepareOKMessage), }, .sender_idx = self_idx(state), .log_index = state->log.count - 1, .view_number = state->view_number, }; if (send_to_peer(state, leader_idx(state), &ok_msg.base) < 0) { assert(0); } } // Execute all operations known to be committed that haven't // been executed previously while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); state->commit_index++; } Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } break; case MESSAGE_TYPE_RECOVERY: { if (state->status != STATUS_NORMAL) break; // Ignore message. RecoveryMessage recovery_message; if (msg.len != sizeof(RecoveryMessage)) return -1; memcpy(&recovery_message, msg.ptr, sizeof(recovery_message)); { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): RECOVERY from node %d (nonce=%lu), sending RECOVERY_RESPONSE", TIME_VAL(now), self_idx(state), recovery_message.sender_idx, (unsigned long)recovery_message.nonce); } RecoveryResponseMessage recovery_response_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_RECOVERY_RESPONSE, .length = sizeof(RecoveryResponseMessage), }, .view_number = state->view_number, .op_number = state->log.count-1, // TODO: What if the log is empty? .nonce = recovery_message.nonce, .commit_index = state->commit_index, .sender_idx = self_idx(state), }; if (send_to_peer(state, recovery_message.sender_idx, &recovery_response_message.base) < 0) { assert(0); } } break; case MESSAGE_TYPE_RECOVERY_RESPONSE: { RecoveryResponseMessage message; if (msg.len < sizeof(message)) return -1; memcpy(&message, msg.ptr, sizeof(message)); // 1. Only process responses if we are actually in the recovering state [cite: 237] // 2. Ensure the nonce matches the one we sent to prevent accepting // delayed responses from previous recovery attempts[cite: 253]. if (state->status != STATUS_RECOVERY || message.nonce != state->recovery_nonce) { break; } // Track this response (e.g., in a bitmask or array) state->recovery_votes |= (1 << message.sender_idx); { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (RECOVERY): RECOVERY_RESPONSE from node %d, view=%lu op_number=%d commit_index=%d", TIME_VAL(now), self_idx(state), message.sender_idx, (unsigned long)message.view_number, message.op_number, message.commit_index); } // Track the highest view number we've seen and who the primary is for it if (message.view_number > state->max_view_seen) { state->max_view_seen = message.view_number; state->latest_primary_idx = (message.view_number % state->num_nodes); } // Store the state from the primary if this message is from them if (message.sender_idx == (int) (message.view_number % state->num_nodes)) { // Copy the log from the variable-sized portion of the message int num_entries = (msg.len - sizeof(RecoveryResponseMessage)) / sizeof(LogEntry); LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); if (entries == NULL) { assert(0); } memcpy(entries, (uint8_t*)msg.ptr + sizeof(RecoveryResponseMessage), num_entries * sizeof(LogEntry)); log_free(&state->potential_primary_log); state->potential_primary_log.count = num_entries; state->potential_primary_log.capacity = num_entries; state->potential_primary_log.entries = entries; state->received_primary_state = true; } // Check if we have f + 1 responses int response_count = 0; for (int i = 0; i < state->num_nodes; i++) { if (state->recovery_votes & (1 << i)) response_count++; } // The threshold f is derived from 2f + 1 = num_nodes [cite: 66, 73] int f = (state->num_nodes - 1) / 2; if (response_count >= f + 1 && state->received_primary_state) { { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d: recovery complete (responses=%d, f+1=%d), adopting view %lu", TIME_VAL(now), self_idx(state), response_count, f + 1, (unsigned long)state->max_view_seen); } // Update state using information from the primary state->view_number = state->max_view_seen; // Move the log (no need to copy since we're transferring ownership) log_free(&state->log); state->log = state->potential_primary_log; state->potential_primary_log = (Log){0}; // Clear to avoid double-free state->commit_index = message.commit_index; // Change status to normal; the recovery protocol is complete state->status = STATUS_NORMAL; // Update heartbeat to avoid immediate view change timeout Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); } state->last_heartbeat_time = now; } } break; default: break; } return 0; } static int process_message(NodeState *state, int conn_idx, uint8_t type, ByteView msg) { Time now = get_current_time(); NODE_TRACE("[" TIME_FMT "] node %d (%s, %s) <- conn %d: recv %s (%d bytes)", TIME_VAL(now), self_idx(state), is_leader(state) ? "LEADER" : "REPLICA", status_name(state->status), conn_idx, message_type_name(type), (int)msg.len); // Tag incoming connections with the sender's node index so that // the connection can be used bidirectionally. Without this, when // node A connects to node B and sends a message, node B can't // send back to node A through the same connection (the tag is // only set on the connector's side). { int sender_idx = -1; switch (type) { case MESSAGE_TYPE_PREPARE: if (msg.len >= sizeof(PrepareMessage)) { PrepareMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; case MESSAGE_TYPE_PREPARE_OK: if (msg.len >= sizeof(PrepareOKMessage)) { PrepareOKMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: if (msg.len >= sizeof(BeginViewChangeMessage)) { BeginViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; case MESSAGE_TYPE_DO_VIEW_CHANGE: if (msg.len >= sizeof(DoViewChangeMessage)) { DoViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; case MESSAGE_TYPE_RECOVERY: if (msg.len >= sizeof(RecoveryMessage)) { RecoveryMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; case MESSAGE_TYPE_RECOVERY_RESPONSE: if (msg.len >= sizeof(RecoveryResponseMessage)) { RecoveryResponseMessage m; memcpy(&m, msg.ptr, sizeof(m)); sender_idx = m.sender_idx; } break; } if (sender_idx >= 0 && sender_idx < state->num_nodes) { int existing = tcp_index_from_tag(&state->tcp, sender_idx); if (existing < 0) { // No connection tagged with this peer yet, tag this one tcp_set_tag(&state->tcp, conn_idx, sender_idx, false); } else if (existing != conn_idx) { // A different (possibly stale) connection has this tag. // Close the old one and tag the current one. tcp_close(&state->tcp, existing); tcp_set_tag(&state->tcp, conn_idx, sender_idx, false); } // If existing == conn_idx, already tagged correctly } } if (is_leader(state)) { return process_message_as_leader(state, conn_idx, type, msg); } else { return process_message_as_replica(state, conn_idx, type, msg); } } int node_init(void *state_, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout) { NodeState *state = state_; Time now = get_current_time(); if (now == INVALID_TIME) { fprintf(stderr, "Node :: Couldn't get current time\n"); return -1; } state->num_nodes = 0; bool self_addr_set = false; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--addr")) { if (self_addr_set) { fprintf(stderr, "Option --addr specified twice\n"); return -1; } self_addr_set = true; i++; if (i == argc) { fprintf(stderr, "Option --addr missing value. Usage is --addr :\n"); return -1; } // TODO: Check address is not duplicated int ret = parse_addr_arg(argv[i], &state->self_addr); if (ret < 0) { fprintf(stderr, "Malformed : pair for --addr option\n"); return -1; } if (state->num_nodes == NODE_LIMIT) { fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); return -1; } state->node_addrs[state->num_nodes++] = state->self_addr; } else if (!strcmp(argv[i], "--peer")) { i++; if (i == argc) { fprintf(stderr, "Option --peer missing value. Usage is --peer :\n"); return -1; } if (state->num_nodes == NODE_LIMIT) { fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); return -1; } // TODO: Check address is not duplicated int ret = parse_addr_arg(argv[i], &state->node_addrs[state->num_nodes]); if (ret < 0) { fprintf(stderr, "Malformed : pair for --peer option\n"); return -1; } state->num_nodes++; } else { printf("Ignoring option '%s'\n", argv[i]); } } // Now sort the addresses addr_sort(state->node_addrs, state->num_nodes); Time deadline = INVALID_TIME; state->view_number = 0; state->last_heartbeat_time = now; state->commit_index = 0; state->num_future = 0; // View change state state->begin_view_change_votes = 0; state->do_view_change_votes = 0; state->do_view_change_best_old_view = 0; state->do_view_change_best_commit = 0; log_init(&state->do_view_change_best_log); // Recovery state state->recovery_votes = 0; state->max_view_seen = 0; state->latest_primary_idx = 0; state->received_primary_state = false; state->recovery_attempt_count = 0; log_init(&state->potential_primary_log); // Detect whether this is a restart after a crash by checking for a // boot marker file on disk. The disk persists across crashes, so if // the marker exists, this node previously ran and crashed. In that // case, enter recovery mode to learn the current view from peers // before participating in the protocol. // // We use open() directly (without O_CREAT) instead of file_exists() // because access() is not available in the simulation environment. int marker_fd = open("vsr_boot_marker", O_RDONLY, 0); bool previously_crashed = (marker_fd >= 0); if (previously_crashed) close(marker_fd); if (previously_crashed) { state->status = STATUS_RECOVERY; state->recovery_nonce = now; state->recovery_start_time = now; } else { state->status = STATUS_NORMAL; } client_table_init(&state->client_table); state_machine_init(&state->state_machine); if (tcp_context_init(&state->tcp) < 0) { fprintf(stderr, "Node :: Couldn't setup TCP context\n"); return -1; } int ret = tcp_listen(&state->tcp, state->self_addr); if (ret < 0) { fprintf(stderr, "Node :: Couldn't setup TCP listener\n"); tcp_context_free(&state->tcp); return -1; } log_init(&state->log); // Write the boot marker to disk so that future restarts can detect // a previous crash. This must happen after TCP init so that the // marker is only written if the node successfully started. if (!previously_crashed) { int fd = open("vsr_boot_marker", O_WRONLY | O_CREAT, 0644); if (fd >= 0) close(fd); } if (previously_crashed) { // Broadcast RECOVERY to all peers to learn the current view RecoveryMessage recovery_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_RECOVERY, .length = sizeof(RecoveryMessage), }, .sender_idx = self_idx(state), .nonce = state->recovery_nonce, }; if (broadcast_to_peers(state, &recovery_message.base) < 0) { assert(0); // TODO } nearest_deadline(&deadline, state->recovery_start_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL); } NODE_TRACE("[" TIME_FMT "] node %d initialized: %s, view=%lu, num_nodes=%d, status=%s", TIME_VAL(now), self_idx(state), is_leader(state) ? "LEADER" : "REPLICA", (unsigned long)state->view_number, state->num_nodes, status_name(state->status)); *timeout = deadline_to_timeout(deadline, now); if (pcap < TCP_POLL_CAPACITY) { fprintf(stderr, "Node :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); return -1; } *pnum = tcp_register_events(&state->tcp, ctxs, pdata); return 0; } int node_tick(void *state_, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout) { NodeState *state = state_; Time now = get_current_time(); if (now == INVALID_TIME) { assert(0); // TODO } ///////////////////////////////////////////////////////////////// // NETWORK EVENTS ///////////////////////////////////////////////////////////////// Event events[TCP_EVENT_CAPACITY]; int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); for (int i = 0; i < num_events; i++) { if (events[i].type == EVENT_DISCONNECT) { tcp_close(&state->tcp, events[i].conn_idx); continue; } if (events[i].type != EVENT_MESSAGE) continue; int conn_idx = events[i].conn_idx; for (;;) { ByteView msg; uint16_t msg_type; int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); if (ret == 0) break; if (ret < 0) { tcp_close(&state->tcp, conn_idx); break; } ret = process_message(state, conn_idx, msg_type, msg); if (ret < 0) { tcp_close(&state->tcp, conn_idx); break; } tcp_consume_message(&state->tcp, conn_idx); } } ///////////////////////////////////////////////////////////////// // TIME EVENTS ///////////////////////////////////////////////////////////////// Time deadline = INVALID_TIME; if (state->status == STATUS_RECOVERY) { // Recovery handling runs regardless of leader/replica position, // since a recovering node must not act as leader until it learns // the current view from its peers. Time recovery_deadline = state->recovery_start_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL; if (recovery_deadline <= now) { state->recovery_attempt_count++; if (state->recovery_attempt_count >= RECOVERY_ATTEMPT_LIMIT) { assert(0); // TODO } else { RecoveryMessage recovery_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_RECOVERY, .length = sizeof(RecoveryMessage), }, .sender_idx = self_idx(state), .nonce = state->recovery_nonce, }; if (broadcast_to_peers(state, &recovery_message.base) < 0) { assert(0); // TODO } state->recovery_start_time = now; } } else { nearest_deadline(&deadline, recovery_deadline); } } else if (is_leader(state)) { Time heartbeat_deadline = state->last_heartbeat_time + HEARTBEAT_INTERVAL_SEC * 1000000000ULL; if (heartbeat_deadline <= now) { // TODO: check the time conversion here NODE_TRACE("[" TIME_FMT "] node %d (LEADER): heartbeat timeout, sending COMMIT to all peers (commit_index=%d)", TIME_VAL(now), self_idx(state), state->commit_index); CommitMessage commit_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_COMMIT, .length = sizeof(CommitMessage), }, .commit_index = state->commit_index, }; if (broadcast_to_peers(state, &commit_message.base)) { assert(0); // TODO } state->last_heartbeat_time = now; } else { nearest_deadline(&deadline, heartbeat_deadline); } } else { Time death_deadline = state->last_heartbeat_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; if (death_deadline <= now) { NODE_TRACE("[" TIME_FMT "] node %d (REPLICA, %s): primary death timeout, initiating view change to view %lu", TIME_VAL(now), self_idx(state), status_name(state->status), (unsigned long)(state->view_number + 1)); BeginViewChangeMessage begin_view_change_message = { .base = { .version = MESSAGE_VERSION, .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, .length = sizeof(BeginViewChangeMessage), }, .view_number = state->view_number + 1, .sender_idx = self_idx(state), }; if (broadcast_to_peers(state, &begin_view_change_message.base)) { assert(0); // TODO } state->status = STATUS_CHANGE_VIEW; state->last_heartbeat_time = now; } else { nearest_deadline(&deadline, death_deadline); } } *timeout = deadline_to_timeout(deadline, now); if (pcap < TCP_POLL_CAPACITY) return -1; *pnum = tcp_register_events(&state->tcp, ctxs, pdata); return 0; } int node_free(void *state_) { NodeState *state = state_; log_free(&state->log); log_free(&state->potential_primary_log); log_free(&state->do_view_change_best_log); tcp_context_free(&state->tcp); client_table_free(&state->client_table); state_machine_free(&state->state_machine); return 0; } void check_vsr_invariants(NodeState **nodes, int num_nodes) { for (int i = 0; i < num_nodes; i++) { NodeState *s = nodes[i]; if (s == NULL) continue; // 1. commit_index <= log.count // A node cannot have committed more entries than it has in its log. if (s->commit_index > s->log.count) { fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n", i, s->commit_index, s->log.count); __builtin_trap(); } // 2. commit_index >= 0 if (s->commit_index < 0) { fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n", i, s->commit_index); __builtin_trap(); } // 4. Future buffer count is in valid range. if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) { fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n", i, s->num_future, FUTURE_LIMIT); __builtin_trap(); } } // Cross-node invariants // 5. At most one leader in normal status per view. for (int i = 0; i < num_nodes; i++) { if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i])) continue; for (int j = i + 1; j < num_nodes; j++) { if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j])) continue; if (nodes[i]->view_number == nodes[j]->view_number) { fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n", (unsigned long)nodes[i]->view_number, i, j); __builtin_trap(); } } } // 6. Committed prefix agreement (State Machine Safety). // For any two nodes, their logs must agree on all entries up to // min(commit_index_i, commit_index_j). This is the core safety // property of VSR: all committed operations are identical across // replicas. for (int i = 0; i < num_nodes; i++) { if (nodes[i] == NULL) continue; for (int j = i + 1; j < num_nodes; j++) { if (nodes[j] == NULL) continue; int min_commit = nodes[i]->commit_index; if (nodes[j]->commit_index < min_commit) min_commit = nodes[j]->commit_index; for (int k = 0; k < min_commit; k++) { if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(Operation)) != 0) { fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d " "between node %d and node %d\n", k, i, j); __builtin_trap(); } } } } }