Version 0.8
This commit is contained in:
+273
@@ -0,0 +1,273 @@
|
||||
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <stdint.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include <lib/file_system.h>
|
||||
|
||||
#include "node.h"
|
||||
#include "client.h"
|
||||
|
||||
//#define CLIENT_TRACE(fmt, ...) {}
|
||||
#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
|
||||
|
||||
// Format time as seconds with 3 decimal places for trace output
|
||||
#define TIME_FMT "%.3fs"
|
||||
#define TIME_VAL(t) ((double)(t) / 1000000000.0)
|
||||
|
||||
static int leader_idx(ClientState *state)
|
||||
{
|
||||
return state->view_number % state->num_servers;
|
||||
}
|
||||
|
||||
static int
|
||||
process_message(ClientState *state,
|
||||
int conn_idx, uint8_t type, ByteView msg)
|
||||
{
|
||||
(void) conn_idx;
|
||||
|
||||
if (!state->pending)
|
||||
return -1;
|
||||
|
||||
if (type != MESSAGE_TYPE_REPLY)
|
||||
return -1;
|
||||
|
||||
ReplyMessage reply_message;
|
||||
if (msg.len != sizeof(ReplyMessage))
|
||||
return -1;
|
||||
memcpy(&reply_message, msg.ptr, sizeof(reply_message));
|
||||
|
||||
{
|
||||
Time now = get_current_time();
|
||||
CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)",
|
||||
TIME_VAL(now),
|
||||
reply_message.rejected ? "true" : "false");
|
||||
}
|
||||
|
||||
state->pending = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int client_init(void *state_, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout)
|
||||
{
|
||||
ClientState *state = state_;
|
||||
|
||||
state->num_servers = 0;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!strcmp(argv[i], "--server")) {
|
||||
i++;
|
||||
if (i == argc) {
|
||||
fprintf(stderr, "Option --server missing value. Usage is --server <addr>:<port>\n");
|
||||
return -1;
|
||||
}
|
||||
if (state->num_servers == NODE_LIMIT) {
|
||||
fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT);
|
||||
return -1;
|
||||
}
|
||||
// TODO: Check address is not duplicated
|
||||
if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) {
|
||||
fprintf(stderr, "Malformed <addr>:<port> pair for --server option\n");
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
printf("Ignoring option '%s'\n", argv[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Now sort the addresses
|
||||
addr_sort(state->server_addrs, state->num_servers);
|
||||
|
||||
if (tcp_context_init(&state->tcp) < 0) {
|
||||
fprintf(stderr, "Client :: Couldn't setup TCP context\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
state->pending = false;
|
||||
|
||||
state->view_number = 0;
|
||||
state->request_id = 0;
|
||||
|
||||
// Load or generate a persistent client_id from disk.
|
||||
// This ensures the client_id survives process restarts.
|
||||
{
|
||||
string path = S("client_id");
|
||||
Handle fd;
|
||||
bool loaded = false;
|
||||
if (file_exists(path)) {
|
||||
if (file_open(path, &fd) == 0) {
|
||||
file_set_offset(fd, 0);
|
||||
uint64_t saved_id;
|
||||
if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) {
|
||||
state->client_id = saved_id;
|
||||
loaded = true;
|
||||
}
|
||||
file_close(fd);
|
||||
}
|
||||
}
|
||||
if (!loaded) {
|
||||
Time now = get_current_time();
|
||||
state->client_id = (uint64_t)now;
|
||||
if (file_open(path, &fd) == 0) {
|
||||
file_set_offset(fd, 0);
|
||||
file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id));
|
||||
file_sync(fd);
|
||||
file_close(fd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to all known servers
|
||||
for (int i = 0; i < state->num_servers; i++) {
|
||||
if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) {
|
||||
fprintf(stderr, "Client :: Couldn't connect to server %d\n", i);
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
Time now = get_current_time();
|
||||
CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d",
|
||||
TIME_VAL(now), state->num_servers, leader_idx(state));
|
||||
}
|
||||
|
||||
*timeout = 0;
|
||||
if (pcap < TCP_POLL_CAPACITY) {
|
||||
fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
|
||||
return -1;
|
||||
}
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int client_tick(void *state_, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
|
||||
{
|
||||
ClientState *state = state_;
|
||||
|
||||
Event events[TCP_EVENT_CAPACITY];
|
||||
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
|
||||
|
||||
for (int i = 0; i < num_events; i++) {
|
||||
if (events[i].type == EVENT_DISCONNECT) {
|
||||
int conn_idx = events[i].conn_idx;
|
||||
int tag = tcp_get_tag(&state->tcp, conn_idx);
|
||||
if (tag == leader_idx(state) && state->pending) {
|
||||
Time now = get_current_time();
|
||||
CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request",
|
||||
TIME_VAL(now), leader_idx(state));
|
||||
state->pending = false;
|
||||
}
|
||||
tcp_close(&state->tcp, conn_idx);
|
||||
continue;
|
||||
}
|
||||
if (events[i].type != EVENT_MESSAGE)
|
||||
continue;
|
||||
int conn_idx = events[i].conn_idx;
|
||||
|
||||
for (;;) {
|
||||
|
||||
ByteView msg;
|
||||
uint16_t msg_type;
|
||||
int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type);
|
||||
if (ret == 0)
|
||||
break;
|
||||
if (ret < 0) {
|
||||
tcp_close(&state->tcp, conn_idx);
|
||||
break;
|
||||
}
|
||||
|
||||
ret = process_message(state, conn_idx, msg_type, msg);
|
||||
if (ret < 0) {
|
||||
tcp_close(&state->tcp, conn_idx);
|
||||
break;
|
||||
}
|
||||
|
||||
tcp_consume_message(&state->tcp, conn_idx);
|
||||
}
|
||||
}
|
||||
|
||||
Time now = get_current_time();
|
||||
|
||||
// If we've been waiting too long for a response, give up and
|
||||
// try the next server (the current leader may have crashed and
|
||||
// a view change may have happened)
|
||||
if (state->pending) {
|
||||
Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
|
||||
if (request_deadline <= now) {
|
||||
CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server",
|
||||
TIME_VAL(now), leader_idx(state),
|
||||
(unsigned long)state->view_number);
|
||||
state->view_number++;
|
||||
state->pending = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!state->pending) {
|
||||
|
||||
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
|
||||
if (conn_idx < 0) {
|
||||
// Leader connection not available, try reconnecting
|
||||
{
|
||||
CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting",
|
||||
TIME_VAL(now), leader_idx(state));
|
||||
}
|
||||
tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
|
||||
} else {
|
||||
// Now start a new operation
|
||||
state->request_id++;
|
||||
|
||||
RequestMessage request_message = {
|
||||
.base = {
|
||||
.version = MESSAGE_VERSION,
|
||||
.type = MESSAGE_TYPE_REQUEST,
|
||||
.length = sizeof(RequestMessage),
|
||||
},
|
||||
.oper = OPERATION_A,
|
||||
.client_id = state->client_id,
|
||||
.request_id = state->request_id,
|
||||
};
|
||||
|
||||
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
|
||||
assert(output);
|
||||
|
||||
byte_queue_write(output, &request_message, request_message.base.length);
|
||||
|
||||
{
|
||||
CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)",
|
||||
TIME_VAL(now), leader_idx(state),
|
||||
(unsigned long)state->view_number,
|
||||
(unsigned long)state->client_id,
|
||||
(unsigned long)state->request_id);
|
||||
}
|
||||
|
||||
state->pending = true;
|
||||
state->request_time = now;
|
||||
}
|
||||
}
|
||||
|
||||
// Set timeout based on pending request deadline
|
||||
Time deadline = INVALID_TIME;
|
||||
if (state->pending) {
|
||||
nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
|
||||
}
|
||||
*timeout = deadline_to_timeout(deadline, now);
|
||||
if (pcap < TCP_POLL_CAPACITY)
|
||||
return -1;
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int client_free(void *state_)
|
||||
{
|
||||
ClientState *state = state_;
|
||||
|
||||
tcp_context_free(&state->tcp);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#ifndef CLIENT_INCLUDED
|
||||
#define CLIENT_INCLUDED
|
||||
|
||||
#include <lib/tcp.h>
|
||||
#include <lib/basic.h>
|
||||
|
||||
#include "config.h"
|
||||
|
||||
typedef struct {
|
||||
|
||||
TCP tcp;
|
||||
|
||||
// True if we are waiting for a response
|
||||
bool pending;
|
||||
Time request_time; // When the current request was sent
|
||||
|
||||
Address server_addrs[NODE_LIMIT];
|
||||
int num_servers;
|
||||
|
||||
uint64_t view_number;
|
||||
|
||||
uint64_t client_id;
|
||||
uint64_t request_id;
|
||||
|
||||
} ClientState;
|
||||
|
||||
struct pollfd;
|
||||
|
||||
int client_init(void *state, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int client_tick(void *state, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
|
||||
int client_free(void *state);
|
||||
|
||||
#endif // CLIENT_INCLUDED
|
||||
@@ -0,0 +1,70 @@
|
||||
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "client_table.h"
|
||||
|
||||
void client_table_init(ClientTable *client_table)
|
||||
{
|
||||
client_table->count = 0;
|
||||
client_table->capacity = 0;
|
||||
client_table->entries = NULL;
|
||||
}
|
||||
|
||||
void client_table_free(ClientTable *client_table)
|
||||
{
|
||||
free(client_table->entries);
|
||||
}
|
||||
|
||||
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id)
|
||||
{
|
||||
for (int i = 0; i < client_table->count; i++) {
|
||||
if (client_table->entries[i].client_id == client_id)
|
||||
return &client_table->entries[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx)
|
||||
{
|
||||
if (client_table->count == client_table->capacity) {
|
||||
int n = 2 * client_table->capacity;
|
||||
if (n == 0) n = 8;
|
||||
void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
|
||||
if (p == NULL)
|
||||
return -1;
|
||||
client_table->capacity = n;
|
||||
client_table->entries = p;
|
||||
}
|
||||
|
||||
client_table->entries[client_table->count++] = (ClientTableEntry) {
|
||||
.client_id = client_id,
|
||||
.last_request_id = request_id,
|
||||
.pending = true,
|
||||
.conn_idx = conn_idx,
|
||||
};
|
||||
return 0;
|
||||
}
|
||||
|
||||
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id)
|
||||
{
|
||||
if (client_table->count == client_table->capacity) {
|
||||
int n = 2 * client_table->capacity;
|
||||
if (n == 0) n = 8;
|
||||
void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
|
||||
if (p == NULL)
|
||||
return -1;
|
||||
client_table->capacity = n;
|
||||
client_table->entries = p;
|
||||
}
|
||||
|
||||
client_table->entries[client_table->count++] = (ClientTableEntry) {
|
||||
.client_id=client_id,
|
||||
.last_request_id=request_id,
|
||||
.pending=true,
|
||||
};
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
#ifndef CLIENT_TABLE_INCLUDED
|
||||
#define CLIENT_TABLE_INCLUDED
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <state_machine/state_machine.h>
|
||||
|
||||
typedef struct {
|
||||
uint64_t client_id;
|
||||
uint64_t last_request_id;
|
||||
OperationResult last_result;
|
||||
bool pending;
|
||||
int conn_idx;
|
||||
} ClientTableEntry;
|
||||
|
||||
typedef struct {
|
||||
int count;
|
||||
int capacity;
|
||||
ClientTableEntry *entries;
|
||||
} ClientTable;
|
||||
|
||||
void client_table_init(ClientTable *client_table);
|
||||
void client_table_free(ClientTable *client_table);
|
||||
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
|
||||
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx);
|
||||
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
|
||||
|
||||
#endif // CLIENT_TABLE_INCLUDED
|
||||
@@ -0,0 +1,11 @@
|
||||
#ifndef CONFIG_INCLUDED
|
||||
#define CONFIG_INCLUDED
|
||||
|
||||
#define NODE_LIMIT 32
|
||||
#define FUTURE_LIMIT 32
|
||||
#define HEARTBEAT_INTERVAL_SEC 1
|
||||
#define PRIMARY_DEATH_TIMEOUT_SEC 5
|
||||
#define RECOVERY_TIMEOUT_SEC 3
|
||||
#define RECOVERY_ATTEMPT_LIMIT 10
|
||||
|
||||
#endif // CONFIG_INCLUDED
|
||||
@@ -0,0 +1,37 @@
|
||||
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
|
||||
#include "log.h"
|
||||
|
||||
void log_init(Log *log)
|
||||
{
|
||||
log->count = 0;
|
||||
log->capacity = 0;
|
||||
log->entries = NULL;
|
||||
}
|
||||
|
||||
void log_free(Log *log)
|
||||
{
|
||||
free(log->entries);
|
||||
}
|
||||
|
||||
int log_append(Log *log, LogEntry entry)
|
||||
{
|
||||
if (log->count == log->capacity) {
|
||||
int n= 2 * log->capacity;
|
||||
if (n < 8)
|
||||
n = 8;
|
||||
LogEntry *p = realloc(log->entries, n * sizeof(LogEntry));
|
||||
if (p == NULL)
|
||||
return -1;
|
||||
|
||||
log->entries = p;
|
||||
log->capacity = n;
|
||||
}
|
||||
|
||||
log->entries[log->count++] = entry;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#ifndef LOG_INCLUDED
|
||||
#define LOG_INCLUDED
|
||||
|
||||
#include <state_machine/state_machine.h>
|
||||
|
||||
#include "config.h"
|
||||
|
||||
typedef struct {
|
||||
Operation oper;
|
||||
uint32_t votes;
|
||||
int view_number;
|
||||
uint64_t client_id;
|
||||
} LogEntry;
|
||||
|
||||
_Static_assert(NODE_LIMIT <= 32, "");
|
||||
|
||||
typedef struct {
|
||||
int count;
|
||||
int capacity;
|
||||
LogEntry *entries;
|
||||
} Log;
|
||||
|
||||
void log_init(Log *log);
|
||||
void log_free(Log *log);
|
||||
int log_append(Log *log, LogEntry entry);
|
||||
|
||||
#endif // LOG_INCLUDED
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
|
||||
#include <signal.h>
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
|
||||
#include "node.h"
|
||||
#include "client.h"
|
||||
|
||||
static volatile int simulation_running = 1;
|
||||
|
||||
static void sigint_handler(int sig)
|
||||
{
|
||||
(void)sig;
|
||||
simulation_running = 0;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
signal(SIGINT, sigint_handler);
|
||||
|
||||
Quakey *quakey;
|
||||
int ret = quakey_init(&quakey, 1);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
QuakeyNode cli_1;
|
||||
QuakeyNode cli_2;
|
||||
QuakeyNode node_1;
|
||||
QuakeyNode node_2;
|
||||
QuakeyNode node_3;
|
||||
|
||||
// Client 1
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.name = "rndcli1",
|
||||
.state_size = sizeof(ClientState),
|
||||
.init_func = client_init,
|
||||
.tick_func = client_tick,
|
||||
.free_func = client_free,
|
||||
.addrs = (char*[]) { "127.0.0.2" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 10<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
|
||||
}
|
||||
|
||||
// Client 2
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.name = "rndcli2",
|
||||
.state_size = sizeof(ClientState),
|
||||
.init_func = client_init,
|
||||
.tick_func = client_tick,
|
||||
.free_func = client_free,
|
||||
.addrs = (char*[]) { "127.0.0.3" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 10<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
|
||||
}
|
||||
|
||||
// Node 1
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.name = "node1",
|
||||
.state_size = sizeof(NodeState),
|
||||
.init_func = node_init,
|
||||
.tick_func = node_tick,
|
||||
.free_func = node_free,
|
||||
.addrs = (char*[]) { "127.0.0.4" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 10<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
|
||||
}
|
||||
|
||||
// Node 2
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.name = "node2",
|
||||
.state_size = sizeof(NodeState),
|
||||
.init_func = node_init,
|
||||
.tick_func = node_tick,
|
||||
.free_func = node_free,
|
||||
.addrs = (char*[]) { "127.0.0.5" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 10<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
|
||||
}
|
||||
|
||||
// Node 3
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.name = "node3",
|
||||
.state_size = sizeof(NodeState),
|
||||
.init_func = node_init,
|
||||
.tick_func = node_tick,
|
||||
.free_func = node_free,
|
||||
.addrs = (char*[]) { "127.0.0.6" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 10<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
|
||||
}
|
||||
|
||||
while (simulation_running) {
|
||||
quakey_schedule_one(quakey);
|
||||
|
||||
NodeState *arr[] = {
|
||||
quakey_node_state(node_1),
|
||||
quakey_node_state(node_2),
|
||||
quakey_node_state(node_3),
|
||||
};
|
||||
check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0]));
|
||||
}
|
||||
|
||||
quakey_free(quakey);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // MAIN_SIMULATION
|
||||
////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////
|
||||
#ifdef MAIN_CLIENT
|
||||
|
||||
#include <poll.h>
|
||||
|
||||
#include "client.h"
|
||||
|
||||
#define POLL_CAPACITY 1024
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret;
|
||||
ClientState state;
|
||||
|
||||
void* poll_ctxs[POLL_CAPACITY];
|
||||
struct pollfd poll_array[POLL_CAPACITY];
|
||||
int poll_count;
|
||||
int poll_timeout;
|
||||
|
||||
ret = client_init(
|
||||
&state,
|
||||
argc,
|
||||
argv,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
for (;;) {
|
||||
|
||||
#ifdef _WIN32
|
||||
WSAPoll(poll_array, poll_count, poll_timeout);
|
||||
#else
|
||||
poll(poll_array, poll_count, poll_timeout);
|
||||
#endif
|
||||
|
||||
ret = client_tick(
|
||||
&state,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
client_free(&state);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // MAIN_CLIENT
|
||||
////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////
|
||||
#ifdef MAIN_SERVER
|
||||
|
||||
#include <poll.h>
|
||||
|
||||
#include "node.h"
|
||||
|
||||
#define POLL_CAPACITY 1024
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret;
|
||||
NodeState state;
|
||||
|
||||
void* poll_ctxs[POLL_CAPACITY];
|
||||
struct pollfd poll_array[POLL_CAPACITY];
|
||||
int poll_count;
|
||||
int poll_timeout;
|
||||
|
||||
ret = node_init(
|
||||
&state,
|
||||
argc,
|
||||
argv,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
for (;;) {
|
||||
|
||||
#ifdef _WIN32
|
||||
WSAPoll(poll_array, poll_count, poll_timeout);
|
||||
#else
|
||||
poll(poll_array, poll_count, poll_timeout);
|
||||
#endif
|
||||
|
||||
ret = node_tick(
|
||||
&state,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
node_free(&state);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // MAIN_SERVER
|
||||
+1745
File diff suppressed because it is too large
Load Diff
+176
@@ -0,0 +1,176 @@
|
||||
#ifndef NODE_INCLUDED
|
||||
#define NODE_INCLUDED
|
||||
|
||||
#include <lib/tcp.h>
|
||||
#include <lib/basic.h>
|
||||
#include <lib/message.h>
|
||||
|
||||
#include <state_machine/state_machine.h>
|
||||
|
||||
#include "log.h"
|
||||
#include "config.h"
|
||||
#include "client_table.h"
|
||||
|
||||
enum {
|
||||
|
||||
// Normal Protocol
|
||||
MESSAGE_TYPE_REQUEST,
|
||||
MESSAGE_TYPE_REPLY,
|
||||
MESSAGE_TYPE_PREPARE,
|
||||
MESSAGE_TYPE_PREPARE_OK,
|
||||
MESSAGE_TYPE_COMMIT,
|
||||
|
||||
// View Change Protocol
|
||||
MESSAGE_TYPE_BEGIN_VIEW_CHANGE,
|
||||
MESSAGE_TYPE_DO_VIEW_CHANGE,
|
||||
MESSAGE_TYPE_BEGIN_VIEW,
|
||||
|
||||
// Recovery Protocol
|
||||
MESSAGE_TYPE_RECOVERY,
|
||||
MESSAGE_TYPE_RECOVERY_RESPONSE,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
Operation oper;
|
||||
uint64_t client_id;
|
||||
uint64_t request_id;
|
||||
} RequestMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
Operation oper;
|
||||
int sender_idx;
|
||||
int log_index;
|
||||
int commit_index;
|
||||
uint64_t view_number;
|
||||
} PrepareMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
int sender_idx;
|
||||
int log_index;
|
||||
uint64_t view_number;
|
||||
} PrepareOKMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
int commit_index;
|
||||
} CommitMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
bool rejected;
|
||||
OperationResult result;
|
||||
} ReplyMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
uint64_t view_number;
|
||||
int sender_idx;
|
||||
} BeginViewChangeMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
uint64_t view_number; // The new view number
|
||||
uint64_t old_view_number; // Last view number when replica was in normal status
|
||||
int op_number; // Number of entries in the log
|
||||
int commit_index;
|
||||
int sender_idx;
|
||||
// Followed by: LogEntry log[op_number]
|
||||
} DoViewChangeMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
uint64_t view_number;
|
||||
int commit_index;
|
||||
int op_number; // Number of log entries that follow
|
||||
// Followed by: LogEntry log[op_number]
|
||||
} BeginViewMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
int sender_idx;
|
||||
uint64_t nonce;
|
||||
} RecoveryMessage;
|
||||
|
||||
typedef struct {
|
||||
MessageHeader base;
|
||||
uint64_t view_number;
|
||||
int op_number;
|
||||
uint64_t nonce;
|
||||
int commit_index;
|
||||
int sender_idx;
|
||||
} RecoveryResponseMessage;
|
||||
|
||||
typedef enum {
|
||||
STATUS_NORMAL,
|
||||
STATUS_CHANGE_VIEW,
|
||||
STATUS_RECOVERY,
|
||||
} Status;
|
||||
|
||||
typedef struct {
|
||||
|
||||
TCP tcp;
|
||||
|
||||
Address self_addr;
|
||||
Address node_addrs[NODE_LIMIT];
|
||||
int num_nodes;
|
||||
|
||||
Status status;
|
||||
|
||||
ClientTable client_table;
|
||||
|
||||
uint64_t view_number;
|
||||
|
||||
// These fields are used in recovery mode
|
||||
uint32_t recovery_votes;
|
||||
uint64_t recovery_nonce;
|
||||
uint64_t max_view_seen;
|
||||
int latest_primary_idx;
|
||||
bool received_primary_state;
|
||||
Log potential_primary_log;
|
||||
Time recovery_start_time;
|
||||
int recovery_attempt_count;
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// VIEW CHANGE
|
||||
|
||||
uint32_t begin_view_change_votes; // Bitmask of received BeginViewChange messages
|
||||
uint32_t do_view_change_votes; // Bitmask of received DoViewChange messages
|
||||
uint64_t do_view_change_best_old_view; // Best old_view_number seen in DoViewChange
|
||||
int do_view_change_best_commit; // Best commit_index seen
|
||||
Log do_view_change_best_log; // Best log seen
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
|
||||
// If this is the leader, commit_index is the index of the next uncommitted element
|
||||
// of the log (if no uncommitted elements are present, it's equal to the total number
|
||||
// of log elements)
|
||||
int commit_index;
|
||||
|
||||
PrepareMessage future[FUTURE_LIMIT];
|
||||
int num_future;
|
||||
|
||||
Log log;
|
||||
|
||||
Time last_heartbeat_time;
|
||||
|
||||
StateMachine state_machine;
|
||||
|
||||
} NodeState;
|
||||
|
||||
struct pollfd;
|
||||
|
||||
int node_init(void *state, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int node_tick(void *state, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
|
||||
int node_free(void *state);
|
||||
|
||||
void check_vsr_invariants(NodeState **nodes, int num_nodes);
|
||||
|
||||
#endif // NODE_INCLUDED
|
||||
Reference in New Issue
Block a user