commit 868312edf870f44a63e924c86c6beffb5eca2b20 Author: Francesco Cozzuto Date: Sun Feb 8 20:42:10 2026 +0100 Version 0.8 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..09594c7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Enforce LF line endings for all text files +* text eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c86dac3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +vsr_client +vsr_server +vsr_simulation +raft_client +raft_server +raft_simulation +raft_simulation_coverage +*.gcno +*.gcda +*.gcov +*.info +coverage_html/ +*.log \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..0a1a46e --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT \ No newline at end of file diff --git a/lib/basic.c b/lib/basic.c new file mode 100644 index 0000000..c15078b --- /dev/null +++ b/lib/basic.c @@ -0,0 +1,232 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "basic.h" + +bool streq(string s1, string s2) +{ + if (s1.len != s2.len) + return false; + for (int i = 0; i < s1.len; i++) + if (s1.ptr[i] != s2.ptr[i]) + return false; + return true; +} + +// Returns the current time in nanoseconds since +// an unspecified time in the past (useful to calculate +// elapsed time intervals) +Time get_current_time(void) +{ +#ifdef _WIN32 + { + int64_t count; + int64_t freq; + int ok; + + ok = QueryPerformanceCounter((LARGE_INTEGER*) &count); + if (!ok) return INVALID_TIME; + + ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq); + if (!ok) return INVALID_TIME; + + uint64_t res = 1000000000 * (double) count / freq; + return res; + } +#else + { + struct timespec time; + + if (clock_gettime(CLOCK_REALTIME, &time)) + return INVALID_TIME; + + uint64_t res; + + uint64_t sec = time.tv_sec; + if (sec > UINT64_MAX / 1000000000) + return INVALID_TIME; + res = sec * 1000000000; + + uint64_t nsec = time.tv_nsec; + if (res > UINT64_MAX - nsec) + return INVALID_TIME; + res += nsec; + + return res; + } +#endif +} + +void nearest_deadline(Time *a, Time b) +{ + if (*a == INVALID_TIME || *a > b) + *a = b; +} + +int deadline_to_timeout(Time deadline, Time current_time) +{ + if (deadline == INVALID_TIME) + return -1; + return (deadline - current_time) / 1000000; +} + +bool getargb(int argc, char **argv, char *name) +{ + for (int i = 0; i < argc; i++) + if (!strcmp(argv[i], name)) + return true; + return false; +} + +string getargs(int argc, char **argv, char *name, char *fallback) +{ + for (int i = 0; i < argc; i++) + if (!strcmp(argv[i], name)) { + i++; + if (i == argc) + break; + return (string) { argv[i], strlen(argv[i]) }; + } + return (string) { fallback, strlen(fallback) }; +} + +int getargi(int argc, char **argv, char *name, int fallback) +{ + for (int i = 0; i < argc; i++) + if (!strcmp(argv[i], name)) { + + i++; + if (i == argc) + break; + + errno = 0; + char *end; + long val = strtol(argv[i], &end, 10); + + if (end == argv[i] || *end != '\0' || errno == ERANGE) + break; + + if (val < INT_MIN || val > INT_MAX) + break; + + return (int) val; + } + return fallback; +} + +void append_hex_as_str(char *out, SHA256 hash) +{ + char table[] = "0123456789abcdef"; + for (int i = 0; i < (int) sizeof(hash); i++) { + out[(i << 1) + 0] = table[(uint8_t) hash.data[i] >> 4]; + out[(i << 1) + 1] = table[(uint8_t) hash.data[i] & 0xF]; + } +} + +// TODO: check this function +bool addr_lower(Address a, Address b) +{ + if (a.is_ipv4) { + + if (!b.is_ipv4) + return true; + + if (a.ipv4.data < b.ipv4.data) + return true; + + if (a.ipv4.data == b.ipv4.data && + a.port < b.port) + return true; + + return false; + + } else { + + if (b.is_ipv4) + return false; + + for (int i = 0; i < 8; i++) { + + if (a.ipv6.data[i] < b.ipv6.data[i]) + return true; + + if (a.ipv6.data[i] > b.ipv6.data[i]) + return false; + } + + if (a.port < b.port) + return true; + + return false; + } +} + +bool addr_eql(Address a, Address b) +{ + if (a.is_ipv4 != b.is_ipv4) + return false; + + if (a.port != b.port) + return false; + + if (a.is_ipv4) { + if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4))) + return false; + } else { + if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6))) + return false; + } + + return true; +} + +int parse_addr_arg(char *arg, Address *out) +{ + int len = strlen(arg); + + int i = 0; + while (i < len && arg[i] != ':') + i++; + + if (i == len) + return -1; // No ':' character. + arg[i] = '\0'; + + IPv4 ipv4; + int ret = inet_pton(AF_INET, arg, &ipv4); + arg[i] = ':'; + + if (ret != 1) + return -1; + + errno = 0; + ret = atoi(arg + i + 1); + if (ret == 0 && errno != 0) + return -1; + + out->ipv4 = ipv4; + out->is_ipv4 = true; + out->port = ret; + + return 0; +} + +void addr_sort(Address *addrs, int count) +{ + for (int i = 0; i < count; i++) { + + int k = i; // Index of the lowest address in [i, num_nodes-1] + for (int j = i+1; j < count; j++) { + if (addr_lower(addrs[j], addrs[k])) + k = j; + } + + Address tmp = addrs[i]; + addrs[i] = addrs[k]; + addrs[k] = tmp; + } +} \ No newline at end of file diff --git a/lib/basic.h b/lib/basic.h new file mode 100644 index 0000000..dfb641f --- /dev/null +++ b/lib/basic.h @@ -0,0 +1,61 @@ +#ifndef BASIC_INCLUDED +#define BASIC_INCLUDED + +#include +#include + +typedef struct { + char data[32]; +} SHA256; + +typedef struct { + uint32_t data; +} IPv4; + +typedef struct { + uint16_t data[8]; +} IPv6; + +typedef struct { + union { + IPv4 ipv4; + IPv6 ipv6; + }; + bool is_ipv4; + uint16_t port; +} Address; + +typedef struct { + char *ptr; + int len; +} string; + +typedef uint64_t Time; +#define INVALID_TIME ((Time) -1) + +#define S(X) ((string) { (X), (int) sizeof(X)-1 }) + +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) + +#define UNREACHABLE __builtin_trap(); + +bool streq(string s1, string s2); + +Time get_current_time(void); +void nearest_deadline(Time *a, Time b); +int deadline_to_timeout(Time deadline, Time current_time); + +bool getargb(int argc, char **argv, char *name); +string getargs(int argc, char **argv, char *name, char *fallback); +int getargi(int argc, char **argv, char *name, int fallback); + +void append_hex_as_str(char *out, SHA256 hash); + +bool addr_eql(Address a, Address b); +bool addr_lower(Address a, Address b); + +int parse_addr_arg(char *arg, Address *out); +void addr_sort(Address *addrs, int count); + +#endif // BASIC_INCLUDED diff --git a/lib/byte_queue.c b/lib/byte_queue.c new file mode 100644 index 0000000..bb807d6 --- /dev/null +++ b/lib/byte_queue.c @@ -0,0 +1,306 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "byte_queue.h" + +// This is the implementation of a byte queue useful +// for systems that need to process engs of bytes. +// +// It features sticky errors, a zero-copy interface, +// and a safe mechanism to patch previously written +// bytes. +// +// Only up to 4GB of data can be stored at once. + +// Initialize the queue +void byte_queue_init(ByteQueue *queue, uint32_t limit) +{ + queue->flags = 0; + queue->head = 0; + queue->size = 0; + queue->used = 0; + queue->curs = 0; + queue->limit = limit; + queue->data = NULL; + queue->read_target = NULL; +} + +// Deinitialize the queue +void byte_queue_free(ByteQueue *queue) +{ + if (queue->read_target) { + if (queue->read_target != queue->data) + free(queue->read_target); + queue->read_target = NULL; + queue->read_target_size = 0; + } + + free(queue->data); + queue->data = NULL; +} + +int byte_queue_error(ByteQueue *queue) +{ + return queue->flags & BYTE_QUEUE_ERROR; +} + +int byte_queue_empty(ByteQueue *queue) +{ + return queue->used == 0; +} + +int byte_queue_full(ByteQueue *queue) +{ + return queue->used == queue->limit; +} + +// Start a read operation on the queue. +// +// This function returnes the pointer to the memory region containing the bytes +// to read. Callers can't read more than [*len] bytes from it. To complete the +// read, the [byte_queue_read_ack] function must be called with the number of +// bytes that were acknowledged by the caller. +// +// Note: +// - You can't have more than one pending read. +ByteView byte_queue_read_buf(ByteQueue *queue) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return (ByteView) {NULL, 0}; + + assert((queue->flags & BYTE_QUEUE_READ) == 0); + queue->flags |= BYTE_QUEUE_READ; + queue->read_target = queue->data; + queue->read_target_size = queue->size; + + if (queue->data == NULL) + return (ByteView) {NULL, 0}; + + return (ByteView) { queue->data + queue->head, queue->used }; +} + +// Complete a previously started operation on the queue. +void byte_queue_read_ack(ByteQueue *queue, uint32_t num) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return; + + if ((queue->flags & BYTE_QUEUE_READ) == 0) + return; + + queue->flags &= ~BYTE_QUEUE_READ; + + assert((uint32_t) num <= queue->used); + queue->head += (uint32_t) num; + queue->used -= (uint32_t) num; + queue->curs += (uint32_t) num; + + if (queue->read_target) { + if (queue->read_target != queue->data) + free(queue->read_target); + queue->read_target = NULL; + queue->read_target_size = 0; + } +} + +ByteView byte_queue_write_buf(ByteQueue *queue) +{ + if ((queue->flags & BYTE_QUEUE_ERROR) || queue->data == NULL) + return (ByteView) {NULL, 0}; + + assert((queue->flags & BYTE_QUEUE_WRITE) == 0); + queue->flags |= BYTE_QUEUE_WRITE; + + return (ByteView) { + queue->data + (queue->head + queue->used), + queue->size - (queue->head + queue->used), + }; +} + +void byte_queue_write_ack(ByteQueue *queue, uint32_t num) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return; + + if ((queue->flags & BYTE_QUEUE_WRITE) == 0) + return; + + queue->flags &= ~BYTE_QUEUE_WRITE; + queue->used += num; +} + +// Sets the minimum capacity for the next write operation +// and returns 1 if the content of the queue was moved, else +// 0 is returned. +// +// You must not call this function while a write is pending. +// In other words, you must do this: +// +// byte_queue_write_setmincap(queue, mincap); +// dst = byte_queue_write_buf(queue, &cap); +// ... +// byte_queue_write_ack(num); +// +// And NOT this: +// +// dst = byte_queue_write_buf(queue, &cap); +// byte_queue_write_setmincap(queue, mincap); <-- BAD +// ... +// byte_queue_write_ack(num); +// +int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap) +{ + // Sticky error + if (queue->flags & BYTE_QUEUE_ERROR) + return 0; + + // In general, the queue's contents look like this: + // + // size + // v + // [___xxxxxxxxxxxx________] + // ^ ^ ^ + // 0 head head + used + // + // This function needs to make sure that at least [mincap] + // bytes are available on the right side of the content. + // + // We have 3 cases: + // + // 1) If there is enough memory already, this function doesn't + // need to do anything. + // + // 2) If there isn't enough memory on the right but there is + // enough free memory if we cound the left unused region, + // then the content is moved back to the + // start of the buffer. + // + // 3) If there isn't enough memory considering both sides, this + // function needs to allocate a new buffer. + // + // If there are pending read or write operations, the application + // is holding pointers to the buffer, so we need to make sure + // to not invalidate them. The only real problem is pending reads + // since this function can only be called before starting a write + // opearation. + // + // To avoid invalidating the read pointer when we allocate a new + // buffer, we don't free the old buffer. Instead, we store the + // pointer in the "old" field so that the read ack function can + // free it. + // + // To avoid invalidating the pointer when we are moving back the + // content since there is enough memory at the start of the buffer, + // we just avoid that. Even if there is enough memory considering + // left and right free regions, we allocate a new buffer. + + assert((queue->flags & BYTE_QUEUE_WRITE) == 0); + + uint32_t total_free_space = queue->size - queue->used; + uint32_t free_space_after_data = queue->size - queue->used - queue->head; + + int moved = 0; + if (free_space_after_data < mincap) { + + if (total_free_space < mincap || (queue->read_target == queue->data)) { + // Resize required + + if (queue->used + mincap > queue->limit) { + queue->flags |= BYTE_QUEUE_ERROR; + return 0; + } + + uint32_t size; + if (queue->size > UINT32_MAX / 2) + size = UINT32_MAX; + else + size = 2 * queue->size; + + if (size < queue->used + mincap) + size = queue->used + mincap; + + if (size > queue->limit) + size = queue->limit; + + uint8_t *data = malloc(size); + if (!data) { + queue->flags |= BYTE_QUEUE_ERROR; + return 0; + } + + if (queue->used > 0) + memcpy(data, queue->data + queue->head, queue->used); + + if (queue->read_target != queue->data) + free(queue->data); + + queue->data = data; + queue->head = 0; + queue->size = size; + + } else { + // Move required + memmove(queue->data, queue->data + queue->head, queue->used); + queue->head = 0; + } + + moved = 1; + } + + return moved; +} + +void byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len) +{ + byte_queue_write_setmincap(queue, len); + ByteView dst = byte_queue_write_buf(queue); + if (dst.ptr) { + memcpy(dst.ptr, ptr, len); + byte_queue_write_ack(queue, len); + } +} + +ByteQueueOffset byte_queue_offset(ByteQueue *queue) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return (ByteQueueOffset) { 0 }; + return (ByteQueueOffset) { queue->curs + queue->used }; +} + +void byte_queue_patch(ByteQueue *queue, ByteQueueOffset off, + void *src, uint32_t len) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return; + + // Check that the offset is in range + assert(off >= queue->curs && off - queue->curs < queue->used); + + // Check that the length is in range + assert(len <= queue->used - (off - queue->curs)); + + // Perform the patch + uint8_t *dst = queue->data + queue->head + (off - queue->curs); + memcpy(dst, src, len); +} + +uint32_t byte_queue_size_from_offset(ByteQueue *queue, ByteQueueOffset off) +{ + return queue->curs + queue->used - off; +} + +void byte_queue_remove_from_offset(ByteQueue *queue, ByteQueueOffset offset) +{ + if (queue->flags & BYTE_QUEUE_ERROR) + return; + + uint64_t num = (queue->curs + queue->used) - offset; + assert(num <= queue->used); + + queue->used -= num; +} diff --git a/lib/byte_queue.h b/lib/byte_queue.h new file mode 100644 index 0000000..5330dce --- /dev/null +++ b/lib/byte_queue.h @@ -0,0 +1,53 @@ +#ifndef BYTE_QUEUE_INCLUDED +#define BYTE_QUEUE_INCLUDED + +#include + +#include "basic.h" + +typedef struct { + uint8_t *ptr; + size_t len; +} ByteView; + +typedef struct { + uint64_t curs; + uint8_t* data; + uint32_t head; + uint32_t size; + uint32_t used; + uint32_t limit; + uint8_t* read_target; + uint32_t read_target_size; + int flags; +} ByteQueue; + +typedef uint64_t ByteQueueOffset; + +enum { + BYTE_QUEUE_ERROR = 1 << 0, + BYTE_QUEUE_READ = 1 << 1, + BYTE_QUEUE_WRITE = 1 << 2, +}; + +void byte_queue_init(ByteQueue *queue, uint32_t limit); +void byte_queue_free(ByteQueue *queue); + +int byte_queue_error(ByteQueue *queue); +int byte_queue_empty(ByteQueue *queue); +int byte_queue_full(ByteQueue *queue); + +ByteView byte_queue_read_buf(ByteQueue *queue); +void byte_queue_read_ack(ByteQueue *queue, uint32_t num); + +ByteView byte_queue_write_buf(ByteQueue *queue); +void byte_queue_write_ack(ByteQueue *queue, uint32_t num); +int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap); +void byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len); + +ByteQueueOffset byte_queue_offset(ByteQueue *queue); +void byte_queue_patch(ByteQueue *queue, ByteQueueOffset off, void *src, uint32_t len); +uint32_t byte_queue_size_from_offset(ByteQueue *queue, ByteQueueOffset off); +void byte_queue_remove_from_offset(ByteQueue *queue, ByteQueueOffset offset); + +#endif // BYTE_QUEUE_INCLUDED diff --git a/lib/file_system.c b/lib/file_system.c new file mode 100644 index 0000000..8323eb0 --- /dev/null +++ b/lib/file_system.c @@ -0,0 +1,459 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "file_system.h" + +int rename_file_or_dir(string oldpath, string newpath); + +bool file_exists(string path) +{ + char zt[1<<10]; + if (path.len >= (int) sizeof(zt)) + return false; + memcpy(zt, path.ptr, path.len); + zt[path.len] = '\0'; + +#ifdef __linux__ + return access(zt, F_OK) == 0; +#endif + +#ifdef _WIN32 + DWORD attrs = GetFileAttributesA(zt); + return attrs != INVALID_FILE_ATTRIBUTES; +#endif +} + +int file_open(string path, Handle *fd) +{ +#ifdef __linux__ + char zt[1<<10]; + if (path.len >= (int) sizeof(zt)) + return -1; + memcpy(zt, path.ptr, path.len); + zt[path.len] = '\0'; + + int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644); + if (ret < 0) + return -1; + + *fd = (Handle) { (uint64_t) ret }; + return 0; +#endif + +#ifdef _WIN32 + WCHAR wpath[MAX_PATH]; + MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH); + wpath[path.len] = L'\0'; + + HANDLE h = CreateFileW( + wpath, + GENERIC_WRITE | GENERIC_READ, + 0, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH, + NULL + ); + if (h == INVALID_HANDLE_VALUE) + return -1; + + *fd = (Handle) { (uint64_t) h }; + return 0; +#endif +} + +void file_close(Handle fd) +{ +#ifdef __linux__ + close((int) fd.data); +#endif + +#ifdef _WIN32 + CloseHandle((HANDLE) fd.data); +#endif +} + +int file_truncate(Handle fd, size_t new_size) +{ +#ifdef __linux__ + if (ftruncate((int) fd.data, new_size) < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + return -1; // TODO: Not implemented +#endif +} + +int file_set_offset(Handle fd, int off) +{ +#ifdef __linux__ + off_t ret = lseek((int) fd.data, off, SEEK_SET); + if (ret < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + LARGE_INTEGER distance; + distance.QuadPart = off; + if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN)) + if (GetLastError() != 0) + return -1; + return 0; +#endif +} + +int file_get_offset(Handle fd, int *off) +{ +#ifdef __linux__ + off_t ret = lseek((int) fd.data, 0, SEEK_CUR); + if (ret < 0) + return -1; + *off = (int) ret; + return 0; +#endif + +#ifdef _WIN32 + DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT); + if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0) + return -1; + *off = (int) pos; + return 0; +#endif +} + +int file_lock(Handle fd) +{ +#ifdef __linux__ + if (flock((int) fd.data, LOCK_EX) < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + return 0; +#endif +} + +int file_unlock(Handle fd) +{ +#ifdef __linux__ + if (flock((int) fd.data, LOCK_UN) < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + return 0; +#endif +} + +int file_sync(Handle fd) +{ +#ifdef __linux__ + if (fsync((int) fd.data) < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + if (!FlushFileBuffers((HANDLE) fd.data)) + return -1; + return 0; +#endif +} + +int file_read(Handle fd, char *dst, int max) +{ +#ifdef __linux__ + return read((int) fd.data, dst, max); +#endif + +#ifdef _WIN32 + DWORD num; + if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL)) + return -1; + if (num > INT_MAX) + return -1; + return num; +#endif +} + +int file_write(Handle fd, char *src, int len) +{ +#ifdef __linux__ + return write((int) fd.data, src, len); +#endif + +#ifdef _WIN32 + DWORD num; + if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL)) + return -1; + if (num > INT_MAX) + return -1; + return num; +#endif +} + +int file_size(Handle fd, size_t *len) +{ +#ifdef __linux__ + struct stat buf; + if (fstat((int) fd.data, &buf) < 0) + return -1; + if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX) + return -1; + *len = (size_t) buf.st_size; + return 0; +#endif + +#ifdef _WIN32 + LARGE_INTEGER buf; + if (!GetFileSizeEx((HANDLE) fd.data, &buf)) + return -1; + if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX) + return -1; + *len = buf.QuadPart; + return 0; +#endif +} + +int create_dir(string path) +{ + char zt[PATH_MAX]; + if (path.len >= (int) sizeof(zt)) + return -1; + memcpy(zt, path.ptr, path.len); + zt[path.len] = '\0'; + +#ifdef _WIN32 + if (_mkdir(zt) < 0) + return -1; +#else + if (mkdir(zt, 0766)) + return -1; +#endif + + return 0; +} + +int rename_file_or_dir(string oldpath, string newpath) +{ + char oldpath_zt[PATH_MAX]; + if (oldpath.len >= (int) sizeof(oldpath_zt)) + return -1; + memcpy(oldpath_zt, oldpath.ptr, oldpath.len); + oldpath_zt[oldpath.len] = '\0'; + + char newpath_zt[PATH_MAX]; + if (newpath.len >= (int) sizeof(newpath_zt)) + return -1; + memcpy(newpath_zt, newpath.ptr, newpath.len); + newpath_zt[newpath.len] = '\0'; + + if (rename(oldpath_zt, newpath_zt)) + return -1; + return 0; +} + +int remove_file_or_dir(string path) +{ + char path_zt[PATH_MAX]; + if (path.len >= (int) sizeof(path_zt)) + return -1; + memcpy(path_zt, path.ptr, path.len); + path_zt[path.len] = '\0'; + + if (remove(path_zt)) + return -1; + return 0; +} + +int get_full_path(string path, char *dst) +{ + char path_zt[PATH_MAX]; + if (path.len >= (int) sizeof(path_zt)) + return -1; + memcpy(path_zt, path.ptr, path.len); + path_zt[path.len] = '\0'; + +#ifdef __linux__ + if (realpath(path_zt, dst) == NULL) + return -1; +#endif + +#ifdef _WIN32 + if (_fullpath(path_zt, dst, PATH_MAX) == NULL) + return -1; +#endif + + size_t path_len = strlen(dst); + if (path_len > 0 && dst[path_len-1] == '/') + dst[path_len-1] = '\0'; + + return 0; +} + +int file_read_all(string path, string *data) +{ + Handle fd; + int ret = file_open(path, &fd); + if (ret < 0) + return -1; + + size_t len; + ret = file_size(fd, &len); + if (ret < 0) { + file_close(fd); + return -1; + } + + char *dst = malloc(len); + if (dst == NULL) { + file_close(fd); + return -1; + } + + int copied = 0; + while ((size_t) copied < len) { + ret = file_read(fd, dst + copied, len - copied); + if (ret < 0) { + free(dst); + file_close(fd); + return -1; + } + copied += ret; + } + + *data = (string) { dst, len }; + file_close(fd); + return 0; +} + +#ifdef _WIN32 + +int directory_scanner_init(DirectoryScanner *scanner, string path) +{ + char pattern[PATH_MAX]; + int ret = snprintf(pattern, sizeof(pattern), "%.*s\\*", path.len, path.ptr); + if (ret < 0 || ret >= (int) sizeof(pattern)) + return -1; + + scanner->handle = FindFirstFileA(pattern, &scanner->find_data); + if (scanner->handle == INVALID_HANDLE_VALUE) { + if (GetLastError() == ERROR_FILE_NOT_FOUND) { + scanner->done = true; + return 0; + } + return -1; + } + + scanner->done = false; + scanner->first = true; + return 0; +} + +int directory_scanner_next(DirectoryScanner *scanner, string *name) +{ + if (scanner->done) + return 1; + + if (!scanner->first) { + BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data); + if (!ok) { + scanner->done = true; + if (GetLastError() == ERROR_NO_MORE_FILES) + return 1; + return -1; + } + } else { + scanner->first = false; + } + + char *p = scanner->find_data.cFileName; + *name = (string) { p, strlen(p) }; + return 0; +} + +void directory_scanner_free(DirectoryScanner *scanner) +{ + FindClose(scanner->handle); +} + +#else + +int directory_scanner_init(DirectoryScanner *scanner, string path) +{ + char path_copy[PATH_MAX]; + if (path.len >= PATH_MAX) + return -1; + memcpy(path_copy, path.ptr, path.len); + path_copy[path.len] = '\0'; + + scanner->d = opendir(path_copy); + if (scanner->d == NULL) { + scanner->done = true; + return -1; + } + + scanner->done = false; + return 0; +} + +int directory_scanner_next(DirectoryScanner *scanner, string *name) +{ + if (scanner->done) + return 1; + + scanner->e = readdir(scanner->d); + if (scanner->e == NULL) { + scanner->done = true; + return 1; + } + + *name = (string) { scanner->e->d_name, strlen(scanner->e->d_name) }; + return 0; +} + +void directory_scanner_free(DirectoryScanner *scanner) +{ + closedir(scanner->d); +} + +#endif + +int file_read_exact(Handle handle, char *dst, int len) +{ + int copied = 0; + while (copied < len) { + int ret = file_read(handle, dst + copied, len - copied); + if (ret < 0) + return -1; + if (ret == 0) + return 0; // EOF + copied += ret; + } + return copied; +} + +int file_write_exact(Handle handle, char *src, int len) +{ + int copied = 0; + while (copied < len) { + int ret = file_write(handle, src + copied, len - copied); + if (ret < 0) + return -1; + copied += ret; + } + return 0; +} \ No newline at end of file diff --git a/lib/file_system.h b/lib/file_system.h new file mode 100644 index 0000000..916200e --- /dev/null +++ b/lib/file_system.h @@ -0,0 +1,58 @@ +#ifndef FILE_SYSTEM_INCLUDED +#define FILE_SYSTEM_INCLUDED + +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "basic.h" + +typedef struct { + uint64_t data; +} Handle; + +#ifdef _WIN32 +typedef struct { + HANDLE handle; + WIN32_FIND_DATA find_data; + bool first; + bool done; +} DirectoryScanner; +#else +typedef struct { + DIR *d; + struct dirent *e; + bool done; +} DirectoryScanner; +#endif + +bool file_exists(string path); +int file_open(string path, Handle *fd); +void file_close(Handle fd); +int file_truncate(Handle fd, size_t new_size); +int file_set_offset(Handle fd, int off); +int file_get_offset(Handle fd, int *off); +int file_lock(Handle fd); +int file_unlock(Handle fd); +int file_sync(Handle fd); +int file_read(Handle fd, char *dst, int max); +int file_write(Handle fd, char *src, int len); +int file_size(Handle fd, size_t *len); +int file_write_atomic(string path, string content); +int create_dir(string path); +int rename_file_or_dir(string oldpath, string newpath); +int remove_file_or_dir(string path); +int get_full_path(string path, char *dst); +int file_read_all(string path, string *data); + +int directory_scanner_init(DirectoryScanner *scanner, string path); +int directory_scanner_next(DirectoryScanner *scanner, string *name); +void directory_scanner_free(DirectoryScanner *scanner); + +int file_read_exact(Handle handle, char *dst, int len); +int file_write_exact(Handle handle, char *src, int len); + +#endif // FILE_SYSTEM_INCLUDED \ No newline at end of file diff --git a/lib/message.c b/lib/message.c new file mode 100644 index 0000000..790d120 --- /dev/null +++ b/lib/message.c @@ -0,0 +1,81 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "message.h" + +bool binary_read(BinaryReader *reader, void *dst, int len) +{ + if (reader->len - reader->cur < len) + return false; + if (dst) + memcpy(dst, reader->src + reader->cur, len); + reader->cur += len; + return true; +} + +void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type) +{ + uint16_t version = MESSAGE_VERSION; + uint32_t dummy = 0; // Dummy value + writer->output = output; + writer->start = byte_queue_offset(output); + byte_queue_write(output, &version, sizeof(version)); + byte_queue_write(output, &type, sizeof(type)); + writer->patch = byte_queue_offset(output); + byte_queue_write(output, &dummy, sizeof(dummy)); +} + +bool message_writer_free(MessageWriter *writer) +{ + uint32_t length = byte_queue_size_from_offset(writer->output, writer->start); + byte_queue_patch(writer->output, writer->patch, &length, sizeof(length)); + if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure? + return false; + return true; +} + +void message_write(MessageWriter *writer, void *mem, int len) +{ + byte_queue_write(writer->output, mem, len); +} + +void message_write_u8(MessageWriter *writer, uint8_t value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +void message_write_u32(MessageWriter *writer, uint32_t value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +void message_write_hash(MessageWriter *writer, SHA256 value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +int message_peek(ByteView msg, uint16_t *type, uint32_t *len) +{ + if (msg.len < (int) sizeof(MessageHeader)) + return 0; + + MessageHeader header; + memcpy(&header, msg.ptr, sizeof(header)); + + // (We ignore endianess for now) + + if (header.version != MESSAGE_VERSION) + return -1; + + if (header.length > msg.len) + return 0; + + if (type) *type = header.type; + if (len) *len = header.length; + + return 1; +} diff --git a/lib/message.h b/lib/message.h new file mode 100644 index 0000000..0c9ec11 --- /dev/null +++ b/lib/message.h @@ -0,0 +1,46 @@ +#ifndef MESSAGE_INCLUDED +#define MESSAGE_INCLUDED + +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "basic.h" +#include "byte_queue.h" + +#define MESSAGE_VERSION 1 + +typedef struct { + uint8_t *src; + int len; + int cur; +} BinaryReader; + +typedef struct { + uint16_t version; + uint16_t type; + uint32_t length; +} MessageHeader; + +typedef struct { + ByteQueue *output; + ByteQueueOffset start; + ByteQueueOffset patch; +} MessageWriter; + +bool binary_read(BinaryReader *reader, void *dst, int len); + +void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type); +bool message_writer_free(MessageWriter *writer); +void message_write(MessageWriter *writer, void *mem, int len); +void message_write_u8(MessageWriter *writer, uint8_t value); +void message_write_u32(MessageWriter *writer, uint32_t value); +void message_write_hash(MessageWriter *writer, SHA256 value); + +int message_peek(ByteView msg, uint16_t *type, uint32_t *len); +void message_dump(FILE *stream, ByteView msg); + +#endif // MESSAGE_INCLUDED diff --git a/lib/tcp.c b/lib/tcp.c new file mode 100644 index 0000000..72ef728 --- /dev/null +++ b/lib/tcp.c @@ -0,0 +1,501 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "tcp.h" +#include "message.h" + +static int set_socket_blocking(SOCKET sock, bool value) +{ +#ifdef _WIN32 + u_long mode = !value; + if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR) + return -1; +#else + int flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + return -1; + if (value) flags &= ~O_NONBLOCK; + else flags |= O_NONBLOCK; + if (fcntl(sock, F_SETFL, flags) < 0) + return -1; +#endif + + return 0; +} + +static SOCKET create_listen_socket(Address addr) +{ + SOCKET fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == INVALID_SOCKET) + return INVALID_SOCKET; + + if (set_socket_blocking(fd, false) < 0) { + CLOSE_SOCKET(fd); + return INVALID_SOCKET; + } + + // TODO: mark address as reusable in debug builds + + if (!addr.is_ipv4) { + assert(0); // TODO + } + + struct sockaddr_in bind_buf; + bind_buf.sin_family = AF_INET; + bind_buf.sin_port = htons(addr.port); + memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4)); + if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) { + CLOSE_SOCKET(fd); + return INVALID_SOCKET; + } + + int backlog = 32; + if (listen(fd, backlog) < 0) { + CLOSE_SOCKET(fd); + return INVALID_SOCKET; + } + + return fd; +} + +static int create_socket_pair(SOCKET *a, SOCKET *b) +{ +#ifdef _WIN32 + SOCKET sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (sock == INVALID_SOCKET) + return -1; + + // Bind to loopback address with port 0 (dynamic port assignment) + struct sockaddr_in addr; + int addr_len = sizeof(addr); + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 + addr.sin_port = 0; // Let system choose port + + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) { + closesocket(sock); + return -1; + } + + if (getsockname(sock, (struct sockaddr*)&addr, &addr_len) == SOCKET_ERROR) { + closesocket(sock); + return -1; + } + + if (connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) { + closesocket(sock); + return -1; + } + + *a = sock; + *b = sock; + + // Optional: Set socket to non-blocking mode + // This prevents send() from blocking if the receive buffer is full + u_long mode = 1; + ioctlsocket(sock, FIONBIO, &mode); // TODO: does this fail? + return 0; +#else + int fds[2]; + if (pipe(fds) < 0) + return -1; + *a = fds[0]; + *b = fds[1]; + return 0; +#endif +} + +static void close_socket_pair(SOCKET a, SOCKET b) +{ +#ifdef _WIN32 + closesocket(a); + (void) b; +#else + close(a); + close(b); +#endif +} + +static void conn_init(Connection *conn, SOCKET fd, bool connecting) +{ + conn->fd = fd; + conn->tag = -1; + conn->connecting = connecting; + conn->closing = false; + conn->msglen = 0; + byte_queue_init(&conn->input, 1<<20); + byte_queue_init(&conn->output, 1<<20); +} + +static void conn_free(Connection *conn) +{ + CLOSE_SOCKET(conn->fd); + byte_queue_free(&conn->input); + byte_queue_free(&conn->output); +} + +static int conn_events(Connection *conn) +{ + int events = 0; + + if (conn->connecting) + events |= POLLOUT; + else { + + assert(!byte_queue_full(&conn->input)); + if (!conn->closing) + events |= POLLIN; + + if (!byte_queue_empty(&conn->output)) + events |= POLLOUT; + } + return events; +} + +int tcp_context_init(TCP *tcp) +{ + tcp->listen_fd = INVALID_SOCKET; + tcp->num_conns = 0; + + if (create_socket_pair(&tcp->wait_fd, &tcp->signal_fd) < 0) + return -1; + + return 0; +} + +void tcp_context_free(TCP *tcp) +{ + // Free all connection byte queues without closing sockets + // (sockets are managed by the simulation and will be cleaned up separately) + for (int i = 0; i < tcp->num_conns; i++) { + byte_queue_free(&tcp->conns[i].input); + byte_queue_free(&tcp->conns[i].output); + } + tcp->num_conns = 0; + + if (tcp->listen_fd != INVALID_SOCKET) + CLOSE_SOCKET(tcp->listen_fd); + + close_socket_pair(tcp->wait_fd, tcp->signal_fd); +} + +int tcp_wakeup(TCP *tcp) +{ + send(tcp->signal_fd, "0", 1, 0); // TODO: Handle error + return 0; +} + +int tcp_index_from_tag(TCP *tcp, int tag) +{ + for (int i = 0; i < tcp->num_conns; i++) + if (tcp->conns[i].tag == tag) + return i; + return -1; +} + +int tcp_listen(TCP *tcp, Address addr) +{ + SOCKET listen_fd = create_listen_socket(addr); + if (listen_fd == INVALID_SOCKET) + return -1; + + tcp->listen_fd = listen_fd; + return 0; +} + +int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type) +{ + *msg = byte_queue_read_buf(&tcp->conns[conn_idx].input); + + uint32_t len; + int ret = message_peek(*msg, type, &len); + + // Invalid message? + if (ret < 0) { + byte_queue_read_ack(&tcp->conns[conn_idx].input, 0); + return -1; + } + + // Still buffering header? + if (ret == 0) { + byte_queue_read_ack(&tcp->conns[conn_idx].input, 0); + if (byte_queue_full(&tcp->conns[conn_idx].input)) + return -1; + return 0; + } + + // Message received + assert(ret > 0); + msg->len = len; + tcp->conns[conn_idx].msglen = len; + + return 1; +} + +void tcp_consume_message(TCP *tcp, int conn_idx) +{ + byte_queue_read_ack(&tcp->conns[conn_idx].input, tcp->conns[conn_idx].msglen); + tcp->conns[conn_idx].msglen = 0; +} + +int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled) +{ + int num_polled = 0; + + polled[num_polled].fd = tcp->wait_fd; + polled[num_polled].events = POLLIN; + polled[num_polled].revents = 0; + contexts[num_polled] = NULL; + num_polled++; + + if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < TCP_CONNECTION_LIMIT) { + polled[num_polled].fd = tcp->listen_fd; + polled[num_polled].events = POLLIN; + polled[num_polled].revents = 0; + contexts[num_polled] = NULL; + num_polled++; + } + + for (int i = 0; i < tcp->num_conns; i++) { + int events = conn_events(&tcp->conns[i]); + if (events) { + polled[num_polled].fd = tcp->conns[i].fd; + polled[num_polled].events = events; + polled[num_polled].revents = 0; + contexts[num_polled] = &tcp->conns[i]; + num_polled++; + } + } + + return num_polled; +} + +// The "events" array must be an array of capacity TCP_EVENT_CAPACITY, +// while "contexts" and "polled" must have capacity TCP_POLL_CAPACITY. +int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled) +{ + bool removed[TCP_POLL_CAPACITY]; + for (int i = 0; i < TCP_POLL_CAPACITY; i++) + removed[i] = false; + + int num_events = 0; + for (int i = 1; i < num_polled; i++) { + + if (polled[i].fd == tcp->wait_fd) { + + if (polled[i].revents & POLLIN) { + char buf[100]; + recv(tcp->wait_fd, buf, sizeof(buf), 0); // TODO: Make sure all bytes are consumed + events[num_events++] = (Event) { EVENT_WAKEUP, -1, -1 }; + } + + } else if (polled[i].fd == tcp->listen_fd) { + + assert(contexts[i] == NULL); + + if (polled[i].revents & POLLIN) { + SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL); + if (new_fd != INVALID_SOCKET) { + + if (set_socket_blocking(new_fd, false) < 0) + CLOSE_SOCKET(new_fd); + else { + conn_init(&tcp->conns[tcp->num_conns++], new_fd, false); + events[num_events++] = (Event) { EVENT_CONNECT, tcp->num_conns-1, tcp->conns[tcp->num_conns-1].tag }; + } + } + } + removed[i] = false; + + } else { + + Connection *conn = contexts[i]; + bool defer_close = false; + bool defer_ready = false; + + if (conn->connecting) { + + // Check for error conditions on the socket + if (polled[i].revents & (POLLERR | POLLHUP | POLLNVAL)) { + defer_close = true; + } else if (polled[i].revents & POLLOUT) { + + int err = 0; + socklen_t len = sizeof(err); + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0) + defer_close = true; + else { + conn->connecting = false; + events[num_events++] = (Event) { EVENT_CONNECT, conn - tcp->conns, conn->tag }; + } + } + + } else { + + if (polled[i].revents & POLLIN) { + byte_queue_write_setmincap(&conn->input, 1<<9); + ByteView buf = byte_queue_write_buf(&conn->input); + int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0); + if (num == 0) + defer_close = true; + else if (num < 0) { + if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) // TODO: does Windows return these error codes or not? + defer_close = true; + num = 0; + } + byte_queue_write_ack(&conn->input, num); + ByteView msg = byte_queue_read_buf(&conn->input); + int ret = message_peek(msg, NULL, NULL); + byte_queue_read_ack(&conn->input, 0); + if (ret < 0) { + // Invalid message + defer_close = true; + } else if (ret == 0) { + // Still buffering + if (byte_queue_full(&conn->input)) + defer_close = true; + } else { + // Message received + assert(ret > 0); + defer_ready = true; + } + } + + if (polled[i].revents & POLLOUT) { + ByteView buf = byte_queue_read_buf(&conn->output); + int num = send(conn->fd, (char*) buf.ptr, buf.len, 0); + if (num < 0) { + if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) + defer_close = true; + num = 0; + } + byte_queue_read_ack(&conn->output, num); + if (conn->closing && byte_queue_empty(&conn->output)) + defer_close = true; + } + } + + // TODO: byte_queue_error here? + + removed[i] = defer_close; + if (0) {} + else if (defer_close) events[num_events++] = (Event) { EVENT_DISCONNECT, conn - tcp->conns, conn->tag }; + else if (defer_ready) events[num_events++] = (Event) { EVENT_MESSAGE, conn - tcp->conns, conn->tag }; + } + } + + for (int i = 1; i < num_polled; i++) { + if (removed[i]) { + Connection *conn = contexts[i]; + assert(conn); + int removed_idx = conn - tcp->conns; + conn_free(conn); + int last_idx = --tcp->num_conns; + if (removed_idx != last_idx) { + *conn = tcp->conns[last_idx]; + // Update event conn_idx values to reflect the swap + for (int j = 0; j < num_events; j++) { + if (events[j].conn_idx == last_idx) + events[j].conn_idx = removed_idx; + } + // Update contexts pointers for remaining iterations + for (int j = i + 1; j < num_polled; j++) { + if (contexts[j] == &tcp->conns[last_idx]) + contexts[j] = conn; + } + } + } + } + + return num_events; +} + +ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx) +{ + return &tcp->conns[conn_idx].output; +} + +int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output) +{ + if (tcp->num_conns == TCP_CONNECTION_LIMIT) + return -1; + int conn_idx = tcp->num_conns; + + SOCKET fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == INVALID_SOCKET) + return -1; + + if (set_socket_blocking(fd, false) < 0) { + CLOSE_SOCKET(fd); + return -1; + } + + int ret; + if (addr.is_ipv4) { + struct sockaddr_in buf; + buf.sin_family = AF_INET; + buf.sin_port = htons(addr.port); + memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4)); + ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf)); + } else { + struct sockaddr_in6 buf; + buf.sin6_family = AF_INET6; + buf.sin6_port = htons(addr.port); + memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6)); + ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf)); + } + + bool connecting; + if (ret == 0) { + connecting = false; + } else { + if (errno != EINPROGRESS) { + CLOSE_SOCKET(fd); + return -1; + } + connecting = true; + } + + // Check that this tag wasn't already used + for (int i = 0; i < tcp->num_conns; i++) + assert(tcp->conns[i].tag != tag); + + conn_init(&tcp->conns[conn_idx], fd, connecting); + tcp->conns[conn_idx].tag = tag; + + if (output) + *output = &tcp->conns[conn_idx].output; + + tcp->num_conns++; + return 0; +} + +void tcp_close(TCP *tcp, int conn_idx) +{ + tcp->conns[conn_idx].closing = true; + tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection + // TODO: if no event will be triggered, the connection will not be closed + // if the output buffer is empty, the connection should be closed here. +} + +void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique) +{ + assert(tag != -1); + + if (unique) { + for (int i = 0; i < tcp->num_conns; i++) + assert(tcp->conns[i].tag != tag); + } + + tcp->conns[conn_idx].tag = tag; +} + +int tcp_get_tag(TCP *tcp, int conn_idx) +{ + return tcp->conns[conn_idx].tag; +} diff --git a/lib/tcp.h b/lib/tcp.h new file mode 100644 index 0000000..8ccaa84 --- /dev/null +++ b/lib/tcp.h @@ -0,0 +1,87 @@ +#ifndef TCP_INCLUDED +#define TCP_INCLUDED + +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +# define QUAKEY_ENABLE_MOCKS +# include +#else +# ifdef _WIN32 +# include +# endif +#endif + +#include "byte_queue.h" + +#ifdef _WIN32 +#define CLOSE_SOCKET closesocket +#else +#define SOCKET int +#define INVALID_SOCKET -1 +#define CLOSE_SOCKET close +#endif + +#ifndef TCP_CONNECTION_LIMIT +// Maximum number of connections that can be managed +// simultaneously. +#define TCP_CONNECTION_LIMIT 512 +#endif + +// This is the maximum number of descriptors that the +// TCP system will want to wait at any given time. +// One descriptor per connection plus a listener socket +// and a self-pipe handle for wakeup. +#define TCP_POLL_CAPACITY (TCP_CONNECTION_LIMIT+2) + +// Number of TCP events that can be returned at a given +// time by "tcp_translate_events". There may be a single +// event per connection (MESSAGE, CONNECT, DISCONNECT) +// plus a general WAKEUP event. +#define TCP_EVENT_CAPACITY (TCP_CONNECTION_LIMIT+1) + +typedef enum { + EVENT_WAKEUP, + EVENT_MESSAGE, + EVENT_CONNECT, + EVENT_DISCONNECT, +} EventType; + +typedef struct { + EventType type; + int conn_idx; + int tag; +} Event; + +typedef struct { + SOCKET fd; + int tag; + bool connecting; + bool closing; + uint32_t msglen; + ByteQueue input; + ByteQueue output; +} Connection; + +typedef struct { + SOCKET listen_fd; + SOCKET wait_fd; + SOCKET signal_fd; + int num_conns; + Connection conns[TCP_CONNECTION_LIMIT]; +} TCP; + +int tcp_context_init(TCP *tcp); +void tcp_context_free(TCP *tcp); +int tcp_wakeup(TCP *tcp); +int tcp_index_from_tag(TCP *tcp, int tag); +int tcp_listen(TCP *tcp, Address addr); +int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type); +void tcp_consume_message(TCP *tcp, int conn_idx); +int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled); +int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled); +ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx); +int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output); +void tcp_close(TCP *tcp, int conn_idx); +void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique); +int tcp_get_tag(TCP *tcp, int conn_idx); + +#endif // TCP_INCLUDED diff --git a/quakey/include/quakey.h b/quakey/include/quakey.h new file mode 100644 index 0000000..8978642 --- /dev/null +++ b/quakey/include/quakey.h @@ -0,0 +1,223 @@ +#ifndef QUAKEY_INCLUDED +#define QUAKEY_INCLUDED + +#define _GNU_SOURCE +#include +#include +#ifdef _WIN32 +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct {} Quakey; + +// Function pointers to a simulated program's code +typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout); +typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout); +typedef int (*QuakeyFreeFunc)(void *state); + +typedef enum { + QUAKEY_LINUX, + QUAKEY_WINDOWS, +} QuakeyPlatform; + +typedef struct { + + // Label associated to the process for debugging + // The string must have global lifetime + char *name; + + // Size of the opaque state struct + int state_size; + + // Pointers to program code + QuakeyInitFunc init_func; + QuakeyTickFunc tick_func; + QuakeyFreeFunc free_func; + + // Network addresses enabled on the process + char **addrs; + int num_addrs; + + // Disk size for the process + int disk_size; + + // Platform used by this program + QuakeyPlatform platform; + +} QuakeySpawn; + +typedef unsigned long long QuakeyUInt64; + +// Start a simulation +int quakey_init(Quakey **quakey, QuakeyUInt64 seed); + +// Stop a simulation +void quakey_free(Quakey *quakey); + +typedef unsigned long long QuakeyNode; + +// Add a program to the simulation +QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg); + +void *quakey_node_state(QuakeyNode node); + +// Schedule and executes one program until it would block, then returns +int quakey_schedule_one(Quakey *quakey); + +// Generate a random u64 +QuakeyUInt64 quakey_random(void); + +typedef struct { + char name[32]; + int name_len; +} QuakeySignal; + +void quakey_signal(char *name); +int quakey_get_signal(Quakey *quakey, QuakeySignal *signal); + +// Access spawned host information +int quakey_num_hosts(Quakey *quakey); +void *quakey_host_state(Quakey *quakey, int idx); // Returns NULL if host is dead +int quakey_host_is_dead(Quakey *quakey, int idx); +const char *quakey_host_name(Quakey *quakey, int idx); + +int *mock_errno_ptr(void); + +#ifdef _WIN32 +BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount); +BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency); +HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile); +BOOL mock_CloseHandle(HANDLE handle); +BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov); +BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov); +BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf); +DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod); +BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh); +BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh); +BOOL mock_FlushFileBuffers(HANDLE handle); +BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags); +char* mock__fullpath(char *path, char *dst, int cap); +HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData); +BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData); +BOOL mock_FindClose(HANDLE hFindFile); +int mock__mkdir(char *path); +#else +int mock_socket(int domain, int type, int protocol); +int mock_closesocket(int fd); +int mock_ioctlsocket(int fd, long cmd, unsigned long *argp); +int mock_bind(int fd, void *addr, unsigned long addr_len); +int mock_connect(int fd, void *addr, unsigned long addr_len); +int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen); +int mock_listen(int fd, int backlog); +int mock_accept(int fd, void *addr, socklen_t *addr_len); +int mock_pipe(int *fds); +int mock_recv(int fd, char *dst, int len, int flags); +int mock_send(int fd, char *src, int len, int flags); +int mock_clock_gettime(clockid_t clockid, struct timespec *tp); +int mock_open(char *path, int flags, int mode); +int mock_fcntl(int fd, int cmd, int flags); +int mock_close(int fd); +int mock_ftruncate(int fd, size_t new_size); +int mock_fstat(int fd, struct stat *buf); +int mock_read(int fd, char *dst, int len); +int mock_write(int fd, char *src, int len); +off_t mock_lseek(int fd, off_t offset, int whence); +int mock_flock(int fd, int op); +int mock_fsync(int fd); +int mock_mkstemp(char *path); +int mock_mkdir(char *path, mode_t mode); +int mock_remove(char *path); +int mock_rename(char *oldpath, char *newpath); +char* mock_realpath(char *path, char *dst); +DIR* mock_opendir(char *name); +struct dirent* mock_readdir(DIR *dirp); +int mock_closedir(DIR *dirp); +#endif + +void *mock_malloc(size_t size); +void *mock_realloc(void *ptr, size_t size); +void mock_free(void *ptr); + +#ifdef QUAKEY_ENABLE_MOCKS + +#define QUAKEY_SIGNAL(name) quakey_signal(name) + +#undef errno +#define errno (*mock_errno_ptr()) + +#define malloc mock_malloc +#define realloc mock_realloc +#define free mock_free +#define socket mock_socket +#define closesocket mock_closesocket +#define ioctlsocket mock_ioctlsocket +#define bind mock_bind +#define connect mock_connect +#define getsockopt mock_getsockopt +#define listen mock_listen +#define accept mock_accept +#define pipe mock_pipe +#define recv mock_recv +#define send mock_send +#define clock_gettime mock_clock_gettime +#define QueryPerformanceCounter mock_QueryPerformanceCounter +#define QueryPerformanceFrequency mock_QueryPerformanceFrequency +#define open mock_open +#define fcntl mock_fcntl +#define close mock_close +#define ftruncate mock_ftruncate +#define CreateFileW mock_CreateFileW +#define CloseHandle mock_CloseHandle +#define ReadFile mock_ReadFile +#define WriteFile mock_WriteFile +#define read mock_read +#define write mock_write +#define fstat mock_fstat +#define GetFileSizeEx mock_GetFileSizeEx +#define lseek mock_lseek +#define SetFilePointer mock_SetFilePointer +#define flock mock_flock +#define LockFile mock_LockFile +#define UnlockFile mock_UnlockFile +#define fsync mock_fsync +#define FlushFileBuffers mock_FlushFileBuffers +#define mkstemp mock_mkstemp +#define mkdir mock_mkdir +#define _mkdir mock__mkdir +#define remove mock_remove +#define rename mock_rename +#define MoveFileExW mock_MoveFileExW +#define realpath mock_realpath +#define _fullpath mock__fullpath +#define opendir mock_opendir +#define readdir mock_readdir +#define closedir mock_closedir +#define FindFirstFileA mock_FindFirstFileA +#define FindNextFileA mock_FindNextFileA +#define FindClose mock_FindClose + +#else + +#define QUAKEY_SIGNAL(name) ((void) (name)) + +#endif + +#endif // QUAKEY_INCLUDED \ No newline at end of file diff --git a/quakey/src/mockfs.c b/quakey/src/mockfs.c new file mode 100644 index 0000000..a6cc262 --- /dev/null +++ b/quakey/src/mockfs.c @@ -0,0 +1,985 @@ +#include +#include +#include +#include + +#include "mockfs.h" + +typedef struct { + char *ptr; + int len; +} Slice; + +#define S(X) (Slice) { (X), (int) sizeof(X)-1 } + +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) + +#define MOCKFS_COMP_LIMIT 32 + +struct MockFS_Entity { + MockFS_Entity *parent; + MockFS_Entity *prev; + MockFS_Entity *next; + char name[MOCKFS_NAME_SIZE]; + int name_len; + bool is_dir; + int refcount; // Number of open file handles pointing to this entity + union { + struct { + MockFS_Entity *head_child; + MockFS_Entity *tail_child; + }; + ByteBuffer byte_buffer; + }; +}; + +static bool slice_eq(Slice s1, Slice s2) +{ + if (s1.len != s2.len) + return false; + return !memcmp(s1.ptr, s2.ptr, s1.len); +} + +static void *alloc(MockFS *mfs, int len, int align) +{ + int pad = -(unsigned long long) (mfs->mem + mfs->off) & (align - 1); + if (mfs->len - mfs->off < pad + len) + return NULL; + void *p = mfs->mem + mfs->off + pad; + mfs->off += pad + len; + return p; +} + +static void byte_buffer_init(ByteBuffer *byte_buffer) +{ + byte_buffer->used = 0; + byte_buffer->tail_used = 0; + byte_buffer->head = NULL; + byte_buffer->tail = NULL; +} + +static void byte_buffer_free(ByteBuffer *byte_buffer, ByteChunk **free_list) +{ + if (byte_buffer->head) { + byte_buffer->tail->next = *free_list; + *free_list = byte_buffer->head; + } +} + +static int convert_offset(ByteBuffer *byte_buffer, int off, ByteChunk **pchunk, int *poffset) +{ + assert(off > -1); + + int skipped = 0; + ByteChunk *chunk = byte_buffer->head; + while (chunk) { + int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used); + if (off >= skipped && off < skipped + chunk_used) { + *pchunk = chunk; + *poffset = off - skipped; + return 1; + } + chunk = chunk->next; + skipped += chunk_used; + } + + if (off == skipped) { + *pchunk = byte_buffer->tail; + *poffset = byte_buffer->tail ? byte_buffer->tail_used : 0; + return 1; + } + + return 0; +} + +static int byte_buffer_read(ByteBuffer *byte_buffer, int off, char *dst, int cap) +{ + int rel_off; + ByteChunk *chunk; + convert_offset(byte_buffer, off, &chunk, &rel_off); + + int copied = 0; + while (copied < cap) { + if (chunk == byte_buffer->tail) { + if (rel_off == byte_buffer->tail_used) + break; + int cpy = MIN(byte_buffer->tail_used - rel_off, cap - copied); + memcpy(dst + copied, chunk->data + rel_off, cpy); + copied += cpy; + break; // No more chunks after tail + } else { + int cpy = MIN(BYTE_CHUNK_SIZE - rel_off, cap - copied); + memcpy(dst + copied, chunk->data + rel_off, cpy); + copied += cpy; + chunk = chunk->next; + rel_off = 0; + } + } + + return copied; +} + +// Calculate total used bytes in the buffer +static int byte_buffer_size(ByteBuffer *byte_buffer) +{ + int size = 0; + ByteChunk *chunk = byte_buffer->head; + while (chunk) { + if (chunk->next) { + size += BYTE_CHUNK_SIZE; + } else { + size += byte_buffer->tail_used; + } + chunk = chunk->next; + } + return size; +} + +// Extend buffer to given size by filling with zeros +static int byte_buffer_extend(ByteBuffer *byte_buffer, int target_size, MockFS *mfs) +{ + int current_size = byte_buffer_size(byte_buffer); + int to_write = target_size - current_size; + + while (to_write > 0) { + // Get or create tail chunk + if (byte_buffer->tail == NULL || byte_buffer->tail_used == BYTE_CHUNK_SIZE) { + ByteChunk *tmp = mfs->chunk_free_list; + if (tmp == NULL) { + tmp = alloc(mfs, sizeof(ByteChunk), _Alignof(ByteChunk)); + if (tmp == NULL) + return MOCKFS_ERRNO_NOSPC; + } else { + mfs->chunk_free_list = tmp->next; + } + tmp->next = NULL; + + if (byte_buffer->head == NULL) { + byte_buffer->head = tmp; + } else { + byte_buffer->tail->next = tmp; + } + byte_buffer->tail = tmp; + byte_buffer->tail_used = 0; + } + + // Fill remaining space in tail chunk with zeros + int space = BYTE_CHUNK_SIZE - byte_buffer->tail_used; + int fill = MIN(space, to_write); + memset(byte_buffer->tail->data + byte_buffer->tail_used, 0, fill); + byte_buffer->tail_used += fill; + to_write -= fill; + } + + return 0; +} + +static int byte_buffer_write(ByteBuffer *byte_buffer, int off, char *src, int len, MockFS *mfs) +{ + int rel_off; + ByteChunk *chunk; + if (!convert_offset(byte_buffer, off, &chunk, &rel_off)) { + // Offset is beyond end of buffer - extend with zeros + int ret = byte_buffer_extend(byte_buffer, off, mfs); + if (ret < 0) + return ret; + convert_offset(byte_buffer, off, &chunk, &rel_off); + } + + int copied = 0; + while (copied < len) { + + if (chunk == byte_buffer->tail) { + if (chunk == NULL || rel_off == BYTE_CHUNK_SIZE) { + + ByteChunk *tmp = mfs->chunk_free_list; + if (tmp == NULL) { + tmp = alloc(mfs, sizeof(ByteChunk), _Alignof(ByteChunk)); + if (tmp == NULL) + return MOCKFS_ERRNO_NOSPC; + } else { + mfs->chunk_free_list = tmp->next; + } + tmp->next = NULL; + + if (byte_buffer->head == NULL) { + byte_buffer->head = tmp; + } else { + byte_buffer->tail->next = tmp; + } + byte_buffer->tail = tmp; + byte_buffer->tail_used = 0; + + rel_off = 0; + + int cpy = MIN(BYTE_CHUNK_SIZE, len - copied); + assert(cpy > 0); + + memcpy(tmp->data, src + copied, cpy); + copied += cpy; + chunk = tmp; + rel_off += cpy; + + byte_buffer->tail_used = cpy; + + } else if (rel_off == byte_buffer->tail_used) { + + assert(chunk); + + int cpy = MIN(BYTE_CHUNK_SIZE - byte_buffer->tail_used, len - copied); + assert(cpy > 0); + + memcpy(chunk->data + byte_buffer->tail_used, src + copied, cpy); + + copied += cpy; + rel_off += cpy; + byte_buffer->tail_used += cpy; + + } else { + + assert(rel_off < byte_buffer->tail_used); + + assert(chunk); + + int cpy = MIN(byte_buffer->tail_used - rel_off, len - copied); + assert(cpy > 0); + + memcpy(chunk->data + rel_off, src + copied, cpy); + copied += cpy; + rel_off += cpy; + } + + } else { + + assert(chunk); + + int cpy = MIN(BYTE_CHUNK_SIZE - rel_off, len - copied); + assert(cpy > 0); + + memcpy(chunk->data + rel_off, src + copied, cpy); + copied += cpy; + chunk = chunk->next; + rel_off = 0; + } + } + + return 0; +} + +static int parse_path(char *src, int len, Slice *buf, int cap) +{ + int cur = 0; + int ret = 0; + + if (len > 0 && src[0] == '/') + cur++; + + for (;;) { + + int off = cur; + while (cur < len && src[cur] != '/') + cur++; + Slice s = { src + off, cur - off }; + + if (s.len > 0) { + if (ret == cap) + return -1; // TODO: proper error code + buf[ret++] = s; + } + + if (cur == len) + break; + assert(src[cur] == '/'); + cur++; + } + + return ret; +} + +static int resolve_path(MockFS *mfs, Slice *comps, int num_comps, + MockFS_Entity **stack, int cap) +{ + int ret = 0; + stack[ret++] = mfs->root; + + for (int i = 0; i < num_comps; i++) { + + if (!stack[ret-1]->is_dir) + return MOCKFS_ERRNO_NOTDIR; + + if (slice_eq(comps[i], S(".."))) { + ret--; + continue; + } + + if (slice_eq(comps[i], S("."))) + continue; + + MockFS_Entity *child = stack[ret-1]->head_child; + while (child) { + if (slice_eq(comps[i], (Slice) { child->name, child->name_len })) + break; + child = child->next; + } + + if (child == NULL) + return MOCKFS_ERRNO_NOENT; + + if (ret == cap) + return -1; // TODO: return proper error code + stack[ret++] = child; + } + + return ret; +} + +static int entity_init(MockFS_Entity *entity, Slice name, bool is_dir) +{ + entity->parent = NULL; + entity->prev = NULL; + entity->next = NULL; + entity->is_dir = is_dir; + entity->refcount = 0; + entity->head_child = NULL; + entity->tail_child = NULL; + + if (name.len > (int) sizeof(entity->name)) + return -1; // TODO: proper error code + memcpy(entity->name, name.ptr, name.len); + entity->name_len = name.len; + + return 0; +} + +int mockfs_init(MockFS **pmfs, char *mem, int len) +{ + int off = -(unsigned long long) mem & (_Alignof(MockFS)-1); + if (off + sizeof(MockFS) > (unsigned long long)len) + return MOCKFS_ERRNO_NOMEM; + MockFS *mfs = (MockFS *)(mem + off); + + mfs->mem = mem; + mfs->len = len; + mfs->off = off + sizeof(MockFS); + mfs->root = NULL; + mfs->entity_free_list = NULL; + mfs->chunk_free_list = NULL; + + MockFS_Entity *entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity)); + if (entity == NULL) + return MOCKFS_ERRNO_NOMEM; + entity_init(entity, (Slice) { "", 0 }, true); + mfs->root = entity; + + *pmfs = mfs; + return 0; +} + +void mockfs_free(MockFS *mfs) +{ + (void) mfs; +} + +static bool find_child(MockFS_Entity *entity, Slice child_name) +{ + assert(entity->is_dir); + MockFS_Entity *child = entity->head_child; + while (child) { + if (slice_eq(child_name, (Slice) { child->name, child->name_len })) + return true; + child = child->next; + } + return false; +} + +static int create_entity(MockFS *mfs, MockFS_Entity *parent, Slice name, bool is_dir) +{ + MockFS_Entity *entity = mfs->entity_free_list; + if (entity == NULL) { + entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity)); + if (entity == NULL) + return MOCKFS_ERRNO_NOMEM; + } else { + mfs->entity_free_list = entity->next; + } + entity->next = NULL; + + int ret = entity_init(entity, name, is_dir); + if (ret < 0) { + entity->next = mfs->entity_free_list; + mfs->entity_free_list = entity; + return ret; + } + + // Initialize file-specific fields + byte_buffer_init(&entity->byte_buffer); + + // Link to parent directory + entity->parent = parent; + entity->next = NULL; + entity->prev = parent->tail_child; + + if (parent->tail_child) { + parent->tail_child->next = entity; + } else { + parent->head_child = entity; + } + parent->tail_child = entity; + + return 0; +} + +int mockfs_open(MockFS *mfs, char *path, int path_len, int flags, MockFS_OpenFile *open_file) +{ + Slice comps[MOCKFS_COMP_LIMIT]; + int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + int num_comps = ret; + + MockFS_Entity *stack[MOCKFS_COMP_LIMIT]; + bool file_was_created = false; + ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT); + if (ret < 0) { + // If file doesn't exist AND O_CREAT is specified, try to create it + if (ret == MOCKFS_ERRNO_NOENT && (flags & MOCKFS_O_CREAT)) { + // Resolve parent directory + ret = resolve_path(mfs, comps, num_comps-1, stack, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + assert(ret > 0); + MockFS_Entity *parent = stack[ret-1]; + + // Path ending with '/' implies directory, but you can't create + // a directory with open() - that's what mkdir() is for + if (path_len > 1 && path[path_len-1] == '/') { + return MOCKFS_ERRNO_ISDIR; + } + + // Create the file + ret = create_entity(mfs, parent, comps[num_comps-1], false); + if (ret < 0) + return ret; + + file_was_created = true; + + // Retry full path resolution (now should succeed) + ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT); + assert(ret > 0); + } else { + return ret; + } + } + assert(ret > 0); + + // Check for trailing slash (but "/" alone is not a "trailing slash" case) + bool has_trailing_slash = (path_len > 1 && path[path_len-1] == '/'); + + if ((flags & MOCKFS_O_CREAT) && has_trailing_slash) + return MOCKFS_ERRNO_ISDIR; + + // O_EXCL check: if file already existed and O_EXCL is set, fail + if ((flags & MOCKFS_O_EXCL) && (flags & MOCKFS_O_CREAT) && !file_was_created) + return MOCKFS_ERRNO_EXIST; + + if (stack[ret-1]->is_dir) + return MOCKFS_ERRNO_ISDIR; + + if (has_trailing_slash) { + // If target is not a directory, return ENOTDIR + if (!stack[ret-1]->is_dir) + return MOCKFS_ERRNO_NOTDIR; + } + + // O_TRUNC: truncate file to zero length if opened for writing + if ((flags & MOCKFS_O_TRUNC) && (flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) { + byte_buffer_free(&stack[ret-1]->byte_buffer, &mfs->chunk_free_list); + byte_buffer_init(&stack[ret-1]->byte_buffer); + } + + open_file->mfs = mfs; + open_file->entity = stack[ret-1]; + open_file->offset = 0; + open_file->flags = flags; + open_file->entity->refcount++; + return 0; +} + +int mockfs_open_dir(MockFS *mfs, char *path, int path_len, MockFS_OpenDir *open_dir) +{ + Slice comps[MOCKFS_COMP_LIMIT]; + int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + int num_comps = ret; + + MockFS_Entity *stack[MOCKFS_COMP_LIMIT]; + ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + assert(ret > 0); + + if (!stack[ret-1]->is_dir) + return MOCKFS_ERRNO_NOTDIR; + + open_dir->mfs = mfs; + open_dir->entity = stack[ret-1]; + open_dir->child = stack[ret-1]->head_child; + open_dir->idx = 0; + return 0; +} + +int mockfs_file_size(MockFS_OpenFile *open_file) +{ + return byte_buffer_size(&open_file->entity->byte_buffer); +} + +void mockfs_close_file(MockFS_OpenFile *open_file) +{ + MockFS_Entity *entity = open_file->entity; + entity->refcount--; + + // If refcount drops to 0 and entity was unlinked (removed while open), + // now we can actually free it + if (entity->refcount == 0 && entity->parent == NULL) { + // Free the byte buffer chunks + byte_buffer_free(&entity->byte_buffer, &open_file->mfs->chunk_free_list); + + // Add entity to free list + entity->next = open_file->mfs->entity_free_list; + open_file->mfs->entity_free_list = entity; + } +} + +void mockfs_close_dir(MockFS_OpenDir *open_dir) +{ + (void) open_dir; +} + +int mockfs_read(MockFS_OpenFile *open_file, char *dst, int len) +{ + if (open_file->flags & MOCKFS_O_WRONLY) { + return MOCKFS_ERRNO_BADF; + } + + int copied = byte_buffer_read(&open_file->entity->byte_buffer, open_file->offset, dst, len); + open_file->offset += copied; + return copied; +} + +int mockfs_write(MockFS_OpenFile *open_file, char *src, int len) +{ + if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) { + return MOCKFS_ERRNO_BADF; + } + + if ((open_file->flags & MOCKFS_O_WRONLY) && (open_file->flags & MOCKFS_O_RDWR)) { + return MOCKFS_ERRNO_BADF; + } + + // If O_APPEND is set, seek to end before writing + if (open_file->flags & MOCKFS_O_APPEND) { + open_file->offset = byte_buffer_size(&open_file->entity->byte_buffer); + } + + int ret = byte_buffer_write(&open_file->entity->byte_buffer, open_file->offset, src, len, open_file->mfs); + if (ret < 0) + return ret; + + open_file->offset += len; + return len; +} + +int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent) +{ + if (open_dir->idx == 0) { + if (sizeof(dirent->name) < 1) + return -1; // TODO: proper error code + dirent->name[0] = '.'; + dirent->name_len = 1; + dirent->is_dir = true; + open_dir->idx++; + return 0; + } + + if (open_dir->idx == 1) { + if (sizeof(dirent->name) < 2) + return -1; // TODO: proper error code + dirent->name[0] = '.'; + dirent->name[1] = '.'; + dirent->name_len = 2; + dirent->is_dir = true; + open_dir->idx++; + return 0; + } + + if (open_dir->child == NULL) + return MOCKFS_ERRNO_NOENT; + + memcpy(dirent->name, open_dir->child->name, open_dir->child->name_len); + dirent->name_len = open_dir->child->name_len; + dirent->is_dir = open_dir->child->is_dir; + + open_dir->child = open_dir->child->next; + open_dir->idx++; + return 0; +} + +int mockfs_sync(MockFS_OpenFile *open_file) +{ + // TODO + (void) open_file; + return 0; +} + +int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence) +{ + int new_offset; + + switch (whence) { + case MOCKFS_SEEK_SET: + new_offset = offset; + break; + case MOCKFS_SEEK_CUR: + new_offset = open_file->offset + offset; + break; + case MOCKFS_SEEK_END: + new_offset = byte_buffer_size(&open_file->entity->byte_buffer) + offset; + break; + default: + return MOCKFS_ERRNO_INVAL; + } + + if (new_offset < 0) + return MOCKFS_ERRNO_INVAL; + + open_file->offset = new_offset; + return new_offset; +} + +static void byte_buffer_truncate(ByteBuffer *byte_buffer, int new_size, ByteChunk **free_list) +{ + if (new_size == 0) { + byte_buffer_free(byte_buffer, free_list); + byte_buffer_init(byte_buffer); + return; + } + + // Walk through chunks to find the one containing the new end + int remaining = new_size; + ByteChunk *chunk = byte_buffer->head; + while (chunk) { + int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used); + if (remaining <= chunk_used) { + // This chunk becomes the new tail + // Free all subsequent chunks + ByteChunk *to_free = chunk->next; + if (to_free) { + // Find end of chain to free + ByteChunk *last = to_free; + while (last->next) + last = last->next; + last->next = *free_list; + *free_list = to_free; + } + chunk->next = NULL; + byte_buffer->tail = chunk; + byte_buffer->tail_used = remaining; + return; + } + remaining -= BYTE_CHUNK_SIZE; + chunk = chunk->next; + } +} + +int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size) +{ + if (new_size < 0) + return MOCKFS_ERRNO_INVAL; + + if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) + return MOCKFS_ERRNO_BADF; + + ByteBuffer *bb = &open_file->entity->byte_buffer; + int current_size = byte_buffer_size(bb); + + if (new_size < current_size) { + byte_buffer_truncate(bb, new_size, &open_file->mfs->chunk_free_list); + } else if (new_size > current_size) { + int ret = byte_buffer_extend(bb, new_size, open_file->mfs); + if (ret < 0) + return ret; + } + + return 0; +} + +static int remove_inner(MockFS *mfs, MockFS_Entity *entity, bool recursive) +{ + if (entity->parent == NULL) + return MOCKFS_ERRNO_BUSY; + + if (entity->is_dir) { + // Remove children + if (entity->head_child) { + if (!recursive) + return MOCKFS_ERRNO_NOTEMPTY; + MockFS_Entity *child = entity->head_child; + while (child) { + MockFS_Entity *next = child->next; + remove_inner(mfs, child, true); + child = next; + } + } + } + + // Unlink entity node from parent + if (entity->prev) { + entity->prev->next = entity->next; + } else { + entity->parent->head_child = entity->next; + } + + if (entity->next) { + entity->next->prev = entity->prev; + } else { + entity->parent->tail_child = entity->prev; + } + + // Mark as unlinked + entity->parent = NULL; + + // Only fully free the entity if no open handles + if (entity->refcount == 0) { + if (!entity->is_dir) { + // Append chunks to the free list + byte_buffer_free(&entity->byte_buffer, &mfs->chunk_free_list); + } + + // Append entity to the free list + entity->next = mfs->entity_free_list; + mfs->entity_free_list = entity; + } + + return 0; +} + +int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive) +{ + Slice comps[MOCKFS_COMP_LIMIT]; + int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + int num_comps = ret; + + if (num_comps > 0) { + if (slice_eq(comps[num_comps-1], S("."))) + return MOCKFS_ERRNO_INVAL; + + if (slice_eq(comps[num_comps-1], S(".."))) + return MOCKFS_ERRNO_INVAL; + } + + MockFS_Entity *stack[MOCKFS_COMP_LIMIT]; + ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + assert(ret > 0); + + if (path_len > 0 && path[path_len-1] == '/') { + if (!stack[ret-1]->is_dir) + return MOCKFS_ERRNO_NOTDIR; + } + + return remove_inner(mfs, stack[ret-1], recursive); +} + +int mockfs_mkdir(MockFS *mfs, char *path, int path_len) +{ + Slice comps[MOCKFS_COMP_LIMIT]; + int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + if (ret == 0) + return MOCKFS_ERRNO_EXIST; + int num_comps = ret; + + if (slice_eq(comps[num_comps-1], S("."))) + return MOCKFS_ERRNO_EXIST; + + if (slice_eq(comps[num_comps-1], S(".."))) + return MOCKFS_ERRNO_INVAL; + + MockFS_Entity *stack[MOCKFS_COMP_LIMIT]; + ret = resolve_path(mfs, comps, num_comps-1, stack, MOCKFS_COMP_LIMIT); + if (ret < 0) + return ret; + assert(ret > 0); + MockFS_Entity *parent = stack[ret-1]; + + if (!parent->is_dir) + return MOCKFS_ERRNO_NOTDIR; + + if (find_child(parent, comps[num_comps-1])) + return MOCKFS_ERRNO_EXIST; + + MockFS_Entity *entity = mfs->entity_free_list; + if (entity == NULL) { + entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity)); + if (entity == NULL) + return MOCKFS_ERRNO_NOMEM; + } else { + mfs->entity_free_list = entity->next; + } + entity->next = NULL; + + ret = entity_init(entity, comps[num_comps-1], true); + if (ret < 0) { + entity->next = mfs->entity_free_list; + mfs->entity_free_list = entity; + return ret; + } + + // Initialize byte_buffer to clear all 24 bytes of the union, including + // byte_buffer.tail (bytes 16-23) which would otherwise retain garbage + // when reusing an entity from the free list + byte_buffer_init(&entity->byte_buffer); + + entity->parent = parent; + entity->next = NULL; + entity->prev = parent->tail_child; + + if (parent->tail_child) { + parent->tail_child->next = entity; + } else { + parent->head_child = entity; + } + parent->tail_child = entity; + return 0; +} + +int mockfs_rename(MockFS *mfs, char *old_path, int old_path_len, char *new_path, int new_path_len) +{ + Slice new_comps[MOCKFS_COMP_LIMIT]; + Slice old_comps[MOCKFS_COMP_LIMIT]; + int num_new_comps = parse_path(new_path, new_path_len, new_comps, MOCKFS_COMP_LIMIT); + int num_old_comps = parse_path(old_path, old_path_len, old_comps, MOCKFS_COMP_LIMIT); + + if (num_new_comps < 0) return num_new_comps; + if (num_old_comps < 0) return num_old_comps; + + MockFS_Entity *new_stack[MOCKFS_COMP_LIMIT]; + MockFS_Entity *old_stack[MOCKFS_COMP_LIMIT]; + int num_new_stack = resolve_path(mfs, new_comps, num_new_comps-1, new_stack, MOCKFS_COMP_LIMIT); + int num_old_stack = resolve_path(mfs, old_comps, num_old_comps, old_stack, MOCKFS_COMP_LIMIT); + + if (num_new_stack == MOCKFS_ERRNO_NOTDIR || + num_old_stack == MOCKFS_ERRNO_NOTDIR) + return MOCKFS_ERRNO_NOTDIR; + + if (num_new_stack < 0) + return num_new_stack; + assert(num_new_stack > 0); + + if (!new_stack[num_new_stack-1]->is_dir) + return MOCKFS_ERRNO_NOTDIR; + + if (num_old_stack < 0) + return num_old_stack; + + assert(num_old_stack > 0); + MockFS_Entity *source = old_stack[num_old_stack-1]; + + if (source->parent == NULL) + return MOCKFS_ERRNO_BUSY; + + if (old_path_len > 0 && old_path[old_path_len-1] == '/') { + if (!source->is_dir) + return MOCKFS_ERRNO_NOTDIR; + } + + if (new_path_len > 0 && new_path[new_path_len-1] == '/') { + if (!source->is_dir) + return MOCKFS_ERRNO_NOTDIR; + } + + if (num_new_comps == 0) + return MOCKFS_ERRNO_BUSY; + + // Make sure the entity isn't being moved inside itself, + // by checking that the last element of the old stack isn't + // in the new stack. + for (int i = 0; i < num_new_stack; i++) + if (new_stack[i] == source) + return MOCKFS_ERRNO_INVAL; + + // Check if new path exists + Slice new_name = new_comps[num_new_comps-1]; + MockFS_Entity *target = NULL; + MockFS_Entity *child = new_stack[num_new_stack-1]->head_child; + while (child) { + if (slice_eq(new_name, (Slice) { child->name, child->name_len })) { + target = child; + break; + } + child = child->next; + } + + if (target) { + + if (target == source) + return 0; + + if (target->is_dir) { + + for (int i = 0; i < num_old_stack; i++) + if (old_stack[i] == target) + return MOCKFS_ERRNO_NOTEMPTY; + + if (!source->is_dir) + return MOCKFS_ERRNO_ISDIR; + + if (target->head_child) + return MOCKFS_ERRNO_NOTEMPTY; + + } else { + if (source->is_dir) + return MOCKFS_ERRNO_NOTDIR; + } + + remove_inner(mfs, target, false); + } + + // Unlink source from old parent + if (source->prev) { + source->prev->next = source->next; + } else { + source->parent->head_child = source->next; + } + + if (source->next) { + source->next->prev = source->prev; + } else { + source->parent->tail_child = source->prev; + } + + // Update source's name + if (new_name.len > (int) sizeof(source->name)) + return MOCKFS_ERRNO_INVAL; // Name too long + memcpy(source->name, new_name.ptr, new_name.len); + source->name_len = new_name.len; + + // Link source to new parent + source->parent = new_stack[num_new_stack-1]; + source->next = NULL; + source->prev = new_stack[num_new_stack-1]->tail_child; + + if (new_stack[num_new_stack-1]->tail_child) { + new_stack[num_new_stack-1]->tail_child->next = source; + } else { + new_stack[num_new_stack-1]->head_child = source; + } + new_stack[num_new_stack-1]->tail_child = source; + + return 0; +} diff --git a/quakey/src/mockfs.h b/quakey/src/mockfs.h new file mode 100644 index 0000000..6953075 --- /dev/null +++ b/quakey/src/mockfs.h @@ -0,0 +1,110 @@ +#ifndef MOCKFS_INCLUDED +#define MOCKFS_INCLUDED + +#include + +#define MOCKFS_NAME_SIZE (1<<7) + +enum { + MOCKFS_ERRNO_SUCCESS = 0, + MOCKFS_ERRNO_NOENT = -1, // No such file or directory + MOCKFS_ERRNO_PERM = -2, // Operation not permitted + MOCKFS_ERRNO_NOMEM = -3, // Out of memory + MOCKFS_ERRNO_NOTDIR = -4, // Not a directory + MOCKFS_ERRNO_ISDIR = -5, // Is a directory + MOCKFS_ERRNO_INVAL = -6, // Invalid argument + MOCKFS_ERRNO_NOTEMPTY = -7, // Directory not empty + MOCKFS_ERRNO_NOSPC = -8, // No space left on device + MOCKFS_ERRNO_EXIST = -9, // File exists + MOCKFS_ERRNO_BUSY = -10, + MOCKFS_ERRNO_BADF = -11, +}; + +enum { + MOCKFS_O_RDONLY = 0x00, + MOCKFS_O_WRONLY = 0x01, + MOCKFS_O_RDWR = 0x02, + MOCKFS_O_CREAT = 0x40, + MOCKFS_O_EXCL = 0x80, + MOCKFS_O_TRUNC = 0x200, + MOCKFS_O_APPEND = 0x400, +}; + +enum { + MOCKFS_SEEK_SET = 0, + MOCKFS_SEEK_CUR = 1, + MOCKFS_SEEK_END = 2, +}; + +#define BYTE_CHUNK_SIZE 128 + +typedef struct ByteChunk ByteChunk; +struct ByteChunk { + ByteChunk *next; + char data[BYTE_CHUNK_SIZE]; +}; + +typedef struct { + int used; + int tail_used; + ByteChunk *head; + ByteChunk *tail; +} ByteBuffer; + +typedef struct MockFS_Entity MockFS_Entity; + +typedef struct { + char *mem; + int len; + int off; + MockFS_Entity *root; + MockFS_Entity *entity_free_list; + ByteChunk *chunk_free_list; +} MockFS; + +typedef struct { + MockFS* mfs; + MockFS_Entity* entity; + int offset; + int flags; +} MockFS_OpenFile; + +typedef struct { + MockFS* mfs; + MockFS_Entity* entity; + MockFS_Entity* child; + int idx; +} MockFS_OpenDir; + +typedef struct { + char name[MOCKFS_NAME_SIZE]; + int name_len; + bool is_dir; +} MockFS_Dirent; + +int mockfs_init(MockFS **mfs, char *mem, int len); +void mockfs_free(MockFS *mfs); + +int mockfs_open(MockFS *mfs, char *path, int path_len, int flags, MockFS_OpenFile *open_file); +int mockfs_open_dir(MockFS *mfs, char *path, int path_len, MockFS_OpenDir *open_dir); + +int mockfs_file_size(MockFS_OpenFile *open_file); + +void mockfs_close_file(MockFS_OpenFile *open_file); +void mockfs_close_dir(MockFS_OpenDir *open_dir); + +int mockfs_read(MockFS_OpenFile *open_file, char *dst, int len); +int mockfs_write(MockFS_OpenFile *open_file, char *src, int len); +int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent); + +int mockfs_sync(MockFS_OpenFile *open_file); +int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence); +int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size); + +int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive); + +int mockfs_mkdir(MockFS *mfs, char *path, int path_len); + +int mockfs_rename(MockFS *mfs, char *old_path, int old_path_len, char *new_path, int new_path_len); + +#endif // MOCKFS_INCLUDED \ No newline at end of file diff --git a/quakey/src/quakey.c b/quakey/src/quakey.c new file mode 100644 index 0000000..c8e3720 --- /dev/null +++ b/quakey/src/quakey.c @@ -0,0 +1,4774 @@ +///////////////////////////////////////////////////////////////// +// Includes + +#include "mockfs.h" +#include +#include +#include + +///////////////////////////////////////////////////////////////// +// Utilities + +#define TODO __builtin_trap() + +#ifdef NDEBUG +#define UNREACHABLE {} +#define ASSERT(X) {} +#else +#define UNREACHABLE __builtin_trap() +#define ASSERT(X) {if (!(X)) __builtin_trap();} +#endif + +///////////////////////////////////////////////////////////////// +// Basic Types + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; + +typedef u32 b32; + +typedef u64 Nanos; + +///////////////////////////////////////////////////////////////// +// Network Types + +typedef struct { + u32 data; +} AddrIPv4; + +typedef struct { + u16 data[8]; +} AddrIPv6; + +typedef enum { + ADDR_FAMILY_IPV4, + ADDR_FAMILY_IPV6, +} AddrFamily; + +typedef struct { + AddrFamily family; + union { + AddrIPv4 ipv4; + AddrIPv6 ipv6; + }; +} Addr; + +///////////////////////////////////////////////////////////////// +// Descriptor Type + +typedef struct Desc Desc; +typedef struct Host Host; + +typedef struct { + int head; + int count; + int capacity; + Desc **entries; +} AcceptQueue; + +typedef struct { + char *data; + int head; + int used; + int size; + int refs; +} SocketQueue; + +typedef enum { + // The Desc structure is unused + DESC_EMPTY, + + // The Desc represents a socket created + // with the socket() system call. The + // specific type of socket depends on + // how it's configured + DESC_SOCKET, + + // The Desc represents a listening socket, + // one created with socket() on which listen() + // is used. + DESC_SOCKET_L, + + // The Desc represents a connection socket, + // one created using accept() or with socket() + // and then configured using listen() + DESC_SOCKET_C, + + // The Desc represents an opened file + DESC_FILE, + + // The Desc represent an open directory + DESC_DIRECTORY, + + DESC_PIPE, +} DescType; + +typedef enum { + CONNECT_STATUS_WAIT, + CONNECT_STATUS_DONE, + CONNECT_STATUS_RESET, + CONNECT_STATUS_CLOSE, + CONNECT_STATUS_NOHOST, +} ConnectStatus; + +struct Desc { + + // Parent host object + // This is set once at startup + Host *host; + + ///////////////////////////////////////// + // General descriptor fields + + DescType type; + b32 non_blocking; + + ///////////////////////////////////////// + // General socket fields + + // True if bind() has been called on this socket + b32 is_explicitly_bound; + + // Address and port bound to this socket (either + // implicitly or explicitly). + // + // An implicit bind occurs when listen() or connect() + // are called on a socket that was never bound to an + // interface using bind(). + // + // The address starts out with family equal to the + // first argument of socket() and address of 0. The + // port starts from 0. + Addr bound_addr; + u16 bound_port; + + ///////////////////////////////////////// + // Listen socket fields + + AcceptQueue accept_queue; + + ///////////////////////////////////////// + // Pipe or Connection socket fields + + // Status code of the connecting process + ConnectStatus connect_status; + + // These are used when the socket is still connecting + // (only used by sockets, not pipes) + Addr connect_addr; + u16 connect_port; + + // When connected this refers to the peer socket, else it's NULL. + // + // The peer is either a listen socket if the connection wasn't + // accepted yet, or a connection socket if it was. Note that this + // means the references to this descriptor must be removed from + // the peer's accept queue if it's freed abruptly. + Desc *peer; + + // Bytes received from/about to get sent to the peer + SocketQueue *input; + SocketQueue *output; + + ///////////////////////////////////////// + // File fields + + MockFS_OpenFile file; + + ///////////////////////////////////////// + // Directory fields + + MockFS_OpenDir dir; + + ///////////////////////////////////////// +}; + +///////////////////////////////////////////////////////////////// +// Host Type + +#define HOST_ADDR_LIMIT 2 +#define HOST_DESC_LIMIT 1024 +#define HOST_ARGC_LIMIT 128 + +typedef struct Sim Sim; + +enum { + HOST_ERROR_OTHER = -1, + HOST_ERROR_FULL = -2, + HOST_ERROR_BADIDX = -3, + HOST_ERROR_NOTSOCK = -4, + HOST_ERROR_CANTBIND = -5, + HOST_ERROR_NOTAVAIL = -6, + HOST_ERROR_ADDRUSED = -7, + HOST_ERROR_BADARG = -8, + HOST_ERROR_BADFAM = -9, + HOST_ERROR_RESET = -10, + HOST_ERROR_HANGUP = -11, + HOST_ERROR_NOTCONN = -12, + HOST_ERROR_IO = -13, + HOST_ERROR_ISDIR = -14, + HOST_ERROR_WOULDBLOCK = -15, + HOST_ERROR_NOMEM = -16, + HOST_ERROR_NOENT = -17, + HOST_ERROR_NOTEMPTY = -18, + HOST_ERROR_EXIST = -19, + HOST_ERROR_EXISTS = -19, // Alias for HOST_ERROR_EXIST + HOST_ERROR_PERM = -20, + HOST_ERROR_NOTDIR = -21, + HOST_ERROR_NOSPC = -22, + HOST_ERROR_BUSY = -23, + HOST_ERROR_BADF = -24, +}; + +// lseek whence values for host_lseek +enum { + HOST_SEEK_SET = 0, + HOST_SEEK_CUR = 1, + HOST_SEEK_END = 2, +}; + +enum { + HOST_FLAG_NONBLOCK = 1, +}; + +struct Host { + + // Pointer to the parent simulation object + Sim *sim; + + char *name; + + // Platform used by this host + QuakeyPlatform platform; + + // State of the ephimeral port allocation + u16 next_ephemeral_port; + + // Dynamic dopy of the argument string used + // to setup this host. Null bytes were injected + // to terminate each argument + char *arg; + + // Pointers into the argument string to make + // a list of individual arguments + int argc; + char *argv[HOST_ARGC_LIMIT]; + + // Opaque program state + void *state; + int state_size; + + // Pointers to program code + QuakeyInitFunc init_func; + QuakeyTickFunc tick_func; + QuakeyFreeFunc free_func; + + // Descriptor table + int num_desc; + Desc desc[HOST_DESC_LIMIT]; + + // Addresses bound to this host + Addr addrs[HOST_ADDR_LIMIT]; + int num_addrs; + + // Argument for poll() + void* poll_ctxs[HOST_DESC_LIMIT]; + struct pollfd poll_array[HOST_DESC_LIMIT]; + int poll_count; + int poll_timeout; + + b32 timedout; + b32 blocked; + b32 dead; + + // Current error number set by system call mocks + int errno_; + + // Raw disk bytes + int disk_size; + char *disk_data; + + // MockFS instance managing the disk bytes + MockFS *mfs; + + // Saved spawn configuration for restart after crash + QuakeySpawn spawn_config; + char *spawn_arg; +}; + +typedef struct { + char name[256]; + bool is_dir; +} DirEntry; + +typedef struct { + int64_t size; + bool is_dir; +} FileInfo; + +///////////////////////////////////////////////////////////////// +// Simulation Type + +typedef enum { + EVENT_TYPE_CONNECT, + EVENT_TYPE_DISCONNECT, + EVENT_TYPE_DATA, + EVENT_TYPE_WAKEUP, + EVENT_TYPE_RESTART, +} TimeEventType; + +typedef struct { + + TimeEventType type; + + // Time when the event should happen + Nanos time; + + // When type=WAKEUP, refers to the host that needs + // to be woken up + Host *host; + + // When type=CONNECT or type=DATA, refers to the + // descriptor that initiated the connection + Desc *src_desc; + + // When type=DISCONNECT, refers to the descriptor + // that is receiving the disconnection message + Desc *dst_desc; + + // When type=DATA, this refers to the output buffer + // of the socket sending the data. The count variable + // dictates how many bytes can be read from the queue + // + // The queue pointer is reference counted to allow + // descriptors to be deinitialized without creating + // dangling DATA events + int data_count; + SocketQueue *data_queue; + + // If set, the DISCONNECT event is to be intended as + // a forceful shutdown + b32 rst; +} TimeEvent; + +#define SIM_SIGNAL_LIMIT 8 + +struct Sim { + + uint64_t seed; + + // Current simulated time in nanoseconds + Nanos current_time; + + // List of simulated hosts. It's an array of pointers + // so that host structures are not moved when the array + // is resized. This allows safely holding references to + // hosts. + int num_hosts; + int max_hosts; + Host **hosts; + + // Array of timed events + int num_events; + int max_events; + TimeEvent *events; + + int num_signals; + int head_signal; + QuakeySignal signals[SIM_SIGNAL_LIMIT]; +}; + +static void time_event_wakeup(Sim *sim, Nanos time, Host *host); +static void time_event_connect(Sim *sim, Nanos time, Desc *desc); +static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst); +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc); +static void time_event_free(TimeEvent *event); +static void time_event_restart(Sim *sim, Nanos time, Host *host); +static void remove_events_targeting_desc(Sim *sim, Desc *desc); + +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// + +///////////////////////////////////////////////////////////////// +// Address Code + +static bool is_digit(char c) +{ + return c >= '0' && c <= '9'; +} + +static bool is_hex_digit(char c) +{ + return (c >= '0' && c <= '9') + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'); +} + +static int parse_ipv4(char *src, int len, int *pcur, AddrIPv4 *ipv4) +{ + int cur = *pcur; + + unsigned int out = 0; + int i = 0; + for (;;) { + + if (cur == len || !is_digit(src[cur])) + return -1; + + int b = 0; + do { + int x = src[cur++] - '0'; + if (b > (UINT8_MAX - x) / 10) + return -1; + b = b * 10 + x; + } while (cur < len && is_digit(src[cur])); + + out <<= 8; + out |= (unsigned char) b; + + i++; + if (i == 4) + break; + + if (cur == len || src[cur] != '.') + return -1; + cur++; + } + + ipv4->data = out; + + *pcur = cur; + return 0; +} + +static int hex_digit_to_int(char c) +{ + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= '0' && c <= '9') return c - '0'; + return -1; +} + +static int parse_ipv6_comp(char *src, int len, int *pcur) +{ + int cur = *pcur; + + unsigned short buf; + if (cur == len || !is_hex_digit(src[cur])) + return -1; + buf = hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + *pcur = cur; + return (int) buf; +} + +static int parse_ipv6(char *src, int len, int *pcur, AddrIPv6 *ipv6) +{ + int cur = *pcur; + + unsigned short head[8]; + unsigned short tail[8]; + int head_len = 0; + int tail_len = 0; + + if (len - cur > 1 + && src[cur+0] == ':' + && src[cur+1] == ':') + cur += 2; + else { + + for (;;) { + + int ret = parse_ipv6_comp(src, len, &cur); + if (ret < 0) return ret; + + head[head_len++] = (unsigned short) ret; + if (head_len == 8) break; + + if (cur == len || src[cur] != ':') + return -1; + cur++; + + if (cur < len && src[cur] == ':') { + cur++; + break; + } + } + } + + if (head_len < 8) { + while (cur < len && is_hex_digit(src[cur])) { + + int ret = parse_ipv6_comp(src, len, &cur); + if (ret < 0) return ret; + + tail[tail_len++] = (unsigned short) ret; + if (head_len + tail_len == 8) break; + + if (cur == len || src[cur] != ':') + break; + cur++; + } + } + + for (int i = 0; i < head_len; i++) + ipv6->data[i] = head[i]; + + for (int i = 0; i < 8 - head_len - tail_len; i++) + ipv6->data[head_len + i] = 0; + + for (int i = 0; i < tail_len; i++) + ipv6->data[8 - tail_len + i] = tail[i]; + + *pcur = cur; + return 0; +} + +static int addr_parse(char *src, Addr *dst) +{ + int cur = 0; + int len = strlen(src); + + if (parse_ipv4(src, len, &cur, &dst->ipv4) == 0) { + dst->family = ADDR_FAMILY_IPV4; + return 0; + } + + cur = 0; + if (parse_ipv6(src, len, &cur, &dst->ipv6) == 0) { + dst->family = ADDR_FAMILY_IPV6; + return 0; + } + + return -1; +} + +static bool addr_eql(Addr a1, Addr a2) +{ + if (a1.family != a2.family) + return false; + if (a1.family == ADDR_FAMILY_IPV4) + return !memcmp(&a1.ipv4, &a2.ipv4, sizeof(AddrIPv4)); + ASSERT(a1.family == ADDR_FAMILY_IPV6); + return !memcmp(&a1.ipv6, &a2.ipv6, sizeof(AddrIPv6)); +} + +static bool is_zero_addr(Addr addr) +{ + char *p; + int n; + if (addr.family == ADDR_FAMILY_IPV4) { + p = (char*) &addr.ipv4; + n = sizeof(addr.ipv4); + } else { + ASSERT(addr.family == ADDR_FAMILY_IPV6); + p = (char*) &addr.ipv6; + n = sizeof(addr.ipv6); + } + for (int i = 0; i < n; i++) { + if (p[i] != 0) + return false; + } + return true; +} + +///////////////////////////////////////////////////////////////// +// Socket Queue Code + +static SocketQueue *socket_queue_init(int size) +{ + void *mem = malloc(sizeof(SocketQueue) + size); + if (mem == NULL) { + TODO; + } + SocketQueue *queue = mem; + queue->head = 0; + queue->used = 0; + queue->size = size; + queue->data = (char*) (queue + 1); + queue->refs = 1; + return queue; +} + +static SocketQueue *socket_queue_ref(SocketQueue *queue) +{ + queue->refs++; + return queue; +} + +static void socket_queue_unref(SocketQueue *queue) +{ + ASSERT(queue->refs > 0); + queue->refs--; + if (queue->refs == 0) + free(queue); +} + +static char *socket_queue_read_buf(SocketQueue *queue, int *num) +{ + *num = queue->used; + return queue->data + queue->head; +} + +static void socket_queue_read_ack(SocketQueue *queue, int num) +{ + queue->head += num; + queue->used -= num; +} + +static int socket_queue_read(SocketQueue *queue, char *dst, int max) +{ + int num; + char *src = socket_queue_read_buf(queue, &num); + + int copy = max; + if (copy > num) + copy = num; + memcpy(dst, src, copy); + + socket_queue_read_ack(queue, copy); + return copy; +} + +static char *socket_queue_write_buf(SocketQueue *queue, int *cap) +{ + // Only write up to the free space + if (*cap > queue->size - queue->used) + *cap = queue->size - queue->used; + + int last = queue->head + queue->used; + if (queue->size - last < *cap) { + memmove(queue->data, queue->data + queue->head, queue->used); + queue->head = 0; + } + + return queue->data + queue->head + queue->used; +} + +static void socket_queue_write_ack(SocketQueue *queue, int num) +{ + queue->used += num; +} + +static int socket_queue_write(SocketQueue *queue, char *src, int len) +{ + char *dst = socket_queue_write_buf(queue, &len); + memcpy(dst, src, len); + socket_queue_write_ack(queue, len); + return len; +} + +static int socket_queue_move(SocketQueue *dst_queue, SocketQueue *src_queue, int max) +{ + int srclen; + char *src = socket_queue_read_buf(src_queue, &srclen); + + if (srclen > max) + srclen = max; + + int dstlen = srclen; + char *dst = socket_queue_write_buf(dst_queue, &dstlen); + + memcpy(dst, src, dstlen); + + socket_queue_write_ack(dst_queue, dstlen); + socket_queue_read_ack(src_queue, dstlen); + return dstlen; +} + +static b32 socket_queue_full(SocketQueue *queue) +{ + return queue->used == queue->size; +} + +static b32 socket_queue_empty(SocketQueue *queue) +{ + return queue->used == 0; +} + +static b32 socket_queue_used(SocketQueue *queue) +{ + return queue->used; +} + +///////////////////////////////////////////////////////////////// +// Accept Queue Code + +static int accept_queue_init(AcceptQueue *queue, int capacity) +{ + Desc **entries = malloc(capacity * sizeof(Desc*)); + if (entries == NULL) + return -1; + queue->head = 0; + queue->count = 0; + queue->capacity = capacity; + queue->entries = entries; + return 0; +} + +static void accept_queue_free(AcceptQueue *queue) +{ + free(queue->entries); +} + +static Desc **accept_queue_peek(AcceptQueue *queue, int idx) +{ + if (idx >= queue->count) + return NULL; + return &queue->entries[(queue->head + idx) % queue->capacity]; +} + +static void accept_queue_remove(AcceptQueue *queue, Desc *desc) +{ + int i = 0; + while (i < queue->count && desc != *accept_queue_peek(queue, i)) + i++; + + if (i == queue->count) + return; // Not found + + while (i < queue->count-1) { + *accept_queue_peek(queue, i) = *accept_queue_peek(queue, i+1); + i++; + } + + queue->count--; +} + +static int accept_queue_push(AcceptQueue *queue, Desc *desc) +{ + if (queue->count == queue->capacity) + return -1; + int tail = (queue->head + queue->count) % queue->capacity; + queue->entries[tail] = desc; + queue->count++; + return 0; +} + +static int accept_queue_pop(AcceptQueue *queue, Desc **desc) +{ + if (queue->count == 0) + return -1; + *desc = queue->entries[queue->head]; + queue->head = (queue->head + 1) % queue->capacity; + queue->count--; + return 0; +} + +static bool accept_queue_empty(AcceptQueue *queue) +{ + return queue->count == 0; +} + +///////////////////////////////////////////////////////////////// +// Descriptor Code + +// If the descriptor is a connection socket and rst=true, +// the peer connection will be marked as "reset" instead +// of simply closed. +static void desc_free(Sim *sim, Desc *desc, bool rst) +{ + switch (desc->type) { + case DESC_EMPTY: + break; + case DESC_SOCKET: + break; + case DESC_SOCKET_L: + // Update the other ends of the connections waiting to be accepted + for (int i = 0; i < desc->accept_queue.count; i++) { + Desc *peer = *accept_queue_peek(&desc->accept_queue, i); + peer->peer = NULL; + peer->connect_status = CONNECT_STATUS_RESET; + } + accept_queue_free(&desc->accept_queue); + break; + case DESC_PIPE: + case DESC_SOCKET_C: + // Remove any pending events targeting this descriptor before freeing it + if (sim) + remove_events_targeting_desc(sim, desc); + if (desc->peer) { + // A connection was previously established. + // We need to update the other end of the connection. + Desc *peer = desc->peer; + if (peer->type == DESC_SOCKET_L) { + // Connection was waiting to be accepted + accept_queue_remove(&peer->accept_queue, desc); + } else { + ASSERT(peer->type == DESC_SOCKET_C || peer->type == DESC_PIPE); + peer->peer = NULL; + peer->connect_status = rst ? CONNECT_STATUS_RESET : CONNECT_STATUS_CLOSE; + } + } + socket_queue_unref(desc->input); + socket_queue_unref(desc->output); + break; + case DESC_FILE: + mockfs_close_file(&desc->file); + break; + case DESC_DIRECTORY: + mockfs_close_dir(&desc->dir); + break; + default: + UNREACHABLE; + break; + } + desc->type = DESC_EMPTY; +} + +///////////////////////////////////////////////////////////////// +// Host Code + +// Backlog size used by listen() when the provided +// backlog argument is non-positive +#define DEFAULT_BACKLOG 128 + +#define FIRST_EPHEMERAL_PORT 10000 +#define LAST_EPHEMERAL_PORT 50000 + +// Current schedulated host +static Host *host___; + +int errno___; + +static uint64_t sim_random(Sim *sim); + +static void abort_(char *str) +{ +#ifdef _WIN32 + WriteFile(GetStdHandle((DWORD)-11), str, strlen(str), NULL, NULL); +#else + long ret; + __asm__ volatile ( + "syscall" + : "=a" (ret) + : "a" (1), + "D" (1), + "S" (str), + "d" (strlen(str)) + : "rcx", "r11", "memory" + ); + (void) ret; +#endif + __builtin_trap(); +} + +static int split_args(char *arg, char **argv, int max_argc) +{ + int argc = 0; + for (int cur = 0, len = strlen(arg);; ) { + + while (cur < len && (arg[cur] == ' ' || arg[cur] == '\t')) + cur++; + + if (cur == len) + break; + + int off = cur; + + while (cur < len && arg[cur] != ' ' && arg[cur] != '\t') + cur++; + + if (cur < len) { + arg[cur] = '\0'; + cur++; + } + + if (argc == max_argc) + return -1; + argv[argc++] = arg + off; + } + + return argc; +} + +static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) +{ + host->sim = sim; + host->name = config.name; + host->platform = config.platform; + host->next_ephemeral_port = FIRST_EPHEMERAL_PORT; + + // Save spawn config for restart after crash + host->spawn_config = config; + int spawn_arg_len = strlen(arg); + host->spawn_arg = malloc(spawn_arg_len+1); + if (host->spawn_arg == NULL) { + TODO; + } + memcpy(host->spawn_arg, arg, spawn_arg_len+1); + + int arg_len = strlen(arg); + host->arg = malloc(arg_len+1); + if (host->arg == NULL) { + TODO; + } + memcpy(host->arg, arg, arg_len+1); // +1 for \0 + + host->argc = split_args(host->arg, host->argv, HOST_ARGC_LIMIT); + if (host->argc < 0) { + TODO; + } + + if (config.num_addrs > HOST_ADDR_LIMIT) { + TODO; + } + for (int i = 0; i < config.num_addrs; i++) { + if (addr_parse(config.addrs[i], &host->addrs[i]) < 0) { + TODO; + } + } + host->num_addrs = config.num_addrs; + + host->state_size = config.state_size; + host->state = calloc(1, config.state_size); + if (host->state == NULL) { + TODO; + } + + host->init_func = config.init_func; + host->tick_func = config.tick_func; + host->free_func = config.free_func; + + host->num_desc = 0; + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + host->desc[i].host = host; + host->desc[i].type = DESC_EMPTY; + } + + host->errno_ = 0; + + host->disk_size = config.disk_size; + host->disk_data = malloc(config.disk_size); + if (host->disk_data == NULL) { + TODO; + } + // Zero out memory to make sure operations are deterministic + memset(host->disk_data, 0, config.disk_size); + + int ret = mockfs_init(&host->mfs, host->disk_data, config.disk_size); + if (ret < 0) { + TODO; + } + + host->dead = false; + host->timedout = false; + host->blocked = false; + host->poll_count = 0; + host->poll_timeout = -1; + + host___ = host; + ret = config.init_func( + host->state, + host->argc, + host->argv, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + // TODO: if the node fails to initialize the simulation should continue + // running without it + TODO; + } + + if (host->poll_timeout > 0) + time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); + else if (host->poll_timeout == 0) + host->timedout = true; // Immediate timeout, don't create event at current_time +} + +static void host_free(Host *host) +{ + if (!host->dead) { + host___ = host; + int ret = host->free_func(host->state); + host___ = NULL; + if (ret < 0) { + TODO; + } + + mockfs_free(host->mfs); + free(host->disk_data); + + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(host->sim, &host->desc[i], true); + } + + free(host->state); + free(host->arg); + } + + free(host->spawn_arg); +} + +// Remove all events associated with a host (WAKEUP events and events +// whose source or destination descriptors belong to this host) +static void remove_events_for_host(Sim *sim, Host *host) +{ + int i = 0; + while (i < sim->num_events) { + TimeEvent *event = &sim->events[i]; + bool remove = false; + + if (event->type == EVENT_TYPE_WAKEUP && event->host == host) + remove = true; + if (event->type == EVENT_TYPE_RESTART && event->host == host) + remove = true; + if (event->src_desc && event->src_desc->host == host) + remove = true; + if (event->dst_desc && event->dst_desc->host == host) + remove = true; + + if (remove) { + time_event_free(event); + sim->events[i] = sim->events[--sim->num_events]; + } else { + i++; + } + } +} + +static int count_dead(Sim *sim) +{ + int n = 0; + for (int i = 0; i < sim->num_hosts; i++) + if (sim->hosts[i]->dead) + n++; + return n; +} + +// Simulate a crash: tear down all connections and wipe application +// state. Everything else (MockFS, arg, addresses, etc.) remains +// initialized until restart. +static void host_crash(Host *host) +{ + assert(!host->dead); + Sim *sim = host->sim; + + // Close all descriptors (RST connections to notify peers) + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(sim, &host->desc[i], true); + } + host->num_desc = 0; + + // Remove all pending events for this host + remove_events_for_host(sim, host); + + // Free application state + free(host->state); + host->state = NULL; + + host->dead = true; + host->timedout = false; + host->blocked = false; + host->poll_count = 0; + host->poll_timeout = -1; + + printf("Quakey: Host crash (%d/%d dead)\n", count_dead(sim), sim->num_hosts); +} + +// Restart a crashed host: re-initialize from saved config, +// preserving the disk contents +static void host_restart(Host *host) +{ + assert(host->dead); + Sim *sim = host->sim; + + // Re-parse arguments (split_args mutates the string with null bytes, + // so we need a fresh copy from spawn_arg) + free(host->arg); + int arg_len = strlen(host->spawn_arg); + host->arg = malloc(arg_len+1); + if (host->arg == NULL) { + TODO; + } + memcpy(host->arg, host->spawn_arg, arg_len+1); + + host->argc = split_args(host->arg, host->argv, HOST_ARGC_LIMIT); + if (host->argc < 0) { + TODO; + } + + // Re-allocate application state (zeroed, as on a fresh boot) + host->state_size = host->spawn_config.state_size; + host->state = calloc(1, host->state_size); + if (host->state == NULL) { + TODO; + } + + // Re-initialize descriptors + host->num_desc = 0; + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + host->desc[i].host = host; + host->desc[i].type = DESC_EMPTY; + } + + host->errno_ = 0; + host->next_ephemeral_port = FIRST_EPHEMERAL_PORT; + + // Keep existing MockFS - persistent state (files) survives crash. + // All file descriptors were closed during host_crash via desc_free, + // so entity refcounts are already correct. + + host->dead = false; + host->timedout = false; + host->blocked = false; + host->poll_count = 0; + host->poll_timeout = -1; + + host___ = host; + int ret = host->init_func( + host->state, + host->argc, + host->argv, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + // If the node fails to restart, mark it as dead again + host->dead = true; + return; + } + + if (host->poll_timeout > 0) + time_event_wakeup(sim, sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); + else if (host->poll_timeout == 0) + host->timedout = true; + + printf("Quakey: Host restart (%d/%d dead)\n", count_dead(sim), sim->num_hosts); +} + +static b32 host_is_linux(Host *host) +{ + return host->platform == QUAKEY_LINUX; +} + +static b32 host_is_windows(Host *host) +{ + return host->platform == QUAKEY_WINDOWS; +} + +static Nanos host_time(Host *host) +{ + return host->sim->current_time; +} + +static int *host_errno_ptr(Host *host) +{ + return &host->errno_; +} + +static bool is_connected_and_accepted(Desc *desc) +{ + assert(desc->type == DESC_SOCKET_C); + return desc->peer && desc->peer->type != DESC_SOCKET_L; +} + +static bool is_desc_idx_valid(Host *host, int desc_idx); + +static bool host_ready(Host *host) +{ + if (host->dead) + return false; + + if (host->blocked) + return false; + + if (host->timedout) + return true; + + // Check if any polled descriptors have pending events + for (int i = 0; i < host->poll_count; i++) { + + int fd = host->poll_array[i].fd; + int events = host->poll_array[i].events; + + if (!is_desc_idx_valid(host, fd)) + continue; + + Desc *desc = &host->desc[fd]; + + switch (desc->type) { + case DESC_SOCKET_L: + if (events & POLLIN) { + if (!accept_queue_empty(&desc->accept_queue)) + return true; + } + break; + case DESC_PIPE: + case DESC_SOCKET_C: + if (desc->connect_status == CONNECT_STATUS_NOHOST || + desc->connect_status == CONNECT_STATUS_RESET) + return true; + if (events & POLLIN) { + if (!socket_queue_empty(desc->input) + || desc->connect_status == CONNECT_STATUS_RESET + || desc->connect_status == CONNECT_STATUS_CLOSE) + return true; + } + if (events & POLLOUT) { + if (!socket_queue_full(desc->output) && is_connected_and_accepted(desc)) + return true; + } + break; + case DESC_FILE: + case DESC_DIRECTORY: + // Files and directories are always ready + if (events & (POLLIN | POLLOUT)) + return true; + break; + default: + UNREACHABLE; + break; + } + } + + return false; +} + +static void set_revents_in_poll_array(Host *host) +{ + for (int i = 0; i < host->poll_count; i++) { + + int fd = host->poll_array[i].fd; + int events = host->poll_array[i].events; + + if (!is_desc_idx_valid(host, fd)) + continue; // TODO: is this ok? + Desc *desc = &host->desc[fd]; + + int revents = 0; + switch (desc->type) { + case DESC_SOCKET: + // TODO + break; + case DESC_SOCKET_L: + if (events & POLLIN) { + if (!accept_queue_empty(&desc->accept_queue)) + revents = POLLIN; + } + break; + case DESC_PIPE: + case DESC_SOCKET_C: + if (desc->connect_status == CONNECT_STATUS_NOHOST || + desc->connect_status == CONNECT_STATUS_RESET) + revents |= POLLERR; + if (events & POLLIN) { + // TODO: should report prover events when hup and rst are set + if (!socket_queue_empty(desc->input)) + revents |= POLLIN; + if (desc->connect_status == CONNECT_STATUS_RESET || + desc->connect_status == CONNECT_STATUS_CLOSE) + revents |= POLLIN; + } + if (events & POLLOUT) { + if (!socket_queue_full(desc->output) && is_connected_and_accepted(desc)) + revents |= POLLOUT; + } + break; + case DESC_FILE: + if (events & POLLIN) { + revents |= POLLIN; + } + if (events & POLLOUT) { + revents |= POLLOUT; + } + break; + case DESC_DIRECTORY: + if (events & POLLIN) { + revents |= POLLIN; + } + if (events & POLLOUT) { + revents |= POLLOUT; + } + break; + default: + UNREACHABLE; + } + + host->poll_array[i].revents = revents; + } +} + +static void host_update(Host *host) +{ + host->timedout = false; + set_revents_in_poll_array(host); + host___ = host; + int ret = host->tick_func( + host->state, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + TODO; + } + + if (host->poll_timeout > 0) + time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); + else if (host->poll_timeout == 0) + host->timedout = true; // Immediate timeout, don't create event at current_time +} + +static bool host_has_addr(Host *host, Addr addr) +{ + for (int i = 0; i < host->num_addrs; i++) { + if (addr_eql(host->addrs[i], addr)) + return true; + } + return false; +} + +// Returns true if the descriptor is a socket bound +// (implicitly or explicitly) to an address +static bool is_bound(Desc *desc) +{ + if (desc->type == DESC_SOCKET) + return desc->is_explicitly_bound; + + if (desc->type == DESC_SOCKET_C || + desc->type == DESC_SOCKET_L) + return true; + + return false; +} + +// Note that this is assumed to work on empty +// descriptors too +static bool is_bound_to(Desc *desc, Addr addr, uint16_t port) +{ + if (!is_bound(desc)) + return false; + + if (!is_zero_addr(desc->bound_addr) && !addr_eql(addr, desc->bound_addr)) + return false; + + if (port != desc->bound_port) + return false; + + return true; +} + +static Desc *host_find_desc_bound_to(Host *host, Addr addr, uint16_t port) +{ + for (int i = 0, j = 0; j < host->num_desc; i++) { + + Desc *desc = &host->desc[i]; + if (desc->type == DESC_EMPTY) + continue; + j++; + + if (is_bound_to(desc, addr, port)) + return desc; + } + + return NULL; +} + +static int find_empty_desc_struct(Host *host) +{ + if (host->num_desc == HOST_DESC_LIMIT) + return -1; + + int i = 0; + while (host->desc[i].type != DESC_EMPTY) { + i++; + ASSERT(i < HOST_DESC_LIMIT); + } + + return i; +} + +static int host_create_socket(Host *host, AddrFamily family) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + desc->type = DESC_SOCKET; + desc->non_blocking = false; + desc->is_explicitly_bound = false; + desc->bound_addr = (Addr) { .family=family }; + desc->bound_port = 0; + + host->num_desc++; + return desc_idx; +} + +static int host_create_pipe(Host *host, int *desc_idxs) +{ + int desc_idx_1 = find_empty_desc_struct(host); + if (desc_idx_1 < 0) + return HOST_ERROR_FULL; + Desc *desc_1 = &host->desc[desc_idx_1]; + + int desc_idx_2 = find_empty_desc_struct(host); + if (desc_idx_2 < 0) + return HOST_ERROR_FULL; + Desc *desc_2 = &host->desc[desc_idx_2]; + + desc_1->type = DESC_PIPE; + desc_1->non_blocking = false; + desc_1->connect_status = CONNECT_STATUS_DONE; + desc_1->peer = desc_2; + desc_1->input = socket_queue_init(1<<12); + desc_1->output = socket_queue_init(1<<12); + + desc_2->type = DESC_PIPE; + desc_2->non_blocking = false; + desc_2->connect_status = CONNECT_STATUS_DONE; + desc_2->peer = desc_1; + desc_2->input = socket_queue_init(1<<12); + desc_2->output = socket_queue_init(1<<12); + + desc_idxs[0] = desc_idx_1; + desc_idxs[1] = desc_idx_2; + + host->num_desc += 2; + return 0; +} + +static bool is_desc_idx_valid(Host *host, int desc_idx) +{ + // Out of bounds + if (desc_idx < 0 || desc_idx >= HOST_DESC_LIMIT) + return false; + + // Not in use + if (host->desc[desc_idx].type == DESC_EMPTY) + return false; + + return true; +} + +static bool is_socket(Desc *desc) +{ + return desc->type == DESC_SOCKET + || desc->type == DESC_SOCKET_L + || desc->type == DESC_SOCKET_C; +} + +static int host_close(Host *host, int desc_idx, bool expect_socket) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + + if (expect_socket) { + if (!is_socket(&host->desc[desc_idx])) + return HOST_ERROR_NOTSOCK; + } + + if (host->desc[desc_idx].type == DESC_SOCKET_C) + time_event_disconnect(host->sim, host->sim->current_time + 10000000, &host->desc[desc_idx], false); + + desc_free(host->sim, &host->desc[desc_idx], false); + host->num_desc--; + return 0; +} + +static bool interf_exists_locally(Host *host, Addr addr) +{ + for (int i = 0; i < host->num_addrs; i++) + if (addr_eql(host->addrs[i], addr)) + return true; + return false; +} + +static bool addr_in_use(Host *host, Addr addr, uint16_t port) +{ + ASSERT(port != 0); + + if (is_zero_addr(addr)) { + // Any address may conflict with the zero address, + // which means we only need to compare ports. + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (is_socket(&host->desc[i])) { + if (host->desc[i].bound_port == port) + return true; + } + } + } else { + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (is_bound_to(&host->desc[i], addr, port)) + return true; + } + } + + return false; +} + +// Returns 0 on error +static uint16_t choose_ephemeral_port(Host *host, Addr addr) +{ + uint16_t first = host->next_ephemeral_port; + uint16_t *next = &host->next_ephemeral_port; + do { + uint16_t port = *next; + if (*next == LAST_EPHEMERAL_PORT) { + *next = FIRST_EPHEMERAL_PORT; + } else { + (*next)++; + } + if (!addr_in_use(host, addr, port)) + return port; + } while (*next != first); + return 0; +} + +static int host_bind(Host *host, int desc_idx, Addr addr, uint16_t port) +{ + ///////////////////////////////////////////////////////// + // Check index + + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + ///////////////////////////////////////////////////////// + // Check descriptor + + if (!is_socket(desc)) + return HOST_ERROR_NOTSOCK; + + if (host->desc[desc_idx].type != DESC_SOCKET) + return HOST_ERROR_CANTBIND; + + if (desc->is_explicitly_bound) + return HOST_ERROR_CANTBIND; + + ///////////////////////////////////////////////////////// + // Check address + + if (addr.family != desc->bound_addr.family) + return HOST_ERROR_BADFAM; + + if (!is_zero_addr(addr)) { + if (!interf_exists_locally(host, addr)) + return HOST_ERROR_NOTAVAIL; + } + + ///////////////////////////////////////////////////////// + // Check port + + if (port == 0) { + port = choose_ephemeral_port(host, addr); + if (port == 0) + return HOST_ERROR_NOTAVAIL; + } else { + if (addr_in_use(host, addr, port)) + return HOST_ERROR_ADDRUSED; + } + + ///////////////////////////////////////////////////////// + // Perform the binding + + desc->is_explicitly_bound = true; + desc->bound_addr = addr; + desc->bound_port = port; + + ///////////////////////////////////////////////////////// + return 0; +} + +static int host_listen(Host *host, int desc_idx, int backlog) +{ + if (backlog <= 0) + backlog = DEFAULT_BACKLOG; + + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->is_explicitly_bound) { + // We need to bind implicitly + // + // The bound_addr field already contains the right + // family and a zero address. The port is 0, which + // is not a valid value. + desc->bound_port = choose_ephemeral_port(host, desc->bound_addr); + if (desc->bound_port == 0) + return HOST_ERROR_ADDRUSED; + } + + if (accept_queue_init(&desc->accept_queue, backlog) < 0) { + TODO; + } + + desc->type = DESC_SOCKET_L; + return 0; +} + +static int host_accept(Host *host, int desc_idx, Addr *addr, uint16_t *port) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET_L) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before accept()\n"); + + int new_desc_idx = find_empty_desc_struct(host); + if (new_desc_idx < 0) + return HOST_ERROR_FULL; + Desc *new_desc = &host->desc[new_desc_idx]; + + Desc *peer; + if (accept_queue_pop(&desc->accept_queue, &peer) < 0) + return HOST_ERROR_WOULDBLOCK; + + *addr = peer->bound_addr; + *port = peer->bound_port; + + Addr local_addr = desc->bound_addr; + if (is_zero_addr(local_addr)) { + assert(host->num_addrs > 0); + local_addr = host->addrs[0]; + } + + new_desc->type = DESC_SOCKET_C; + new_desc->non_blocking = false; + new_desc->bound_addr = local_addr; + new_desc->bound_port = desc->bound_port; + new_desc->connect_addr = peer->bound_addr; + new_desc->connect_port = peer->bound_port; + new_desc->connect_status = CONNECT_STATUS_DONE; + new_desc->peer = peer; + new_desc->input = socket_queue_init(1<<12); + new_desc->output = socket_queue_init(1<<12); + + // Update the peer's end of the connection + peer->peer = new_desc; + + host->num_desc++; + return new_desc_idx; +} + +// TODO: check error codes returned by this function +static int host_connect(Host *host, int desc_idx, + Addr addr, uint16_t port) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before connect()\n"); + + if (!desc->is_explicitly_bound) { + // We need to bind implicitly + // + // The bound_addr field already contains the right + // family and a zero address. The port is 0, which + // is not a valid value. + desc->bound_port = choose_ephemeral_port(host, desc->bound_addr); + if (desc->bound_port == 0) + return HOST_ERROR_ADDRUSED; + } + + // TODO: some percent of times connect() should resolve immediately + + Nanos latency = 10000000; +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + latency = 1000000 + (rng % 99000000); // between 1ms and 100ms +#endif + + time_event_connect(host->sim, host->sim->current_time + latency, desc); + + desc->connect_addr = addr; + desc->connect_port = port; + desc->connect_status = CONNECT_STATUS_WAIT; + desc->peer = NULL; + desc->input = socket_queue_init(1<<12); + desc->output = socket_queue_init(1<<12); + + desc->type = DESC_SOCKET_C; + return 0; +} + +static int host_connect_status(Host *host, int desc_idx, ConnectStatus *status) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET_C) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (status) + *status = desc->connect_status; + return 0; +} + +static int mockfs_to_quakey_error(int err) +{ + switch (err) { + case 0: return 0; + case MOCKFS_ERRNO_NOENT : return HOST_ERROR_NOENT; + case MOCKFS_ERRNO_PERM : return HOST_ERROR_PERM; + case MOCKFS_ERRNO_NOMEM : return HOST_ERROR_NOMEM; + case MOCKFS_ERRNO_NOTDIR : return HOST_ERROR_NOTDIR; + case MOCKFS_ERRNO_ISDIR : return HOST_ERROR_ISDIR; + case MOCKFS_ERRNO_INVAL : return HOST_ERROR_BADARG; + case MOCKFS_ERRNO_NOTEMPTY : return HOST_ERROR_NOTEMPTY; + case MOCKFS_ERRNO_NOSPC : return HOST_ERROR_NOSPC; + case MOCKFS_ERRNO_EXIST : return HOST_ERROR_EXIST; + case MOCKFS_ERRNO_BUSY : return HOST_ERROR_BUSY; + case MOCKFS_ERRNO_BADF : return HOST_ERROR_BADF; + default: + printf("Unexpected mockfs errno %d\n", err); + assert(0); + } +} + +static int host_open_file(Host *host, char *path, int flags) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + int ret = mockfs_open(host->mfs, path, strlen(path), flags, &desc->file); + if (ret < 0) + return mockfs_to_quakey_error(ret); + + desc->type = DESC_FILE; + desc->non_blocking = false; + + host->num_desc++; + return desc_idx; +} + +static int host_open_dir(Host *host, char *path) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + int ret = mockfs_open_dir(host->mfs, path, strlen(path), &desc->dir); + if (ret < 0) + return mockfs_to_quakey_error(ret); + + desc->type = DESC_DIRECTORY; + desc->non_blocking = false; + + host->num_desc++; + return desc_idx; +} + +static int host_read_dir(Host *host, int desc_idx, DirEntry *entry) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_DIRECTORY) + return HOST_ERROR_BADARG; + + MockFS_Dirent buf; + int ret = mockfs_read_dir(&desc->dir, &buf); + if (ret < 0) { + if (ret == MOCKFS_ERRNO_NOENT) + return 0; + return mockfs_to_quakey_error(ret); + } + + // Copy entry information + int i = 0; + while (i < buf.name_len && i < 255) { + entry->name[i] = buf.name[i]; + i++; + } + entry->name[i] = '\0'; + entry->is_dir = buf.is_dir; + + return 1; +} + +static int recv_inner(Desc *desc, char *dst, int len) +{ + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before recv()\n"); + + if (desc->peer == NULL) { + if (desc->connect_status == CONNECT_STATUS_RESET) + return HOST_ERROR_RESET; + if (desc->connect_status == CONNECT_STATUS_CLOSE) + return HOST_ERROR_HANGUP; + return HOST_ERROR_NOTCONN; + } + + int ret = socket_queue_read(desc->input, dst, len); + if (ret == 0) + return HOST_ERROR_WOULDBLOCK; + + return ret; +} + +static int send_inner(Desc *desc, char *src, int len) +{ + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before send()\n"); + + if (desc->peer == NULL) { + if (desc->connect_status == CONNECT_STATUS_RESET) + return HOST_ERROR_RESET; + if (desc->connect_status == CONNECT_STATUS_CLOSE) + return HOST_ERROR_HANGUP; + return HOST_ERROR_NOTCONN; + } + + int ret = socket_queue_write(desc->output, src, len); + if (ret == 0) + return HOST_ERROR_WOULDBLOCK; + + Nanos latency = 10000000; +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + latency = 1000000 + (rng % 99000000); // between 1ms and 100ms +#endif + time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc); + + return ret; +} + +static int host_read(Host *host, int desc_idx, char *dst, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); +#endif + + int num = 0; + if (desc->type == DESC_SOCKET_C || + desc->type == DESC_PIPE) { + num = recv_inner(desc, dst, len); + } else if (desc->type == DESC_FILE) { +#ifdef FAULT_INJECTION + uint64_t roll = sim_random(host->sim) % 1000; + if (roll == 0) return HOST_ERROR_IO; +#endif + int ret = mockfs_read(&desc->file, dst, len); + if (ret < 0) + return mockfs_to_quakey_error(ret); +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) + if (ret > 0) { + // 1 in 10,000 reads gets a bit flip + if ((sim_random(host->sim) % 10000) == 0) { + int byte_idx = sim_random(host->sim) % ret; + int bit_idx = sim_random(host->sim) % 8; + dst[byte_idx] ^= (1 << bit_idx); + } + } +#endif + num = ret; + } else { + if (desc->type == DESC_DIRECTORY) + return HOST_ERROR_ISDIR; + return HOST_ERROR_BADARG; + } + + return num; +} + +static int host_write(Host *host, int desc_idx, char *src, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); +#endif + + int num = 0; + if (desc->type == DESC_SOCKET_C || + desc->type == DESC_PIPE) { + num = send_inner(desc, src, len); + } else if (desc->type == DESC_FILE) { +#ifdef FAULT_INJECTION + uint64_t roll = sim_random(host->sim) % 1000; + if (roll == 0) return HOST_ERROR_IO; + if (roll == 1) return HOST_ERROR_NOSPC; +#endif +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) + int byte_idx = -1; + int bit_idx; + if (len > 0) { + // 1 in 100,000 reads gets a bit flip + if ((sim_random(host->sim) % 100000) == 0) { + byte_idx = sim_random(host->sim) % len; + bit_idx = sim_random(host->sim) % 8; + src[byte_idx] ^= (1 << bit_idx); + } + } +#endif + int ret = mockfs_write(&desc->file, src, len); +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) + if (byte_idx > -1) { + src[byte_idx] ^= (1 << bit_idx); + } +#endif + if (ret < 0) + return mockfs_to_quakey_error(ret); + num = ret; + } else { + return HOST_ERROR_BADIDX; + } + + return num; +} + +static int host_recv(Host *host, int desc_idx, char *dst, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); +#endif + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = recv_inner(desc, dst, len); + } else { + if (!is_socket(desc)) + return HOST_ERROR_NOTSOCK; + TODO; + } + + return num; +} + +static int host_send(Host *host, int desc_idx, char *src, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + +#ifdef FAULT_INJECTION + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); +#endif + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = send_inner(desc, src, len); + } else { + TODO; + } + + return num; +} + +static int host_mkdir(Host *host, char *path) +{ + int ret = mockfs_mkdir(host->mfs, path, strlen(path)); + if (ret < 0) + return mockfs_to_quakey_error(ret); + return 0; +} + +static int host_remove(Host *host, char *path) +{ + int ret = mockfs_remove(host->mfs, path, strlen(path), false); + if (ret < 0) + return mockfs_to_quakey_error(ret); + return 0; +} + +static int host_rename(Host *host, char *oldpath, char *newpath) +{ + int ret = mockfs_rename(host->mfs, oldpath, strlen(oldpath), newpath, strlen(newpath)); + if (ret < 0) + return mockfs_to_quakey_error(ret); + return 0; +} + +static int host_fileinfo(Host *host, int desc_idx, FileInfo *info) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + switch (desc->type) { + case DESC_FILE: + { + int size = mockfs_file_size(&desc->file); + if (size < 0) + return HOST_ERROR_IO; + info->size = size; + info->is_dir = false; + } + break; + case DESC_DIRECTORY: + { + info->size = 0; + info->is_dir = true; + } + break; + default: + return HOST_ERROR_BADIDX; + } + + return 0; +} + +static int host_lseek(Host *host, int desc_idx, int64_t offset, int whence) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + + int lfs_whence; + switch (whence) { + case HOST_SEEK_SET: + lfs_whence = MOCKFS_SEEK_SET; + break; + case HOST_SEEK_CUR: + lfs_whence = MOCKFS_SEEK_CUR; + break; + case HOST_SEEK_END: + lfs_whence = MOCKFS_SEEK_END; + break; + default: + return HOST_ERROR_BADARG; + } + + int ret = mockfs_lseek(&desc->file, offset, lfs_whence); + if (ret < 0) + return HOST_ERROR_BADARG; + + return ret; +} + +static int host_fsync(Host *host, int desc_idx) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + +#ifdef FAULT_INJECTION + uint64_t roll = sim_random(host->sim) % 100; + if (roll == 0) + return HOST_ERROR_IO; +#endif + + int ret = mockfs_sync(&desc->file); + if (ret < 0) + return mockfs_to_quakey_error(ret); + + return 0; +} + +static int host_ftruncate(Host *host, int desc_idx, int new_size) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + + int ret = mockfs_ftruncate(&desc->file, new_size); + if (ret < 0) + return mockfs_to_quakey_error(ret); + + return 0; +} + +static int host_setdescflags(Host *host, int desc_idx, int flags) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + // TODO: check the descriptor type + + desc->non_blocking = (flags & HOST_FLAG_NONBLOCK) != 0; + return 0; +} + +static int host_getdescflags(Host *host, int desc_idx) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + // TODO: check the descriptor type + + int flags = 0; + if (desc->non_blocking) + flags |= HOST_FLAG_NONBLOCK; + + return flags; +} + +///////////////////////////////////////////////////////////////// +// Time Event Code + +static void append_event(Sim *sim, TimeEvent event) +{ + if (sim->num_events == sim->max_events) { + int n = 2 * sim->max_events; + if (n == 0) + n = 8; + TimeEvent *p = realloc(sim->events, n * sizeof(TimeEvent)); + if (p == NULL) { + TODO; + } + sim->events = p; + sim->max_events = n; + } + + sim->events[sim->num_events++] = event; +} + +static void time_event_wakeup(Sim *sim, Nanos time, Host *host) +{ + TimeEvent event = { + .type = EVENT_TYPE_WAKEUP, + .time = time, + .host = host, + .src_desc = NULL, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static void time_event_connect(Sim *sim, Nanos time, Desc *desc) +{ + TimeEvent event = { + .type = EVENT_TYPE_CONNECT, + .time = time, + .host = NULL, + .src_desc = desc, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static b32 remove_connect_event(Sim *sim, Desc *desc) +{ + int i = 0; + while (i < sim->num_events && (sim->events[i].type != EVENT_TYPE_CONNECT || sim->events[i].src_desc != desc)) + i++; + + if (i == sim->num_events) + return false; + + sim->events[i] = sim->events[--sim->num_events]; + return true; +} + +// Remove all events that target a specific descriptor (DISCONNECT and DATA events) +static void remove_events_targeting_desc(Sim *sim, Desc *desc) +{ + int i = 0; + while (i < sim->num_events) { + TimeEvent *event = &sim->events[i]; + if (event->dst_desc == desc) { + // Free any resources associated with the event + if (event->type == EVENT_TYPE_DATA && event->data_queue) { + socket_queue_unref(event->data_queue); + } + // Remove by swapping with last element + sim->events[i] = sim->events[--sim->num_events]; + // Don't increment i - need to check the swapped element + } else { + i++; + } + } +} + +static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst) +{ + if (remove_connect_event(sim, desc)) + return; + + if (desc->peer == NULL) + return; + + TimeEvent event = { + .type = EVENT_TYPE_DISCONNECT, + .time = time, + .host = NULL, + .src_desc = NULL, + .dst_desc = desc->peer, + .data_count = 0, + .data_queue = NULL, + .rst = rst, + }; + append_event(sim, event); +} + +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc) +{ + TimeEvent event = { + .type = EVENT_TYPE_DATA, + .time = time, + .host = NULL, + .src_desc = NULL, + .dst_desc = desc->peer, + .data_count = socket_queue_used(desc->output), + .data_queue = socket_queue_ref(desc->output), + .rst = false, + }; + append_event(sim, event); +} + +static void time_event_free(TimeEvent *event) +{ + if (event->data_queue) + socket_queue_unref(event->data_queue); +} + +static void time_event_restart(Sim *sim, Nanos time, Host *host) +{ + TimeEvent event = { + .type = EVENT_TYPE_RESTART, + .time = time, + .host = host, + .src_desc = NULL, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static int sim_find_host(Sim *sim, Addr addr); + +static b32 time_event_process(TimeEvent *event, Sim *sim) +{ + b32 consumed = true; + switch (event->type) { + case EVENT_TYPE_CONNECT: + { + Desc *src_desc = event->src_desc; + assert(event->dst_desc == NULL); + + int idx = sim_find_host(sim, src_desc->connect_addr); + if (idx < 0) { + src_desc->connect_status = CONNECT_STATUS_NOHOST; + break; + } + Host *peer_host = sim->hosts[idx]; + + Desc *peer = host_find_desc_bound_to(peer_host, + src_desc->connect_addr, src_desc->connect_port); + if (peer == NULL) { + // Peer host exists but the port isn't open. Reset the connection. + src_desc->connect_status = CONNECT_STATUS_RESET; + break; + } + + assert(peer->type == DESC_SOCKET_L); + if (accept_queue_push(&peer->accept_queue, src_desc) < 0) { + // Accept queue is full + src_desc->connect_status = CONNECT_STATUS_RESET; + break; + } + + src_desc->connect_status = CONNECT_STATUS_DONE; + src_desc->peer = peer; // Resolved! + } + break; + case EVENT_TYPE_DISCONNECT: + { + Desc *dst_desc = event->dst_desc; + assert(dst_desc->type == DESC_SOCKET_C); + assert(event->src_desc == NULL); + dst_desc->peer = NULL; + dst_desc->connect_status = event->rst + ? CONNECT_STATUS_RESET + : CONNECT_STATUS_CLOSE; + } + break; + case EVENT_TYPE_DATA: + { + Desc *dst_desc = event->dst_desc; + assert(dst_desc->type == DESC_SOCKET_C); + assert(event->src_desc == NULL); + + int num = socket_queue_move(dst_desc->input, event->data_queue, event->data_count); + if (num < 0) { + TODO; + } + + event->data_count -= num; + if (event->data_count == 0) { + socket_queue_unref(event->data_queue); + } else { + // Reschedule to future time so time can advance + event->time = sim->current_time + 10000000; + consumed = false; + } + } + break; + case EVENT_TYPE_WAKEUP: + event->host->timedout = true; + break; + case EVENT_TYPE_RESTART: + host_restart(event->host); + break; + } + return consumed; +} + +///////////////////////////////////////////////////////////////// +// Sim Code + +static void sim_init(Sim *sim, uint64_t seed) +{ + sim->seed = seed; + sim->current_time = 0; + sim->num_hosts = 0; + sim->max_hosts = 0; + sim->hosts = NULL; + sim->num_events = 0; + sim->max_events = 0; + sim->events = NULL; + sim->num_signals = 0; + sim->head_signal = 0; +} + +static void sim_free(Sim *sim) +{ + for (int i = 0; i < sim->num_hosts; i++) + host_free(sim->hosts[i]); + free(sim->hosts); + + for (int i = 0; i < sim->num_events; i++) + time_event_free(&sim->events[i]); + free(sim->events); +} + +static Host *sim_spawn(Sim *sim, QuakeySpawn config, char *arg) +{ + if (sim->num_hosts == sim->max_hosts) { + int n = 2 * sim->max_hosts; + if (n == 0) + n = 8; + Host **p = realloc(sim->hosts, n * sizeof(Host*)); + if (p == NULL) { + TODO; + } + sim->hosts = p; + sim->max_hosts = n; + } + + Host *host = malloc(sizeof(Host)); + if (host == NULL) { + TODO; + } + host_init(host, sim, config, arg); + + sim->hosts[sim->num_hosts++] = host; + return host; +} + +static int sim_find_host(Sim *sim, Addr addr) +{ + for (int i = 0; i < sim->num_hosts; i++) + if (!sim->hosts[i]->dead && host_has_addr(sim->hosts[i], addr)) + return i; + return -1; +} + +static void advance_time_to_next_event(Sim *sim) +{ + if (sim->num_events == 0) + return; + + Nanos lowest_time = sim->events[0].time; + for (int i = 1; i < sim->num_events; i++) + if (lowest_time > sim->events[i].time) + lowest_time = sim->events[i].time; + + sim->current_time = lowest_time; +} + +static void process_events_at_current_time(Sim *sim) +{ + bool deferred_disconnect = false; + for (int i = 0; i < sim->num_events; i++) { + if (sim->events[i].time == sim->current_time) { + if (sim->events[i].type == EVENT_TYPE_DISCONNECT) { + deferred_disconnect = true; + } else { + if (time_event_process(&sim->events[i], sim)) + sim->events[i--] = sim->events[--sim->num_events]; + } + } + } + + if (deferred_disconnect) { + for (int i = 0; i < sim->num_events; i++) { + if (sim->events[i].time == sim->current_time) + if (sim->events[i].type == EVENT_TYPE_DISCONNECT) { + if (time_event_process(&sim->events[i], sim)) + sim->events[i--] = sim->events[--sim->num_events]; + } + } + } +} + +static int find_first_ready_host(Sim *sim) +{ + int i = 0; + while (i < sim->num_hosts && !host_ready(sim->hosts[i])) + i++; + if (i == sim->num_hosts) + return -1; + return i; +} + +static void move_host_to_last(Sim *sim, int idx) +{ + assert(idx > -1 && idx < sim->num_hosts); + + Host *host = sim->hosts[idx]; + for (int i = idx; i < sim->num_hosts-1; i++) + sim->hosts[i] = sim->hosts[i+1]; + sim->hosts[sim->num_hosts-1] = host; +} + +static b32 sim_update(Sim *sim) +{ + if (sim->num_hosts == 0) + return false; + + int host_idx; + for (;;) { + + // Schedule the first host that's ready. + // + // If all host are waiting, advance the time to the + // next timed event and try again. + + host_idx = find_first_ready_host(sim); + if (host_idx > -1) + break; + + for (int i = 0; i < sim->num_hosts; i++) + sim->hosts[i]->blocked = false; + + advance_time_to_next_event(sim); + process_events_at_current_time(sim); + } + + move_host_to_last(sim, host_idx); + + Host *host = sim->hosts[sim->num_hosts-1]; + + host_update(host); + if (host_ready(host)) + host->blocked = true; + +#ifdef FAULT_INJECTION + // Crash failure injection: with a small probability, crash a random + // live host. The host will be restarted after a random delay (1-10 + // seconds), simulating a reboot. The disk contents persist across + // crashes, but all volatile state (memory, connections) is lost. + if (sim->num_hosts > 1) { + uint64_t rng = sim_random(sim); + if ((rng % 10000) == 0) { + // Pick a random live host to crash + int live_count = 0; + for (int i = 0; i < sim->num_hosts; i++) + if (!sim->hosts[i]->dead) + live_count++; + + // Only crash if at least 2 hosts are alive (keep a majority) + if (live_count >= 2) { + // Pick one of the live hosts at random + int target = sim_random(sim) % live_count; + int j = 0; + for (int i = 0; i < sim->num_hosts; i++) { + if (!sim->hosts[i]->dead) { + if (j == target) { + host_crash(sim->hosts[i]); + // Schedule restart after 1-10 seconds + Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL); + time_event_restart(sim, sim->current_time + restart_delay, sim->hosts[i]); + break; + } + j++; + } + } + } + } + } +#endif + + return true; +} + +static uint64_t sim_random(Sim *sim) +{ + uint64_t x = sim->seed; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + sim->seed = x; + return x; +} + +static void sim_signal(Sim *sim, char *name) +{ + int tail_signal = (sim->head_signal + sim->num_signals) % SIM_SIGNAL_LIMIT; + QuakeySignal *signal = &sim->signals[tail_signal]; + + int name_len = strlen(name); + if (name_len > (int) sizeof(sim->signals[sim->num_signals].name)) + abort_("Signal name array is too small"); + + memcpy(signal->name, name, name_len); + signal->name_len = name_len; + + sim->num_signals++; +} + +static int sim_get_signal(Sim *sim, QuakeySignal *out) +{ + if (sim->num_signals == 0) + return 0; + + *out = sim->signals[sim->head_signal]; + + sim->head_signal = (sim->head_signal + 1) % SIM_SIGNAL_LIMIT; + sim->num_signals--; + return 1; +} + +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +// Public Interface + +///////////////////////////////////////////////////////////////// +// Quakey Object + +int quakey_init(Quakey **quakey, QuakeyUInt64 seed) +{ + Sim *sim = malloc(sizeof(Sim)); + if (sim == NULL) + return -1; + sim_init(sim, seed); + if (quakey) { + *quakey = (void*) sim; + } else { + sim_free(sim); + free(sim); + } + return 0; +} + +void quakey_free(Quakey *quakey) +{ + if (quakey) { + sim_free((Sim*) quakey); + free(quakey); + } +} + +QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg) +{ + return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg); +} + +void *quakey_node_state(QuakeyNode node) +{ + Host *host = (void*) node; + return host->state; +} + +int quakey_schedule_one(Quakey *quakey) +{ + return sim_update((Sim*) quakey); +} + +QuakeyUInt64 quakey_random(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to quakey_random() with no node scheduled\n"); + return sim_random(host->sim); +} + +void quakey_signal(char *name) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to quakey_signal() with no node scheduled\n"); + + Sim *sim = host->sim; + if (sim->num_signals == SIM_SIGNAL_LIMIT) + abort_("Signal array is too small"); + + sim_signal(sim, name); +} + +int quakey_get_signal(Quakey *quakey, QuakeySignal *signal) +{ + return sim_get_signal((Sim*) quakey, signal); +} + +int quakey_num_hosts(Quakey *quakey) +{ + Sim *sim = (Sim*) quakey; + return sim->num_hosts; +} + +void *quakey_host_state(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return NULL; + Host *host = sim->hosts[idx]; + if (host->dead) + return NULL; + return host->state; +} + +int quakey_host_is_dead(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return 1; + return sim->hosts[idx]->dead ? 1 : 0; +} + +const char *quakey_host_name(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return NULL; + return sim->hosts[idx]->name; +} + +///////////////////////////////////////////////////////////////// +// Mock System Calls + +int *mock_errno_ptr(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_errno_ptr() with no node scheduled\n"); + + return host_errno_ptr(host); +} + +#ifndef _WIN32 + +int mock_socket(int domain, int type, int protocol) +{ + if (domain != AF_INET && domain != AF_INET6) + abort_("Quakey only supports socket() calls with doman=AF_INET or AF_INET6\n"); + + if (type != SOCK_STREAM) + abort_("Quakey only supports socket() calls with type=SOCK_STREAM\n"); + + if (protocol != 0) + abort_("Quakey only supports socket() calls with protocol=0\n"); + + Host *host = host___; + if (host == NULL) + abort_("Call to mock_socket() with no node scheduled\n"); + + AddrFamily family; + switch (domain) { + case AF_INET: + family = ADDR_FAMILY_IPV4; + break; + case AF_INET6: + family = ADDR_FAMILY_IPV6; + break; + default: + UNREACHABLE; + } + + int ret = host_create_socket(host, family); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + int desc_idx = ret; + + return desc_idx; +} + +int mock_close(int fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_close() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_close() not from Linux\n"); + + int desc_idx = fd; + int ret = host_close(host, desc_idx, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +static int convert_addr(void *addr, size_t addr_len, + Addr *converted_addr, uint16_t *converted_port) +{ + int family = ((struct sockaddr*) addr)->sa_family; + switch (family) { + case AF_INET: + { + if (addr_len != sizeof(struct sockaddr_in)) + return -1; + struct sockaddr_in *p = addr; + converted_addr->family = ADDR_FAMILY_IPV4; + converted_addr->ipv4.data = ntohl(((AddrIPv4*) &p->sin_addr)->data); + *converted_port = ntohs(p->sin_port); + } + break; + case AF_INET6: + { + if (addr_len != sizeof(struct sockaddr_in6)) + return -1; + struct sockaddr_in6 *p = addr; + converted_addr->family = ADDR_FAMILY_IPV6; + converted_addr->ipv6 = *(AddrIPv6*) &p->sin6_addr; // TODO: convert to host byte order + *converted_port = ntohs(p->sin6_port); + } + break; + default: + abort_("Quakey only supports the AF_INET and AF_INET6 address families"); + } + return 0; +} + +int mock_bind(int fd, void *addr, unsigned long addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_bind() with no node scheduled\n"); + + Addr converted_addr; + uint16_t converted_port; + int ret = convert_addr(addr, addr_len, &converted_addr, &converted_port); + if (ret < 0) { + *host_errno_ptr(host) = EINVAL; + return ret; + } + + int desc_idx = fd; + ret = host_bind(host, desc_idx, converted_addr, converted_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_CANTBIND: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_BADFAM: + *host_errno_ptr(host) = EAFNOSUPPORT; + return -1; + case HOST_ERROR_NOTAVAIL: + *host_errno_ptr(host) = EADDRNOTAVAIL; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +int mock_listen(int fd, int backlog) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_listen() with no node scheduled\n"); + + int desc_idx = fd; + int ret = host_listen(host, desc_idx, backlog); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +int mock_connect(int fd, void *addr, unsigned long addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_connect() with no node scheduled\n"); + + Addr converted_addr; + uint16_t converted_port; + int ret = convert_addr(addr, addr_len, &converted_addr, &converted_port); + if (ret < 0) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // TODO: connect() operations are only allowed on non-blocking + // sockets + + int desc_idx = fd; + ret = host_connect(host, desc_idx, converted_addr, converted_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EISCONN; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EINPROGRESS; + return -1; + } + + *host_errno_ptr(host) = EINPROGRESS; + return -1; +} + +int mock_pipe(int *fds) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_pipe() with no node scheduled\n"); + + int ret = host_create_pipe(host, fds); + if (ret < 0) + return EIO; + + return 0; +} + +static int convert_linux_open_flags_to_mockfs(int flags) +{ + int lfs_flags = 0; + + // Convert other flags + if (flags & O_RDWR) + lfs_flags |= MOCKFS_O_RDWR; + if (flags & O_WRONLY) + lfs_flags |= MOCKFS_O_WRONLY; + if (flags & O_CREAT) + lfs_flags |= MOCKFS_O_CREAT; + if (flags & O_EXCL) + lfs_flags |= MOCKFS_O_EXCL; + if (flags & O_TRUNC) + lfs_flags |= MOCKFS_O_TRUNC; + if (flags & O_APPEND) + lfs_flags |= MOCKFS_O_APPEND; + + return lfs_flags; +} + +int mock_open(char *path, int flags, int mode) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_open() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_open() not from Linux\n"); + + int converted_flags = convert_linux_open_flags_to_mockfs(flags); + + int ret = host_open_file(host, path, converted_flags); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + default: + break; + } + *host_errno_ptr(host) = ENOENT; + return -1; + } + int desc_idx = ret; + + return desc_idx; +} + +int mock_read(int fd, char *dst, int len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_read() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_read() not from Linux\n"); + + int ret = host_read(host, fd, dst, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = EISDIR; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_write(int fd, char *src, int len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_write() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_write() not from Linux\n"); + + int ret = host_write(host, fd, src, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_recv(int fd, char *dst, int len, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_recv() with no node scheduled\n"); + + if (flags) + abort_("Call to mock_recv() with non-zero flags\n"); + + int ret = host_recv(host, fd, dst, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_NOTCONN: + *host_errno_ptr(host) = ENOTCONN; + return -1; + case HOST_ERROR_RESET: + *host_errno_ptr(host) = ECONNRESET; + return -1; + case HOST_ERROR_HANGUP: + *host_errno_ptr(host) = 0; + return 0; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + ASSERT(ret > 0); + return ret; +} + +int mock_send(int fd, char *src, int len, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_send() with no node scheduled\n"); + + if (flags) + abort_("Call to mock_send() with non-zero flags\n"); + + int ret = host_send(host, fd, src, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_RESET: + *host_errno_ptr(host) = ECONNRESET; + return -1; + case HOST_ERROR_HANGUP: + *host_errno_ptr(host) = EPIPE; + return -1; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_accept(int fd, void *addr, socklen_t *addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_accept() with no node scheduled\n"); + + Addr peer_addr; + uint16_t peer_port; + int ret = host_accept(host, fd, &peer_addr, &peer_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + int new_fd = ret; + + // Fill in the address if provided + if (addr != NULL && addr_len != NULL) { + if (peer_addr.family == ADDR_FAMILY_IPV4) { + struct sockaddr_in *sin = addr; + if (*addr_len >= sizeof(struct sockaddr_in)) { + sin->sin_family = AF_INET; + sin->sin_port = peer_port; + memcpy(&sin->sin_addr, &peer_addr.ipv4, sizeof(peer_addr.ipv4)); + *addr_len = sizeof(struct sockaddr_in); + } + } else { + struct sockaddr_in6 *sin6 = addr; + if (*addr_len >= sizeof(struct sockaddr_in6)) { + sin6->sin6_family = AF_INET6; + sin6->sin6_port = peer_port; + memcpy(&sin6->sin6_addr, &peer_addr.ipv6, sizeof(peer_addr.ipv6)); + *addr_len = sizeof(struct sockaddr_in6); + } + } + } + + return new_fd; +} + +int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen) +{ + if (level != SOL_SOCKET) + abort_("Call to mock_getsockopt() with level other than SOL_SOCKET\n"); + + if (optname != SO_ERROR) + abort_("Call to mock_getsockopt() with option other than SO_ERROR\n"); + + Host *host = host___; + if (host == NULL) + abort_("Call to mock_getsockopt() with no node scheduled\n"); + + ConnectStatus status; + int ret = host_connect_status(host, fd, &status); + if (ret < 0) { + TODO; + } + + int out; + switch (status) { + case CONNECT_STATUS_WAIT: + out = 0; + break; + case CONNECT_STATUS_DONE: + out = 0; + break; + case CONNECT_STATUS_RESET: + out = ECONNRESET; + break; + case CONNECT_STATUS_CLOSE: + out = ECONNRESET; + break; + case CONNECT_STATUS_NOHOST: + out = ETIMEDOUT; + break; + default: + UNREACHABLE; + break; + } + + if (*optlen < sizeof(out)) + memcpy(optval, &out, *optlen); + else { + memcpy(optval, &out, sizeof(out)); + *optlen = sizeof(out); + } + return 0; +} + +int mock_remove(char *path) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_remove() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_remove() not from Linux\n"); + + int ret = host_remove(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ENOTEMPTY; + break; + default: + *host_errno_ptr(host) = EIO; + break; + } + return -1; + } + + return 0; +} + +int mock_rename(char *oldpath, char *newpath) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_rename() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_rename() not from Linux\n"); + + int ret = host_rename(host, oldpath, newpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + break; + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ENOTEMPTY; + break; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = EISDIR; + break; + default: + *host_errno_ptr(host) = EIO; + break; + } + return -1; + } + + return 0; +} + +int mock_clock_gettime(clockid_t clockid, struct timespec *tp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_clock_gettime() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_clock_gettime() not from Linux\n"); + + if (tp == NULL) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // Both CLOCK_REALTIME and CLOCK_MONOTONIC use the same + // simulated time. In simulation, they're equivalent since + // we don't model wall-clock vs monotonic differences. + if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // Get current time + Nanos now = host_time(host); + + // Convert nanoseconds to timespec + // 1 second = 1,000,000,000 nanoseconds + tp->tv_sec = (time_t) (now / 1000000000ULL); + tp->tv_nsec = (int64_t) (now % 1000000000ULL); + + return 0; +} + +int mock_flock(int fd, int op) +{ + // TODO + return 0; +} + +int mock_fsync(int fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fsync() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fsync() not from Linux\n"); + + int ret = host_fsync(host, fd); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) + *host_errno_ptr(host) = EBADF; + else + *host_errno_ptr(host) = EINVAL; + return -1; + } + + return 0; +} + +int mock_ftruncate(int fd, size_t new_size) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_ftruncate() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_ftruncate() not from Linux\n"); + + int ret = host_ftruncate(host, fd, (int) new_size); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX || ret == HOST_ERROR_BADF) + *host_errno_ptr(host) = EBADF; + else if (ret == HOST_ERROR_NOSPC) + *host_errno_ptr(host) = ENOSPC; + else + *host_errno_ptr(host) = EINVAL; + return -1; + } + + return 0; +} + +off_t mock_lseek(int fd, off_t offset, int whence) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_lseek() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_lseek() not from Linux\n"); + + // Convert POSIX whence to HOST whence + int host_whence; + switch (whence) { + case SEEK_SET: + host_whence = HOST_SEEK_SET; + break; + case SEEK_CUR: + host_whence = HOST_SEEK_CUR; + break; + case SEEK_END: + host_whence = HOST_SEEK_END; + break; + default: + *host_errno_ptr(host) = EINVAL; + return (off_t)-1; + } + + int ret = host_lseek(host, fd, offset, host_whence); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) + *host_errno_ptr(host) = EBADF; + else + *host_errno_ptr(host) = EINVAL; + return (off_t)-1; + } + + return (off_t)ret; +} + +int mock_fstat(int fd, struct stat *buf) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fstat() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fstat() not from Linux\n"); + + if (buf == NULL) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + FileInfo info; + int ret = host_fileinfo(host, fd, &info); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) { + *host_errno_ptr(host) = EBADF; + } else { + *host_errno_ptr(host) = EIO; + } + return -1; + } + + memset(buf, 0, sizeof(*buf)); + + if (info.is_dir) { + buf->st_mode = S_IFDIR | 0755; // Directory with rwxr-xr-x permissions + buf->st_size = 0; + } else { + buf->st_mode = S_IFREG | 0644; // Regular file with rw-r--r-- permissions + buf->st_size = (off_t) info.size; + } + + return 0; +} + +int mock_mkstemp(char *path) +{ + TODO; +} + +char *mock_realpath(char *path, char *dst) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_realpath() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_realpath() not from Linux\n"); + + if (path == NULL) { + *host_errno_ptr(host) = EINVAL; + return NULL; + } + + // Temporary buffer for path normalization + char temp[4096]; + int temp_len = 0; + + // Copy path to temp + for (int i = 0; path[i] != '\0' && temp_len < (int)sizeof(temp) - 1; i++) { + temp[temp_len++] = path[i]; + } + temp[temp_len] = '\0'; + + // Result buffer for the normalized absolute path + char result[4096]; + int result_len = 0; + + // If path doesn't start with '/', prepend '/' (mock has no CWD, uses root) + const char *src = temp; + if (temp[0] != '/') { + result[result_len++] = '/'; + } + + // Parse path components and resolve . and .. + while (*src != '\0') { + // Skip consecutive slashes + while (*src == '/') src++; + + if (*src == '\0') break; + + // Find end of this component + const char *end = src; + while (*end != '\0' && *end != '/') end++; + + int comp_len = (int)(end - src); + + if (comp_len == 1 && src[0] == '.') { + // Current directory - skip it + } else if (comp_len == 2 && src[0] == '.' && src[1] == '.') { + // Parent directory - remove last component from result + if (result_len > 1) { + // Find the last slash before the current position + result_len--; // Move back from current position + while (result_len > 0 && result[result_len - 1] != '/') { + result_len--; + } + if (result_len == 0) { + result_len = 1; // Keep the root slash + } + } + } else { + // Regular component - add it + if (result_len > 1 || (result_len == 1 && result[0] != '/')) { + if (result_len < (int)sizeof(result) - 1) + result[result_len++] = '/'; + } + for (int i = 0; i < comp_len && result_len < (int)sizeof(result) - 1; i++) { + result[result_len++] = src[i]; + } + } + + src = end; + } + + // Ensure we have at least root + if (result_len == 0) { + result[result_len++] = '/'; + } + result[result_len] = '\0'; + + // Unlike _fullpath, realpath requires the path to exist + // Try to open as file first, then as directory + int fd = host_open_file(host, result, MOCKFS_O_RDONLY); + if (fd >= 0) { + host_close(host, fd, false); + } else { + // Try as directory + fd = host_open_dir(host, result); + if (fd >= 0) { + host_close(host, fd, false); + } else { + // Path doesn't exist + *host_errno_ptr(host) = ENOENT; + return NULL; + } + } + + // Allocate buffer if dst is NULL + if (dst == NULL) { + dst = malloc(result_len + 1); + if (dst == NULL) { + *host_errno_ptr(host) = ENOMEM; + return NULL; + } + } + + // Copy result to destination + for (int i = 0; i <= result_len; i++) { + dst[i] = result[i]; + } + + return dst; +} + +int mock_mkdir(char *path, mode_t mode) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_mkdir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_mkdir() not from Linux\n"); + + // LittleFS doesn't use mode, but we accept it for API compatibility + (void) mode; + + int ret = host_mkdir(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + return -1; + case HOST_ERROR_NOENT: + // Parent directory doesn't exist + *host_errno_ptr(host) = ENOENT; + return -1; + default: + *host_errno_ptr(host) = EIO; + return -1; + } + } + + return 0; +} + +int mock_fcntl(int fd, int cmd, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fcntl() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fcntl() not from Linux\n"); + + switch (cmd) { + + case F_GETFL: + { + int ret = host_getdescflags(host, fd); + if (ret < 0) { + *host_errno_ptr(host) = EBADF; + return -1; + } + + int flags = 0; + if (ret & HOST_FLAG_NONBLOCK) + flags |= O_NONBLOCK; + + return flags; + } + break; + + case F_SETFL: + { + int host_flags = 0; + if (flags & O_NONBLOCK) + host_flags |= HOST_FLAG_NONBLOCK; + + int ret = host_setdescflags(host, fd, host_flags); + + if (ret < 0) { + *host_errno_ptr(host) = EBADF; + return -1; + } + return 0; + } + break; + + default: + *host_errno_ptr(host) = EINVAL; + return -1; + } +} + +typedef struct { + int fd; // Descriptor index + struct dirent entry; // Current entry (returned by readdir) +} DIR_; + +DIR *mock_opendir(char *name) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_opendir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_opendir() not from Linux\n"); + + int ret = host_open_dir(host, name); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return NULL; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + return NULL; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = EIO; + return NULL; + } + } + + // Allocate DIR structure + DIR_ *dirp = malloc(sizeof(DIR_)); + if (dirp == NULL) { + // Close the descriptor since we can't return it + host_close(host, ret, false); + *host_errno_ptr(host) = EMFILE; + return NULL; + } + + dirp->fd = ret; + return (DIR*) dirp; +} + +struct dirent* mock_readdir(DIR *dirp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_readdir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_readdir() not from Linux\n"); + + DIR_ *dirp_ = (DIR_*) dirp; + + if (dirp_ == NULL) { + *host_errno_ptr(host) = EBADF; + return NULL; + } + + DirEntry entry; + int ret = host_read_dir(host, dirp_->fd, &entry); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return NULL; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EBADF; + return NULL; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = EIO; + return NULL; + } + } + + if (ret == 0) { + // End of directory - return NULL without setting errno + return NULL; + } + + // Copy to the DIR's entry buffer + int i = 0; + while (entry.name[i] != '\0' && i < 255) { + dirp_->entry.d_name[i] = entry.name[i]; + i++; + } + dirp_->entry.d_name[i] = '\0'; + dirp_->entry.d_type = entry.is_dir ? DT_DIR : DT_REG; + + return &dirp_->entry; +} + +int mock_closedir(DIR *dirp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_closedir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_closedir() not from Linux\n"); + + DIR_ *dirp_ = (DIR_*) dirp; + + if (dirp_ == NULL) { + *host_errno_ptr(host) = EBADF; + return -1; + } + + int ret = host_close(host, dirp_->fd, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + free(dirp_); + return -1; + default: + *host_errno_ptr(host) = EIO; + free(dirp_); + return -1; + } + } + + free(dirp_); + return 0; +} + +#else + +int mock_GetLastError(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_GetLastError() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_GetLastError() not from Windows\n"); + + // Note that technically on windows errno and GetLastError + // are different things. Here we use errno_ to store the + // GetLastError value and assume the user will not access + // errno. + return *host_errno_ptr(host); +} + +int mock_WSAGetLastError(void) +{ + return mock_GetLastError(); +} + +void mock_SetLastError(int err) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_SetLastError() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_SetLastError() not from Windows\n"); + + *host_errno_ptr(host) = err; +} + +void mock_WSASetLastError(int err) +{ + return mock_SetLastError(err); +} + +int mock_closesocket(SOCKET fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_closesocket() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_closesocket() not from Windows\n"); + + int desc_idx = fd; + int ret = host_close(host, desc_idx, true); // expect_socket = true + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_NOTSOCK: + // Windows uses WSAGetLastError(), but for simplicity we just return error + return -1; + default: + break; + } + return -1; + } + + return 0; +} + +int mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp) +{ + TODO; +} + +// Helper function to convert wide string to narrow string (ASCII subset) +static int wchar_to_char(WCHAR *src, char *dst, int dst_size) +{ + int i = 0; + while (src[i] != 0) { + if (i >= dst_size - 1) + return -1; // Buffer too small + if (src[i] > 127) + return -1; // Non-ASCII character + dst[i] = (char) src[i]; + i++; + } + dst[i] = '\0'; + return i; // Return length +} + +// Convert Windows access flags and creation disposition to LFS flags +static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess, + DWORD dwCreationDisposition, bool *truncate) +{ + int lfs_flags = 0; + + // Convert access mode + if ((dwDesiredAccess & GENERIC_READ) && (dwDesiredAccess & GENERIC_WRITE)) + lfs_flags = LFS_O_RDWR; + else if (dwDesiredAccess & GENERIC_WRITE) + lfs_flags = LFS_O_WRONLY; + else + lfs_flags = LFS_O_RDONLY; + + *truncate = false; + + // Convert creation disposition + switch (dwCreationDisposition) { + case CREATE_NEW: + // Creates a new file, fails if file exists + lfs_flags |= LFS_O_CREAT | LFS_O_EXCL; + break; + case CREATE_ALWAYS: + // Creates a new file, always (truncates if exists) + lfs_flags |= LFS_O_CREAT | LFS_O_TRUNC; + *truncate = true; + break; + case OPEN_EXISTING: + // Opens file only if it exists, fails otherwise + // No extra flags needed - LFS will fail if file doesn't exist + break; + case OPEN_ALWAYS: + // Opens file if it exists, creates if it doesn't + lfs_flags |= LFS_O_CREAT; + break; + case TRUNCATE_EXISTING: + // Opens and truncates, fails if file doesn't exist + lfs_flags |= LFS_O_TRUNC; + *truncate = true; + break; + default: + return -1; // Invalid creation disposition + } + + return lfs_flags; +} + +HANDLE mock_CreateFileW(WCHAR *lpFileName, + DWORD dwDesiredAccess, DWORD dwShareMode, + LPSECURITY_ATTRIBUTES lpSecurityAttributes, + DWORD dwCreationDisposition, + DWORD dwFlagsAndAttributes, HANDLE hTemplateFile) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_CreateFileW() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_CreateFileW() not from Windows\n"); + + // lpSecurityAttributes and hTemplateFile are typically NULL + (void) lpSecurityAttributes; + (void) hTemplateFile; + (void) dwShareMode; // Share mode not implemented in simulation + (void) dwFlagsAndAttributes; // Attributes not implemented in simulation + + // Convert wide string path to narrow string + char path[MAX_PATH]; + if (wchar_to_char(lpFileName, path, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + // Convert Windows flags to LFS flags + bool truncate; + int lfs_flags = convert_windows_flags_to_lfs(dwDesiredAccess, dwCreationDisposition, &truncate); + if (lfs_flags < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + int ret = host_open_file(host, path, lfs_flags); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_EXISTS: + // CREATE_NEW with existing file + *host_errno_ptr(host) = ERROR_FILE_EXISTS; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + return INVALID_HANDLE_VALUE; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + int desc_idx = ret; + + // For OPEN_ALWAYS, if the file already existed, set ERROR_ALREADY_EXISTS + // (but still return success). This is Windows behavior. + if (dwCreationDisposition == OPEN_ALWAYS) { + // We can't easily detect this case here, so we skip it for now + // A full implementation would check if the file was newly created + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (HANDLE)(long long)desc_idx; +} + +BOOL mock_CloseHandle(HANDLE handle) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_CloseHandle() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_CloseHandle() not from Windows\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + // CloseHandle is for file handles, not sockets + // (sockets use closesocket on Windows) + int ret = host_close(host, desc_idx, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh) +{ + TODO; +} + +BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh) +{ + TODO; +} + +BOOL mock_FlushFileBuffers(HANDLE handle) +{ + TODO; +} + +BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_ReadFile() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_ReadFile() not from Windows\n"); + + // We don't support overlapped (async) I/O + if (ov != NULL) + abort_("Quakey does not support overlapped I/O in ReadFile\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (dst == NULL && len > 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + int ret = host_read(host, desc_idx, dst, (int)len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_BADARG: + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (num != NULL) + *num = (DWORD)ret; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_WriteFile() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_WriteFile() not from Windows\n"); + + // We don't support overlapped (async) I/O + if (ov != NULL) + abort_("Quakey does not support overlapped I/O in WriteFile\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (src == NULL && len > 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + int ret = host_write(host, desc_idx, src, (int)len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (num != NULL) + *num = (DWORD)ret; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_SetFilePointer() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_SetFilePointer() not from Windows\n"); + + if (hFile == INVALID_HANDLE_VALUE || hFile == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_SET_FILE_POINTER; + } + + int desc_idx = (int)(long long)hFile; + + // Convert Windows move method to HOST whence + int host_whence; + switch (dwMoveMethod) { + case FILE_BEGIN: + host_whence = HOST_SEEK_SET; + break; + case FILE_CURRENT: + host_whence = HOST_SEEK_CUR; + break; + case FILE_END: + host_whence = HOST_SEEK_END; + break; + default: + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_SET_FILE_POINTER; + } + + // Build 64-bit offset + int64_t offset; + if (lpDistanceToMoveHigh != NULL) { + // 64-bit seek: combine high and low parts + offset = ((int64_t)(*lpDistanceToMoveHigh) << 32) | ((uint32_t)lDistanceToMove); + } else { + // 32-bit seek: use signed extension + offset = (int64_t)lDistanceToMove; + } + + int ret = host_lseek(host, desc_idx, offset, host_whence); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_SET_FILE_POINTER; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_NEGATIVE_SEEK; + return INVALID_SET_FILE_POINTER; + default: + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_SET_FILE_POINTER; + } + } + + int64_t new_pos = (int64_t)ret; + + // Set high part if requested + if (lpDistanceToMoveHigh != NULL) + *lpDistanceToMoveHigh = (LONG)(new_pos >> 32); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (DWORD)(new_pos & 0xFFFFFFFF); +} + +BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_GetFileSizeEx() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_GetFileSizeEx() not from Windows\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (buf == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + FileInfo info; + int ret = host_fileinfo(host, desc_idx, &info); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + buf->QuadPart = (LONGLONG)info.size; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_QueryPerformanceCounter() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_QueryPerformanceCounter() not from Windows\n"); + + if (lpPerformanceCount == NULL) + return 0; // FALSE + + // Get current time in nanoseconds and convert to performance counter units + // We use nanoseconds directly as the counter value (frequency = 1,000,000,000) + Nanos now = host_time(host); + lpPerformanceCount->QuadPart = (LONGLONG)now; + + return 1; // TRUE +} + +BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_QueryPerformanceFrequency() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_QueryPerformanceFrequency() not from Windows\n"); + + if (lpFrequency == NULL) + return 0; // FALSE + + // Frequency is 1 billion (nanoseconds per second) + // This matches our counter which counts in nanoseconds + lpFrequency->QuadPart = 1000000000LL; + + return 1; // TRUE +} + +char *mock__fullpath(char *path, char *dst, int cap) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock__fullpath() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock__fullpath() not from Windows\n"); + + if (path == NULL) { + *host_errno_ptr(host) = EINVAL; + return NULL; + } + + // Temporary buffer for path normalization + // We'll build the absolute path here + char temp[4096]; + int temp_len = 0; + + // Copy path to temp, converting backslashes to forward slashes + for (int i = 0; path[i] != '\0' && temp_len < (int)sizeof(temp) - 1; i++) { + if (path[i] == '\\') { + temp[temp_len++] = '/'; + } else { + temp[temp_len++] = path[i]; + } + } + temp[temp_len] = '\0'; + + // Result buffer for the normalized absolute path + char result[4096]; + int result_len = 0; + + // If path doesn't start with '/', prepend '/' (mock has no CWD, uses root) + const char *src = temp; + if (temp[0] != '/') { + result[result_len++] = '/'; + } + + // Parse path components and resolve . and .. + while (*src != '\0') { + // Skip consecutive slashes + while (*src == '/') src++; + + if (*src == '\0') break; + + // Find end of this component + const char *end = src; + while (*end != '\0' && *end != '/') end++; + + int comp_len = (int)(end - src); + + if (comp_len == 1 && src[0] == '.') { + // Current directory - skip it + } else if (comp_len == 2 && src[0] == '.' && src[1] == '.') { + // Parent directory - remove last component from result + if (result_len > 1) { + // Find the last slash before the current position + result_len--; // Move back from current position + while (result_len > 0 && result[result_len - 1] != '/') { + result_len--; + } + if (result_len == 0) { + result_len = 1; // Keep the root slash + } + } + } else { + // Regular component - add it + if (result_len > 1 || (result_len == 1 && result[0] != '/')) { + if (result_len < (int)sizeof(result) - 1) + result[result_len++] = '/'; + } + for (int i = 0; i < comp_len && result_len < (int)sizeof(result) - 1; i++) { + result[result_len++] = src[i]; + } + } + + src = end; + } + + // Ensure we have at least root + if (result_len == 0) { + result[result_len++] = '/'; + } + result[result_len] = '\0'; + + // Allocate buffer if dst is NULL + if (dst == NULL) { + dst = malloc(result_len + 1); + if (dst == NULL) { + *host_errno_ptr(host) = ENOMEM; + return NULL; + } + } else { + // Check if result fits in the provided buffer + if (result_len + 1 > cap) { + *host_errno_ptr(host) = ERANGE; + return NULL; + } + } + + // Copy result to destination + for (int i = 0; i <= result_len; i++) { + dst[i] = result[i]; + } + + return dst; +} + +int mock__mkdir(char *path) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock__mkdir() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock__mkdir() not from Windows\n"); + + int ret = host_mkdir(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + return -1; + case HOST_ERROR_NOENT: + // Parent directory doesn't exist + *host_errno_ptr(host) = ENOENT; + return -1; + default: + *host_errno_ptr(host) = EIO; + return -1; + } + } + + return 0; +} + +// Structure to track Windows find handle state +typedef struct { + int fd; // Descriptor index for the directory +} FindHandle; + +// Helper function to populate WIN32_FIND_DATAA from a DirEntry +static void populate_find_data(WIN32_FIND_DATAA *data, DirEntry *entry) +{ + // Clear the structure + for (int i = 0; i < (int)sizeof(WIN32_FIND_DATAA); i++) + ((char *)data)[i] = 0; + + // Set file attributes + data->dwFileAttributes = entry->is_dir ? FILE_ATTRIBUTE_DIRECTORY : FILE_ATTRIBUTE_NORMAL; + + // Copy filename + int i = 0; + while (entry->name[i] != '\0' && i < MAX_PATH - 1) { + data->cFileName[i] = entry->name[i]; + i++; + } + data->cFileName[i] = '\0'; +} + +HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindFirstFileA() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindFirstFileA() not from Windows\n"); + + if (lpFileName == NULL || lpFindFileData == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + // Extract directory path from the search pattern + // The pattern is typically "path\*" or "path\*.ext" + // We need to find the last path separator and extract the directory + char dirpath[MAX_PATH]; + int len = 0; + while (lpFileName[len] != '\0' && len < MAX_PATH - 1) { + dirpath[len] = lpFileName[len]; + len++; + } + dirpath[len] = '\0'; + + // Find the last path separator (either '/' or '\') + int last_sep = -1; + for (int i = 0; i < len; i++) { + if (dirpath[i] == '/' || dirpath[i] == '\\') + last_sep = i; + } + + // If we found a separator, truncate to get the directory path + // If the pattern is just "*", use "." as the directory + if (last_sep >= 0) { + dirpath[last_sep] = '\0'; + } else { + // No separator found - use current directory + dirpath[0] = '.'; + dirpath[1] = '\0'; + } + + // Open the directory + int ret = host_open_dir(host, dirpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_PATH_NOT_FOUND; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + // Allocate find handle structure + FindHandle *fh = malloc(sizeof(FindHandle)); + if (fh == NULL) { + host_close(host, ret, false); + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + } + fh->fd = ret; + + // Read the first entry + DirEntry entry; + int read_ret = host_read_dir(host, fh->fd, &entry); + if (read_ret < 0) { + host_close(host, fh->fd, false); + free(fh); + switch (read_ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_HANDLE_VALUE; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + if (read_ret == 0) { + // Empty directory - no files found + host_close(host, fh->fd, false); + free(fh); + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + return INVALID_HANDLE_VALUE; + } + + // Populate the find data structure + populate_find_data(lpFindFileData, &entry); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (HANDLE)fh; +} + +BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindNextFileA() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindNextFileA() not from Windows\n"); + + if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL || lpFindFileData == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + FindHandle *fh = (FindHandle *)hFindFile; + + // Read the next entry + DirEntry entry; + int ret = host_read_dir(host, fh->fd, &entry); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (ret == 0) { + // No more files + *host_errno_ptr(host) = ERROR_NO_MORE_FILES; + return 0; // FALSE + } + + // Populate the find data structure + populate_find_data(lpFindFileData, &entry); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_FindClose(HANDLE hFindFile) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindClose() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindClose() not from Windows\n"); + + if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + FindHandle *fh = (FindHandle *)hFindFile; + + int ret = host_close(host, fh->fd, false); + if (ret < 0) { + free(fh); + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + free(fh); + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_MoveFileExW() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_MoveFileExW() not from Windows\n"); + + // Validate parameters + if (lpExistingFileName == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // lpNewFileName can be NULL only with MOVEFILE_DELAY_UNTIL_REBOOT + // (marks file for deletion on reboot), but we don't support that + if (lpNewFileName == NULL) { + if (dwFlags & MOVEFILE_DELAY_UNTIL_REBOOT) { + // We don't simulate reboot, so just succeed without doing anything + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE + } + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // Convert wide string paths to narrow strings + char oldpath[MAX_PATH]; + char newpath[MAX_PATH]; + + if (wchar_to_char(lpExistingFileName, oldpath, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + if (wchar_to_char(lpNewFileName, newpath, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // If MOVEFILE_REPLACE_EXISTING is not set and destination exists, fail + // We need to check this before calling host_rename + if (!(dwFlags & MOVEFILE_REPLACE_EXISTING)) { + // Try to check if destination exists by attempting to open it + int check = host_open_file(host, newpath, LFS_O_RDONLY); + if (check >= 0) { + // File exists, close it and return error + host_close(host, check, false); + *host_errno_ptr(host) = ERROR_ALREADY_EXISTS; + return 0; // FALSE + } + } + + int ret = host_rename(host, oldpath, newpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + break; + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = ERROR_ALREADY_EXISTS; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + } + return 0; // FALSE + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +#endif + +void *mock_malloc(size_t size) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_malloc() with no node scheduled\n"); +#ifdef FAULT_INJECTION + if ((sim_random(host->sim) % 1000) == 0) + return NULL; +#endif + return malloc(size); +} + +void *mock_realloc(void *ptr, size_t size) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_realloc() with no node scheduled\n"); +#ifdef FAULT_INJECTION + if ((sim_random(host->sim) % 1000) == 0) + return NULL; +#endif + return realloc(ptr, size); +} + +void mock_free(void *ptr) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_free() with no node scheduled\n"); + free(ptr); +} diff --git a/raft/client.c b/raft/client.c new file mode 100644 index 0000000..92211a1 --- /dev/null +++ b/raft/client.c @@ -0,0 +1,225 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "node.h" +#include "client.h" + +#define CLIENT_TRACE(fmt, ...) {} +//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__); + +#define CLIENT_REQUEST_TIMEOUT_SEC 3 + +static uint64_t next_client_id = 1; + +static int +process_message(ClientState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + (void) conn_idx; + + if (type == MESSAGE_TYPE_REDIRECT) { + RedirectMessage redirect_message; + if (msg.len != sizeof(RedirectMessage)) + return -1; + memcpy(&redirect_message, msg.ptr, sizeof(redirect_message)); + + if (redirect_message.leader_idx >= 0 && redirect_message.leader_idx < state->num_servers) { + CLIENT_TRACE("Redirected to leader %d", redirect_message.leader_idx); + state->current_leader = redirect_message.leader_idx; + // Retry immediately with the correct leader + state->pending = false; + } + return 0; + } + + if (!state->pending) + return 0; + + if (type != MESSAGE_TYPE_REPLY) + return 0; + + ReplyMessage reply_message; + if (msg.len != sizeof(ReplyMessage)) + return -1; + memcpy(&reply_message, msg.ptr, sizeof(reply_message)); + + CLIENT_TRACE("Received reply for request %lu", (unsigned long)state->request_id); + + state->pending = false; + return 0; +} + +int client_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + ClientState *state = state_; + + state->num_servers = 0; + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--server")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --server missing value. Usage is --server :\n"); + return -1; + } + if (state->num_servers == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) { + fprintf(stderr, "Malformed : pair for --server option\n"); + return -1; + } + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } + + // Now sort the addresses + addr_sort(state->server_addrs, state->num_servers); + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Client :: Couldn't setup TCP context\n"); + return -1; + } + + state->pending = false; + state->client_id = next_client_id++; + state->request_id = 0; + state->current_leader = 0; + + Time now = get_current_time(); + if (now == INVALID_TIME) { + fprintf(stderr, "Client :: Couldn't get current time\n"); + tcp_context_free(&state->tcp); + return -1; + } + state->last_request_time = now; + + // Connect to all known servers + for (int i = 0; i < state->num_servers; i++) { + if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) { + fprintf(stderr, "Client :: Couldn't connect to server %d\n", i); + tcp_context_free(&state->tcp); + return -1; + } + } + + *timeout = 0; + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int client_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + ClientState *state = state_; + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + if (events[i].type != EVENT_MESSAGE) + continue; + int conn_idx = events[i].conn_idx; + + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + // Timeout: if pending request has no response, try next server + if (state->pending) { + Time request_deadline = state->last_request_time + CLIENT_REQUEST_TIMEOUT_SEC * 1000000000ULL; + if (now >= request_deadline) { + CLIENT_TRACE("Request %lu timed out, trying next server", + (unsigned long)state->request_id); + state->pending = false; + state->current_leader = (state->current_leader + 1) % state->num_servers; + } + } + + // Send a new request if not currently waiting for a response + if (!state->pending) { + int leader = state->current_leader; + int conn_idx = tcp_index_from_tag(&state->tcp, leader); + if (conn_idx < 0) { + // Connection lost, try reconnecting + tcp_connect(&state->tcp, state->server_addrs[leader], leader, NULL); + } else { + state->request_id++; + + RequestMessage request_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REQUEST, + .length = sizeof(RequestMessage), + }, + .oper = OPERATION_A, + .client_id = state->client_id, + .request_id = state->request_id, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &request_message, sizeof(request_message)); + + state->pending = true; + state->last_request_time = now; + } + } + + // Set timeout for next tick + if (state->pending) { + Time request_deadline = state->last_request_time + CLIENT_REQUEST_TIMEOUT_SEC * 1000000000ULL; + *timeout = deadline_to_timeout(request_deadline, now); + } else { + *timeout = 0; // Send next request immediately + } + + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int client_free(void *state_) +{ + ClientState *state = state_; + + tcp_context_free(&state->tcp); + return 0; +} diff --git a/raft/client.h b/raft/client.h new file mode 100644 index 0000000..9f049b8 --- /dev/null +++ b/raft/client.h @@ -0,0 +1,39 @@ +#ifndef CLIENT_INCLUDED +#define CLIENT_INCLUDED + +#include +#include + +#include "config.h" + +typedef struct { + + TCP tcp; + + // True if we are waiting for a response + bool pending; + + Address server_addrs[NODE_LIMIT]; + int num_servers; + + uint64_t client_id; + uint64_t request_id; + + int current_leader; + + Time last_request_time; + +} ClientState; + +struct pollfd; + +int client_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int client_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int client_free(void *state); + +#endif // CLIENT_INCLUDED diff --git a/raft/config.h b/raft/config.h new file mode 100644 index 0000000..811b1ef --- /dev/null +++ b/raft/config.h @@ -0,0 +1,8 @@ +#ifndef CONFIG_INCLUDED +#define CONFIG_INCLUDED + +#define NODE_LIMIT 32 +#define HEARTBEAT_INTERVAL_SEC 1 +#define PRIMARY_DEATH_TIMEOUT_SEC 2 + +#endif // CONFIG_INCLUDED \ No newline at end of file diff --git a/raft/main.c b/raft/main.c new file mode 100644 index 0000000..5a8bf64 --- /dev/null +++ b/raft/main.c @@ -0,0 +1,234 @@ +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS + +#include +#include +#include +#include + +#include "node.h" +#include "client.h" + +static volatile int simulation_running = 1; + +static void sigint_handler(int sig) +{ + (void)sig; + simulation_running = 0; +} + +int main(void) +{ + signal(SIGINT, sigint_handler); + + Quakey *quakey; + int ret = quakey_init(&quakey, 2); + if (ret < 0) + return -1; + + // Client 1 + { + QuakeySpawn config = { + .name = "rndcli1", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, + .addrs = (char*[]) { "127.0.0.2" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); + } + + // Client 2 + { + QuakeySpawn config = { + .name = "rndcli2", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, + .addrs = (char*[]) { "127.0.0.3" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); + } + + // Node 1 + { + QuakeySpawn config = { + .name = "node1", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.4" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080"); + } + + // Node 2 + { + QuakeySpawn config = { + .name = "node2", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.5" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080"); + } + + // Node 3 + { + QuakeySpawn config = { + .name = "node3", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.6" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080"); + } + + while (simulation_running) + quakey_schedule_one(quakey); + + quakey_free(quakey); + return 0; +} + +#endif // MAIN_SIMULATION +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_CLIENT + +#include + +#include "client.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + int ret; + ClientState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = client_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = client_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + client_free(&state); + return 0; +} + +#endif // MAIN_CLIENT +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_SERVER + +#include + +#include "node.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + int ret; + NodeState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = node_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = node_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + node_free(&state); + return 0; +} + +#endif // MAIN_SERVER \ No newline at end of file diff --git a/raft/node.c b/raft/node.c new file mode 100644 index 0000000..3117d9e --- /dev/null +++ b/raft/node.c @@ -0,0 +1,1265 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "node.h" + +//#define NODE_TRACE(fmt, ...) {} +#define NODE_TRACE(fmt, ...) fprintf(stderr, "NODE: " fmt "\n", ##__VA_ARGS__); + +// Format time as seconds with 3 decimal places for trace output +#define TIME_FMT "%.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static const char *message_type_name(uint8_t type) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST_VOTE: return "REQUEST_VOTE"; + case MESSAGE_TYPE_VOTED: return "VOTED"; + case MESSAGE_TYPE_APPEND_ENTRIES: return "APPEND_ENTRIES"; + case MESSAGE_TYPE_APPENDED: return "APPENDED"; + case MESSAGE_TYPE_REQUEST: return "REQUEST"; + case MESSAGE_TYPE_REPLY: return "REPLY"; + case MESSAGE_TYPE_REDIRECT: return "REDIRECT"; + default: return "UNKNOWN"; + } +} + +static const char *role_name(Role role) +{ + switch (role) { + case ROLE_LEADER : return "LEADER"; + case ROLE_FOLLOWER : return "FOLLOWER"; + case ROLE_CANDIDATE: return "CANDIDATE"; + } + return "UNKNOWN"; +} + +static void client_table_init(ClientTable *ct) +{ + ct->count = 0; + ct->capacity = 0; + ct->entries = NULL; +} + +static void client_table_free(ClientTable *ct) +{ + free(ct->entries); +} + +static ClientTableEntry *client_table_find(ClientTable *ct, uint64_t client_id) +{ + for (int i = 0; i < ct->count; i++) + if (ct->entries[i].client_id == client_id) + return &ct->entries[i]; + return NULL; +} + +static int client_table_add(ClientTable *ct, uint64_t client_id, uint64_t request_id, int conn_tag) +{ + if (ct->count == ct->capacity) { + int n = ct->capacity ? 2 * ct->capacity : 8; + void *p = realloc(ct->entries, n * sizeof(ClientTableEntry)); + if (p == NULL) return -1; + ct->capacity = n; + ct->entries = p; + } + ct->entries[ct->count++] = (ClientTableEntry) { + .client_id = client_id, + .last_request_id = request_id, + .pending = true, + .conn_tag = conn_tag, + }; + return 0; +} + +static int self_idx(NodeState *state) +{ + for (int i = 0; i < state->num_nodes; i++) + if (addr_eql(state->node_addrs[i], state->self_addr)) + return i; + UNREACHABLE; +} + +static uint64_t choose_election_timeout(void) +{ + uint64_t base = PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) + return base + quakey_random() % base; +#else + return base + (uint64_t)rand() % base; +#endif +} + +static int set_term_and_vote(NodeState *state, uint64_t term, int voted_for) +{ + state->term = term; + state->voted_for = voted_for; + if (file_set_offset(state->term_and_vote_handle, 0) < 0) + return -1; + if (file_write_exact(state->term_and_vote_handle, (char*) &term, sizeof(term))) + return -1; + if (file_write_exact(state->term_and_vote_handle, (char*) &voted_for, sizeof(voted_for))) + return -1; + if (file_sync(state->term_and_vote_handle) < 0) + return -1; + return 0; +} + +static int send_to_peer_ex(NodeState *state, int peer_idx, MessageHeader *msg, void *extra, int extra_len) +{ + ByteQueue *output; + int conn_idx = tcp_index_from_tag(&state->tcp, peer_idx); + if (conn_idx < 0) { + int ret = tcp_connect(&state->tcp, state->node_addrs[peer_idx], peer_idx, &output); + if (ret < 0) + return -1; + } else { + output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) { + assert(0); + } + } + int header_len = msg->length - extra_len; + byte_queue_write(output, msg, header_len); + if (extra_len > 0) + byte_queue_write(output, extra, extra_len); + return 0; +} + +static int broadcast_to_peers_ex(NodeState *state, MessageHeader *msg, void *extra, int extra_len) +{ + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + if (send_to_peer_ex(state, i, msg, extra, extra_len) < 0) + return -1; + } + return 0; +} + +static int broadcast_to_peers(NodeState *state, MessageHeader *msg) +{ + return broadcast_to_peers_ex(state, msg, NULL, 0); +} + +static void send_vote_response(NodeState *state, int conn_idx, bool value, int candidate_idx) +{ + VotedMessage voted_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_VOTED, + .length = sizeof(VotedMessage), + }, + .term = state->term, + .value = (value == true) ? 1 : 0, + }; + + { + Time t = get_current_time(); + int peer = tcp_get_tag(&state->tcp, conn_idx); + NODE_TRACE("[" TIME_FMT "] node %d (%s) -> node %d: VOTED term=%lu granted=%s", + TIME_VAL(t), self_idx(state), role_name(state->role), + peer, (unsigned long)state->term, value ? "yes" : "no"); + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &voted_message, voted_message.base.length); + + if (value) { + + if (set_term_and_vote(state, state->term, candidate_idx) < 0) { + // I/O error persisting vote; proceed anyway + } + + // Reset election timer when granting a vote (Raft Section 5.2) + Time now = get_current_time(); + if (now == INVALID_TIME) + return; + state->watchdog = now; + state->election_timeout = choose_election_timeout(); + } +} + +static void send_appended_response(NodeState *state, int conn_idx, bool success, int match_index) +{ + AppendedMessage appended_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_APPENDED, + .length = sizeof(AppendedMessage), + }, + .sender_idx = self_idx(state), + .term = state->term, + .success = success ? 1 : 0, + .match_index = match_index, + }; + + { + Time t = get_current_time(); + int peer = tcp_get_tag(&state->tcp, conn_idx); + NODE_TRACE("[" TIME_FMT "] node %d (%s) -> node %d: APPENDED term=%lu success=%s match_index=%d", + TIME_VAL(t), self_idx(state), role_name(state->role), + peer, (unsigned long)state->term, success ? "yes" : "no", match_index); + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &appended_message, appended_message.base.length); +} + +static void send_redirect(NodeState *state, int conn_idx) +{ + RedirectMessage redirect_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REDIRECT, + .length = sizeof(RedirectMessage), + }, + .leader_idx = state->leader_idx, + }; + + { + Time t = get_current_time(); + int tag = tcp_get_tag(&state->tcp, conn_idx); + NODE_TRACE("[" TIME_FMT "] node %d (%s) -> conn %d: REDIRECT leader_idx=%d", + TIME_VAL(t), self_idx(state), role_name(state->role), + tag, state->leader_idx); + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &redirect_message, redirect_message.base.length); +} + +// Apply all committed but not-yet-applied entries to the state machine. +// If this node is the leader, also reply to waiting clients. +static void apply_committed(NodeState *state) +{ + while (state->last_applied < state->commit_index) { + + state->last_applied++; + + { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (%s): applying log entry %d (term %lu)", + TIME_VAL(t), self_idx(state), role_name(state->role), + state->last_applied, + (unsigned long) wal_peek_entry(&state->wal, state->last_applied)->term); + } + + OperationResult result = state_machine_update(&state->state_machine, wal_peek_entry(&state->wal, state->last_applied)->oper); + + if (state->role == ROLE_LEADER) { + + ClientTableEntry *entry = client_table_find(&state->client_table, wal_peek_entry(&state->wal, state->last_applied)->client_id); + if (entry && entry->pending) { + + entry->pending = false; + entry->last_result = result; + + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .result = result, + }; + int ci = tcp_index_from_tag(&state->tcp, entry->conn_tag); + if (ci >= 0) { + + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER) -> client %lu: REPLY for log entry %d", + TIME_VAL(t), self_idx(state), + (unsigned long)entry->client_id, state->last_applied); + + ByteQueue *output = tcp_output_buffer(&state->tcp, ci); + if (output) + byte_queue_write(output, &reply_message, sizeof(reply_message)); + } + } + } + } +} + +static int handle_append_entries(NodeState *state, int conn_idx, + AppendEntriesMessage *append_entries_message, WALEntry *entries) +{ + // Reset election timer + Time now = get_current_time(); + if (now == INVALID_TIME) + return -1; + state->watchdog = now; + state->election_timeout = choose_election_timeout(); + + state->leader_idx = append_entries_message->leader_idx; + + int prev_log_index = append_entries_message->prev_log_index; + uint64_t prev_log_term = append_entries_message->prev_log_term; + + // Log consistency check: verify prev_log_index/prev_log_term + if (prev_log_index >= 0) { + + if (prev_log_index >= wal_entry_count(&state->wal)) { + // We don't have the entry at prev_log_index + send_appended_response(state, conn_idx, false, -1); + return 0; + } + + if (wal_peek_entry(&state->wal, prev_log_index)->term != prev_log_term) { + // Conflicting entry at prev_log_index + send_appended_response(state, conn_idx, false, -1); + return 0; + } + } + + int insert_idx = prev_log_index+1; + if (insert_idx < wal_entry_count(&state->wal)) { + // TODO: What if we're truncating operations that were already applied to the state machine? + if (wal_truncate(&state->wal, insert_idx) < 0) { + send_appended_response(state, conn_idx, false, -1); + return 0; + } + } + + for (int i = 0; i < append_entries_message->entry_count; i++) { + + // Copy in case it's unaligned + WALEntry entry; + memcpy(&entry, &entries[i], sizeof(WALEntry)); + if (wal_append(&state->wal, &entry) < 0) { + send_appended_response(state, conn_idx, false, -1); + return 0; + } + } + + // Now we need to advance the local commit index + // to sync with the leader. + // + // Usually the leader's commit index is greater or equal + // to followers, in which case the follower will just need + // to advance its own to match the leader. But in general, + // it is possible for a follower to receive a greater commit + // index + // + // Say we have a cluster of nodes A, B, C where A is the + // leader: + // A: Replicates log entry 100 to a majority of nodes including B, but not C. + // A: Sets commit_index to 100 and sends AppendEntries to B + // B: Receives AppendEntries and sets commit_index to 100 + // Now A crashes and C is elected: + // C: Entry 100 must be in the log to win the election, but + // it is not committed yet. + // C: Sends AppendEntries message to B with commit_index of 99 + // B: Receives a commit_index of 99 while its own is at 100 + // + // Note that this is handled gracefully as B will + // gradually commit messages until it's up to date + // with other nodes. + + int leader_commit = append_entries_message->leader_commit; + + if (state->commit_index < leader_commit) + state->commit_index = MIN(leader_commit, wal_entry_count(&state->wal)-1); + + apply_committed(state); + + send_appended_response(state, conn_idx, true, wal_entry_count(&state->wal)-1); + return 0; +} + +static void start_election(NodeState *state) +{ + { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: starting election for term %lu", + TIME_VAL(t), self_idx(state), (unsigned long)state->term); + } + + state->role = ROLE_CANDIDATE; + state->votes_received = 1; // Vote for self + + if (set_term_and_vote(state, state->term+1, self_idx(state)) < 0) { + return; // I/O error; retry on next election timeout + } + + Time now = get_current_time(); + if (now != INVALID_TIME) { + state->watchdog = now; + state->election_timeout = choose_election_timeout(); + } + + RequestVoteMessage request_vote_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REQUEST_VOTE, + .length = sizeof(RequestVoteMessage), + }, + .term = state->term, + .sender_idx = self_idx(state), + .last_log_index = wal_entry_count(&state->wal)-1, + .last_log_term = wal_last_term(&state->wal), + }; + + { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (%s) -> ALL: REQUEST_VOTE term=%lu last_log_index=%d last_log_term=%lu", + TIME_VAL(t), self_idx(state), role_name(state->role), + (unsigned long) state->term, wal_entry_count(&state->wal)-1, + (unsigned long) wal_last_term(&state->wal)); + } + + broadcast_to_peers(state, &request_vote_message.base); +} + +// Common pattern: step down to follower when we see a higher term. +static void step_down(NodeState *state, uint64_t new_term) +{ + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (%s): stepping down to FOLLOWER, term %lu -> %lu", + TIME_VAL(t), self_idx(state), role_name(state->role), + (unsigned long) state->term, (unsigned long) new_term); + + state->role = ROLE_FOLLOWER; + + if (set_term_and_vote(state, new_term, -1) < 0) { + // I/O error persisting term; in-memory state already updated + } +} + +// Send AppendEntries to a specific follower, including +// any log entries from next_indices[peer] onward. +static void send_append_entries_to_peer(NodeState *state, int peer_idx) +{ + int next = state->next_indices[peer_idx]; + int prev_index = next - 1; + + uint64_t prev_term = 0; + if (prev_index >= 0 && prev_index < wal_entry_count(&state->wal)) + prev_term = wal_peek_entry(&state->wal, prev_index)->term; + + int count = MAX(wal_entry_count(&state->wal) - next, 0); + + AppendEntriesMessage append_entries_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_APPEND_ENTRIES, + .length = sizeof(AppendEntriesMessage) + count * sizeof(WALEntry), + }, + .term = state->term, + .leader_idx = self_idx(state), + .prev_log_index = prev_index, + .prev_log_term = prev_term, + .leader_commit = state->commit_index, + .entry_count = count, + }; + + { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (%s) -> node %d: APPEND_ENTRIES term=%lu prev_log_index=%d prev_log_term=%lu leader_commit=%d entries=%d", + TIME_VAL(t), self_idx(state), role_name(state->role), + peer_idx, (unsigned long)state->term, prev_index, (unsigned long)prev_term, + state->commit_index, count); + } + + WALEntry *entries = (count > 0) ? wal_peek_entry(&state->wal, next) : NULL; + send_to_peer_ex(state, peer_idx, &append_entries_message.base, entries, count * sizeof(WALEntry)); +} + +static void become_leader(NodeState *state) +{ + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: became LEADER for term %lu (votes=%d/%d)", + TIME_VAL(t), self_idx(state), (unsigned long)state->term, + state->votes_received, state->num_nodes); + + state->role = ROLE_LEADER; + state->leader_idx = self_idx(state); + + // Initialize volatile leader state (Raft Section 5.3) + for (int i = 0; i < state->num_nodes; i++) { + state->next_indices[i] = wal_entry_count(&state->wal); + state->match_indices[i] = -1; + } + + // Append a no-op entry for the current term (Raft dissertation §6.4). + // This ensures entries from previous terms can be committed, since + // Section 5.4.2 only allows committing entries from the current term. + WALEntry noop = { + .term = state->term, + .oper = OPERATION_NOOP, + .client_id = 0, + }; + if (wal_append(&state->wal, &noop) < 0) { + // I/O error; step down and let another node become leader + state->role = ROLE_FOLLOWER; + return; + } + + state->match_indices[self_idx(state)] = wal_entry_count(&state->wal)-1; + + // Send AppendEntries (including the no-op) to establish authority + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + send_append_entries_to_peer(state, i); + } + + Time now = get_current_time(); + if (now == INVALID_TIME) + return; + state->watchdog = now; +} + +// A candidate's log is "at least as up-to-date" if its last +// entry has a higher term, or the same term but equal or +// greater index. +static bool remote_has_recent_state(NodeState *state, + int peer_index, uint64_t peer_term) +{ + uint64_t term = wal_last_term(&state->wal); + + if (peer_term != term) + return peer_term > term; + + return peer_index >= wal_entry_count(&state->wal)-1; +} + +static int +process_message_as_follower(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST_VOTE: + { + RequestVoteMessage request_vote_message; + if (msg.len != sizeof(request_vote_message)) + return -1; + memcpy(&request_vote_message, msg.ptr, sizeof(request_vote_message)); + + // If the request's term is old, the peer is stale + // and we let it know by replying with our own term + // number. + if (request_vote_message.term < state->term) { + send_vote_response(state, conn_idx, false, -1); + break; + } + + // If the request's term is newer, we are staled + // and need to move forward. Then, we can procede + // with evaluating the peer for a vote. + if (request_vote_message.term > state->term) { + if (set_term_and_vote(state, request_vote_message.term, -1) < 0) + break; // I/O error; ignore this request + } + + // Grant vote if we haven't voted yet (or already + // voted for this candidate) and the candidate's + // log is at least as recent as our own. + if (state->voted_for == -1 || state->voted_for == request_vote_message.sender_idx) { + if (remote_has_recent_state(state, + request_vote_message.last_log_index, + request_vote_message.last_log_term)) { + send_vote_response(state, conn_idx, true, request_vote_message.sender_idx); + break; + } + } + + send_vote_response(state, conn_idx, false, -1); + } + break; + case MESSAGE_TYPE_VOTED: + { + // Followers don't expect vote responses. Ignore. + } + break; + case MESSAGE_TYPE_APPEND_ENTRIES: + { + AppendEntriesMessage append_entries_message; + if (msg.len < (int)sizeof(append_entries_message)) + return -1; + memcpy(&append_entries_message, msg.ptr, sizeof(append_entries_message)); + + if (append_entries_message.term < state->term) { + // Stale leader + send_appended_response(state, conn_idx, false, -1); + break; + } + + if (append_entries_message.term > state->term) { + if (set_term_and_vote(state, append_entries_message.term, -1) < 0) + break; // I/O error; ignore this message + } + + return handle_append_entries(state, conn_idx, &append_entries_message, (WALEntry*) (msg.ptr + sizeof(AppendEntriesMessage))); + } + break; + case MESSAGE_TYPE_APPENDED: + { + // Followers don't expect append responses. Ignore. + } + break; + case MESSAGE_TYPE_REQUEST: + { + // Redirect client to the current leader. + // If no leader exists at this time, we tell the client. + send_redirect(state, conn_idx); + } + break; + case MESSAGE_TYPE_REPLY: + { + return -1; + } + break; + case MESSAGE_TYPE_REDIRECT: + { + return -1; + } + break; + } + + return 0; +} + +static int +process_message_as_candidate(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST_VOTE: + { + RequestVoteMessage request_vote_message; + if (msg.len != sizeof(request_vote_message)) + return -1; + memcpy(&request_vote_message, msg.ptr, sizeof(request_vote_message)); + + // If same term, we already voted for ourselves; deny + if (request_vote_message.term == state->term) { + send_vote_response(state, conn_idx, false, -1); + break; + } + + // Stale candidate + if (request_vote_message.term < state->term) { + send_vote_response(state, conn_idx, false, -1); + break; + } + + // Higher term: step down and consider the vote + step_down(state, request_vote_message.term); + + if (remote_has_recent_state(state, + request_vote_message.last_log_index, + request_vote_message.last_log_term)) { + send_vote_response(state, conn_idx, true, request_vote_message.sender_idx); + } else { + send_vote_response(state, conn_idx, false, -1); + } + } + break; + case MESSAGE_TYPE_VOTED: + { + VotedMessage voted_message; + if (msg.len != sizeof(voted_message)) + return -1; + memcpy(&voted_message, msg.ptr, sizeof(voted_message)); + + // Local state is stale + if (voted_message.term > state->term) { + step_down(state, voted_message.term); + break; + } + + // Ignore votes from old terms + if (voted_message.term < state->term) + break; + + if (voted_message.value) { + { + Time t = get_current_time(); + int sender = tcp_get_tag(&state->tcp, conn_idx); + NODE_TRACE("[" TIME_FMT "] node %d (CANDIDATE): received vote from node %d (%d/%d votes for term %lu)", + TIME_VAL(t), self_idx(state), sender, + state->votes_received+1, state->num_nodes, + (unsigned long)state->term); + } + state->votes_received++; + if (state->votes_received > state->num_nodes/2) { + become_leader(state); + } + } + } + break; + case MESSAGE_TYPE_APPEND_ENTRIES: + { + AppendEntriesMessage append_entries_message; + if (msg.len < (int)sizeof(append_entries_message)) + return -1; + memcpy(&append_entries_message, msg.ptr, sizeof(append_entries_message)); + + if (append_entries_message.term < state->term) { + // Stale leader; reject + send_appended_response(state, conn_idx, false, -1); + break; + } + + // A valid leader exists for this or a higher term; step down + step_down(state, append_entries_message.term); + + return handle_append_entries(state, conn_idx, &append_entries_message, (WALEntry*) (msg.ptr + sizeof(AppendEntriesMessage))); + } + break; + case MESSAGE_TYPE_APPENDED: + { + // Candidates don't expect append responses. Ignore. + } + break; + case MESSAGE_TYPE_REQUEST: + { + // Redirect client to the current leader. + // If no leader exists at this time, we tell the client. + send_redirect(state, conn_idx); + } + break; + case MESSAGE_TYPE_REPLY: + { + return -1; + } + break; + case MESSAGE_TYPE_REDIRECT: + { + return -1; + } + break; + } + + return 0; +} + +// Leader: advance commit_index to the highest index replicated +// on a majority of servers, provided that entry's term matches +// the current term (Raft Section 5.4.2). +static void advance_commit_index(NodeState *state) +{ + int arr[NODE_LIMIT]; + for (int i = 0; i < state->num_nodes; i++) + arr[i] = state->match_indices[i]; + + // Simple insertion sort (ascending) + for (int i = 1; i < state->num_nodes; i++) { + int key = arr[i]; + int j = i - 1; + while (j >= 0 && arr[j] > key) { + arr[j + 1] = arr[j]; + j--; + } + arr[j + 1] = key; + } + + // The median value is the highest index replicated on a majority. + // For num_nodes=3: arr[1], for num_nodes=5: arr[2], etc. + int candidate = arr[state->num_nodes / 2]; + + if (candidate <= state->commit_index) + return; + + assert(candidate < wal_entry_count(&state->wal)); + + if (wal_peek_entry(&state->wal, candidate)->term == state->term) + state->commit_index = candidate; +} + +static int +process_message_as_leader(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST_VOTE: + { + RequestVoteMessage request_vote_message; + if (msg.len != sizeof(request_vote_message)) + return -1; + memcpy(&request_vote_message, msg.ptr, sizeof(request_vote_message)); + + if (request_vote_message.term > state->term) { + + step_down(state, request_vote_message.term); + + if (remote_has_recent_state(state, + request_vote_message.last_log_index, + request_vote_message.last_log_term)) { + send_vote_response(state, conn_idx, true, request_vote_message.sender_idx); + } else { + send_vote_response(state, conn_idx, false, -1); + } + + } else { + + // Our term is at least as high; reject + send_vote_response(state, conn_idx, false, -1); + } + } + break; + case MESSAGE_TYPE_VOTED: + { + // Already leader, ignore stray vote responses + } + break; + case MESSAGE_TYPE_APPEND_ENTRIES: + { + AppendEntriesMessage append_entries_message; + if (msg.len < (int)sizeof(append_entries_message)) + return -1; + memcpy(&append_entries_message, msg.ptr, sizeof(append_entries_message)); + + if (append_entries_message.term > state->term) { + // A leader with a higher term exists; step down + step_down(state, append_entries_message.term); + return handle_append_entries(state, conn_idx, &append_entries_message, (WALEntry*) (msg.ptr + sizeof(AppendEntriesMessage))); + } + + // Same or lower term: reject (two leaders in the same term is impossible) + send_appended_response(state, conn_idx, false, -1); + } + break; + case MESSAGE_TYPE_APPENDED: + { + AppendedMessage appended_message; + if (msg.len != sizeof(appended_message)) + return -1; + memcpy(&appended_message, msg.ptr, sizeof(appended_message)); + + // Our state is stale + if (appended_message.term > state->term) { + step_down(state, appended_message.term); + break; + } + + int follower_idx = appended_message.sender_idx; + assert(follower_idx > -1); + assert(follower_idx < state->num_nodes); + + if (appended_message.success) { + + state->match_indices[follower_idx] = appended_message.match_index; + state->next_indices[follower_idx] = appended_message.match_index + 1; + + int old_commit_index = state->commit_index; + + advance_commit_index(state); + + if (state->commit_index > old_commit_index) { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): commit_index advanced %d -> %d", + TIME_VAL(t), self_idx(state), old_commit_index, state->commit_index); + } + + apply_committed(state); + + } else { + + // Log inconsistency: decrement nextIndex and retry + + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): log inconsistency with node %d, decrementing next_index to %d", + TIME_VAL(t), self_idx(state), follower_idx, + state->next_indices[follower_idx] > 0 ? state->next_indices[follower_idx] - 1 : 0); + + state->next_indices[follower_idx] = MAX(0, state->next_indices[follower_idx]-1); + send_append_entries_to_peer(state, follower_idx); + } + } + break; + case MESSAGE_TYPE_REQUEST: + { + RequestMessage request_message; + if (msg.len != sizeof(request_message)) + return -1; + memcpy(&request_message, msg.ptr, sizeof(request_message)); + + // Assign a unique tag to this client connection so we can + // find it later even if the connection array is reordered. + int tag = tcp_get_tag(&state->tcp, conn_idx); + if (tag == -1) { + tag = state->next_client_tag++; + tcp_set_tag(&state->tcp, conn_idx, tag, true); + } + + // Client table deduplication (same pattern as VSR) + ClientTableEntry *entry = client_table_find(&state->client_table, request_message.client_id); + if (entry == NULL) { + if (client_table_add(&state->client_table, request_message.client_id, request_message.request_id, tag) < 0) + break; + } else { + if (entry->pending) + break; // Already processing a request for this client + + if (entry->last_request_id > request_message.request_id) + break; // Stale request + + if (entry->last_request_id == request_message.request_id) { + // Return cached result + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .result = entry->last_result, + }; + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &reply_message, sizeof(reply_message)); + break; + } + + entry->last_request_id = request_message.request_id; + entry->pending = true; + entry->conn_tag = tag; + } + + WALEntry wal_entry = { + .term = state->term, + .client_id = request_message.client_id, + .oper = request_message.oper, + }; + if (wal_append(&state->wal, &wal_entry) < 0) + break; // I/O error; client will retry + + // Update own match index + state->match_indices[self_idx(state)] = wal_entry_count(&state->wal)-1; + + // Replicate to all followers + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + send_append_entries_to_peer(state, i); + } + } + break; + case MESSAGE_TYPE_REPLY: + { + return -1; + } + break; + case MESSAGE_TYPE_REDIRECT: + { + return -1; + } + break; + } + + return 0; +} + +static int +process_message(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + { + Time t = get_current_time(); + int sender = tcp_get_tag(&state->tcp, conn_idx); + NODE_TRACE("[" TIME_FMT "] node %d (%s) <- node/conn %d: recv %s (%d bytes)", + TIME_VAL(t), self_idx(state), role_name(state->role), + sender, message_type_name(type), (int)msg.len); + } + + switch (state->role) { + case ROLE_LEADER: + return process_message_as_leader(state, conn_idx, type, msg); + case ROLE_FOLLOWER: + return process_message_as_follower(state, conn_idx, type, msg); + case ROLE_CANDIDATE: + return process_message_as_candidate(state, conn_idx, type, msg); + } + UNREACHABLE; +} + +int node_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + NodeState *state = state_; + + string wal_file = S("raft.wal"); + string term_and_vote_file = S("term_and_vote.wal"); + + Time now = get_current_time(); + if (now == INVALID_TIME) { + fprintf(stderr, "Node :: Couldn't get current time\n"); + return -1; + } + + /////////////////////////////////////////////////////////////// + // Parse arguments + + bool self_addr_set = false; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--addr")) { + if (self_addr_set) { + fprintf(stderr, "Option --addr specified twice\n"); + return -1; + } + self_addr_set = true; + i++; + if (i == argc) { + fprintf(stderr, "Option --addr missing value. Usage is --addr :\n"); + return -1; + } + int ret = parse_addr_arg(argv[i], &state->self_addr); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --addr option\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + state->node_addrs[state->num_nodes++] = state->self_addr; + } else if (!strcmp(argv[i], "--peer")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --peer missing value. Usage is --peer :\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + int ret = parse_addr_arg(argv[i], &state->node_addrs[state->num_nodes]); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --peer option\n"); + return -1; + } + state->num_nodes++; + } else if (!strcmp(argv[i], "--wal-file")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --wal-file missing value. Usage is --wal-file \n"); + return -1; + } + wal_file = (string) { argv[i], strlen(argv[i]) }; + } else if (!strcmp(argv[i], "--term-and-vote-file")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --term-and-vote-file missing value. Usage is --term-and-vote-file \n"); + return -1; + } + term_and_vote_file = (string) { argv[i], strlen(argv[i]) }; + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } + + if (!self_addr_set) { + printf("Option --addr not specified\n"); + return -1; + } + + // Sort cluster addresses. This allows us to + // globally refer to nodes by their index. + addr_sort(state->node_addrs, state->num_nodes); + + /////////////////////////////////////////////////////////////// + // Initialize term and vote + + bool existed = false; + if (file_exists(term_and_vote_file)) + existed = true; + + Handle term_and_vote_handle; + if (file_open(term_and_vote_file, &term_and_vote_handle) < 0) + return -1; + + uint64_t term = 0; + int voted_for = -1; + if (existed) { + if (file_read_exact(term_and_vote_handle, (char*) &term, sizeof(term)) < 0) { + file_close(term_and_vote_handle); + return -1; + } + if (file_read_exact(term_and_vote_handle, (char*) &voted_for, sizeof(voted_for)) < 0) { + file_close(term_and_vote_handle); + return -1; + } + } + + state->term_and_vote_handle = term_and_vote_handle; + state->term = term; + state->voted_for = voted_for; + + /////////////////////////////////////////////////////////////// + // Initialize WAL and state machine + + state_machine_init(&state->state_machine); + + if (wal_init(&state->wal, wal_file) < 0) { + printf("Couldn't initialize the WAL"); + return -1; + } + + WALReplay wal_replay; + wal_replay_init(&wal_replay, &state->wal); + for (WALEntry *entry; (entry = wal_replay_next(&wal_replay)); ) { + state_machine_update(&state->state_machine, entry->oper); + } + wal_replay_free(&wal_replay); + + /////////////////////////////////////////////////////////////// + // Initialize volatile state + + // Current role of the node + state->role = ROLE_FOLLOWER; + + // The time an AppendEntries was last sent or received + state->watchdog = now; + + // The index of the current leader + state->leader_idx = -1; + + // Index of the last committed operation + state->commit_index = -1; + + // Index of the last operation applied to the state machine + state->last_applied = -1; + + // Number of votes received in the current term + state->votes_received = 0; + + // Nodes pick different election timeouts to reduce the risk + // of split votes + state->election_timeout = choose_election_timeout(); + + for (int i = 0; i < NODE_LIMIT; i++) { + state->next_indices[i] = 0; + state->match_indices[i] = -1; + } + + /////////////////////////////////////////////////////////////// + // Initialize client table + + client_table_init(&state->client_table); + state->next_client_tag = NODE_LIMIT; + + /////////////////////////////////////////////////////////////// + // Initialize networking + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP context\n"); + wal_free(&state->wal); + return -1; + } + + int ret = tcp_listen(&state->tcp, state->self_addr); + if (ret < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP listener\n"); + tcp_context_free(&state->tcp); + wal_free(&state->wal); + return -1; + } + + *timeout = -1; // No timeout until we have chunk servers + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Node :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int node_free(void *state_) +{ + NodeState *state = state_; + + tcp_context_free(&state->tcp); + client_table_free(&state->client_table); + wal_free(&state->wal); + state_machine_free(&state->state_machine); + return 0; +} + +int node_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + NodeState *state = state_; + + Time now = get_current_time(); + if (now == INVALID_TIME) + return -1; + + ///////////////////////////////////////////////////////////////// + // Network events + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + + if (events[i].type != EVENT_MESSAGE) + continue; + int conn_idx = events[i].conn_idx; + + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + ///////////////////////////////////////////////////////////////// + // Time events + + Time deadline = INVALID_TIME; + + if (state->role == ROLE_LEADER) { + Time watchdog_deadline = state->watchdog + HEARTBEAT_INTERVAL_SEC * 1000000000ULL; + if (now >= watchdog_deadline) { + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): heartbeat timeout, sending APPEND_ENTRIES to all peers", + TIME_VAL(now), self_idx(state)); + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + send_append_entries_to_peer(state, i); + } + state->watchdog = now; + } else { + nearest_deadline(&deadline, watchdog_deadline); + } + } else { + // Follower/Candidate: start election on leader timeout + Time death_deadline = state->watchdog + state->election_timeout; + if (now >= death_deadline) { + NODE_TRACE("[" TIME_FMT "] node %d (%s): election timeout expired, triggering election", + TIME_VAL(now), self_idx(state), role_name(state->role)); + start_election(state); + } else { + nearest_deadline(&deadline, death_deadline); + } + } + + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} \ No newline at end of file diff --git a/raft/node.h b/raft/node.h new file mode 100644 index 0000000..80853b1 --- /dev/null +++ b/raft/node.h @@ -0,0 +1,146 @@ +#ifndef NODE_INCLUDED +#define NODE_INCLUDED + +#include +#include +#include + +#include "wal.h" +#include "config.h" + +enum { + MESSAGE_TYPE_REQUEST_VOTE, + MESSAGE_TYPE_VOTED, + MESSAGE_TYPE_APPEND_ENTRIES, + MESSAGE_TYPE_APPENDED, + MESSAGE_TYPE_REQUEST, + MESSAGE_TYPE_REPLY, + MESSAGE_TYPE_REDIRECT, +}; + +typedef struct { + MessageHeader base; + uint64_t term; + int sender_idx; + int last_log_index; + uint64_t last_log_term; +} RequestVoteMessage; + +typedef struct { + MessageHeader base; + uint64_t term; + uint8_t value; +} VotedMessage; + +typedef struct { + MessageHeader base; + uint64_t term; + int leader_idx; + int prev_log_index; + uint64_t prev_log_term; + int leader_commit; + int entry_count; + // Followed by: LogEntry entries[entry_count] +} AppendEntriesMessage; + +typedef struct { + MessageHeader base; + int sender_idx; + uint64_t term; + uint8_t success; + int match_index; +} AppendedMessage; + +typedef struct { + MessageHeader base; + Operation oper; + uint64_t client_id; + uint64_t request_id; +} RequestMessage; + +typedef struct { + MessageHeader base; + OperationResult result; +} ReplyMessage; + +typedef struct { + MessageHeader base; + int leader_idx; +} RedirectMessage; + +typedef struct { + uint64_t client_id; + uint64_t last_request_id; + OperationResult last_result; + bool pending; + int conn_tag; +} ClientTableEntry; + +typedef struct { + int count; + int capacity; + ClientTableEntry *entries; +} ClientTable; + +typedef enum { + ROLE_FOLLOWER, + ROLE_CANDIDATE, + ROLE_LEADER, +} Role; + +typedef struct { + + TCP tcp; + WAL wal; + + Address self_addr; + Address node_addrs[NODE_LIMIT]; + int num_nodes; + + Role role; + + uint64_t term; + int voted_for; + Handle term_and_vote_handle; + + // Index of the current leader + int leader_idx; + + int commit_index; + int last_applied; + + // When CANDIDATE + int votes_received; + + // When LEADER + int next_indices[NODE_LIMIT]; + int match_indices[NODE_LIMIT]; + + // Relative timeout in nanoseconds + uint64_t election_timeout; + + // Last heartbeat time + uint64_t watchdog; + + // Keep track of client request order to enforce + // linearizability + ClientTable client_table; + int next_client_tag; + + // The state machine to be replicated + StateMachine state_machine; + +} NodeState; + +struct pollfd; + +int node_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int node_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int node_free(void *state); + +#endif // NODE_INCLUDED diff --git a/raft/wal.c b/raft/wal.c new file mode 100644 index 0000000..984211f --- /dev/null +++ b/raft/wal.c @@ -0,0 +1,127 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "wal.h" + +int wal_init(WAL *wal, string file) +{ + Handle handle; + if (file_open(file, &handle) < 0) + return -1; + wal->handle = handle; + + size_t size; + if (file_size(handle, &size) < 0) { + file_close(handle); + return -1; + } + + if (size % sizeof(WALEntry) != 0) { + file_close(handle); + return -1; + } + int count = size / sizeof(WALEntry); + + wal->count = count; + wal->capacity = count; + wal->entries = malloc(count * sizeof(WALEntry)); + if (wal->entries == NULL) { + file_close(handle); + return -1; + } + + if (file_read_exact(handle, (char*) wal->entries, count * sizeof(WALEntry)) < 0) { + file_close(handle); + return -1; + } + + return 0; +} + +void wal_free(WAL *wal) +{ + free(wal->entries); + file_close(wal->handle); +} + +int wal_append(WAL *wal, WALEntry *entry) +{ + if (wal->count == wal->capacity) { + int n = 2 * wal->capacity; + if (n < 8) n = 8; + void *p = realloc(wal->entries, n * sizeof(WALEntry)); + if (p == NULL) + return -1; + wal->capacity = n; + wal->entries = p; + } + + // TODO: Should truncate file on partial writes + if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0) + return -1; + + if (file_sync(wal->handle) < 0) + return -1; + + wal->entries[wal->count++] = *entry; + return 0; +} + +int wal_truncate(WAL *wal, int new_count) +{ + assert(new_count <= wal->count); + if (wal->count == new_count) + return 0; + + if (file_truncate(wal->handle, new_count * sizeof(WALEntry)) < 0) + return -1; + + if (file_set_offset(wal->handle, new_count * sizeof(WALEntry)) < 0) + return -1; + + wal->count = new_count; + return 0; +} + +uint64_t wal_last_term(WAL *wal) +{ + if (wal->count == 0) + return 0; + return wal->entries[wal->count-1].term; +} + +int wal_entry_count(WAL *wal) +{ + return wal->count; +} + +WALEntry *wal_peek_entry(WAL *wal, int idx) +{ + assert(idx > -1); + assert(idx < wal->count); + return &wal->entries[idx]; +} + +void wal_replay_init(WALReplay *wal_replay, WAL *wal) +{ + wal_replay->count = wal->count; + wal_replay->current = 0; + wal_replay->entries = wal->entries; +} + +void wal_replay_free(WALReplay *wal_replay) +{ + (void) wal_replay; +} + +WALEntry *wal_replay_next(WALReplay *wal_replay) +{ + if (wal_replay->current == wal_replay->count) + return NULL; + return &wal_replay->entries[wal_replay->current++]; +} diff --git a/raft/wal.h b/raft/wal.h new file mode 100644 index 0000000..498edfb --- /dev/null +++ b/raft/wal.h @@ -0,0 +1,38 @@ +#ifndef WAL_INCLUDED +#define WAL_INCLUDED + +#include + +#include + +typedef struct { + uint64_t term; + uint64_t client_id; + Operation oper; +} WALEntry; + +typedef struct { + int count; + int capacity; + WALEntry *entries; + Handle handle; +} WAL; + +typedef struct { + int count; + int current; + WALEntry *entries; +} WALReplay; + +int wal_init(WAL *wal, string file); +void wal_free(WAL *wal); +int wal_append(WAL *wal, WALEntry *entry); +int wal_truncate(WAL *wal, int new_count); +uint64_t wal_last_term(WAL *wal); +int wal_entry_count(WAL *wal); +WALEntry *wal_peek_entry(WAL *wal, int idx); +void wal_replay_init(WALReplay *wal_replay, WAL *wal); +void wal_replay_free(WALReplay *wal_replay); +WALEntry *wal_replay_next(WALReplay *wal_replay); + +#endif // WAL_INCLUDED \ No newline at end of file diff --git a/state_machine/state_machine.c b/state_machine/state_machine.c new file mode 100644 index 0000000..d7bb9e5 --- /dev/null +++ b/state_machine/state_machine.c @@ -0,0 +1,25 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "state_machine.h" + +void state_machine_init(StateMachine *sm) +{ + (void)sm; +} + +void state_machine_free(StateMachine *sm) +{ + (void)sm; +} + +OperationResult state_machine_update(StateMachine *sm, Operation op) +{ + (void)sm; + (void)op; + return OPERATION_RESULT_X; +} diff --git a/state_machine/state_machine.h b/state_machine/state_machine.h new file mode 100644 index 0000000..959369f --- /dev/null +++ b/state_machine/state_machine.h @@ -0,0 +1,26 @@ +#ifndef STATE_MACHINE_INCLUDED +#define STATE_MACHINE_INCLUDED + +typedef enum { + OPERATION_NOOP, + OPERATION_A, + OPERATION_B, + OPERATION_C, +} Operation; + +typedef enum { + OPERATION_RESULT_X, + OPERATION_RESULT_Y, + OPERATION_RESULT_Z, +} OperationResult; + +typedef struct { + +} StateMachine; + +void state_machine_init(StateMachine *sm); +void state_machine_free(StateMachine *sm); + +OperationResult state_machine_update(StateMachine *sm, Operation op); + +#endif // STATE_MACHINE_INCLUDED \ No newline at end of file diff --git a/vsr/client.c b/vsr/client.c new file mode 100644 index 0000000..870f831 --- /dev/null +++ b/vsr/client.c @@ -0,0 +1,273 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include + +#include "node.h" +#include "client.h" + +//#define CLIENT_TRACE(fmt, ...) {} +#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__); + +// Format time as seconds with 3 decimal places for trace output +#define TIME_FMT "%.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static int leader_idx(ClientState *state) +{ + return state->view_number % state->num_servers; +} + +static int +process_message(ClientState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + (void) conn_idx; + + if (!state->pending) + return -1; + + if (type != MESSAGE_TYPE_REPLY) + return -1; + + ReplyMessage reply_message; + if (msg.len != sizeof(ReplyMessage)) + return -1; + memcpy(&reply_message, msg.ptr, sizeof(reply_message)); + + { + Time now = get_current_time(); + CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)", + TIME_VAL(now), + reply_message.rejected ? "true" : "false"); + } + + state->pending = false; + return 0; +} + +int client_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + ClientState *state = state_; + + state->num_servers = 0; + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--server")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --server missing value. Usage is --server :\n"); + return -1; + } + if (state->num_servers == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + // TODO: Check address is not duplicated + if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) { + fprintf(stderr, "Malformed : pair for --server option\n"); + return -1; + } + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } + + // Now sort the addresses + addr_sort(state->server_addrs, state->num_servers); + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Client :: Couldn't setup TCP context\n"); + return -1; + } + + state->pending = false; + + state->view_number = 0; + state->request_id = 0; + + // Load or generate a persistent client_id from disk. + // This ensures the client_id survives process restarts. + { + string path = S("client_id"); + Handle fd; + bool loaded = false; + if (file_exists(path)) { + if (file_open(path, &fd) == 0) { + file_set_offset(fd, 0); + uint64_t saved_id; + if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) { + state->client_id = saved_id; + loaded = true; + } + file_close(fd); + } + } + if (!loaded) { + Time now = get_current_time(); + state->client_id = (uint64_t)now; + if (file_open(path, &fd) == 0) { + file_set_offset(fd, 0); + file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id)); + file_sync(fd); + file_close(fd); + } + } + } + + // Connect to all known servers + for (int i = 0; i < state->num_servers; i++) { + if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) { + fprintf(stderr, "Client :: Couldn't connect to server %d\n", i); + tcp_context_free(&state->tcp); + return -1; + } + } + + { + Time now = get_current_time(); + CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d", + TIME_VAL(now), state->num_servers, leader_idx(state)); + } + + *timeout = 0; + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int client_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + ClientState *state = state_; + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + if (events[i].type == EVENT_DISCONNECT) { + int conn_idx = events[i].conn_idx; + int tag = tcp_get_tag(&state->tcp, conn_idx); + if (tag == leader_idx(state) && state->pending) { + Time now = get_current_time(); + CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request", + TIME_VAL(now), leader_idx(state)); + state->pending = false; + } + tcp_close(&state->tcp, conn_idx); + continue; + } + if (events[i].type != EVENT_MESSAGE) + continue; + int conn_idx = events[i].conn_idx; + + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + Time now = get_current_time(); + + // If we've been waiting too long for a response, give up and + // try the next server (the current leader may have crashed and + // a view change may have happened) + if (state->pending) { + Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (request_deadline <= now) { + CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server", + TIME_VAL(now), leader_idx(state), + (unsigned long)state->view_number); + state->view_number++; + state->pending = false; + } + } + + if (!state->pending) { + + int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state)); + if (conn_idx < 0) { + // Leader connection not available, try reconnecting + { + CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting", + TIME_VAL(now), leader_idx(state)); + } + tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL); + } else { + // Now start a new operation + state->request_id++; + + RequestMessage request_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REQUEST, + .length = sizeof(RequestMessage), + }, + .oper = OPERATION_A, + .client_id = state->client_id, + .request_id = state->request_id, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &request_message, request_message.base.length); + + { + CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)", + TIME_VAL(now), leader_idx(state), + (unsigned long)state->view_number, + (unsigned long)state->client_id, + (unsigned long)state->request_id); + } + + state->pending = true; + state->request_time = now; + } + } + + // Set timeout based on pending request deadline + Time deadline = INVALID_TIME; + if (state->pending) { + nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL); + } + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int client_free(void *state_) +{ + ClientState *state = state_; + + tcp_context_free(&state->tcp); + return 0; +} diff --git a/vsr/client.h b/vsr/client.h new file mode 100644 index 0000000..40daccc --- /dev/null +++ b/vsr/client.h @@ -0,0 +1,38 @@ +#ifndef CLIENT_INCLUDED +#define CLIENT_INCLUDED + +#include +#include + +#include "config.h" + +typedef struct { + + TCP tcp; + + // True if we are waiting for a response + bool pending; + Time request_time; // When the current request was sent + + Address server_addrs[NODE_LIMIT]; + int num_servers; + + uint64_t view_number; + + uint64_t client_id; + uint64_t request_id; + +} ClientState; + +struct pollfd; + +int client_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int client_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int client_free(void *state); + +#endif // CLIENT_INCLUDED \ No newline at end of file diff --git a/vsr/client_table.c b/vsr/client_table.c new file mode 100644 index 0000000..1094734 --- /dev/null +++ b/vsr/client_table.c @@ -0,0 +1,70 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "client_table.h" + +void client_table_init(ClientTable *client_table) +{ + client_table->count = 0; + client_table->capacity = 0; + client_table->entries = NULL; +} + +void client_table_free(ClientTable *client_table) +{ + free(client_table->entries); +} + +ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id) +{ + for (int i = 0; i < client_table->count; i++) { + if (client_table->entries[i].client_id == client_id) + return &client_table->entries[i]; + } + return NULL; +} + +int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx) +{ + if (client_table->count == client_table->capacity) { + int n = 2 * client_table->capacity; + if (n == 0) n = 8; + void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry)); + if (p == NULL) + return -1; + client_table->capacity = n; + client_table->entries = p; + } + + client_table->entries[client_table->count++] = (ClientTableEntry) { + .client_id = client_id, + .last_request_id = request_id, + .pending = true, + .conn_idx = conn_idx, + }; + return 0; +} + +int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id) +{ + if (client_table->count == client_table->capacity) { + int n = 2 * client_table->capacity; + if (n == 0) n = 8; + void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry)); + if (p == NULL) + return -1; + client_table->capacity = n; + client_table->entries = p; + } + + client_table->entries[client_table->count++] = (ClientTableEntry) { + .client_id=client_id, + .last_request_id=request_id, + .pending=true, + }; + return 0; +} diff --git a/vsr/client_table.h b/vsr/client_table.h new file mode 100644 index 0000000..4c39f2b --- /dev/null +++ b/vsr/client_table.h @@ -0,0 +1,28 @@ +#ifndef CLIENT_TABLE_INCLUDED +#define CLIENT_TABLE_INCLUDED + +#include + +#include + +typedef struct { + uint64_t client_id; + uint64_t last_request_id; + OperationResult last_result; + bool pending; + int conn_idx; +} ClientTableEntry; + +typedef struct { + int count; + int capacity; + ClientTableEntry *entries; +} ClientTable; + +void client_table_init(ClientTable *client_table); +void client_table_free(ClientTable *client_table); +ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id); +int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx); +int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id); + +#endif // CLIENT_TABLE_INCLUDED \ No newline at end of file diff --git a/vsr/config.h b/vsr/config.h new file mode 100644 index 0000000..09d352c --- /dev/null +++ b/vsr/config.h @@ -0,0 +1,11 @@ +#ifndef CONFIG_INCLUDED +#define CONFIG_INCLUDED + +#define NODE_LIMIT 32 +#define FUTURE_LIMIT 32 +#define HEARTBEAT_INTERVAL_SEC 1 +#define PRIMARY_DEATH_TIMEOUT_SEC 5 +#define RECOVERY_TIMEOUT_SEC 3 +#define RECOVERY_ATTEMPT_LIMIT 10 + +#endif // CONFIG_INCLUDED \ No newline at end of file diff --git a/vsr/log.c b/vsr/log.c new file mode 100644 index 0000000..276da6d --- /dev/null +++ b/vsr/log.c @@ -0,0 +1,37 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include + +#include "log.h" + +void log_init(Log *log) +{ + log->count = 0; + log->capacity = 0; + log->entries = NULL; +} + +void log_free(Log *log) +{ + free(log->entries); +} + +int log_append(Log *log, LogEntry entry) +{ + if (log->count == log->capacity) { + int n= 2 * log->capacity; + if (n < 8) + n = 8; + LogEntry *p = realloc(log->entries, n * sizeof(LogEntry)); + if (p == NULL) + return -1; + + log->entries = p; + log->capacity = n; + } + + log->entries[log->count++] = entry; + return 0; +} \ No newline at end of file diff --git a/vsr/log.h b/vsr/log.h new file mode 100644 index 0000000..23d0521 --- /dev/null +++ b/vsr/log.h @@ -0,0 +1,27 @@ +#ifndef LOG_INCLUDED +#define LOG_INCLUDED + +#include + +#include "config.h" + +typedef struct { + Operation oper; + uint32_t votes; + int view_number; + uint64_t client_id; +} LogEntry; + +_Static_assert(NODE_LIMIT <= 32, ""); + +typedef struct { + int count; + int capacity; + LogEntry *entries; +} Log; + +void log_init(Log *log); +void log_free(Log *log); +int log_append(Log *log, LogEntry entry); + +#endif // LOG_INCLUDED \ No newline at end of file diff --git a/vsr/main.c b/vsr/main.c new file mode 100644 index 0000000..c8ea3bd --- /dev/null +++ b/vsr/main.c @@ -0,0 +1,247 @@ +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS + +#include +#include +#include + +#include "node.h" +#include "client.h" + +static volatile int simulation_running = 1; + +static void sigint_handler(int sig) +{ + (void)sig; + simulation_running = 0; +} + +int main(void) +{ + signal(SIGINT, sigint_handler); + + Quakey *quakey; + int ret = quakey_init(&quakey, 1); + if (ret < 0) + return -1; + + QuakeyNode cli_1; + QuakeyNode cli_2; + QuakeyNode node_1; + QuakeyNode node_2; + QuakeyNode node_3; + + // Client 1 + { + QuakeySpawn config = { + .name = "rndcli1", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, + .addrs = (char*[]) { "127.0.0.2" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); + } + + // Client 2 + { + QuakeySpawn config = { + .name = "rndcli2", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, + .addrs = (char*[]) { "127.0.0.3" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); + } + + // Node 1 + { + QuakeySpawn config = { + .name = "node1", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.4" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080"); + } + + // Node 2 + { + QuakeySpawn config = { + .name = "node2", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.5" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080"); + } + + // Node 3 + { + QuakeySpawn config = { + .name = "node3", + .state_size = sizeof(NodeState), + .init_func = node_init, + .tick_func = node_tick, + .free_func = node_free, + .addrs = (char*[]) { "127.0.0.6" }, + .num_addrs = 1, + .disk_size = 10<<20, + .platform = QUAKEY_LINUX, + }; + node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080"); + } + + while (simulation_running) { + quakey_schedule_one(quakey); + + NodeState *arr[] = { + quakey_node_state(node_1), + quakey_node_state(node_2), + quakey_node_state(node_3), + }; + check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0])); + } + + quakey_free(quakey); + return 0; +} + +#endif // MAIN_SIMULATION +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_CLIENT + +#include + +#include "client.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + int ret; + ClientState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = client_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = client_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + client_free(&state); + return 0; +} + +#endif // MAIN_CLIENT +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_SERVER + +#include + +#include "node.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + int ret; + NodeState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = node_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = node_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + node_free(&state); + return 0; +} + +#endif // MAIN_SERVER \ No newline at end of file diff --git a/vsr/node.c b/vsr/node.c new file mode 100644 index 0000000..0b19910 --- /dev/null +++ b/vsr/node.c @@ -0,0 +1,1745 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "node.h" + +//#define NODE_TRACE(fmt, ...) {} +#define NODE_TRACE(fmt, ...) fprintf(stderr, "NODE: " fmt "\n", ##__VA_ARGS__); + +// Format time as seconds with 3 decimal places for trace output +#define TIME_FMT "%.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static const char *message_type_name(uint8_t type) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST: return "REQUEST"; + case MESSAGE_TYPE_REPLY: return "REPLY"; + case MESSAGE_TYPE_PREPARE: return "PREPARE"; + case MESSAGE_TYPE_PREPARE_OK: return "PREPARE_OK"; + case MESSAGE_TYPE_COMMIT: return "COMMIT"; + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: return "BEGIN_VIEW_CHANGE"; + case MESSAGE_TYPE_DO_VIEW_CHANGE: return "DO_VIEW_CHANGE"; + case MESSAGE_TYPE_BEGIN_VIEW: return "BEGIN_VIEW"; + case MESSAGE_TYPE_RECOVERY: return "RECOVERY"; + case MESSAGE_TYPE_RECOVERY_RESPONSE: return "RECOVERY_RESPONSE"; + default: return "UNKNOWN"; + } +} + +static const char *status_name(Status status) +{ + switch (status) { + case STATUS_NORMAL: return "NORMAL"; + case STATUS_CHANGE_VIEW: return "CHANGE_VIEW"; + case STATUS_RECOVERY: return "RECOVERY"; + } + return "UNKNOWN"; +} + +static int self_idx(NodeState *state) +{ + for (int i = 0; i < state->num_nodes; i++) + if (addr_eql(state->node_addrs[i], state->self_addr)) + return i; + UNREACHABLE; +} + +static int leader_idx(NodeState *state) +{ + return state->view_number % state->num_nodes; +} + +static bool is_leader(NodeState *state) +{ + return self_idx(state) == leader_idx(state); +} + +static int send_to_peer_ex(NodeState *state, int peer_idx, MessageHeader *msg, void *extra, int extra_len) +{ + ByteQueue *output; + int conn_idx = tcp_index_from_tag(&state->tcp, peer_idx); + if (conn_idx < 0) { + int ret = tcp_connect(&state->tcp, state->node_addrs[peer_idx], peer_idx, &output); + if (ret < 0) + return -1; + } else { + output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) { + assert(0); + } + } + byte_queue_write(output, msg, msg->length - extra_len); + byte_queue_write(output, extra, extra_len); + return 0; +} + +static int send_to_peer(NodeState *state, int peer_idx, MessageHeader *msg) +{ + return send_to_peer_ex(state, peer_idx, msg, NULL, 0); +} + +static int broadcast_to_peers_ex(NodeState *state, MessageHeader *msg, void *extra, int extra_len) +{ + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + if (send_to_peer_ex(state, i, msg, extra, extra_len) < 0) + return -1; + } + return 0; +} + +static int broadcast_to_peers(NodeState *state, MessageHeader *msg) +{ + return broadcast_to_peers_ex(state, msg, NULL, 0); +} + +// TODO: test this function +static int count_set(uint32_t word) +{ + int n = 0; + for (int i = 0; i < sizeof(word) * 8; i++) + if (word & (1 << i)) + n++; + return n; +} + +static int +process_message_as_leader(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST: + { + // Don't accept new requests during a view change. + // The leader must complete the view change first. + if (state->status != STATUS_NORMAL) + break; + + RequestMessage request_message; + + if (msg.len != sizeof(request_message)) + return -1; + memcpy(&request_message, msg.ptr, sizeof(request_message)); + + // We must first add or update the client table to + // invalidate the request ID. This makes it so any + // subsequent requests with the same ID are rejected + // while the first one is in progress. + // + // If the request ID is lower than the one stored in + // the table, the request is rejected. + // + // If the request ID is the same as the one in the table + // but no result was saved as the original one is still + // in progress, the request is rejected. + // + // If the request ID is the same and a result is available, + // it is returned immediately. + { + ClientTableEntry *entry = client_table_find(&state->client_table, request_message.client_id); + if (entry == NULL) { + + if (client_table_add(&state->client_table, request_message.client_id, request_message.request_id, conn_idx) < 0) { + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .rejected = true, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &reply_message, sizeof(reply_message)); + break; + } + + } else { + + if (entry->pending) + break; // Only one pending operation per client is allowed. Ignore the message. + + if (entry->last_request_id > request_message.request_id) + break; // Request is old. Ignore. + + if (entry->last_request_id == request_message.request_id) { + + // This request was already processed and its value was cached. + // Respond with the cached value. + + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .rejected = false, + .result = entry->last_result, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &reply_message, sizeof(reply_message)); + break; + } + + entry->last_request_id = request_message.request_id; + entry->pending = true; + } + } + + LogEntry log_entry = { + .oper = request_message.oper, + .votes = 1 << self_idx(state), + .view_number = state->view_number, + .client_id = request_message.client_id, + }; + if (log_append(&state->log, log_entry) < 0) { + assert(0); // TODO + } + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): REQUEST from client %lu (req_id=%lu), appended to log[%d], view=%lu", + TIME_VAL(now), self_idx(state), + (unsigned long)request_message.client_id, + (unsigned long)request_message.request_id, + state->log.count-1, (unsigned long)state->view_number); + } + + PrepareMessage prepare_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE, + .length = sizeof(PrepareMessage), + }, + .oper = request_message.oper, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .commit_index = state->commit_index, + .view_number = state->view_number, + }; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER) -> ALL: PREPARE log_index=%d commit_index=%d view=%lu", + TIME_VAL(now), self_idx(state), + state->log.count-1, state->commit_index, + (unsigned long)state->view_number); + } + + if (broadcast_to_peers(state, &prepare_message.base) < 0) { + assert(0); + } + + // We forwarded the message to all peers. As soon as + // we get enough PREPARE_OK responses, we'll commit + // and reply to the client. + } + break; + case MESSAGE_TYPE_PREPARE_OK: + { + PrepareOKMessage prepare_ok_message; + + if (msg.len != sizeof(prepare_ok_message)) + return -1; + memcpy(&prepare_ok_message, msg.ptr, sizeof(prepare_ok_message)); + + if (prepare_ok_message.view_number != state->view_number) { + assert(0); + } + + // TODO: check log_index + int log_index = prepare_ok_message.log_index; + if (log_index < 0 || log_index >= state->log.count) { + assert(0); // TODO + } + + // When the primary appends an entry to its log, it sends a + // PREPARE message to all backups. Backups add the entry to + // their own logs and reply with PREPARE_OK messages. When + // the primary receives a quorum of PREPARE_OKs, it commits + // the entry. + // + // In a reliable network and with no node crashes, we would + // expect entries to be committed linearly in the log. If + // log entries A and B are added to the log in that order, + // we expect A to reach PREPARE_OK messages before B. + // + // Unfortunately, we must assume messages will be lost (*). + // If that happens, instead of worrying about resending the + // PREPARE_OK for that entry, we rely on the fact that OK + // messages for future messages imply OK messages for the + // previous ones. The first message for which a quorum of + // OK messages is reached can work as an OK for all previous + // entries. + // + // For this reason, we allow "holes" in the log and if an + // entry reached quorum we advance the commit index to it. + // + // If the log index is lower than the log, it means we + // received an OK message that was not necessary anymore + // so we can ignore it. + // + // (*) This implementation uses TCP as a transport protocol, + // which means messages will be retransmitted if lost on + // the network. Nevertheless, if a node crashes while receiving + // a node or we crash before sending it, the message will + // be lost. + + if (log_index < state->commit_index) + break; // Already processed + LogEntry *entry = &state->log.entries[log_index]; + + entry->votes |= 1 << prepare_ok_message.sender_idx; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): PREPARE_OK from node %d for log[%d], prep_oks=%d/%d view=%lu", + TIME_VAL(now), self_idx(state), + prepare_ok_message.sender_idx, log_index, + count_set(entry->votes), state->num_nodes/2 + 1, + (unsigned long)prepare_ok_message.view_number); + } + + // Quorum reached for this log entry? + if (count_set(entry->votes) > state->num_nodes/2) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): quorum reached for log[%d], advancing commit_index %d -> %d", + TIME_VAL(now), self_idx(state), + log_index, state->commit_index, log_index + 1); + } + + // Okay, we can advance the commit index to here + while (state->commit_index <= log_index) { + + uint64_t client_id = state->log.entries[state->commit_index].client_id; + Operation oper = state->log.entries[state->commit_index].oper; + state->commit_index++; + + OperationResult result = state_machine_update(&state->state_machine, oper); + + ClientTableEntry *table_entry = client_table_find(&state->client_table, client_id); + if (table_entry) { + + assert(table_entry->pending); + table_entry->pending = false; + table_entry->last_result = result; + + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .result = result, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, table_entry->conn_idx); + assert(output); + + byte_queue_write(output, &reply_message, sizeof(reply_message)); + } + } + } + } + break; + case MESSAGE_TYPE_DO_VIEW_CHANGE: + { + DoViewChangeMessage do_view_change_message; + if (msg.len < sizeof(do_view_change_message)) + return -1; + memcpy(&do_view_change_message, msg.ptr, sizeof(do_view_change_message)); + + // Parse the variable-sized log from the message + int num_entries = (msg.len - sizeof(DoViewChangeMessage)) / sizeof(LogEntry); + if (num_entries != do_view_change_message.op_number) + return -1; // Message size mismatch + + // Only process if the view matches what we're transitioning to + if (do_view_change_message.view_number != state->view_number) + break; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: DO_VIEW_CHANGE from node %d, view=%lu old_view=%lu op_number=%d commit_index=%d", + TIME_VAL(now), self_idx(state), + do_view_change_message.sender_idx, + (unsigned long)do_view_change_message.view_number, + (unsigned long)do_view_change_message.old_view_number, + do_view_change_message.op_number, + do_view_change_message.commit_index); + } + + // Track this vote + uint32_t vote_mask = 1 << do_view_change_message.sender_idx; + if ((state->do_view_change_votes & vote_mask) == 0) { + + state->do_view_change_votes |= vote_mask; + + // Check if this log is better than the one we have stored + // Best log: highest old_view_number, then highest op_number + bool is_better = (do_view_change_message.old_view_number > state->do_view_change_best_old_view) || + (do_view_change_message.old_view_number == state->do_view_change_best_old_view && + do_view_change_message.op_number > state->do_view_change_best_log.count); + + if (is_better) { + state->do_view_change_best_old_view = do_view_change_message.old_view_number; + + // Copy the log entries from the message + LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); + if (entries == NULL) { + assert(0); + } + memcpy(entries, (uint8_t*)msg.ptr + sizeof(DoViewChangeMessage), num_entries * sizeof(LogEntry)); + + log_free(&state->do_view_change_best_log); + state->do_view_change_best_log.count = num_entries; + state->do_view_change_best_log.capacity = num_entries; + state->do_view_change_best_log.entries = entries; + } + + // Track the maximum commit index + if (do_view_change_message.commit_index > state->do_view_change_best_commit) { + state->do_view_change_best_commit = do_view_change_message.commit_index; + } + } + + // Count votes + int vote_count = 0; + for (int i = 0; i < state->num_nodes; i++) { + if (state->do_view_change_votes & (1 << i)) vote_count++; + } + + // Need f + 1 DoViewChange messages (including own) + if (vote_count >= state->num_nodes / 2 + 1) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: view change complete, becoming LEADER for view %lu (votes=%d/%d)", + TIME_VAL(now), self_idx(state), + (unsigned long)state->view_number, + vote_count, state->num_nodes); + } + + // Use the best log we collected + log_free(&state->log); + state->log = state->do_view_change_best_log; + state->do_view_change_best_log = (Log){0}; // Clear to avoid double-free + + state->commit_index = state->do_view_change_best_commit; + state->status = STATUS_NORMAL; + + // Reset vote tracking for uncommitted entries. The log + // entries inherited from DO_VIEW_CHANGE have stale + // votes from the previous view. The new leader starts + // with its own vote for each entry. + for (int i = state->commit_index; i < state->log.count; i++) + state->log.entries[i].votes = 1 << self_idx(state); + + BeginViewMessage begin_view_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW, + .length = sizeof(BeginViewMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .commit_index = state->commit_index, + .op_number = state->log.count, + }; + + if (broadcast_to_peers_ex(state, &begin_view_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { + assert(0); + } + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER) -> ALL: BEGIN_VIEW view=%lu commit_index=%d log_count=%d", + TIME_VAL(now), self_idx(state), + (unsigned long)state->view_number, + state->commit_index, state->log.count); + } + + // Reset view change state for next time + state->do_view_change_votes = 0; + state->do_view_change_best_old_view = 0; + state->do_view_change_best_commit = 0; + } + } + break; + case MESSAGE_TYPE_RECOVERY: + { + if (state->status != STATUS_NORMAL) + break; // Ignore message. + + RecoveryMessage recovery_message; + if (msg.len != sizeof(RecoveryMessage)) + return -1; + memcpy(&recovery_message, msg.ptr, sizeof(recovery_message)); + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): RECOVERY from node %d (nonce=%lu), sending RECOVERY_RESPONSE with log_count=%d", + TIME_VAL(now), self_idx(state), + recovery_message.sender_idx, + (unsigned long)recovery_message.nonce, + state->log.count); + } + + RecoveryResponseMessage recovery_response_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY_RESPONSE, + .length = sizeof(RecoveryResponseMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .op_number = state->log.count-1, // TODO: What if the log is empty? + .nonce = recovery_message.nonce, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + + if (send_to_peer_ex(state, recovery_message.sender_idx, &recovery_response_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { + assert(0); + } + } + break; + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + { + if (state->status == STATUS_RECOVERY) break; + + BeginViewChangeMessage begin_view_change_message; + if (msg.len != sizeof(BeginViewChangeMessage)) + return -1; + memcpy(&begin_view_change_message, msg.ptr, sizeof(begin_view_change_message)); + + if (begin_view_change_message.view_number > state->view_number) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): BEGIN_VIEW_CHANGE from node %d, transitioning view %lu -> %lu", + TIME_VAL(now), self_idx(state), + begin_view_change_message.sender_idx, + (unsigned long)state->view_number, + (unsigned long)begin_view_change_message.view_number); + } + + // Reset vote tracking when transitioning to a new view + state->begin_view_change_votes = 0; + state->view_number = begin_view_change_message.view_number; + state->status = STATUS_CHANGE_VIEW; + + // Send our own BeginViewChange to all peers + BeginViewChangeMessage own_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = state->view_number, + .sender_idx = self_idx(state), + }; + + if (broadcast_to_peers(state, &own_message.base) < 0) { + assert(0); + } + // Count our own vote + state->begin_view_change_votes |= (1 << self_idx(state)); + } + + // Track this sender's vote (only if view matches) + if (begin_view_change_message.view_number == state->view_number) { + state->begin_view_change_votes |= (1 << begin_view_change_message.sender_idx); + } + + // Count votes + int vote_count = 0; + for (int i = 0; i < state->num_nodes; i++) { + if (state->begin_view_change_votes & (1 << i)) vote_count++; + } + + // Need f + 1 votes to send DoViewChange + if (vote_count >= state->num_nodes / 2 + 1) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): f+1 BEGIN_VIEW_CHANGE votes reached (%d/%d), sending DO_VIEW_CHANGE to node %d for view %lu", + TIME_VAL(now), self_idx(state), + vote_count, state->num_nodes, + (int)(state->view_number % state->num_nodes), + (unsigned long)state->view_number); + } + + DoViewChangeMessage do_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_DO_VIEW_CHANGE, + .length = sizeof(DoViewChangeMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .old_view_number = state->view_number - 1, // View before we started view change + .op_number = state->log.count, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + if (leader_idx(state) == self_idx(state)) { + // We are the new leader: count our own vote directly + // since send_to_peer_ex skips self-sends. + state->do_view_change_votes |= (1 << self_idx(state)); + } else { + if (send_to_peer_ex(state, leader_idx(state), &do_view_change_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { + assert(0); + } + } + + // Clear the future array since we're changing views + state->num_future = 0; + } + } + break; + case MESSAGE_TYPE_BEGIN_VIEW: + { + BeginViewMessage message; + + if (msg.len < sizeof(message)) + return -1; + memcpy(&message, msg.ptr, sizeof(message)); + + // Only process messages containing a view-number + // that matches or advances the one we know. + if (message.view_number < state->view_number) + break; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): BEGIN_VIEW received, stepping down and adopting view %lu (commit_index=%d, log_entries=%d)", + TIME_VAL(now), self_idx(state), + (unsigned long)message.view_number, + message.commit_index, message.op_number); + } + + // Update view and status + state->view_number = message.view_number; + state->status = STATUS_NORMAL; // Paper state: change status to normal + + // Replace the local log with the authoritative log from the primary + { + int num_entries = (msg.len - sizeof(BeginViewMessage)) / sizeof(LogEntry); + LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); + if (entries == NULL) { + assert(0); + } + memcpy(entries, (uint8_t*)msg.ptr + sizeof(BeginViewMessage), num_entries * sizeof(LogEntry)); + + log_free(&state->log); + state->log.count = num_entries; + state->log.capacity = num_entries; + state->log.entries = entries; + } + + // Reset view change state + state->begin_view_change_votes = 0; + + // If there are non-committed operations in the log, + // send a PREPAREOK to the new primary + if (state->log.count > message.commit_index) { + PrepareOKMessage ok_msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count - 1, + .view_number = state->view_number, + }; + if (send_to_peer(state, leader_idx(state), &ok_msg.base) < 0) { + assert(0); + } + } + + // Execute all operations known to be committed that haven't + // been executed previously + while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { + state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); + state->commit_index++; + } + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + break; + case MESSAGE_TYPE_RECOVERY_RESPONSE: + { + RecoveryResponseMessage message; + + if (msg.len < sizeof(message)) + return -1; + memcpy(&message, msg.ptr, sizeof(message)); + + // 1. Only process responses if we are actually in the recovering state + // 2. Ensure the nonce matches the one we sent to prevent accepting + // delayed responses from previous recovery attempts. + if (state->status != STATUS_RECOVERY || message.nonce != state->recovery_nonce) { + break; + } + + // Track this response + state->recovery_votes |= (1 << message.sender_idx); + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (RECOVERY): RECOVERY_RESPONSE from node %d, view=%lu op_number=%d commit_index=%d", + TIME_VAL(now), self_idx(state), + message.sender_idx, + (unsigned long)message.view_number, + message.op_number, message.commit_index); + } + + // Track the highest view number we've seen and who the primary is for it + if (message.view_number > state->max_view_seen) { + state->max_view_seen = message.view_number; + state->latest_primary_idx = (message.view_number % state->num_nodes); + } + + // Store the state from the primary if this message is from them + if (message.sender_idx == (int) (message.view_number % state->num_nodes)) { + // Copy the log from the variable-sized portion of the message + int num_entries = (msg.len - sizeof(RecoveryResponseMessage)) / sizeof(LogEntry); + LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); + if (entries == NULL) { + assert(0); + } + memcpy(entries, (uint8_t*)msg.ptr + sizeof(RecoveryResponseMessage), num_entries * sizeof(LogEntry)); + + log_free(&state->potential_primary_log); + state->potential_primary_log.count = num_entries; + state->potential_primary_log.capacity = num_entries; + state->potential_primary_log.entries = entries; + + state->received_primary_state = true; + } + + // Check if we have f + 1 responses + int response_count = 0; + for (int i = 0; i < state->num_nodes; i++) { + if (state->recovery_votes & (1 << i)) response_count++; + } + + // The threshold f is derived from 2f + 1 = num_nodes + int f = (state->num_nodes - 1) / 2; + + if (response_count >= f + 1 && state->received_primary_state) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: recovery complete (responses=%d, f+1=%d), adopting view %lu", + TIME_VAL(now), self_idx(state), + response_count, f + 1, + (unsigned long)state->max_view_seen); + } + + // Update state using information from the primary + state->view_number = state->max_view_seen; + + // Move the log (no need to copy since we're transferring ownership) + log_free(&state->log); + state->log = state->potential_primary_log; + state->potential_primary_log = (Log){0}; // Clear to avoid double-free + + state->commit_index = message.commit_index; + + // Change status to normal; the recovery protocol is complete + state->status = STATUS_NORMAL; + + // Update heartbeat to avoid immediate view change timeout + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + } + break; + default: + break; + } + return 0; +} + +static int +process_message_as_replica(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + (void) conn_idx; + + switch (type) { + case MESSAGE_TYPE_REQUEST: + { + // Do nothing. Replicas ignore client requests. + } + break; + case MESSAGE_TYPE_PREPARE: + { + PrepareMessage prepare_message; + if (msg.len != sizeof(prepare_message)) + return -1; + memcpy(&prepare_message, msg.ptr, sizeof(prepare_message)); + + // Stale peer + if (prepare_message.view_number < state->view_number) + break; + + if (prepare_message.view_number > state->view_number) { + // The new leader has started a view we haven't seen. + // Adopt the new view and process the PREPARE normally. + { + Time t = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): adopting view %lu from PREPARE (was view %lu, status %s)", + TIME_VAL(t), self_idx(state), + (unsigned long)prepare_message.view_number, + (unsigned long)state->view_number, + status_name(state->status)); + } + state->view_number = prepare_message.view_number; + state->status = STATUS_NORMAL; + state->begin_view_change_votes = 0; + state->num_future = 0; + } + + if (prepare_message.log_index > state->log.count) { + // The prepare message for a previous log entry was not received yet. + // Buffer this message to process it later + if (state->num_future == FUTURE_LIMIT) { + assert(0); // TODO + } + state->future[state->num_future++] = prepare_message; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): PREPARE log_index=%d buffered (expected %d), num_future=%d", + TIME_VAL(now), self_idx(state), + prepare_message.log_index, state->log.count, + state->num_future); + } + + } else if (prepare_message.log_index < state->log.count) { + // Message refers to an old entry. Ignore. + } else { + + LogEntry log_entry = { + .oper = prepare_message.oper, + .votes = 0, + .view_number = state->view_number, + }; + + if (log_append(&state->log, log_entry) < 0) { + assert(0); // TODO + } + + PrepareOKMessage prepare_ok_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .view_number = state->view_number, + }; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA) -> node %d: PREPARE_OK log_index=%d view=%lu", + TIME_VAL(now), self_idx(state), + prepare_message.sender_idx, + state->log.count-1, + (unsigned long)state->view_number); + } + + if (send_to_peer(state, prepare_message.sender_idx, &prepare_ok_message.base) < 0) { + assert(0); + } + + // Now try to process future log + bool processed_at_least_one; + do { + processed_at_least_one = false; + for (int i = 0; i < state->num_future; i++) { + + if (state->future[i].log_index < state->log.count) { + state->future[i--] = state->future[--state->num_future]; + continue; + } + + if (state->future[i].log_index == state->log.count) { + + LogEntry future_log_entry = { + .oper = state->future[i].oper, + .votes = 0, + .view_number = state->view_number, + }; + if (log_append(&state->log, future_log_entry) < 0) { + assert(0); // TODO + } + + PrepareOKMessage prepare_ok_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .view_number = state->view_number, + }; + if (send_to_peer(state, state->future[i].sender_idx, &prepare_ok_message.base) < 0) { + assert(0); + } + + processed_at_least_one = true; + break; + } + } + } while (processed_at_least_one); + + while (state->commit_index < prepare_message.commit_index && state->commit_index < state->log.count) { + state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); + state->commit_index++; + } + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + } + break; + case MESSAGE_TYPE_COMMIT: + { + CommitMessage message; + + if (msg.len != sizeof(CommitMessage)) + return -1; + memcpy(&message, msg.ptr, sizeof(message)); + + // Don't process heartbeats from a stale leader during + // a view change, as it would reset the retry timer and + // prevent the view change from completing. + if (state->status == STATUS_CHANGE_VIEW) + break; + + if (message.commit_index > state->commit_index) { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): COMMIT advancing commit_index %d -> %d", + TIME_VAL(now), self_idx(state), + state->commit_index, message.commit_index); + } + + while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { + state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); + state->commit_index++; + } + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + break; + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + { + if (state->status == STATUS_RECOVERY) break; + + BeginViewChangeMessage begin_view_change_message; + if (msg.len != sizeof(BeginViewChangeMessage)) + return -1; + memcpy(&begin_view_change_message, msg.ptr, sizeof(begin_view_change_message)); + + if (begin_view_change_message.view_number > state->view_number) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): BEGIN_VIEW_CHANGE from node %d, transitioning view %lu -> %lu", + TIME_VAL(now), self_idx(state), + begin_view_change_message.sender_idx, + (unsigned long)state->view_number, + (unsigned long)begin_view_change_message.view_number); + } + + // Reset vote tracking when transitioning to a new view + state->begin_view_change_votes = 0; + state->view_number = begin_view_change_message.view_number; + state->status = STATUS_CHANGE_VIEW; + + // Send our own BeginViewChange to all peers + BeginViewChangeMessage own_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = state->view_number, + .sender_idx = self_idx(state), + }; + + if (broadcast_to_peers(state, &own_message.base) < 0) { + assert(0); + } + // Count our own vote + state->begin_view_change_votes |= (1 << self_idx(state)); + } + + // Track this sender's vote (only if view matches) + if (begin_view_change_message.view_number == state->view_number) { + state->begin_view_change_votes |= (1 << begin_view_change_message.sender_idx); + } + + // Count votes + int vote_count = 0; + for (int i = 0; i < state->num_nodes; i++) { + if (state->begin_view_change_votes & (1 << i)) vote_count++; + } + + // Need f + 1 votes to send DoViewChange + if (vote_count >= state->num_nodes / 2 + 1) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): f+1 BEGIN_VIEW_CHANGE votes reached (%d/%d), sending DO_VIEW_CHANGE to node %d for view %lu", + TIME_VAL(now), self_idx(state), + vote_count, state->num_nodes, + (int)(state->view_number % state->num_nodes), + (unsigned long)state->view_number); + } + + DoViewChangeMessage do_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_DO_VIEW_CHANGE, + .length = sizeof(DoViewChangeMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .old_view_number = state->view_number - 1, // View before we started view change + .op_number = state->log.count, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + if (leader_idx(state) == self_idx(state)) { + // We are the new leader: count our own vote directly + // since send_to_peer_ex skips self-sends. + state->do_view_change_votes |= (1 << self_idx(state)); + } else { + if (send_to_peer_ex(state, leader_idx(state), &do_view_change_message.base, state->log.entries, state->log.count * sizeof(LogEntry)) < 0) { + assert(0); + } + } + + // Clear the future array since we're changing views + state->num_future = 0; + } + } + break; + case MESSAGE_TYPE_BEGIN_VIEW: + { + BeginViewMessage message; + + if (msg.len < sizeof(message)) + return -1; + memcpy(&message, msg.ptr, sizeof(message)); + + // Replicas only process messages containing a view-number + // that matches or advances the one they know[cite: 738, 812]. + if (message.view_number < state->view_number) + break; + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): BEGIN_VIEW received, adopting view %lu (commit_index=%d, log_entries=%d)", + TIME_VAL(now), self_idx(state), + (unsigned long)message.view_number, + message.commit_index, message.op_number); + } + + // Update view and status + state->view_number = message.view_number; + state->status = STATUS_NORMAL; // Paper state: change status to normal + + // Replace the local log with the authoritative log from the primary + { + int num_entries = (msg.len - sizeof(BeginViewMessage)) / sizeof(LogEntry); + LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); + if (entries == NULL) { + assert(0); + } + memcpy(entries, (uint8_t*)msg.ptr + sizeof(BeginViewMessage), num_entries * sizeof(LogEntry)); + + log_free(&state->log); + state->log.count = num_entries; + state->log.capacity = num_entries; + state->log.entries = entries; + } + + // Reset view change state + state->begin_view_change_votes = 0; + + // If there are non-committed operations in the log, + // send a PREPAREOK to the new primary + if (state->log.count > message.commit_index) { + PrepareOKMessage ok_msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count - 1, + .view_number = state->view_number, + }; + if (send_to_peer(state, leader_idx(state), &ok_msg.base) < 0) { + assert(0); + } + } + + // Execute all operations known to be committed that haven't + // been executed previously + while (state->commit_index < message.commit_index && state->commit_index < state->log.count) { + state_machine_update(&state->state_machine, state->log.entries[state->commit_index].oper); + state->commit_index++; + } + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + break; + case MESSAGE_TYPE_RECOVERY: + { + if (state->status != STATUS_NORMAL) + break; // Ignore message. + + RecoveryMessage recovery_message; + if (msg.len != sizeof(RecoveryMessage)) + return -1; + memcpy(&recovery_message, msg.ptr, sizeof(recovery_message)); + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA): RECOVERY from node %d (nonce=%lu), sending RECOVERY_RESPONSE", + TIME_VAL(now), self_idx(state), + recovery_message.sender_idx, + (unsigned long)recovery_message.nonce); + } + + RecoveryResponseMessage recovery_response_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY_RESPONSE, + .length = sizeof(RecoveryResponseMessage), + }, + .view_number = state->view_number, + .op_number = state->log.count-1, // TODO: What if the log is empty? + .nonce = recovery_message.nonce, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + + if (send_to_peer(state, recovery_message.sender_idx, &recovery_response_message.base) < 0) { + assert(0); + } + } + break; + case MESSAGE_TYPE_RECOVERY_RESPONSE: + { + RecoveryResponseMessage message; + + if (msg.len < sizeof(message)) + return -1; + memcpy(&message, msg.ptr, sizeof(message)); + + // 1. Only process responses if we are actually in the recovering state [cite: 237] + // 2. Ensure the nonce matches the one we sent to prevent accepting + // delayed responses from previous recovery attempts[cite: 253]. + if (state->status != STATUS_RECOVERY || message.nonce != state->recovery_nonce) { + break; + } + + // Track this response (e.g., in a bitmask or array) + state->recovery_votes |= (1 << message.sender_idx); + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (RECOVERY): RECOVERY_RESPONSE from node %d, view=%lu op_number=%d commit_index=%d", + TIME_VAL(now), self_idx(state), + message.sender_idx, + (unsigned long)message.view_number, + message.op_number, message.commit_index); + } + + // Track the highest view number we've seen and who the primary is for it + if (message.view_number > state->max_view_seen) { + state->max_view_seen = message.view_number; + state->latest_primary_idx = (message.view_number % state->num_nodes); + } + + // Store the state from the primary if this message is from them + if (message.sender_idx == (int) (message.view_number % state->num_nodes)) { + // Copy the log from the variable-sized portion of the message + int num_entries = (msg.len - sizeof(RecoveryResponseMessage)) / sizeof(LogEntry); + LogEntry *entries = malloc(num_entries * sizeof(LogEntry)); + if (entries == NULL) { + assert(0); + } + memcpy(entries, (uint8_t*)msg.ptr + sizeof(RecoveryResponseMessage), num_entries * sizeof(LogEntry)); + + log_free(&state->potential_primary_log); + state->potential_primary_log.count = num_entries; + state->potential_primary_log.capacity = num_entries; + state->potential_primary_log.entries = entries; + + state->received_primary_state = true; + } + + // Check if we have f + 1 responses + int response_count = 0; + for (int i = 0; i < state->num_nodes; i++) { + if (state->recovery_votes & (1 << i)) response_count++; + } + + // The threshold f is derived from 2f + 1 = num_nodes [cite: 66, 73] + int f = (state->num_nodes - 1) / 2; + + if (response_count >= f + 1 && state->received_primary_state) { + + { + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d: recovery complete (responses=%d, f+1=%d), adopting view %lu", + TIME_VAL(now), self_idx(state), + response_count, f + 1, + (unsigned long)state->max_view_seen); + } + + // Update state using information from the primary + state->view_number = state->max_view_seen; + + // Move the log (no need to copy since we're transferring ownership) + log_free(&state->log); + state->log = state->potential_primary_log; + state->potential_primary_log = (Log){0}; // Clear to avoid double-free + + state->commit_index = message.commit_index; + + // Change status to normal; the recovery protocol is complete + state->status = STATUS_NORMAL; + + // Update heartbeat to avoid immediate view change timeout + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); + } + state->last_heartbeat_time = now; + } + } + break; + default: + break; + } + return 0; +} + +static int +process_message(NodeState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + Time now = get_current_time(); + NODE_TRACE("[" TIME_FMT "] node %d (%s, %s) <- conn %d: recv %s (%d bytes)", + TIME_VAL(now), self_idx(state), + is_leader(state) ? "LEADER" : "REPLICA", + status_name(state->status), + conn_idx, message_type_name(type), (int)msg.len); + + // Tag incoming connections with the sender's node index so that + // the connection can be used bidirectionally. Without this, when + // node A connects to node B and sends a message, node B can't + // send back to node A through the same connection (the tag is + // only set on the connector's side). + { + int sender_idx = -1; + switch (type) { + case MESSAGE_TYPE_PREPARE: + if (msg.len >= sizeof(PrepareMessage)) { + PrepareMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_PREPARE_OK: + if (msg.len >= sizeof(PrepareOKMessage)) { + PrepareOKMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + if (msg.len >= sizeof(BeginViewChangeMessage)) { + BeginViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_DO_VIEW_CHANGE: + if (msg.len >= sizeof(DoViewChangeMessage)) { + DoViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_RECOVERY: + if (msg.len >= sizeof(RecoveryMessage)) { + RecoveryMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_RECOVERY_RESPONSE: + if (msg.len >= sizeof(RecoveryResponseMessage)) { + RecoveryResponseMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + } + if (sender_idx >= 0 && sender_idx < state->num_nodes) { + int existing = tcp_index_from_tag(&state->tcp, sender_idx); + if (existing < 0) { + // No connection tagged with this peer yet, tag this one + tcp_set_tag(&state->tcp, conn_idx, sender_idx, false); + } else if (existing != conn_idx) { + // A different (possibly stale) connection has this tag. + // Close the old one and tag the current one. + tcp_close(&state->tcp, existing); + tcp_set_tag(&state->tcp, conn_idx, sender_idx, false); + } + // If existing == conn_idx, already tagged correctly + } + } + + if (is_leader(state)) { + return process_message_as_leader(state, conn_idx, type, msg); + } else { + return process_message_as_replica(state, conn_idx, type, msg); + } +} + +int node_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + NodeState *state = state_; + + Time now = get_current_time(); + if (now == INVALID_TIME) { + fprintf(stderr, "Node :: Couldn't get current time\n"); + return -1; + } + + state->num_nodes = 0; + + bool self_addr_set = false; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--addr")) { + if (self_addr_set) { + fprintf(stderr, "Option --addr specified twice\n"); + return -1; + } + self_addr_set = true; + i++; + if (i == argc) { + fprintf(stderr, "Option --addr missing value. Usage is --addr :\n"); + return -1; + } + // TODO: Check address is not duplicated + int ret = parse_addr_arg(argv[i], &state->self_addr); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --addr option\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + state->node_addrs[state->num_nodes++] = state->self_addr; + } else if (!strcmp(argv[i], "--peer")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --peer missing value. Usage is --peer :\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + // TODO: Check address is not duplicated + int ret = parse_addr_arg(argv[i], &state->node_addrs[state->num_nodes]); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --peer option\n"); + return -1; + } + state->num_nodes++; + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } + + // Now sort the addresses + addr_sort(state->node_addrs, state->num_nodes); + + Time deadline = INVALID_TIME; + + state->view_number = 0; + state->last_heartbeat_time = now; + state->commit_index = 0; + state->num_future = 0; + + // View change state + state->begin_view_change_votes = 0; + state->do_view_change_votes = 0; + state->do_view_change_best_old_view = 0; + state->do_view_change_best_commit = 0; + log_init(&state->do_view_change_best_log); + + // Recovery state + state->recovery_votes = 0; + state->max_view_seen = 0; + state->latest_primary_idx = 0; + state->received_primary_state = false; + state->recovery_attempt_count = 0; + log_init(&state->potential_primary_log); + + // Detect whether this is a restart after a crash by checking for a + // boot marker file on disk. The disk persists across crashes, so if + // the marker exists, this node previously ran and crashed. In that + // case, enter recovery mode to learn the current view from peers + // before participating in the protocol. + // + // We use open() directly (without O_CREAT) instead of file_exists() + // because access() is not available in the simulation environment. + int marker_fd = open("vsr_boot_marker", O_RDONLY, 0); + bool previously_crashed = (marker_fd >= 0); + if (previously_crashed) + close(marker_fd); + if (previously_crashed) { + state->status = STATUS_RECOVERY; + state->recovery_nonce = now; + state->recovery_start_time = now; + } else { + state->status = STATUS_NORMAL; + } + + client_table_init(&state->client_table); + + state_machine_init(&state->state_machine); + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP context\n"); + return -1; + } + + int ret = tcp_listen(&state->tcp, state->self_addr); + if (ret < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP listener\n"); + tcp_context_free(&state->tcp); + return -1; + } + + log_init(&state->log); + + // Write the boot marker to disk so that future restarts can detect + // a previous crash. This must happen after TCP init so that the + // marker is only written if the node successfully started. + if (!previously_crashed) { + int fd = open("vsr_boot_marker", O_WRONLY | O_CREAT, 0644); + if (fd >= 0) + close(fd); + } + + if (previously_crashed) { + // Broadcast RECOVERY to all peers to learn the current view + RecoveryMessage recovery_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY, + .length = sizeof(RecoveryMessage), + }, + .sender_idx = self_idx(state), + .nonce = state->recovery_nonce, + }; + + if (broadcast_to_peers(state, &recovery_message.base) < 0) { + assert(0); // TODO + } + + nearest_deadline(&deadline, state->recovery_start_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL); + } + + NODE_TRACE("[" TIME_FMT "] node %d initialized: %s, view=%lu, num_nodes=%d, status=%s", + TIME_VAL(now), self_idx(state), + is_leader(state) ? "LEADER" : "REPLICA", + (unsigned long)state->view_number, + state->num_nodes, + status_name(state->status)); + + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Node :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int node_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + NodeState *state = state_; + + Time now = get_current_time(); + if (now == INVALID_TIME) { + assert(0); // TODO + } + + ///////////////////////////////////////////////////////////////// + // NETWORK EVENTS + ///////////////////////////////////////////////////////////////// + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + + if (events[i].type == EVENT_DISCONNECT) { + tcp_close(&state->tcp, events[i].conn_idx); + continue; + } + if (events[i].type != EVENT_MESSAGE) + continue; + int conn_idx = events[i].conn_idx; + + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + ///////////////////////////////////////////////////////////////// + // TIME EVENTS + ///////////////////////////////////////////////////////////////// + + Time deadline = INVALID_TIME; + + if (state->status == STATUS_RECOVERY) { + + // Recovery handling runs regardless of leader/replica position, + // since a recovering node must not act as leader until it learns + // the current view from its peers. + Time recovery_deadline = state->recovery_start_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL; + if (recovery_deadline <= now) { + + state->recovery_attempt_count++; + if (state->recovery_attempt_count >= RECOVERY_ATTEMPT_LIMIT) { + assert(0); // TODO + } else { + RecoveryMessage recovery_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY, + .length = sizeof(RecoveryMessage), + }, + .sender_idx = self_idx(state), + .nonce = state->recovery_nonce, + }; + + if (broadcast_to_peers(state, &recovery_message.base) < 0) { + assert(0); // TODO + } + + state->recovery_start_time = now; + } + } else { + nearest_deadline(&deadline, recovery_deadline); + } + + } else if (is_leader(state)) { + + Time heartbeat_deadline = state->last_heartbeat_time + HEARTBEAT_INTERVAL_SEC * 1000000000ULL; + if (heartbeat_deadline <= now) { // TODO: check the time conversion here + + NODE_TRACE("[" TIME_FMT "] node %d (LEADER): heartbeat timeout, sending COMMIT to all peers (commit_index=%d)", + TIME_VAL(now), self_idx(state), state->commit_index); + + CommitMessage commit_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_COMMIT, + .length = sizeof(CommitMessage), + }, + .commit_index = state->commit_index, + }; + if (broadcast_to_peers(state, &commit_message.base)) { + assert(0); // TODO + } + state->last_heartbeat_time = now; + } else { + nearest_deadline(&deadline, heartbeat_deadline); + } + + } else { + + Time death_deadline = state->last_heartbeat_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (death_deadline <= now) { + + NODE_TRACE("[" TIME_FMT "] node %d (REPLICA, %s): primary death timeout, initiating view change to view %lu", + TIME_VAL(now), self_idx(state), + status_name(state->status), + (unsigned long)(state->view_number + 1)); + + BeginViewChangeMessage begin_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = state->view_number + 1, + .sender_idx = self_idx(state), + }; + if (broadcast_to_peers(state, &begin_view_change_message.base)) { + assert(0); // TODO + } + state->status = STATUS_CHANGE_VIEW; + state->last_heartbeat_time = now; + } else { + nearest_deadline(&deadline, death_deadline); + } + } + + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int node_free(void *state_) +{ + NodeState *state = state_; + + log_free(&state->log); + log_free(&state->potential_primary_log); + log_free(&state->do_view_change_best_log); + tcp_context_free(&state->tcp); + client_table_free(&state->client_table); + state_machine_free(&state->state_machine); + return 0; +} + +void check_vsr_invariants(NodeState **nodes, int num_nodes) +{ + for (int i = 0; i < num_nodes; i++) { + NodeState *s = nodes[i]; + if (s == NULL) + continue; + + // 1. commit_index <= log.count + // A node cannot have committed more entries than it has in its log. + if (s->commit_index > s->log.count) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n", + i, s->commit_index, s->log.count); + __builtin_trap(); + } + + // 2. commit_index >= 0 + if (s->commit_index < 0) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n", + i, s->commit_index); + __builtin_trap(); + } + + // 4. Future buffer count is in valid range. + if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n", + i, s->num_future, FUTURE_LIMIT); + __builtin_trap(); + } + } + + // Cross-node invariants + + // 5. At most one leader in normal status per view. + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i])) + continue; + for (int j = i + 1; j < num_nodes; j++) { + if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j])) + continue; + if (nodes[i]->view_number == nodes[j]->view_number) { + fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n", + (unsigned long)nodes[i]->view_number, i, j); + __builtin_trap(); + } + } + } + + // 6. Committed prefix agreement (State Machine Safety). + // For any two nodes, their logs must agree on all entries up to + // min(commit_index_i, commit_index_j). This is the core safety + // property of VSR: all committed operations are identical across + // replicas. + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) + continue; + for (int j = i + 1; j < num_nodes; j++) { + if (nodes[j] == NULL) + continue; + + int min_commit = nodes[i]->commit_index; + if (nodes[j]->commit_index < min_commit) + min_commit = nodes[j]->commit_index; + + for (int k = 0; k < min_commit; k++) { + if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(Operation)) != 0) { + fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d " + "between node %d and node %d\n", k, i, j); + __builtin_trap(); + } + } + } + } +} \ No newline at end of file diff --git a/vsr/node.h b/vsr/node.h new file mode 100644 index 0000000..14f8f36 --- /dev/null +++ b/vsr/node.h @@ -0,0 +1,176 @@ +#ifndef NODE_INCLUDED +#define NODE_INCLUDED + +#include +#include +#include + +#include + +#include "log.h" +#include "config.h" +#include "client_table.h" + +enum { + + // Normal Protocol + MESSAGE_TYPE_REQUEST, + MESSAGE_TYPE_REPLY, + MESSAGE_TYPE_PREPARE, + MESSAGE_TYPE_PREPARE_OK, + MESSAGE_TYPE_COMMIT, + + // View Change Protocol + MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + MESSAGE_TYPE_DO_VIEW_CHANGE, + MESSAGE_TYPE_BEGIN_VIEW, + + // Recovery Protocol + MESSAGE_TYPE_RECOVERY, + MESSAGE_TYPE_RECOVERY_RESPONSE, +}; + +typedef struct { + MessageHeader base; + Operation oper; + uint64_t client_id; + uint64_t request_id; +} RequestMessage; + +typedef struct { + MessageHeader base; + Operation oper; + int sender_idx; + int log_index; + int commit_index; + uint64_t view_number; +} PrepareMessage; + +typedef struct { + MessageHeader base; + int sender_idx; + int log_index; + uint64_t view_number; +} PrepareOKMessage; + +typedef struct { + MessageHeader base; + int commit_index; +} CommitMessage; + +typedef struct { + MessageHeader base; + bool rejected; + OperationResult result; +} ReplyMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int sender_idx; +} BeginViewChangeMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; // The new view number + uint64_t old_view_number; // Last view number when replica was in normal status + int op_number; // Number of entries in the log + int commit_index; + int sender_idx; + // Followed by: LogEntry log[op_number] +} DoViewChangeMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int commit_index; + int op_number; // Number of log entries that follow + // Followed by: LogEntry log[op_number] +} BeginViewMessage; + +typedef struct { + MessageHeader base; + int sender_idx; + uint64_t nonce; +} RecoveryMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int op_number; + uint64_t nonce; + int commit_index; + int sender_idx; +} RecoveryResponseMessage; + +typedef enum { + STATUS_NORMAL, + STATUS_CHANGE_VIEW, + STATUS_RECOVERY, +} Status; + +typedef struct { + + TCP tcp; + + Address self_addr; + Address node_addrs[NODE_LIMIT]; + int num_nodes; + + Status status; + + ClientTable client_table; + + uint64_t view_number; + + // These fields are used in recovery mode + uint32_t recovery_votes; + uint64_t recovery_nonce; + uint64_t max_view_seen; + int latest_primary_idx; + bool received_primary_state; + Log potential_primary_log; + Time recovery_start_time; + int recovery_attempt_count; + + /////////////////////////////////////////////////////////// + // VIEW CHANGE + + uint32_t begin_view_change_votes; // Bitmask of received BeginViewChange messages + uint32_t do_view_change_votes; // Bitmask of received DoViewChange messages + uint64_t do_view_change_best_old_view; // Best old_view_number seen in DoViewChange + int do_view_change_best_commit; // Best commit_index seen + Log do_view_change_best_log; // Best log seen + + /////////////////////////////////////////////////////////// + + // If this is the leader, commit_index is the index of the next uncommitted element + // of the log (if no uncommitted elements are present, it's equal to the total number + // of log elements) + int commit_index; + + PrepareMessage future[FUTURE_LIMIT]; + int num_future; + + Log log; + + Time last_heartbeat_time; + + StateMachine state_machine; + +} NodeState; + +struct pollfd; + +int node_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int node_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int node_free(void *state); + +void check_vsr_invariants(NodeState **nodes, int num_nodes); + +#endif // NODE_INCLUDED \ No newline at end of file