Version 0.8

.gitattributes

@@ -0,0 +1,2 @@
+# Enforce LF line endings for all text files
+* text eol=lf

.gitignore

@@ -0,0 +1,13 @@
+vsr_client
+vsr_server
+vsr_simulation
+raft_client
+raft_server
+raft_simulation
+raft_simulation_coverage
+*.gcno
+*.gcda
+*.gcov
+*.info
+coverage_html/
+*.log

build.sh

@@ -0,0 +1,6 @@
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o vsr_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c vsr/node.c vsr/client.c vsr/main.c vsr/log.c vsr/client_table.c state_machine/state_machine.c -o vsr_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c quakey/src/mockfs.c quakey/src/quakey.c -o raft_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -DDISABLE_DISK_CORRUPTION
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_server -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
+gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c raft/node.c raft/client.c raft/wal.c raft/main.c state_machine/state_machine.c -o raft_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT

lib/basic.c

@@ -0,0 +1,232 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "basic.h"
+
+bool streq(string s1, string s2)
+{
+    if (s1.len != s2.len)
+        return false;
+    for (int i = 0; i < s1.len; i++)
+        if (s1.ptr[i] != s2.ptr[i])
+            return false;
+    return true;
+}
+
+// Returns the current time in nanoseconds since
+// an unspecified time in the past (useful to calculate
+// elapsed time intervals)
+Time get_current_time(void)
+{
+#ifdef _WIN32
+    {
+        int64_t count;
+        int64_t freq;
+        int ok;
+
+        ok = QueryPerformanceCounter((LARGE_INTEGER*) &count);
+        if (!ok) return INVALID_TIME;
+
+        ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
+        if (!ok) return INVALID_TIME;
+
+        uint64_t res = 1000000000 * (double) count / freq;
+        return res;
+    }
+#else
+    {
+        struct timespec time;
+
+        if (clock_gettime(CLOCK_REALTIME, &time))
+            return INVALID_TIME;
+
+        uint64_t res;
+
+        uint64_t sec = time.tv_sec;
+        if (sec > UINT64_MAX / 1000000000)
+            return INVALID_TIME;
+        res = sec * 1000000000;
+
+        uint64_t nsec = time.tv_nsec;
+        if (res > UINT64_MAX - nsec)
+            return INVALID_TIME;
+        res += nsec;
+
+        return res;
+    }
+#endif
+}
+
+void nearest_deadline(Time *a, Time b)
+{
+    if (*a == INVALID_TIME || *a > b)
+        *a = b;
+}
+
+int deadline_to_timeout(Time deadline, Time current_time)
+{
+    if (deadline == INVALID_TIME)
+        return -1;
+    return (deadline - current_time) / 1000000;
+}
+
+bool getargb(int argc, char **argv, char *name)
+{
+    for (int i = 0; i < argc; i++)
+        if (!strcmp(argv[i], name))
+            return true;
+    return false;
+}
+
+string getargs(int argc, char **argv, char *name, char *fallback)
+{
+    for (int i = 0; i < argc; i++)
+        if (!strcmp(argv[i], name)) {
+            i++;
+            if (i == argc)
+                break;
+            return (string) { argv[i], strlen(argv[i]) };
+        }
+    return (string) { fallback, strlen(fallback) };
+}
+
+int getargi(int argc, char **argv, char *name, int fallback)
+{
+    for (int i = 0; i < argc; i++)
+        if (!strcmp(argv[i], name)) {
+
+            i++;
+            if (i == argc)
+                break;
+
+            errno = 0;
+            char *end;
+            long val = strtol(argv[i], &end, 10);
+
+            if (end == argv[i] || *end != '\0' || errno == ERANGE)
+                break;
+
+            if (val < INT_MIN || val > INT_MAX)
+                break;
+
+            return (int) val;
+        }
+    return fallback;
+}
+
+void append_hex_as_str(char *out, SHA256 hash)
+{
+    char table[] = "0123456789abcdef";
+    for (int i = 0; i < (int) sizeof(hash); i++) {
+        out[(i << 1) + 0] = table[(uint8_t) hash.data[i] >> 4];
+        out[(i << 1) + 1] = table[(uint8_t) hash.data[i] & 0xF];
+    }
+}
+
+// TODO: check this function
+bool addr_lower(Address a, Address b)
+{
+    if (a.is_ipv4) {
+
+        if (!b.is_ipv4)
+            return true;
+
+        if (a.ipv4.data < b.ipv4.data)
+            return true;
+
+        if (a.ipv4.data == b.ipv4.data &&
+            a.port < b.port)
+            return true;
+
+        return false;
+
+    } else {
+
+        if (b.is_ipv4)
+            return false;
+
+        for (int i = 0; i < 8; i++) {
+
+            if (a.ipv6.data[i] < b.ipv6.data[i])
+                return true;
+
+            if (a.ipv6.data[i] > b.ipv6.data[i])
+                return false;
+        }
+
+        if (a.port < b.port)
+            return true;
+
+        return false;
+    }
+}
+
+bool addr_eql(Address a, Address b)
+{
+    if (a.is_ipv4 != b.is_ipv4)
+        return false;
+
+    if (a.port != b.port)
+        return false;
+
+    if (a.is_ipv4) {
+        if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4)))
+            return false;
+    } else {
+        if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6)))
+            return false;
+    }
+
+    return true;
+}
+
+int parse_addr_arg(char *arg, Address *out)
+{
+    int len = strlen(arg);
+
+    int i = 0;
+    while (i < len && arg[i] != ':')
+        i++;
+
+    if (i == len)
+        return -1; // No ':' character.
+    arg[i] = '\0';
+
+    IPv4 ipv4;
+    int ret = inet_pton(AF_INET, arg, &ipv4);
+    arg[i] = ':';
+
+    if (ret != 1)
+        return -1;
+
+    errno = 0;
+    ret = atoi(arg + i + 1);
+    if (ret == 0 && errno != 0)
+        return -1;
+
+    out->ipv4 = ipv4;
+    out->is_ipv4 = true;
+    out->port = ret;
+
+    return 0;
+}
+
+void addr_sort(Address *addrs, int count)
+{
+    for (int i = 0; i < count; i++) {
+
+        int k = i; // Index of the lowest address in [i, num_nodes-1]
+        for (int j = i+1; j < count; j++) {
+            if (addr_lower(addrs[j], addrs[k]))
+                k = j;
+        }
+
+        Address tmp = addrs[i];
+        addrs[i] = addrs[k];
+        addrs[k] = tmp;
+    }
+}

lib/basic.h

@@ -0,0 +1,61 @@
+#ifndef BASIC_INCLUDED
+#define BASIC_INCLUDED
+
+#include <stdint.h>
+#include <stdbool.h>
+
+typedef struct {
+    char data[32];
+} SHA256;
+
+typedef struct {
+    uint32_t data;
+} IPv4;
+
+typedef struct {
+    uint16_t data[8];
+} IPv6;
+
+typedef struct {
+    union {
+        IPv4 ipv4;
+        IPv6 ipv6;
+    };
+    bool is_ipv4;
+    uint16_t port;
+} Address;
+
+typedef struct {
+    char *ptr;
+    int   len;
+} string;
+
+typedef uint64_t Time;
+#define INVALID_TIME ((Time) -1)
+
+#define S(X) ((string) { (X), (int) sizeof(X)-1 })
+
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+
+#define UNREACHABLE __builtin_trap();
+
+bool streq(string s1, string s2);
+
+Time get_current_time(void);
+void nearest_deadline(Time *a, Time b);
+int  deadline_to_timeout(Time deadline, Time current_time);
+
+bool   getargb(int argc, char **argv, char *name);
+string getargs(int argc, char **argv, char *name, char *fallback);
+int    getargi(int argc, char **argv, char *name, int fallback);
+
+void append_hex_as_str(char *out, SHA256 hash);
+
+bool addr_eql(Address a, Address b);
+bool addr_lower(Address a, Address b);
+
+int parse_addr_arg(char *arg, Address *out);
+void addr_sort(Address *addrs, int count);
+
+#endif // BASIC_INCLUDED

lib/byte_queue.c

@@ -0,0 +1,306 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include "byte_queue.h"
+
+// This is the implementation of a byte queue useful
+// for systems that need to process engs of bytes.
+//
+// It features sticky errors, a zero-copy interface,
+// and a safe mechanism to patch previously written
+// bytes.
+//
+// Only up to 4GB of data can be stored at once.
+
+// Initialize the queue
+void byte_queue_init(ByteQueue *queue, uint32_t limit)
+{
+    queue->flags = 0;
+    queue->head = 0;
+    queue->size = 0;
+    queue->used = 0;
+    queue->curs = 0;
+    queue->limit = limit;
+    queue->data = NULL;
+    queue->read_target = NULL;
+}
+
+// Deinitialize the queue
+void byte_queue_free(ByteQueue *queue)
+{
+    if (queue->read_target) {
+        if (queue->read_target != queue->data)
+            free(queue->read_target);
+        queue->read_target = NULL;
+        queue->read_target_size = 0;
+    }
+
+    free(queue->data);
+    queue->data = NULL;
+}
+
+int byte_queue_error(ByteQueue *queue)
+{
+    return queue->flags & BYTE_QUEUE_ERROR;
+}
+
+int byte_queue_empty(ByteQueue *queue)
+{
+    return queue->used == 0;
+}
+
+int byte_queue_full(ByteQueue *queue)
+{
+    return queue->used == queue->limit;
+}
+
+// Start a read operation on the queue.
+//
+// This function returnes the pointer to the memory region containing the bytes
+// to read. Callers can't read more than [*len] bytes from it. To complete the
+// read, the [byte_queue_read_ack] function must be called with the number of
+// bytes that were acknowledged by the caller.
+//
+// Note:
+//   - You can't have more than one pending read.
+ByteView byte_queue_read_buf(ByteQueue *queue)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return (ByteView) {NULL, 0};
+
+    assert((queue->flags & BYTE_QUEUE_READ) == 0);
+    queue->flags |= BYTE_QUEUE_READ;
+    queue->read_target      = queue->data;
+    queue->read_target_size = queue->size;
+
+    if (queue->data == NULL)
+        return (ByteView) {NULL, 0};
+
+    return (ByteView) { queue->data + queue->head, queue->used };
+}
+
+// Complete a previously started operation on the queue.
+void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return;
+
+    if ((queue->flags & BYTE_QUEUE_READ) == 0)
+        return;
+
+    queue->flags &= ~BYTE_QUEUE_READ;
+
+    assert((uint32_t) num <= queue->used);
+    queue->head += (uint32_t) num;
+    queue->used -= (uint32_t) num;
+    queue->curs += (uint32_t) num;
+
+    if (queue->read_target) {
+        if (queue->read_target != queue->data)
+            free(queue->read_target);
+        queue->read_target = NULL;
+        queue->read_target_size = 0;
+    }
+}
+
+ByteView byte_queue_write_buf(ByteQueue *queue)
+{
+    if ((queue->flags & BYTE_QUEUE_ERROR) || queue->data == NULL)
+        return (ByteView) {NULL, 0};
+
+    assert((queue->flags & BYTE_QUEUE_WRITE) == 0);
+    queue->flags |= BYTE_QUEUE_WRITE;
+
+    return (ByteView) {
+        queue->data + (queue->head + queue->used),
+        queue->size - (queue->head + queue->used),
+    };
+}
+
+void byte_queue_write_ack(ByteQueue *queue, uint32_t num)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return;
+
+    if ((queue->flags & BYTE_QUEUE_WRITE) == 0)
+        return;
+
+    queue->flags &= ~BYTE_QUEUE_WRITE;
+    queue->used += num;
+}
+
+// Sets the minimum capacity for the next write operation
+// and returns 1 if the content of the queue was moved, else
+// 0 is returned.
+//
+// You must not call this function while a write is pending.
+// In other words, you must do this:
+//
+//   byte_queue_write_setmincap(queue, mincap);
+//   dst = byte_queue_write_buf(queue, &cap);
+//   ...
+//   byte_queue_write_ack(num);
+//
+// And NOT this:
+//
+//   dst = byte_queue_write_buf(queue, &cap);
+//   byte_queue_write_setmincap(queue, mincap); <-- BAD
+//   ...
+//   byte_queue_write_ack(num);
+//
+int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
+{
+    // Sticky error
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return 0;
+
+    // In general, the queue's contents look like this:
+    //
+    //                           size
+    //                           v
+    //   [___xxxxxxxxxxxx________]
+    //   ^   ^           ^
+    //   0   head        head + used
+    //
+    // This function needs to make sure that at least [mincap]
+    // bytes are available on the right side of the content.
+    //
+    // We have 3 cases:
+    //
+    //   1) If there is enough memory already, this function doesn't
+    //      need to do anything.
+    //
+    //   2) If there isn't enough memory on the right but there is
+    //      enough free memory if we cound the left unused region,
+    //      then the content is moved back to the
+    //      start of the buffer.
+    //
+    //   3) If there isn't enough memory considering both sides, this
+    //      function needs to allocate a new buffer.
+    //
+    // If there are pending read or write operations, the application
+    // is holding pointers to the buffer, so we need to make sure
+    // to not invalidate them. The only real problem is pending reads
+    // since this function can only be called before starting a write
+    // opearation.
+    //
+    // To avoid invalidating the read pointer when we allocate a new
+    // buffer, we don't free the old buffer. Instead, we store the
+    // pointer in the "old" field so that the read ack function can
+    // free it.
+    //
+    // To avoid invalidating the pointer when we are moving back the
+    // content since there is enough memory at the start of the buffer,
+    // we just avoid that. Even if there is enough memory considering
+    // left and right free regions, we allocate a new buffer.
+
+    assert((queue->flags & BYTE_QUEUE_WRITE) == 0);
+
+    uint32_t total_free_space = queue->size - queue->used;
+    uint32_t free_space_after_data = queue->size - queue->used - queue->head;
+
+    int moved = 0;
+    if (free_space_after_data < mincap) {
+
+        if (total_free_space < mincap || (queue->read_target == queue->data)) {
+            // Resize required
+
+            if (queue->used + mincap > queue->limit) {
+                queue->flags |= BYTE_QUEUE_ERROR;
+                return 0;
+            }
+
+            uint32_t size;
+            if (queue->size > UINT32_MAX / 2)
+                size = UINT32_MAX;
+            else
+                size = 2 * queue->size;
+
+            if (size < queue->used + mincap)
+                size = queue->used + mincap;
+
+            if (size > queue->limit)
+                size = queue->limit;
+
+            uint8_t *data = malloc(size);
+            if (!data) {
+                queue->flags |= BYTE_QUEUE_ERROR;
+                return 0;
+            }
+
+            if (queue->used > 0)
+                memcpy(data, queue->data + queue->head, queue->used);
+
+            if (queue->read_target != queue->data)
+                free(queue->data);
+
+            queue->data = data;
+            queue->head = 0;
+            queue->size = size;
+
+        } else {
+            // Move required
+            memmove(queue->data, queue->data + queue->head, queue->used);
+            queue->head = 0;
+        }
+
+        moved = 1;
+    }
+
+    return moved;
+}
+
+void byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len)
+{
+    byte_queue_write_setmincap(queue, len);
+    ByteView dst = byte_queue_write_buf(queue);
+    if (dst.ptr) {
+        memcpy(dst.ptr, ptr, len);
+        byte_queue_write_ack(queue, len);
+    }
+}
+
+ByteQueueOffset byte_queue_offset(ByteQueue *queue)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return (ByteQueueOffset) { 0 };
+    return (ByteQueueOffset) { queue->curs + queue->used };
+}
+
+void byte_queue_patch(ByteQueue *queue, ByteQueueOffset off,
+    void *src, uint32_t len)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return;
+
+    // Check that the offset is in range
+    assert(off >= queue->curs && off - queue->curs < queue->used);
+
+    // Check that the length is in range
+    assert(len <= queue->used - (off - queue->curs));
+
+    // Perform the patch
+    uint8_t *dst = queue->data + queue->head + (off - queue->curs);
+    memcpy(dst, src, len);
+}
+
+uint32_t byte_queue_size_from_offset(ByteQueue *queue, ByteQueueOffset off)
+{
+    return queue->curs + queue->used - off;
+}
+
+void byte_queue_remove_from_offset(ByteQueue *queue, ByteQueueOffset offset)
+{
+    if (queue->flags & BYTE_QUEUE_ERROR)
+        return;
+
+    uint64_t num = (queue->curs + queue->used) - offset;
+    assert(num <= queue->used);
+
+    queue->used -= num;
+}

lib/byte_queue.h

@@ -0,0 +1,53 @@
+#ifndef BYTE_QUEUE_INCLUDED
+#define BYTE_QUEUE_INCLUDED
+
+#include <stddef.h>
+
+#include "basic.h"
+
+typedef struct {
+    uint8_t *ptr;
+    size_t   len;
+} ByteView;
+
+typedef struct {
+    uint64_t curs;
+    uint8_t* data;
+    uint32_t head;
+    uint32_t size;
+    uint32_t used;
+    uint32_t limit;
+    uint8_t* read_target;
+    uint32_t read_target_size;
+    int flags;
+} ByteQueue;
+
+typedef uint64_t ByteQueueOffset;
+
+enum {
+    BYTE_QUEUE_ERROR = 1 << 0,
+    BYTE_QUEUE_READ  = 1 << 1,
+    BYTE_QUEUE_WRITE = 1 << 2,
+};
+
+void byte_queue_init(ByteQueue *queue, uint32_t limit);
+void byte_queue_free(ByteQueue *queue);
+
+int byte_queue_error(ByteQueue *queue);
+int byte_queue_empty(ByteQueue *queue);
+int byte_queue_full(ByteQueue *queue);
+
+ByteView byte_queue_read_buf(ByteQueue *queue);
+void     byte_queue_read_ack(ByteQueue *queue, uint32_t num);
+
+ByteView byte_queue_write_buf(ByteQueue *queue);
+void     byte_queue_write_ack(ByteQueue *queue, uint32_t num);
+int      byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap);
+void     byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len);
+
+ByteQueueOffset byte_queue_offset(ByteQueue *queue);
+void            byte_queue_patch(ByteQueue *queue, ByteQueueOffset off, void *src, uint32_t len);
+uint32_t        byte_queue_size_from_offset(ByteQueue *queue, ByteQueueOffset off);
+void            byte_queue_remove_from_offset(ByteQueue *queue, ByteQueueOffset offset);
+
+#endif // BYTE_QUEUE_INCLUDED

lib/file_system.c

@@ -0,0 +1,459 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "file_system.h"
+
+int rename_file_or_dir(string oldpath, string newpath);
+
+bool file_exists(string path)
+{
+    char zt[1<<10];
+    if (path.len >= (int) sizeof(zt))
+        return false;
+    memcpy(zt, path.ptr, path.len);
+    zt[path.len] = '\0';
+
+#ifdef __linux__
+    return access(zt, F_OK) == 0;
+#endif
+
+#ifdef _WIN32
+    DWORD attrs = GetFileAttributesA(zt);
+    return attrs != INVALID_FILE_ATTRIBUTES;
+#endif
+}
+
+int file_open(string path, Handle *fd)
+{
+#ifdef __linux__
+    char zt[1<<10];
+    if (path.len >= (int) sizeof(zt))
+        return -1;
+    memcpy(zt, path.ptr, path.len);
+    zt[path.len] = '\0';
+
+    int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
+    if (ret < 0)
+        return -1;
+
+    *fd = (Handle) { (uint64_t) ret };
+    return 0;
+#endif
+
+#ifdef _WIN32
+    WCHAR wpath[MAX_PATH];
+    MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH);
+    wpath[path.len] = L'\0';
+
+    HANDLE h = CreateFileW(
+        wpath,
+        GENERIC_WRITE | GENERIC_READ,
+        0,
+        NULL,
+        OPEN_ALWAYS,
+        FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH,
+        NULL
+    );
+    if (h == INVALID_HANDLE_VALUE)
+        return -1;
+
+    *fd = (Handle) { (uint64_t) h };
+    return 0;
+#endif
+}
+
+void file_close(Handle fd)
+{
+#ifdef __linux__
+    close((int) fd.data);
+#endif
+
+#ifdef _WIN32
+    CloseHandle((HANDLE) fd.data);
+#endif
+}
+
+int file_truncate(Handle fd, size_t new_size)
+{
+#ifdef __linux__
+    if (ftruncate((int) fd.data, new_size) < 0)
+        return -1;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    return -1; // TODO: Not implemented
+#endif
+}
+
+int file_set_offset(Handle fd, int off)
+{
+#ifdef __linux__
+    off_t ret = lseek((int) fd.data, off, SEEK_SET);
+    if (ret < 0)
+        return -1;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    LARGE_INTEGER distance;
+    distance.QuadPart = off;
+    if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
+        if (GetLastError() != 0)
+            return -1;
+    return 0;
+#endif
+}
+
+int file_get_offset(Handle fd, int *off)
+{
+#ifdef __linux__
+    off_t ret = lseek((int) fd.data, 0, SEEK_CUR);
+    if (ret < 0)
+        return -1;
+    *off = (int) ret;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
+    if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0)
+        return -1;
+    *off = (int) pos;
+    return 0;
+#endif
+}
+
+int file_lock(Handle fd)
+{
+#ifdef __linux__
+    if (flock((int) fd.data, LOCK_EX) < 0)
+        return -1;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
+        return -1;
+    return 0;
+#endif
+}
+
+int file_unlock(Handle fd)
+{
+#ifdef __linux__
+    if (flock((int) fd.data, LOCK_UN) < 0)
+        return -1;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
+        return -1;
+    return 0;
+#endif
+}
+
+int file_sync(Handle fd)
+{
+#ifdef __linux__
+    if (fsync((int) fd.data) < 0)
+        return -1;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    if (!FlushFileBuffers((HANDLE) fd.data))
+        return -1;
+    return 0;
+#endif
+}
+
+int file_read(Handle fd, char *dst, int max)
+{
+#ifdef __linux__
+    return read((int) fd.data, dst, max);
+#endif
+
+#ifdef _WIN32
+    DWORD num;
+    if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
+        return -1;
+    if (num > INT_MAX)
+        return -1;
+    return num;
+#endif
+}
+
+int file_write(Handle fd, char *src, int len)
+{
+#ifdef __linux__
+    return write((int) fd.data, src, len);
+#endif
+
+#ifdef _WIN32
+    DWORD num;
+    if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL))
+        return -1;
+    if (num > INT_MAX)
+        return -1;
+    return num;
+#endif
+}
+
+int file_size(Handle fd, size_t *len)
+{
+#ifdef __linux__
+    struct stat buf;
+    if (fstat((int) fd.data, &buf) < 0)
+        return -1;
+    if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX)
+        return -1;
+    *len = (size_t) buf.st_size;
+    return 0;
+#endif
+
+#ifdef _WIN32
+    LARGE_INTEGER buf;
+    if (!GetFileSizeEx((HANDLE) fd.data, &buf))
+        return -1;
+    if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX)
+        return -1;
+    *len = buf.QuadPart;
+    return 0;
+#endif
+}
+
+int create_dir(string path)
+{
+    char zt[PATH_MAX];
+    if (path.len >= (int) sizeof(zt))
+        return -1;
+    memcpy(zt, path.ptr, path.len);
+    zt[path.len] = '\0';
+
+#ifdef _WIN32
+    if (_mkdir(zt) < 0)
+        return -1;
+#else
+    if (mkdir(zt, 0766))
+        return -1;
+#endif
+
+    return 0;
+}
+
+int rename_file_or_dir(string oldpath, string newpath)
+{
+    char oldpath_zt[PATH_MAX];
+    if (oldpath.len >= (int) sizeof(oldpath_zt))
+        return -1;
+    memcpy(oldpath_zt, oldpath.ptr, oldpath.len);
+    oldpath_zt[oldpath.len] = '\0';
+
+    char newpath_zt[PATH_MAX];
+    if (newpath.len >= (int) sizeof(newpath_zt))
+        return -1;
+    memcpy(newpath_zt, newpath.ptr, newpath.len);
+    newpath_zt[newpath.len] = '\0';
+
+    if (rename(oldpath_zt, newpath_zt))
+        return -1;
+    return 0;
+}
+
+int remove_file_or_dir(string path)
+{
+    char path_zt[PATH_MAX];
+    if (path.len >= (int) sizeof(path_zt))
+        return -1;
+    memcpy(path_zt, path.ptr, path.len);
+    path_zt[path.len] = '\0';
+
+    if (remove(path_zt))
+        return -1;
+    return 0;
+}
+
+int get_full_path(string path, char *dst)
+{
+    char path_zt[PATH_MAX];
+    if (path.len >= (int) sizeof(path_zt))
+        return -1;
+    memcpy(path_zt, path.ptr, path.len);
+    path_zt[path.len] = '\0';
+
+#ifdef __linux__
+    if (realpath(path_zt, dst) == NULL)
+        return -1;
+#endif
+
+#ifdef _WIN32
+    if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
+        return -1;
+#endif
+
+    size_t path_len = strlen(dst);
+    if (path_len > 0 && dst[path_len-1] == '/')
+        dst[path_len-1] = '\0';
+
+    return 0;
+}
+
+int file_read_all(string path, string *data)
+{
+    Handle fd;
+    int ret = file_open(path, &fd);
+    if (ret < 0)
+        return -1;
+
+    size_t len;
+    ret = file_size(fd, &len);
+    if (ret < 0) {
+        file_close(fd);
+        return -1;
+    }
+
+    char *dst = malloc(len);
+    if (dst == NULL) {
+        file_close(fd);
+        return -1;
+    }
+
+    int copied = 0;
+    while ((size_t) copied < len) {
+        ret = file_read(fd, dst + copied, len - copied);
+        if (ret < 0) {
+            free(dst);
+            file_close(fd);
+            return -1;
+        }
+        copied += ret;
+    }
+
+    *data = (string) { dst, len };
+    file_close(fd);
+    return 0;
+}
+
+#ifdef _WIN32
+
+int directory_scanner_init(DirectoryScanner *scanner, string path)
+{
+    char pattern[PATH_MAX];
+    int ret = snprintf(pattern, sizeof(pattern), "%.*s\\*", path.len, path.ptr);
+    if (ret < 0 || ret >= (int) sizeof(pattern))
+        return -1;
+
+    scanner->handle = FindFirstFileA(pattern, &scanner->find_data);
+    if (scanner->handle == INVALID_HANDLE_VALUE) {
+        if (GetLastError() == ERROR_FILE_NOT_FOUND) {
+            scanner->done = true;
+            return 0;
+        }
+        return -1;
+    }
+
+    scanner->done = false;
+    scanner->first = true;
+    return 0;
+}
+
+int directory_scanner_next(DirectoryScanner *scanner, string *name)
+{
+    if (scanner->done)
+        return 1;
+
+    if (!scanner->first) {
+        BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data);
+        if (!ok) {
+            scanner->done = true;
+            if (GetLastError() == ERROR_NO_MORE_FILES)
+                return 1;
+            return -1;
+        }
+    } else {
+        scanner->first = false;
+    }
+
+    char *p = scanner->find_data.cFileName;
+    *name = (string) { p, strlen(p) };
+    return 0;
+}
+
+void directory_scanner_free(DirectoryScanner *scanner)
+{
+    FindClose(scanner->handle);
+}
+
+#else
+
+int directory_scanner_init(DirectoryScanner *scanner, string path)
+{
+    char path_copy[PATH_MAX];
+    if (path.len >= PATH_MAX)
+        return -1;
+    memcpy(path_copy, path.ptr, path.len);
+    path_copy[path.len] = '\0';
+
+    scanner->d = opendir(path_copy);
+    if (scanner->d == NULL) {
+        scanner->done = true;
+        return -1;
+    }
+
+    scanner->done = false;
+    return 0;
+}
+
+int directory_scanner_next(DirectoryScanner *scanner, string *name)
+{
+    if (scanner->done)
+        return 1;
+
+    scanner->e = readdir(scanner->d);
+    if (scanner->e == NULL) {
+        scanner->done = true;
+        return 1;
+    }
+
+    *name = (string) { scanner->e->d_name, strlen(scanner->e->d_name) };
+    return 0;
+}
+
+void directory_scanner_free(DirectoryScanner *scanner)
+{
+    closedir(scanner->d);
+}
+
+#endif
+
+int file_read_exact(Handle handle, char *dst, int len)
+{
+    int copied = 0;
+    while (copied < len) {
+        int ret = file_read(handle, dst + copied, len - copied);
+        if (ret < 0)
+            return -1;
+        if (ret == 0)
+            return 0; // EOF
+        copied += ret;
+    }
+    return copied;
+}
+
+int file_write_exact(Handle handle, char *src, int len)
+{
+    int copied = 0;
+    while (copied < len) {
+        int ret = file_write(handle, src + copied, len - copied);
+        if (ret < 0)
+            return -1;
+        copied += ret;
+    }
+    return 0;
+}

lib/file_system.h

@@ -0,0 +1,58 @@
+#ifndef FILE_SYSTEM_INCLUDED
+#define FILE_SYSTEM_INCLUDED
+
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "basic.h"
+
+typedef struct {
+    uint64_t data;
+} Handle;
+
+#ifdef _WIN32
+typedef struct {
+    HANDLE handle;
+    WIN32_FIND_DATA find_data;
+    bool first;
+    bool done;
+} DirectoryScanner;
+#else
+typedef struct {
+    DIR *d;
+    struct dirent *e;
+    bool done;
+} DirectoryScanner;
+#endif
+
+bool file_exists(string path);
+int  file_open(string path, Handle *fd);
+void file_close(Handle fd);
+int  file_truncate(Handle fd, size_t new_size);
+int  file_set_offset(Handle fd, int off);
+int  file_get_offset(Handle fd, int *off);
+int  file_lock(Handle fd);
+int  file_unlock(Handle fd);
+int  file_sync(Handle fd);
+int  file_read(Handle fd, char *dst, int max);
+int  file_write(Handle fd, char *src, int len);
+int  file_size(Handle fd, size_t *len);
+int  file_write_atomic(string path, string content);
+int  create_dir(string path);
+int  rename_file_or_dir(string oldpath, string newpath);
+int  remove_file_or_dir(string path);
+int  get_full_path(string path, char *dst);
+int  file_read_all(string path, string *data);
+
+int  directory_scanner_init(DirectoryScanner *scanner, string path);
+int  directory_scanner_next(DirectoryScanner *scanner, string *name);
+void directory_scanner_free(DirectoryScanner *scanner);
+
+int file_read_exact(Handle handle, char *dst, int len);
+int file_write_exact(Handle handle, char *src, int len);
+
+#endif // FILE_SYSTEM_INCLUDED

lib/message.c

@@ -0,0 +1,81 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "message.h"
+
+bool binary_read(BinaryReader *reader, void *dst, int len)
+{
+    if (reader->len - reader->cur < len)
+        return false;
+    if (dst)
+        memcpy(dst, reader->src + reader->cur, len);
+    reader->cur += len;
+    return true;
+}
+
+void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type)
+{
+    uint16_t version = MESSAGE_VERSION;
+    uint32_t dummy = 0; // Dummy value
+    writer->output = output;
+    writer->start  = byte_queue_offset(output);
+    byte_queue_write(output, &version, sizeof(version));
+    byte_queue_write(output, &type, sizeof(type));
+    writer->patch = byte_queue_offset(output);
+    byte_queue_write(output, &dummy, sizeof(dummy));
+}
+
+bool message_writer_free(MessageWriter *writer)
+{
+    uint32_t length = byte_queue_size_from_offset(writer->output, writer->start);
+    byte_queue_patch(writer->output, writer->patch, &length, sizeof(length));
+    if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure?
+        return false;
+    return true;
+}
+
+void message_write(MessageWriter *writer, void *mem, int len)
+{
+    byte_queue_write(writer->output, mem, len);
+}
+
+void message_write_u8(MessageWriter *writer, uint8_t value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+void message_write_u32(MessageWriter *writer, uint32_t value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+void message_write_hash(MessageWriter *writer, SHA256 value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+int message_peek(ByteView msg, uint16_t *type, uint32_t *len)
+{
+    if (msg.len < (int) sizeof(MessageHeader))
+        return 0;
+
+    MessageHeader header;
+    memcpy(&header, msg.ptr, sizeof(header));
+
+    // (We ignore endianess for now)
+
+    if (header.version != MESSAGE_VERSION)
+        return -1;
+
+    if (header.length > msg.len)
+        return 0;
+
+    if (type) *type = header.type;
+    if (len) *len = header.length;
+
+    return 1;
+}

lib/message.h

@@ -0,0 +1,46 @@
+#ifndef MESSAGE_INCLUDED
+#define MESSAGE_INCLUDED
+
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "basic.h"
+#include "byte_queue.h"
+
+#define MESSAGE_VERSION 1
+
+typedef struct {
+    uint8_t *src;
+    int      len;
+    int      cur;
+} BinaryReader;
+
+typedef struct {
+    uint16_t version;
+    uint16_t type;
+    uint32_t length;
+} MessageHeader;
+
+typedef struct {
+    ByteQueue *output;
+    ByteQueueOffset start;
+    ByteQueueOffset patch;
+} MessageWriter;
+
+bool binary_read(BinaryReader *reader, void *dst, int len);
+
+void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type);
+bool message_writer_free(MessageWriter *writer);
+void message_write(MessageWriter *writer, void *mem, int len);
+void message_write_u8(MessageWriter *writer, uint8_t value);
+void message_write_u32(MessageWriter *writer, uint32_t value);
+void message_write_hash(MessageWriter *writer, SHA256 value);
+
+int  message_peek(ByteView msg, uint16_t *type, uint32_t *len);
+void message_dump(FILE *stream, ByteView msg);
+
+#endif // MESSAGE_INCLUDED

lib/tcp.c

@@ -0,0 +1,501 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+
+#include "tcp.h"
+#include "message.h"
+
+static int set_socket_blocking(SOCKET sock, bool value)
+{
+#ifdef _WIN32
+    u_long mode = !value;
+    if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
+        return -1;
+#else
+    int flags = fcntl(sock, F_GETFL, 0);
+    if (flags < 0)
+        return -1;
+    if (value) flags &= ~O_NONBLOCK;
+    else       flags |= O_NONBLOCK;
+    if (fcntl(sock, F_SETFL, flags) < 0)
+        return -1;
+#endif
+
+    return 0;
+}
+
+static SOCKET create_listen_socket(Address addr)
+{
+    SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd == INVALID_SOCKET)
+        return INVALID_SOCKET;
+
+    if (set_socket_blocking(fd, false) < 0) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+    // TODO: mark address as reusable in debug builds
+
+    if (!addr.is_ipv4) {
+        assert(0); // TODO
+    }
+
+    struct sockaddr_in bind_buf;
+    bind_buf.sin_family = AF_INET;
+    bind_buf.sin_port   = htons(addr.port);
+    memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4));
+    if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+    int backlog = 32;
+    if (listen(fd, backlog) < 0) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+    return fd;
+}
+
+static int create_socket_pair(SOCKET *a, SOCKET *b)
+{
+#ifdef _WIN32
+    SOCKET sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    if (sock == INVALID_SOCKET)
+        return -1;
+
+    // Bind to loopback address with port 0 (dynamic port assignment)
+    struct sockaddr_in addr;
+    int addr_len = sizeof(addr);
+    memset(&addr, 0, sizeof(addr));
+    addr.sin_family = AF_INET;
+    addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1
+    addr.sin_port = 0; // Let system choose port
+
+    if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    if (getsockname(sock, (struct sockaddr*)&addr, &addr_len) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    if (connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    *a = sock;
+    *b = sock;
+
+    // Optional: Set socket to non-blocking mode
+    // This prevents send() from blocking if the receive buffer is full
+    u_long mode = 1;
+    ioctlsocket(sock, FIONBIO, &mode); // TODO: does this fail?
+    return 0;
+#else
+    int fds[2];
+    if (pipe(fds) < 0)
+        return -1;
+    *a = fds[0];
+    *b = fds[1];
+    return 0;
+#endif
+}
+
+static void close_socket_pair(SOCKET a, SOCKET b)
+{
+#ifdef _WIN32
+    closesocket(a);
+    (void) b;
+#else
+    close(a);
+    close(b);
+#endif
+}
+
+static void conn_init(Connection *conn, SOCKET fd, bool connecting)
+{
+    conn->fd = fd;
+    conn->tag = -1;
+    conn->connecting = connecting;
+    conn->closing = false;
+    conn->msglen = 0;
+    byte_queue_init(&conn->input, 1<<20);
+    byte_queue_init(&conn->output, 1<<20);
+}
+
+static void conn_free(Connection *conn)
+{
+    CLOSE_SOCKET(conn->fd);
+    byte_queue_free(&conn->input);
+    byte_queue_free(&conn->output);
+}
+
+static int conn_events(Connection *conn)
+{
+    int events = 0;
+
+    if (conn->connecting)
+        events |= POLLOUT;
+    else {
+
+        assert(!byte_queue_full(&conn->input));
+        if (!conn->closing)
+            events |= POLLIN;
+
+        if (!byte_queue_empty(&conn->output))
+            events |= POLLOUT;
+    }
+    return events;
+}
+
+int tcp_context_init(TCP *tcp)
+{
+    tcp->listen_fd = INVALID_SOCKET;
+    tcp->num_conns = 0;
+
+    if (create_socket_pair(&tcp->wait_fd, &tcp->signal_fd) < 0)
+        return -1;
+
+    return 0;
+}
+
+void tcp_context_free(TCP *tcp)
+{
+    // Free all connection byte queues without closing sockets
+    // (sockets are managed by the simulation and will be cleaned up separately)
+    for (int i = 0; i < tcp->num_conns; i++) {
+        byte_queue_free(&tcp->conns[i].input);
+        byte_queue_free(&tcp->conns[i].output);
+    }
+    tcp->num_conns = 0;
+
+    if (tcp->listen_fd != INVALID_SOCKET)
+        CLOSE_SOCKET(tcp->listen_fd);
+
+    close_socket_pair(tcp->wait_fd, tcp->signal_fd);
+}
+
+int tcp_wakeup(TCP *tcp)
+{
+    send(tcp->signal_fd, "0", 1, 0); // TODO: Handle error
+    return 0;
+}
+
+int tcp_index_from_tag(TCP *tcp, int tag)
+{
+    for (int i = 0; i < tcp->num_conns; i++)
+        if (tcp->conns[i].tag == tag)
+            return i;
+    return -1;
+}
+
+int tcp_listen(TCP *tcp, Address addr)
+{
+    SOCKET listen_fd = create_listen_socket(addr);
+    if (listen_fd == INVALID_SOCKET)
+        return -1;
+
+    tcp->listen_fd = listen_fd;
+    return 0;
+}
+
+int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type)
+{
+    *msg = byte_queue_read_buf(&tcp->conns[conn_idx].input);
+
+    uint32_t len;
+    int ret = message_peek(*msg, type, &len);
+
+    // Invalid message?
+    if (ret < 0) {
+        byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
+        return -1;
+    }
+
+    // Still buffering header?
+    if (ret == 0) {
+        byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
+        if (byte_queue_full(&tcp->conns[conn_idx].input))
+            return -1;
+        return 0;
+    }
+
+    // Message received
+    assert(ret > 0);
+    msg->len = len;
+    tcp->conns[conn_idx].msglen = len;
+
+    return 1;
+}
+
+void tcp_consume_message(TCP *tcp, int conn_idx)
+{
+    byte_queue_read_ack(&tcp->conns[conn_idx].input, tcp->conns[conn_idx].msglen);
+    tcp->conns[conn_idx].msglen = 0;
+}
+
+int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled)
+{
+    int num_polled = 0;
+
+    polled[num_polled].fd = tcp->wait_fd;
+    polled[num_polled].events = POLLIN;
+    polled[num_polled].revents = 0;
+    contexts[num_polled] = NULL;
+    num_polled++;
+
+    if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < TCP_CONNECTION_LIMIT) {
+        polled[num_polled].fd = tcp->listen_fd;
+        polled[num_polled].events = POLLIN;
+        polled[num_polled].revents = 0;
+        contexts[num_polled] = NULL;
+        num_polled++;
+    }
+
+    for (int i = 0; i < tcp->num_conns; i++) {
+        int events = conn_events(&tcp->conns[i]);
+        if (events) {
+            polled[num_polled].fd = tcp->conns[i].fd;
+            polled[num_polled].events = events;
+            polled[num_polled].revents = 0;
+            contexts[num_polled] = &tcp->conns[i];
+            num_polled++;
+        }
+    }
+
+    return num_polled;
+}
+
+// The "events" array must be an array of capacity TCP_EVENT_CAPACITY,
+// while "contexts" and "polled" must have capacity TCP_POLL_CAPACITY.
+int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled)
+{
+    bool removed[TCP_POLL_CAPACITY];
+    for (int i = 0; i < TCP_POLL_CAPACITY; i++)
+        removed[i] = false;
+
+    int num_events = 0;
+    for (int i = 1; i < num_polled; i++) {
+
+        if (polled[i].fd == tcp->wait_fd) {
+
+            if (polled[i].revents & POLLIN) {
+                char buf[100];
+                recv(tcp->wait_fd, buf, sizeof(buf), 0); // TODO: Make sure all bytes are consumed
+                events[num_events++] = (Event) { EVENT_WAKEUP, -1, -1 };
+            }
+
+        } else if (polled[i].fd == tcp->listen_fd) {
+
+            assert(contexts[i] == NULL);
+
+            if (polled[i].revents & POLLIN) {
+                SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
+                if (new_fd != INVALID_SOCKET) {
+
+                    if (set_socket_blocking(new_fd, false) < 0)
+                        CLOSE_SOCKET(new_fd);
+                    else {
+                        conn_init(&tcp->conns[tcp->num_conns++], new_fd, false);
+                        events[num_events++] = (Event) { EVENT_CONNECT, tcp->num_conns-1, tcp->conns[tcp->num_conns-1].tag };
+                    }
+                }
+            }
+            removed[i] = false;
+
+        } else {
+
+            Connection *conn = contexts[i];
+            bool defer_close = false;
+            bool defer_ready = false;
+
+            if (conn->connecting) {
+
+                // Check for error conditions on the socket
+                if (polled[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+                    defer_close = true;
+                } else if (polled[i].revents & POLLOUT) {
+
+                    int err = 0;
+                    socklen_t len = sizeof(err);
+                    if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
+                        defer_close = true;
+                    else {
+                        conn->connecting = false;
+                        events[num_events++] = (Event) { EVENT_CONNECT, conn - tcp->conns, conn->tag };
+                    }
+                }
+
+            } else {
+
+                if (polled[i].revents & POLLIN) {
+                    byte_queue_write_setmincap(&conn->input, 1<<9);
+                    ByteView buf = byte_queue_write_buf(&conn->input);
+                    int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
+                    if (num == 0)
+                        defer_close = true;
+                    else if (num < 0) {
+                        if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) // TODO: does Windows return these error codes or not?
+                            defer_close = true;
+                        num = 0;
+                    }
+                    byte_queue_write_ack(&conn->input, num);
+                    ByteView msg = byte_queue_read_buf(&conn->input);
+                    int ret = message_peek(msg, NULL, NULL);
+                    byte_queue_read_ack(&conn->input, 0);
+                    if (ret < 0) {
+                        // Invalid message
+                        defer_close = true;
+                    } else if (ret == 0) {
+                        // Still buffering
+                        if (byte_queue_full(&conn->input))
+                            defer_close = true;
+                    } else {
+                        // Message received
+                        assert(ret > 0);
+                        defer_ready = true;
+                    }
+                }
+
+                if (polled[i].revents & POLLOUT) {
+                    ByteView buf = byte_queue_read_buf(&conn->output);
+                    int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
+                    if (num < 0) {
+                        if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
+                            defer_close = true;
+                        num = 0;
+                    }
+                    byte_queue_read_ack(&conn->output, num);
+                    if (conn->closing && byte_queue_empty(&conn->output))
+                        defer_close = true;
+                }
+            }
+
+            // TODO: byte_queue_error here?
+
+            removed[i] = defer_close;
+            if (0) {}
+            else if (defer_close) events[num_events++] = (Event) { EVENT_DISCONNECT, conn - tcp->conns, conn->tag };
+            else if (defer_ready) events[num_events++] = (Event) { EVENT_MESSAGE,    conn - tcp->conns, conn->tag };
+        }
+    }
+
+    for (int i = 1; i < num_polled; i++) {
+        if (removed[i]) {
+            Connection *conn = contexts[i];
+            assert(conn);
+            int removed_idx = conn - tcp->conns;
+            conn_free(conn);
+            int last_idx = --tcp->num_conns;
+            if (removed_idx != last_idx) {
+                *conn = tcp->conns[last_idx];
+                // Update event conn_idx values to reflect the swap
+                for (int j = 0; j < num_events; j++) {
+                    if (events[j].conn_idx == last_idx)
+                        events[j].conn_idx = removed_idx;
+                }
+                // Update contexts pointers for remaining iterations
+                for (int j = i + 1; j < num_polled; j++) {
+                    if (contexts[j] == &tcp->conns[last_idx])
+                        contexts[j] = conn;
+                }
+            }
+        }
+    }
+
+    return num_events;
+}
+
+ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx)
+{
+    return &tcp->conns[conn_idx].output;
+}
+
+int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
+{
+    if (tcp->num_conns == TCP_CONNECTION_LIMIT)
+        return -1;
+    int conn_idx = tcp->num_conns;
+
+    SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd == INVALID_SOCKET)
+        return -1;
+
+    if (set_socket_blocking(fd, false) < 0) {
+        CLOSE_SOCKET(fd);
+        return -1;
+    }
+
+    int ret;
+    if (addr.is_ipv4) {
+        struct sockaddr_in buf;
+        buf.sin_family = AF_INET;
+        buf.sin_port = htons(addr.port);
+        memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
+        ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
+    } else {
+        struct sockaddr_in6 buf;
+        buf.sin6_family = AF_INET6;
+        buf.sin6_port = htons(addr.port);
+        memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
+        ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
+    }
+
+    bool connecting;
+    if (ret == 0) {
+        connecting = false;
+    } else {
+        if (errno != EINPROGRESS) {
+            CLOSE_SOCKET(fd);
+            return -1;
+        }
+        connecting = true;
+    }
+
+    // Check that this tag wasn't already used
+    for (int i = 0; i < tcp->num_conns; i++)
+        assert(tcp->conns[i].tag != tag);
+
+    conn_init(&tcp->conns[conn_idx], fd, connecting);
+    tcp->conns[conn_idx].tag = tag;
+
+    if (output)
+        *output = &tcp->conns[conn_idx].output;
+
+    tcp->num_conns++;
+    return 0;
+}
+
+void tcp_close(TCP *tcp, int conn_idx)
+{
+    tcp->conns[conn_idx].closing = true;
+    tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection
+    // TODO: if no event will be triggered, the connection will not be closed
+    //       if the output buffer is empty, the connection should be closed here.
+}
+
+void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique)
+{
+    assert(tag != -1);
+
+    if (unique) {
+        for (int i = 0; i < tcp->num_conns; i++)
+            assert(tcp->conns[i].tag != tag);
+    }
+
+    tcp->conns[conn_idx].tag = tag;
+}
+
+int tcp_get_tag(TCP *tcp, int conn_idx)
+{
+    return tcp->conns[conn_idx].tag;
+}

lib/tcp.h

@@ -0,0 +1,87 @@
+#ifndef TCP_INCLUDED
+#define TCP_INCLUDED
+
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#   define QUAKEY_ENABLE_MOCKS
+#   include <quakey.h>
+#else
+#   ifdef _WIN32
+#       include <winsock2.h>
+#   endif
+#endif
+
+#include "byte_queue.h"
+
+#ifdef _WIN32
+#define CLOSE_SOCKET closesocket
+#else
+#define SOCKET int
+#define INVALID_SOCKET -1
+#define CLOSE_SOCKET close
+#endif
+
+#ifndef TCP_CONNECTION_LIMIT
+// Maximum number of connections that can be managed
+// simultaneously.
+#define TCP_CONNECTION_LIMIT 512
+#endif
+
+// This is the maximum number of descriptors that the
+// TCP system will want to wait at any given time.
+// One descriptor per connection plus a listener socket
+// and a self-pipe handle for wakeup.
+#define TCP_POLL_CAPACITY (TCP_CONNECTION_LIMIT+2)
+
+// Number of TCP events that can be returned at a given
+// time by "tcp_translate_events". There may be a single
+// event per connection (MESSAGE, CONNECT, DISCONNECT)
+// plus a general WAKEUP event.
+#define TCP_EVENT_CAPACITY (TCP_CONNECTION_LIMIT+1)
+
+typedef enum {
+    EVENT_WAKEUP,
+    EVENT_MESSAGE,
+    EVENT_CONNECT,
+    EVENT_DISCONNECT,
+} EventType;
+
+typedef struct {
+    EventType type;
+    int conn_idx;
+    int tag;
+} Event;
+
+typedef struct {
+    SOCKET    fd;
+    int       tag;
+    bool      connecting;
+    bool      closing;
+    uint32_t  msglen;
+    ByteQueue input;
+    ByteQueue output;
+} Connection;
+
+typedef struct {
+    SOCKET listen_fd;
+    SOCKET wait_fd;
+    SOCKET signal_fd;
+    int    num_conns;
+    Connection conns[TCP_CONNECTION_LIMIT];
+} TCP;
+
+int  tcp_context_init(TCP *tcp);
+void tcp_context_free(TCP *tcp);
+int  tcp_wakeup(TCP *tcp);
+int  tcp_index_from_tag(TCP *tcp, int tag);
+int  tcp_listen(TCP *tcp, Address addr);
+int  tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type);
+void tcp_consume_message(TCP *tcp, int conn_idx);
+int  tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled);
+int  tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled);
+ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx);
+int  tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output);
+void tcp_close(TCP *tcp, int conn_idx);
+void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique);
+int  tcp_get_tag(TCP *tcp, int conn_idx);
+
+#endif // TCP_INCLUDED

quakey/include/quakey.h

@@ -0,0 +1,223 @@
+#ifndef QUAKEY_INCLUDED
+#define QUAKEY_INCLUDED
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <direct.h>
+#else
+#include <time.h>
+#include <poll.h>
+#include <errno.h>
+#include <unistd.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <pthread.h>
+#endif
+
+typedef struct {} Quakey;
+
+// Function pointers to a simulated program's code
+typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+typedef int (*QuakeyFreeFunc)(void *state);
+
+typedef enum {
+    QUAKEY_LINUX,
+    QUAKEY_WINDOWS,
+} QuakeyPlatform;
+
+typedef struct {
+
+    // Label associated to the process for debugging
+    // The string must have global lifetime
+    char *name;
+
+    // Size of the opaque state struct
+    int state_size;
+
+    // Pointers to program code
+    QuakeyInitFunc init_func;
+    QuakeyTickFunc tick_func;
+    QuakeyFreeFunc free_func;
+
+    // Network addresses enabled on the process
+    char **addrs;
+    int num_addrs;
+
+    // Disk size for the process
+    int disk_size;
+
+    // Platform used by this program
+    QuakeyPlatform platform;
+
+} QuakeySpawn;
+
+typedef unsigned long long QuakeyUInt64;
+
+// Start a simulation
+int quakey_init(Quakey **quakey, QuakeyUInt64 seed);
+
+// Stop a simulation
+void quakey_free(Quakey *quakey);
+
+typedef unsigned long long QuakeyNode;
+
+// Add a program to the simulation
+QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg);
+
+void *quakey_node_state(QuakeyNode node);
+
+// Schedule and executes one program until it would block, then returns
+int quakey_schedule_one(Quakey *quakey);
+
+// Generate a random u64
+QuakeyUInt64 quakey_random(void);
+
+typedef struct {
+    char name[32];
+    int  name_len;
+} QuakeySignal;
+
+void quakey_signal(char *name);
+int  quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
+
+// Access spawned host information
+int         quakey_num_hosts(Quakey *quakey);
+void       *quakey_host_state(Quakey *quakey, int idx);  // Returns NULL if host is dead
+int         quakey_host_is_dead(Quakey *quakey, int idx);
+const char *quakey_host_name(Quakey *quakey, int idx);
+
+int *mock_errno_ptr(void);
+
+#ifdef _WIN32
+BOOL   mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
+BOOL   mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
+HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
+BOOL   mock_CloseHandle(HANDLE handle);
+BOOL   mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
+BOOL   mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
+BOOL   mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
+DWORD  mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
+BOOL   mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
+BOOL   mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
+BOOL   mock_FlushFileBuffers(HANDLE handle);
+BOOL   mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
+char*  mock__fullpath(char *path, char *dst, int cap);
+HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
+BOOL   mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
+BOOL   mock_FindClose(HANDLE hFindFile);
+int    mock__mkdir(char *path);
+#else
+int   mock_socket(int domain, int type, int protocol);
+int   mock_closesocket(int fd);
+int   mock_ioctlsocket(int fd, long cmd, unsigned long *argp);
+int   mock_bind(int fd, void *addr, unsigned long addr_len);
+int   mock_connect(int fd, void *addr, unsigned long addr_len);
+int   mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
+int   mock_listen(int fd, int backlog);
+int   mock_accept(int fd, void *addr, socklen_t *addr_len);
+int   mock_pipe(int *fds);
+int   mock_recv(int fd, char *dst, int len, int flags);
+int   mock_send(int fd, char *src, int len, int flags);
+int   mock_clock_gettime(clockid_t clockid, struct timespec *tp);
+int   mock_open(char *path, int flags, int mode);
+int   mock_fcntl(int fd, int cmd, int flags);
+int   mock_close(int fd);
+int   mock_ftruncate(int fd, size_t new_size);
+int   mock_fstat(int fd, struct stat *buf);
+int   mock_read(int fd, char *dst, int len);
+int   mock_write(int fd, char *src, int len);
+off_t mock_lseek(int fd, off_t offset, int whence);
+int   mock_flock(int fd, int op);
+int   mock_fsync(int fd);
+int   mock_mkstemp(char *path);
+int   mock_mkdir(char *path, mode_t mode);
+int   mock_remove(char *path);
+int   mock_rename(char *oldpath, char *newpath);
+char* mock_realpath(char *path, char *dst);
+DIR*  mock_opendir(char *name);
+struct dirent* mock_readdir(DIR *dirp);
+int   mock_closedir(DIR *dirp);
+#endif
+
+void *mock_malloc(size_t size);
+void *mock_realloc(void *ptr, size_t size);
+void  mock_free(void *ptr);
+
+#ifdef QUAKEY_ENABLE_MOCKS
+
+#define QUAKEY_SIGNAL(name) quakey_signal(name)
+
+#undef errno
+#define errno (*mock_errno_ptr())
+
+#define malloc           mock_malloc
+#define realloc          mock_realloc
+#define free             mock_free
+#define socket           mock_socket
+#define closesocket      mock_closesocket
+#define ioctlsocket      mock_ioctlsocket
+#define bind             mock_bind
+#define connect          mock_connect
+#define getsockopt       mock_getsockopt
+#define listen           mock_listen
+#define accept           mock_accept
+#define pipe             mock_pipe
+#define recv             mock_recv
+#define send             mock_send
+#define clock_gettime    mock_clock_gettime
+#define QueryPerformanceCounter   mock_QueryPerformanceCounter
+#define QueryPerformanceFrequency mock_QueryPerformanceFrequency
+#define open             mock_open
+#define fcntl            mock_fcntl
+#define close            mock_close
+#define ftruncate        mock_ftruncate
+#define CreateFileW      mock_CreateFileW
+#define CloseHandle      mock_CloseHandle
+#define ReadFile         mock_ReadFile
+#define WriteFile        mock_WriteFile
+#define read             mock_read
+#define write            mock_write
+#define fstat            mock_fstat
+#define GetFileSizeEx    mock_GetFileSizeEx
+#define lseek            mock_lseek
+#define SetFilePointer   mock_SetFilePointer
+#define flock            mock_flock
+#define LockFile         mock_LockFile
+#define UnlockFile       mock_UnlockFile
+#define fsync            mock_fsync
+#define FlushFileBuffers mock_FlushFileBuffers
+#define mkstemp          mock_mkstemp
+#define mkdir            mock_mkdir
+#define _mkdir           mock__mkdir
+#define remove           mock_remove
+#define rename           mock_rename
+#define MoveFileExW      mock_MoveFileExW
+#define realpath         mock_realpath
+#define _fullpath        mock__fullpath
+#define opendir          mock_opendir
+#define readdir          mock_readdir
+#define closedir         mock_closedir
+#define FindFirstFileA   mock_FindFirstFileA
+#define FindNextFileA    mock_FindNextFileA
+#define FindClose        mock_FindClose
+
+#else
+
+#define QUAKEY_SIGNAL(name) ((void) (name))
+
+#endif
+
+#endif // QUAKEY_INCLUDED

quakey/src/mockfs.c

@@ -0,0 +1,985 @@
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "mockfs.h"
+
+typedef struct {
+    char *ptr;
+    int   len;
+} Slice;
+
+#define S(X) (Slice) { (X), (int) sizeof(X)-1 }
+
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+
+#define MOCKFS_COMP_LIMIT  32
+
+struct MockFS_Entity {
+    MockFS_Entity *parent;
+    MockFS_Entity *prev;
+    MockFS_Entity *next;
+    char name[MOCKFS_NAME_SIZE];
+    int  name_len;
+    bool is_dir;
+    int  refcount;  // Number of open file handles pointing to this entity
+    union {
+        struct {
+            MockFS_Entity *head_child;
+            MockFS_Entity *tail_child;
+        };
+        ByteBuffer byte_buffer;
+    };
+};
+
+static bool slice_eq(Slice s1, Slice s2)
+{
+    if (s1.len != s2.len)
+        return false;
+    return !memcmp(s1.ptr, s2.ptr, s1.len);
+}
+
+static void *alloc(MockFS *mfs, int len, int align)
+{
+    int pad = -(unsigned long long) (mfs->mem + mfs->off) & (align - 1);
+    if (mfs->len - mfs->off < pad + len)
+        return NULL;
+    void *p = mfs->mem + mfs->off + pad;
+    mfs->off += pad + len;
+    return p;
+}
+
+static void byte_buffer_init(ByteBuffer *byte_buffer)
+{
+    byte_buffer->used = 0;
+    byte_buffer->tail_used = 0;
+    byte_buffer->head = NULL;
+    byte_buffer->tail = NULL;
+}
+
+static void byte_buffer_free(ByteBuffer *byte_buffer, ByteChunk **free_list)
+{
+    if (byte_buffer->head) {
+        byte_buffer->tail->next = *free_list;
+        *free_list = byte_buffer->head;
+    }
+}
+
+static int convert_offset(ByteBuffer *byte_buffer, int off, ByteChunk **pchunk, int *poffset)
+{
+    assert(off > -1);
+
+    int skipped = 0;
+    ByteChunk *chunk = byte_buffer->head;
+    while (chunk) {
+        int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used);
+        if (off >= skipped && off < skipped + chunk_used) {
+            *pchunk = chunk;
+            *poffset = off - skipped;
+            return 1;
+        }
+        chunk = chunk->next;
+        skipped += chunk_used;
+    }
+
+    if (off == skipped) {
+        *pchunk = byte_buffer->tail;
+        *poffset = byte_buffer->tail ? byte_buffer->tail_used : 0;
+        return 1;
+    }
+
+    return 0;
+}
+
+static int byte_buffer_read(ByteBuffer *byte_buffer, int off, char *dst, int cap)
+{
+    int rel_off;
+    ByteChunk *chunk;
+    convert_offset(byte_buffer, off, &chunk, &rel_off);
+
+    int copied = 0;
+    while (copied < cap) {
+        if (chunk == byte_buffer->tail) {
+            if (rel_off == byte_buffer->tail_used)
+                break;
+            int cpy = MIN(byte_buffer->tail_used - rel_off, cap - copied);
+            memcpy(dst + copied, chunk->data + rel_off, cpy);
+            copied += cpy;
+            break;  // No more chunks after tail
+        } else {
+            int cpy = MIN(BYTE_CHUNK_SIZE - rel_off, cap - copied);
+            memcpy(dst + copied, chunk->data + rel_off, cpy);
+            copied += cpy;
+            chunk = chunk->next;
+            rel_off = 0;
+        }
+    }
+
+    return copied;
+}
+
+// Calculate total used bytes in the buffer
+static int byte_buffer_size(ByteBuffer *byte_buffer)
+{
+    int size = 0;
+    ByteChunk *chunk = byte_buffer->head;
+    while (chunk) {
+        if (chunk->next) {
+            size += BYTE_CHUNK_SIZE;
+        } else {
+            size += byte_buffer->tail_used;
+        }
+        chunk = chunk->next;
+    }
+    return size;
+}
+
+// Extend buffer to given size by filling with zeros
+static int byte_buffer_extend(ByteBuffer *byte_buffer, int target_size, MockFS *mfs)
+{
+    int current_size = byte_buffer_size(byte_buffer);
+    int to_write = target_size - current_size;
+
+    while (to_write > 0) {
+        // Get or create tail chunk
+        if (byte_buffer->tail == NULL || byte_buffer->tail_used == BYTE_CHUNK_SIZE) {
+            ByteChunk *tmp = mfs->chunk_free_list;
+            if (tmp == NULL) {
+                tmp = alloc(mfs, sizeof(ByteChunk), _Alignof(ByteChunk));
+                if (tmp == NULL)
+                    return MOCKFS_ERRNO_NOSPC;
+            } else {
+                mfs->chunk_free_list = tmp->next;
+            }
+            tmp->next = NULL;
+
+            if (byte_buffer->head == NULL) {
+                byte_buffer->head = tmp;
+            } else {
+                byte_buffer->tail->next = tmp;
+            }
+            byte_buffer->tail = tmp;
+            byte_buffer->tail_used = 0;
+        }
+
+        // Fill remaining space in tail chunk with zeros
+        int space = BYTE_CHUNK_SIZE - byte_buffer->tail_used;
+        int fill = MIN(space, to_write);
+        memset(byte_buffer->tail->data + byte_buffer->tail_used, 0, fill);
+        byte_buffer->tail_used += fill;
+        to_write -= fill;
+    }
+
+    return 0;
+}
+
+static int byte_buffer_write(ByteBuffer *byte_buffer, int off, char *src, int len, MockFS *mfs)
+{
+    int rel_off;
+    ByteChunk *chunk;
+    if (!convert_offset(byte_buffer, off, &chunk, &rel_off)) {
+        // Offset is beyond end of buffer - extend with zeros
+        int ret = byte_buffer_extend(byte_buffer, off, mfs);
+        if (ret < 0)
+            return ret;
+        convert_offset(byte_buffer, off, &chunk, &rel_off);
+    }
+
+    int copied = 0;
+    while (copied < len) {
+
+        if (chunk == byte_buffer->tail) {
+            if (chunk == NULL || rel_off == BYTE_CHUNK_SIZE) {
+
+                ByteChunk *tmp = mfs->chunk_free_list;
+                if (tmp == NULL) {
+                    tmp = alloc(mfs, sizeof(ByteChunk), _Alignof(ByteChunk));
+                    if (tmp == NULL)
+                        return MOCKFS_ERRNO_NOSPC;
+                } else {
+                    mfs->chunk_free_list = tmp->next;
+                }
+                tmp->next = NULL;
+
+                if (byte_buffer->head == NULL) {
+                    byte_buffer->head = tmp;
+                } else {
+                    byte_buffer->tail->next = tmp;
+                }
+                byte_buffer->tail = tmp;
+                byte_buffer->tail_used = 0;
+
+                rel_off = 0;
+
+                int cpy = MIN(BYTE_CHUNK_SIZE, len - copied);
+                assert(cpy > 0);
+
+                memcpy(tmp->data, src + copied, cpy);
+                copied += cpy;
+                chunk = tmp;
+                rel_off += cpy;
+
+                byte_buffer->tail_used = cpy;
+
+            } else if (rel_off == byte_buffer->tail_used) {
+
+                assert(chunk);
+
+                int cpy = MIN(BYTE_CHUNK_SIZE - byte_buffer->tail_used, len - copied);
+                assert(cpy > 0);
+
+                memcpy(chunk->data + byte_buffer->tail_used, src + copied, cpy);
+
+                copied += cpy;
+                rel_off += cpy;
+                byte_buffer->tail_used += cpy;
+
+            } else {
+
+                assert(rel_off < byte_buffer->tail_used);
+
+                assert(chunk);
+
+                int cpy = MIN(byte_buffer->tail_used - rel_off, len - copied);
+                assert(cpy > 0);
+
+                memcpy(chunk->data + rel_off, src + copied, cpy);
+                copied += cpy;
+                rel_off += cpy;
+            }
+
+        } else {
+
+            assert(chunk);
+
+            int cpy = MIN(BYTE_CHUNK_SIZE - rel_off, len - copied);
+            assert(cpy > 0);
+
+            memcpy(chunk->data + rel_off, src + copied, cpy);
+            copied += cpy;
+            chunk = chunk->next;
+            rel_off = 0;
+        }
+    }
+
+    return 0;
+}
+
+static int parse_path(char *src, int len, Slice *buf, int cap)
+{
+    int cur = 0;
+    int ret = 0;
+
+    if (len > 0 && src[0] == '/')
+        cur++;
+
+    for (;;) {
+
+        int off = cur;
+        while (cur < len && src[cur] != '/')
+            cur++;
+        Slice s = { src + off, cur - off };
+
+        if (s.len > 0) {
+            if (ret == cap)
+                return -1; // TODO: proper error code
+            buf[ret++] = s;
+        }
+
+        if (cur == len)
+            break;
+        assert(src[cur] == '/');
+        cur++;
+    }
+
+    return ret;
+}
+
+static int resolve_path(MockFS *mfs, Slice *comps, int num_comps,
+    MockFS_Entity **stack, int cap)
+{
+    int ret = 0;
+    stack[ret++] = mfs->root;
+
+    for (int i = 0; i < num_comps; i++) {
+
+        if (!stack[ret-1]->is_dir)
+            return MOCKFS_ERRNO_NOTDIR;
+
+        if (slice_eq(comps[i], S(".."))) {
+            ret--;
+            continue;
+        }
+
+        if (slice_eq(comps[i], S(".")))
+            continue;
+
+        MockFS_Entity *child = stack[ret-1]->head_child;
+        while (child) {
+            if (slice_eq(comps[i], (Slice) { child->name, child->name_len }))
+                break;
+            child = child->next;
+        }
+
+        if (child == NULL)
+            return MOCKFS_ERRNO_NOENT;
+
+        if (ret == cap)
+            return -1; // TODO: return proper error code
+        stack[ret++] = child;
+    }
+
+    return ret;
+}
+
+static int entity_init(MockFS_Entity *entity, Slice name, bool is_dir)
+{
+    entity->parent = NULL;
+    entity->prev = NULL;
+    entity->next = NULL;
+    entity->is_dir = is_dir;
+    entity->refcount = 0;
+    entity->head_child = NULL;
+    entity->tail_child = NULL;
+
+    if (name.len > (int) sizeof(entity->name))
+        return -1; // TODO: proper error code
+    memcpy(entity->name, name.ptr, name.len);
+    entity->name_len = name.len;
+
+    return 0;
+}
+
+int mockfs_init(MockFS **pmfs, char *mem, int len)
+{
+    int off = -(unsigned long long) mem & (_Alignof(MockFS)-1);
+    if (off + sizeof(MockFS) > (unsigned long long)len)
+        return MOCKFS_ERRNO_NOMEM;
+    MockFS *mfs = (MockFS *)(mem + off);
+
+    mfs->mem = mem;
+    mfs->len = len;
+    mfs->off = off + sizeof(MockFS);
+    mfs->root = NULL;
+    mfs->entity_free_list = NULL;
+    mfs->chunk_free_list = NULL;
+
+    MockFS_Entity *entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity));
+    if (entity == NULL)
+        return MOCKFS_ERRNO_NOMEM;
+    entity_init(entity, (Slice) { "", 0 }, true);
+    mfs->root = entity;
+
+    *pmfs = mfs;
+    return 0;
+}
+
+void mockfs_free(MockFS *mfs)
+{
+    (void) mfs;
+}
+
+static bool find_child(MockFS_Entity *entity, Slice child_name)
+{
+    assert(entity->is_dir);
+    MockFS_Entity *child = entity->head_child;
+    while (child) {
+        if (slice_eq(child_name, (Slice) { child->name, child->name_len }))
+            return true;
+        child = child->next;
+    }
+    return false;
+}
+
+static int create_entity(MockFS *mfs, MockFS_Entity *parent, Slice name, bool is_dir)
+{
+    MockFS_Entity *entity = mfs->entity_free_list;
+    if (entity == NULL) {
+        entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity));
+        if (entity == NULL)
+            return MOCKFS_ERRNO_NOMEM;
+    } else {
+        mfs->entity_free_list = entity->next;
+    }
+    entity->next = NULL;
+
+    int ret = entity_init(entity, name, is_dir);
+    if (ret < 0) {
+        entity->next = mfs->entity_free_list;
+        mfs->entity_free_list = entity;
+        return ret;
+    }
+
+    // Initialize file-specific fields
+    byte_buffer_init(&entity->byte_buffer);
+
+    // Link to parent directory
+    entity->parent = parent;
+    entity->next = NULL;
+    entity->prev = parent->tail_child;
+
+    if (parent->tail_child) {
+        parent->tail_child->next = entity;
+    } else {
+        parent->head_child = entity;
+    }
+    parent->tail_child = entity;
+
+    return 0;
+}
+
+int mockfs_open(MockFS *mfs, char *path, int path_len, int flags, MockFS_OpenFile *open_file)
+{
+    Slice comps[MOCKFS_COMP_LIMIT];
+    int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    int num_comps = ret;
+
+    MockFS_Entity *stack[MOCKFS_COMP_LIMIT];
+    bool file_was_created = false;
+    ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT);
+    if (ret < 0) {
+        // If file doesn't exist AND O_CREAT is specified, try to create it
+        if (ret == MOCKFS_ERRNO_NOENT && (flags & MOCKFS_O_CREAT)) {
+            // Resolve parent directory
+            ret = resolve_path(mfs, comps, num_comps-1, stack, MOCKFS_COMP_LIMIT);
+            if (ret < 0)
+                return ret;
+            assert(ret > 0);
+            MockFS_Entity *parent = stack[ret-1];
+
+            // Path ending with '/' implies directory, but you can't create
+            // a directory with open() - that's what mkdir() is for
+            if (path_len > 1 && path[path_len-1] == '/') {
+                return MOCKFS_ERRNO_ISDIR;
+            }
+
+            // Create the file
+            ret = create_entity(mfs, parent, comps[num_comps-1], false);
+            if (ret < 0)
+                return ret;
+
+            file_was_created = true;
+
+            // Retry full path resolution (now should succeed)
+            ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT);
+            assert(ret > 0);
+        } else {
+            return ret;
+        }
+    }
+    assert(ret > 0);
+
+    // Check for trailing slash (but "/" alone is not a "trailing slash" case)
+    bool has_trailing_slash = (path_len > 1 && path[path_len-1] == '/');
+
+    if ((flags & MOCKFS_O_CREAT) && has_trailing_slash)
+        return MOCKFS_ERRNO_ISDIR;
+
+    // O_EXCL check: if file already existed and O_EXCL is set, fail
+    if ((flags & MOCKFS_O_EXCL) && (flags & MOCKFS_O_CREAT) && !file_was_created)
+        return MOCKFS_ERRNO_EXIST;
+
+    if (stack[ret-1]->is_dir)
+        return MOCKFS_ERRNO_ISDIR;
+
+    if (has_trailing_slash) {
+        // If target is not a directory, return ENOTDIR
+        if (!stack[ret-1]->is_dir)
+            return MOCKFS_ERRNO_NOTDIR;
+    }
+
+    // O_TRUNC: truncate file to zero length if opened for writing
+    if ((flags & MOCKFS_O_TRUNC) && (flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) {
+        byte_buffer_free(&stack[ret-1]->byte_buffer, &mfs->chunk_free_list);
+        byte_buffer_init(&stack[ret-1]->byte_buffer);
+    }
+
+    open_file->mfs = mfs;
+    open_file->entity = stack[ret-1];
+    open_file->offset = 0;
+    open_file->flags = flags;
+    open_file->entity->refcount++;
+    return 0;
+}
+
+int mockfs_open_dir(MockFS *mfs, char *path, int path_len, MockFS_OpenDir *open_dir)
+{
+    Slice comps[MOCKFS_COMP_LIMIT];
+    int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    int num_comps = ret;
+
+    MockFS_Entity *stack[MOCKFS_COMP_LIMIT];
+    ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    assert(ret > 0);
+
+    if (!stack[ret-1]->is_dir)
+        return MOCKFS_ERRNO_NOTDIR;
+
+    open_dir->mfs = mfs;
+    open_dir->entity = stack[ret-1];
+    open_dir->child = stack[ret-1]->head_child;
+    open_dir->idx = 0;
+    return 0;
+}
+
+int mockfs_file_size(MockFS_OpenFile *open_file)
+{
+    return byte_buffer_size(&open_file->entity->byte_buffer);
+}
+
+void mockfs_close_file(MockFS_OpenFile *open_file)
+{
+    MockFS_Entity *entity = open_file->entity;
+    entity->refcount--;
+
+    // If refcount drops to 0 and entity was unlinked (removed while open),
+    // now we can actually free it
+    if (entity->refcount == 0 && entity->parent == NULL) {
+        // Free the byte buffer chunks
+        byte_buffer_free(&entity->byte_buffer, &open_file->mfs->chunk_free_list);
+
+        // Add entity to free list
+        entity->next = open_file->mfs->entity_free_list;
+        open_file->mfs->entity_free_list = entity;
+    }
+}
+
+void mockfs_close_dir(MockFS_OpenDir *open_dir)
+{
+    (void) open_dir;
+}
+
+int mockfs_read(MockFS_OpenFile *open_file, char *dst, int len)
+{
+    if (open_file->flags & MOCKFS_O_WRONLY) {
+        return MOCKFS_ERRNO_BADF;
+    }
+
+    int copied = byte_buffer_read(&open_file->entity->byte_buffer, open_file->offset, dst, len);
+    open_file->offset += copied;
+    return copied;
+}
+
+int mockfs_write(MockFS_OpenFile *open_file, char *src, int len)
+{
+    if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) {
+        return MOCKFS_ERRNO_BADF;
+    }
+
+    if ((open_file->flags & MOCKFS_O_WRONLY) && (open_file->flags & MOCKFS_O_RDWR)) {
+        return MOCKFS_ERRNO_BADF;
+    }
+
+    // If O_APPEND is set, seek to end before writing
+    if (open_file->flags & MOCKFS_O_APPEND) {
+        open_file->offset = byte_buffer_size(&open_file->entity->byte_buffer);
+    }
+
+    int ret = byte_buffer_write(&open_file->entity->byte_buffer, open_file->offset, src, len, open_file->mfs);
+    if (ret < 0)
+        return ret;
+
+    open_file->offset += len;
+    return len;
+}
+
+int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent)
+{
+    if (open_dir->idx == 0) {
+        if (sizeof(dirent->name) < 1)
+            return -1; // TODO: proper error code
+        dirent->name[0] = '.';
+        dirent->name_len = 1;
+        dirent->is_dir = true;
+        open_dir->idx++;
+        return 0;
+    }
+
+    if (open_dir->idx == 1) {
+        if (sizeof(dirent->name) < 2)
+            return -1; // TODO: proper error code
+        dirent->name[0] = '.';
+        dirent->name[1] = '.';
+        dirent->name_len = 2;
+        dirent->is_dir = true;
+        open_dir->idx++;
+        return 0;
+    }
+
+    if (open_dir->child == NULL)
+        return MOCKFS_ERRNO_NOENT;
+
+    memcpy(dirent->name, open_dir->child->name, open_dir->child->name_len);
+    dirent->name_len = open_dir->child->name_len;
+    dirent->is_dir = open_dir->child->is_dir;
+
+    open_dir->child = open_dir->child->next;
+    open_dir->idx++;
+    return 0;
+}
+
+int mockfs_sync(MockFS_OpenFile *open_file)
+{
+    // TODO
+    (void) open_file;
+    return 0;
+}
+
+int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence)
+{
+    int new_offset;
+
+    switch (whence) {
+        case MOCKFS_SEEK_SET:
+            new_offset = offset;
+            break;
+        case MOCKFS_SEEK_CUR:
+            new_offset = open_file->offset + offset;
+            break;
+        case MOCKFS_SEEK_END:
+            new_offset = byte_buffer_size(&open_file->entity->byte_buffer) + offset;
+            break;
+        default:
+            return MOCKFS_ERRNO_INVAL;
+    }
+
+    if (new_offset < 0)
+        return MOCKFS_ERRNO_INVAL;
+
+    open_file->offset = new_offset;
+    return new_offset;
+}
+
+static void byte_buffer_truncate(ByteBuffer *byte_buffer, int new_size, ByteChunk **free_list)
+{
+    if (new_size == 0) {
+        byte_buffer_free(byte_buffer, free_list);
+        byte_buffer_init(byte_buffer);
+        return;
+    }
+
+    // Walk through chunks to find the one containing the new end
+    int remaining = new_size;
+    ByteChunk *chunk = byte_buffer->head;
+    while (chunk) {
+        int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used);
+        if (remaining <= chunk_used) {
+            // This chunk becomes the new tail
+            // Free all subsequent chunks
+            ByteChunk *to_free = chunk->next;
+            if (to_free) {
+                // Find end of chain to free
+                ByteChunk *last = to_free;
+                while (last->next)
+                    last = last->next;
+                last->next = *free_list;
+                *free_list = to_free;
+            }
+            chunk->next = NULL;
+            byte_buffer->tail = chunk;
+            byte_buffer->tail_used = remaining;
+            return;
+        }
+        remaining -= BYTE_CHUNK_SIZE;
+        chunk = chunk->next;
+    }
+}
+
+int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size)
+{
+    if (new_size < 0)
+        return MOCKFS_ERRNO_INVAL;
+
+    if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR)))
+        return MOCKFS_ERRNO_BADF;
+
+    ByteBuffer *bb = &open_file->entity->byte_buffer;
+    int current_size = byte_buffer_size(bb);
+
+    if (new_size < current_size) {
+        byte_buffer_truncate(bb, new_size, &open_file->mfs->chunk_free_list);
+    } else if (new_size > current_size) {
+        int ret = byte_buffer_extend(bb, new_size, open_file->mfs);
+        if (ret < 0)
+            return ret;
+    }
+
+    return 0;
+}
+
+static int remove_inner(MockFS *mfs, MockFS_Entity *entity, bool recursive)
+{
+    if (entity->parent == NULL)
+        return MOCKFS_ERRNO_BUSY;
+
+    if (entity->is_dir) {
+        // Remove children
+        if (entity->head_child) {
+            if (!recursive)
+                return MOCKFS_ERRNO_NOTEMPTY;
+            MockFS_Entity *child = entity->head_child;
+            while (child) {
+                MockFS_Entity *next = child->next;
+                remove_inner(mfs, child, true);
+                child = next;
+            }
+        }
+    }
+
+    // Unlink entity node from parent
+    if (entity->prev) {
+        entity->prev->next = entity->next;
+    } else {
+        entity->parent->head_child = entity->next;
+    }
+
+    if (entity->next) {
+        entity->next->prev = entity->prev;
+    } else {
+        entity->parent->tail_child = entity->prev;
+    }
+
+    // Mark as unlinked
+    entity->parent = NULL;
+
+    // Only fully free the entity if no open handles
+    if (entity->refcount == 0) {
+        if (!entity->is_dir) {
+            // Append chunks to the free list
+            byte_buffer_free(&entity->byte_buffer, &mfs->chunk_free_list);
+        }
+
+        // Append entity to the free list
+        entity->next = mfs->entity_free_list;
+        mfs->entity_free_list = entity;
+    }
+
+    return 0;
+}
+
+int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive)
+{
+    Slice comps[MOCKFS_COMP_LIMIT];
+    int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    int num_comps = ret;
+
+    if (num_comps > 0) {
+        if (slice_eq(comps[num_comps-1], S(".")))
+            return MOCKFS_ERRNO_INVAL;
+
+        if (slice_eq(comps[num_comps-1], S("..")))
+            return MOCKFS_ERRNO_INVAL;
+    }
+
+    MockFS_Entity *stack[MOCKFS_COMP_LIMIT];
+    ret = resolve_path(mfs, comps, num_comps, stack, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    assert(ret > 0);
+
+    if (path_len > 0 && path[path_len-1] == '/') {
+        if (!stack[ret-1]->is_dir)
+            return MOCKFS_ERRNO_NOTDIR;
+    }
+
+    return remove_inner(mfs, stack[ret-1], recursive);
+}
+
+int mockfs_mkdir(MockFS *mfs, char *path, int path_len)
+{
+    Slice comps[MOCKFS_COMP_LIMIT];
+    int ret = parse_path(path, path_len, comps, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    if (ret == 0)
+        return MOCKFS_ERRNO_EXIST;
+    int num_comps = ret;
+
+    if (slice_eq(comps[num_comps-1], S(".")))
+        return MOCKFS_ERRNO_EXIST;
+
+    if (slice_eq(comps[num_comps-1], S("..")))
+        return MOCKFS_ERRNO_INVAL;
+
+    MockFS_Entity *stack[MOCKFS_COMP_LIMIT];
+    ret = resolve_path(mfs, comps, num_comps-1, stack, MOCKFS_COMP_LIMIT);
+    if (ret < 0)
+        return ret;
+    assert(ret > 0);
+    MockFS_Entity *parent = stack[ret-1];
+
+    if (!parent->is_dir)
+        return MOCKFS_ERRNO_NOTDIR;
+
+    if (find_child(parent, comps[num_comps-1]))
+        return MOCKFS_ERRNO_EXIST;
+
+    MockFS_Entity *entity = mfs->entity_free_list;
+    if (entity == NULL) {
+        entity = alloc(mfs, sizeof(MockFS_Entity), _Alignof(MockFS_Entity));
+        if (entity == NULL)
+            return MOCKFS_ERRNO_NOMEM;
+    } else {
+        mfs->entity_free_list = entity->next;
+    }
+    entity->next = NULL;
+
+    ret = entity_init(entity, comps[num_comps-1], true);
+    if (ret < 0) {
+        entity->next = mfs->entity_free_list;
+        mfs->entity_free_list = entity;
+        return ret;
+    }
+
+    // Initialize byte_buffer to clear all 24 bytes of the union, including
+    // byte_buffer.tail (bytes 16-23) which would otherwise retain garbage
+    // when reusing an entity from the free list
+    byte_buffer_init(&entity->byte_buffer);
+
+    entity->parent = parent;
+    entity->next = NULL;
+    entity->prev = parent->tail_child;
+
+    if (parent->tail_child) {
+        parent->tail_child->next = entity;
+    } else {
+        parent->head_child = entity;
+    }
+    parent->tail_child = entity;
+    return 0;
+}
+
+int mockfs_rename(MockFS *mfs, char *old_path, int old_path_len, char *new_path, int new_path_len)
+{
+    Slice new_comps[MOCKFS_COMP_LIMIT];
+    Slice old_comps[MOCKFS_COMP_LIMIT];
+    int num_new_comps = parse_path(new_path, new_path_len, new_comps, MOCKFS_COMP_LIMIT);
+    int num_old_comps = parse_path(old_path, old_path_len, old_comps, MOCKFS_COMP_LIMIT);
+
+    if (num_new_comps < 0) return num_new_comps;
+    if (num_old_comps < 0) return num_old_comps;
+
+    MockFS_Entity *new_stack[MOCKFS_COMP_LIMIT];
+    MockFS_Entity *old_stack[MOCKFS_COMP_LIMIT];
+    int num_new_stack = resolve_path(mfs, new_comps, num_new_comps-1, new_stack, MOCKFS_COMP_LIMIT);
+    int num_old_stack = resolve_path(mfs, old_comps, num_old_comps, old_stack, MOCKFS_COMP_LIMIT);
+
+    if (num_new_stack == MOCKFS_ERRNO_NOTDIR ||
+        num_old_stack == MOCKFS_ERRNO_NOTDIR)
+        return MOCKFS_ERRNO_NOTDIR;
+
+    if (num_new_stack < 0)
+        return num_new_stack;
+    assert(num_new_stack > 0);
+
+    if (!new_stack[num_new_stack-1]->is_dir)
+        return MOCKFS_ERRNO_NOTDIR;
+
+    if (num_old_stack < 0)
+        return num_old_stack;
+
+    assert(num_old_stack > 0);
+    MockFS_Entity *source = old_stack[num_old_stack-1];
+
+    if (source->parent == NULL)
+        return MOCKFS_ERRNO_BUSY;
+
+    if (old_path_len > 0 && old_path[old_path_len-1] == '/') {
+        if (!source->is_dir)
+            return MOCKFS_ERRNO_NOTDIR;
+    }
+
+    if (new_path_len > 0 && new_path[new_path_len-1] == '/') {
+        if (!source->is_dir)
+            return MOCKFS_ERRNO_NOTDIR;
+    }
+
+    if (num_new_comps == 0)
+        return MOCKFS_ERRNO_BUSY;
+
+    // Make sure the entity isn't being moved inside itself,
+    // by checking that the last element of the old stack isn't
+    // in the new stack.
+    for (int i = 0; i < num_new_stack; i++)
+        if (new_stack[i] == source)
+            return MOCKFS_ERRNO_INVAL;
+
+    // Check if new path exists
+    Slice new_name = new_comps[num_new_comps-1];
+    MockFS_Entity *target = NULL;
+    MockFS_Entity *child = new_stack[num_new_stack-1]->head_child;
+    while (child) {
+        if (slice_eq(new_name, (Slice) { child->name, child->name_len })) {
+            target = child;
+            break;
+        }
+        child = child->next;
+    }
+
+    if (target) {
+
+        if (target == source)
+            return 0;
+
+        if (target->is_dir) {
+
+            for (int i = 0; i < num_old_stack; i++)
+                if (old_stack[i] == target)
+                    return MOCKFS_ERRNO_NOTEMPTY;
+
+            if (!source->is_dir)
+                return MOCKFS_ERRNO_ISDIR;
+
+            if (target->head_child)
+                return MOCKFS_ERRNO_NOTEMPTY;
+
+        } else {
+            if (source->is_dir)
+                return MOCKFS_ERRNO_NOTDIR;
+        }
+
+        remove_inner(mfs, target, false);
+    }
+
+    // Unlink source from old parent
+    if (source->prev) {
+        source->prev->next = source->next;
+    } else {
+        source->parent->head_child = source->next;
+    }
+
+    if (source->next) {
+        source->next->prev = source->prev;
+    } else {
+        source->parent->tail_child = source->prev;
+    }
+
+    // Update source's name
+    if (new_name.len > (int) sizeof(source->name))
+        return MOCKFS_ERRNO_INVAL;  // Name too long
+    memcpy(source->name, new_name.ptr, new_name.len);
+    source->name_len = new_name.len;
+
+    // Link source to new parent
+    source->parent = new_stack[num_new_stack-1];
+    source->next = NULL;
+    source->prev = new_stack[num_new_stack-1]->tail_child;
+
+    if (new_stack[num_new_stack-1]->tail_child) {
+        new_stack[num_new_stack-1]->tail_child->next = source;
+    } else {
+        new_stack[num_new_stack-1]->head_child = source;
+    }
+    new_stack[num_new_stack-1]->tail_child = source;
+
+    return 0;
+}

quakey/src/mockfs.h

@@ -0,0 +1,110 @@
+#ifndef MOCKFS_INCLUDED
+#define MOCKFS_INCLUDED
+
+#include <stdbool.h>
+
+#define MOCKFS_NAME_SIZE (1<<7)
+
+enum {
+    MOCKFS_ERRNO_SUCCESS = 0,
+    MOCKFS_ERRNO_NOENT = -1,     // No such file or directory
+    MOCKFS_ERRNO_PERM = -2,      // Operation not permitted
+    MOCKFS_ERRNO_NOMEM = -3,     // Out of memory
+    MOCKFS_ERRNO_NOTDIR = -4,    // Not a directory
+    MOCKFS_ERRNO_ISDIR = -5,     // Is a directory
+    MOCKFS_ERRNO_INVAL = -6,     // Invalid argument
+    MOCKFS_ERRNO_NOTEMPTY = -7,  // Directory not empty
+    MOCKFS_ERRNO_NOSPC = -8,     // No space left on device
+    MOCKFS_ERRNO_EXIST = -9,     // File exists
+    MOCKFS_ERRNO_BUSY  = -10,
+    MOCKFS_ERRNO_BADF  = -11,
+};
+
+enum {
+    MOCKFS_O_RDONLY = 0x00,
+    MOCKFS_O_WRONLY = 0x01,
+    MOCKFS_O_RDWR   = 0x02,
+    MOCKFS_O_CREAT  = 0x40,
+    MOCKFS_O_EXCL   = 0x80,
+    MOCKFS_O_TRUNC  = 0x200,
+    MOCKFS_O_APPEND = 0x400,
+};
+
+enum {
+    MOCKFS_SEEK_SET = 0,
+    MOCKFS_SEEK_CUR = 1,
+    MOCKFS_SEEK_END = 2,
+};
+
+#define BYTE_CHUNK_SIZE 128
+
+typedef struct ByteChunk ByteChunk;
+struct ByteChunk {
+    ByteChunk *next;
+    char       data[BYTE_CHUNK_SIZE];
+};
+
+typedef struct {
+    int used;
+    int tail_used;
+    ByteChunk *head;
+    ByteChunk *tail;
+} ByteBuffer;
+
+typedef struct MockFS_Entity MockFS_Entity;
+
+typedef struct {
+    char *mem;
+    int   len;
+    int   off;
+    MockFS_Entity *root;
+    MockFS_Entity *entity_free_list;
+    ByteChunk     *chunk_free_list;
+} MockFS;
+
+typedef struct {
+    MockFS*        mfs;
+    MockFS_Entity* entity;
+    int            offset;
+    int            flags;
+} MockFS_OpenFile;
+
+typedef struct {
+    MockFS*        mfs;
+    MockFS_Entity* entity;
+    MockFS_Entity* child;
+    int idx;
+} MockFS_OpenDir;
+
+typedef struct {
+    char name[MOCKFS_NAME_SIZE];
+    int  name_len;
+    bool is_dir;
+} MockFS_Dirent;
+
+int  mockfs_init(MockFS **mfs, char *mem, int len);
+void mockfs_free(MockFS *mfs);
+
+int  mockfs_open(MockFS *mfs, char *path, int path_len, int flags, MockFS_OpenFile *open_file);
+int  mockfs_open_dir(MockFS *mfs, char *path, int path_len, MockFS_OpenDir *open_dir);
+
+int mockfs_file_size(MockFS_OpenFile *open_file);
+
+void mockfs_close_file(MockFS_OpenFile *open_file);
+void mockfs_close_dir(MockFS_OpenDir *open_dir);
+
+int mockfs_read(MockFS_OpenFile *open_file, char *dst, int len);
+int mockfs_write(MockFS_OpenFile *open_file, char *src, int len);
+int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent);
+
+int mockfs_sync(MockFS_OpenFile *open_file);
+int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence);
+int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size);
+
+int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive);
+
+int mockfs_mkdir(MockFS *mfs, char *path, int path_len);
+
+int mockfs_rename(MockFS *mfs, char *old_path, int old_path_len, char *new_path, int new_path_len);
+
+#endif // MOCKFS_INCLUDED

raft/client.c

@@ -0,0 +1,225 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include "node.h"
+#include "client.h"
+
+#define CLIENT_TRACE(fmt, ...) {}
+//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
+
+#define CLIENT_REQUEST_TIMEOUT_SEC 3
+
+static uint64_t next_client_id = 1;
+
+static int
+process_message(ClientState *state,
+    int conn_idx, uint8_t type, ByteView msg)
+{
+    (void) conn_idx;
+
+    if (type == MESSAGE_TYPE_REDIRECT) {
+        RedirectMessage redirect_message;
+        if (msg.len != sizeof(RedirectMessage))
+            return -1;
+        memcpy(&redirect_message, msg.ptr, sizeof(redirect_message));
+
+        if (redirect_message.leader_idx >= 0 && redirect_message.leader_idx < state->num_servers) {
+            CLIENT_TRACE("Redirected to leader %d", redirect_message.leader_idx);
+            state->current_leader = redirect_message.leader_idx;
+            // Retry immediately with the correct leader
+            state->pending = false;
+        }
+        return 0;
+    }
+
+    if (!state->pending)
+        return 0;
+
+    if (type != MESSAGE_TYPE_REPLY)
+        return 0;
+
+    ReplyMessage reply_message;
+    if (msg.len != sizeof(ReplyMessage))
+        return -1;
+    memcpy(&reply_message, msg.ptr, sizeof(reply_message));
+
+    CLIENT_TRACE("Received reply for request %lu", (unsigned long)state->request_id);
+
+    state->pending = false;
+    return 0;
+}
+
+int client_init(void *state_, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout)
+{
+    ClientState *state = state_;
+
+    state->num_servers = 0;
+
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "--server")) {
+            i++;
+            if (i == argc) {
+                fprintf(stderr, "Option --server missing value. Usage is --server <addr>:<port>\n");
+                return -1;
+            }
+            if (state->num_servers == NODE_LIMIT) {
+                fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT);
+                return -1;
+            }
+            if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) {
+                fprintf(stderr, "Malformed <addr>:<port> pair for --server option\n");
+                return -1;
+            }
+        } else {
+            printf("Ignoring option '%s'\n", argv[i]);
+        }
+    }
+
+    // Now sort the addresses
+    addr_sort(state->server_addrs, state->num_servers);
+
+    if (tcp_context_init(&state->tcp) < 0) {
+        fprintf(stderr, "Client :: Couldn't setup TCP context\n");
+        return -1;
+    }
+
+    state->pending = false;
+    state->client_id = next_client_id++;
+    state->request_id = 0;
+    state->current_leader = 0;
+
+    Time now = get_current_time();
+    if (now == INVALID_TIME) {
+        fprintf(stderr, "Client :: Couldn't get current time\n");
+        tcp_context_free(&state->tcp);
+        return -1;
+    }
+    state->last_request_time = now;
+
+    // Connect to all known servers
+    for (int i = 0; i < state->num_servers; i++) {
+        if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) {
+            fprintf(stderr, "Client :: Couldn't connect to server %d\n", i);
+            tcp_context_free(&state->tcp);
+            return -1;
+        }
+    }
+
+    *timeout = 0;
+    if (pcap < TCP_POLL_CAPACITY) {
+        fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
+        return -1;
+    }
+    *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
+    return 0;
+}
+
+int client_tick(void *state_, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout)
+{
+    ClientState *state = state_;
+
+    Time now = get_current_time();
+    if (now == INVALID_TIME) {
+        assert(0);
+    }
+
+    Event events[TCP_EVENT_CAPACITY];
+    int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
+
+    for (int i = 0; i < num_events; i++) {
+        if (events[i].type != EVENT_MESSAGE)
+            continue;
+        int conn_idx = events[i].conn_idx;
+
+        for (;;) {
+
+            ByteView msg;
+            uint16_t msg_type;
+            int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type);
+            if (ret == 0)
+                break;
+            if (ret < 0) {
+                tcp_close(&state->tcp, conn_idx);
+                break;
+            }
+
+            ret = process_message(state, conn_idx, msg_type, msg);
+            if (ret < 0) {
+                tcp_close(&state->tcp, conn_idx);
+                break;
+            }
+
+            tcp_consume_message(&state->tcp, conn_idx);
+        }
+    }
+
+    // Timeout: if pending request has no response, try next server
+    if (state->pending) {
+        Time request_deadline = state->last_request_time + CLIENT_REQUEST_TIMEOUT_SEC * 1000000000ULL;
+        if (now >= request_deadline) {
+            CLIENT_TRACE("Request %lu timed out, trying next server",
+                (unsigned long)state->request_id);
+            state->pending = false;
+            state->current_leader = (state->current_leader + 1) % state->num_servers;
+        }
+    }
+
+    // Send a new request if not currently waiting for a response
+    if (!state->pending) {
+        int leader = state->current_leader;
+        int conn_idx = tcp_index_from_tag(&state->tcp, leader);
+        if (conn_idx < 0) {
+            // Connection lost, try reconnecting
+            tcp_connect(&state->tcp, state->server_addrs[leader], leader, NULL);
+        } else {
+            state->request_id++;
+
+            RequestMessage request_message = {
+                .base = {
+                    .version = MESSAGE_VERSION,
+                    .type    = MESSAGE_TYPE_REQUEST,
+                    .length  = sizeof(RequestMessage),
+                },
+                .oper = OPERATION_A,
+                .client_id = state->client_id,
+                .request_id = state->request_id,
+            };
+
+            ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
+            if (output)
+                byte_queue_write(output, &request_message, sizeof(request_message));
+
+            state->pending = true;
+            state->last_request_time = now;
+        }
+    }
+
+    // Set timeout for next tick
+    if (state->pending) {
+        Time request_deadline = state->last_request_time + CLIENT_REQUEST_TIMEOUT_SEC * 1000000000ULL;
+        *timeout = deadline_to_timeout(request_deadline, now);
+    } else {
+        *timeout = 0; // Send next request immediately
+    }
+
+    if (pcap < TCP_POLL_CAPACITY)
+        return -1;
+    *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
+    return 0;
+}
+
+int client_free(void *state_)
+{
+    ClientState *state = state_;
+
+    tcp_context_free(&state->tcp);
+    return 0;
+}

raft/client.h

@@ -0,0 +1,39 @@
+#ifndef CLIENT_INCLUDED
+#define CLIENT_INCLUDED
+
+#include <lib/tcp.h>
+#include <lib/basic.h>
+
+#include "config.h"
+
+typedef struct {
+
+    TCP tcp;
+
+    // True if we are waiting for a response
+    bool pending;
+
+    Address server_addrs[NODE_LIMIT];
+    int num_servers;
+
+    uint64_t client_id;
+    uint64_t request_id;
+
+    int current_leader;
+
+    Time last_request_time;
+
+} ClientState;
+
+struct pollfd;
+
+int client_init(void *state, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout);
+
+int client_tick(void *state, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+
+int client_free(void *state);
+
+#endif // CLIENT_INCLUDED

raft/config.h

@@ -0,0 +1,8 @@
+#ifndef CONFIG_INCLUDED
+#define CONFIG_INCLUDED
+
+#define NODE_LIMIT 32
+#define HEARTBEAT_INTERVAL_SEC 1
+#define PRIMARY_DEATH_TIMEOUT_SEC 2
+
+#endif // CONFIG_INCLUDED

raft/main.c

@@ -0,0 +1,234 @@
+#ifdef MAIN_SIMULATION
+#define QUAKEY_ENABLE_MOCKS
+
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <quakey.h>
+
+#include "node.h"
+#include "client.h"
+
+static volatile int simulation_running = 1;
+
+static void sigint_handler(int sig)
+{
+    (void)sig;
+    simulation_running = 0;
+}
+
+int main(void)
+{
+    signal(SIGINT, sigint_handler);
+
+    Quakey *quakey;
+    int ret = quakey_init(&quakey, 2);
+    if (ret < 0)
+        return -1;
+
+    // Client 1
+    {
+        QuakeySpawn config = {
+            .name       = "rndcli1",
+            .state_size = sizeof(ClientState),
+            .init_func  = client_init,
+            .tick_func  = client_tick,
+            .free_func  = client_free,
+            .addrs      = (char*[]) { "127.0.0.2" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+    }
+
+    // Client 2
+    {
+        QuakeySpawn config = {
+            .name       = "rndcli2",
+            .state_size = sizeof(ClientState),
+            .init_func  = client_init,
+            .tick_func  = client_tick,
+            .free_func  = client_free,
+            .addrs      = (char*[]) { "127.0.0.3" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+    }
+
+    // Node 1
+    {
+        QuakeySpawn config = {
+            .name       = "node1",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.4" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
+    }
+
+    // Node 2
+    {
+        QuakeySpawn config = {
+            .name       = "node2",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.5" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
+    }
+
+    // Node 3
+    {
+        QuakeySpawn config = {
+            .name       = "node3",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.6" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
+    }
+
+    while (simulation_running)
+        quakey_schedule_one(quakey);
+
+    quakey_free(quakey);
+    return 0;
+}
+
+#endif // MAIN_SIMULATION
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+#ifdef MAIN_CLIENT
+
+#include <poll.h>
+
+#include "client.h"
+
+#define POLL_CAPACITY 1024
+
+int main(int argc, char **argv)
+{
+    int ret;
+    ClientState state;
+
+    void*         poll_ctxs[POLL_CAPACITY];
+    struct pollfd poll_array[POLL_CAPACITY];
+    int poll_count;
+    int poll_timeout;
+
+    ret = client_init(
+        &state,
+        argc,
+        argv,
+        poll_ctxs,
+        poll_array,
+        POLL_CAPACITY,
+        &poll_count,
+        &poll_timeout
+    );
+    if (ret < 0)
+        return -1;
+
+    for (;;) {
+
+#ifdef _WIN32
+        WSAPoll(poll_array, poll_count, poll_timeout);
+#else
+        poll(poll_array, poll_count, poll_timeout);
+#endif
+
+        ret = client_tick(
+            &state,
+            poll_ctxs,
+            poll_array,
+            POLL_CAPACITY,
+            &poll_count,
+            &poll_timeout
+        );
+        if (ret < 0)
+            return -1;
+    }
+
+    client_free(&state);
+    return 0;
+}
+
+#endif // MAIN_CLIENT
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+#ifdef MAIN_SERVER
+
+#include <poll.h>
+
+#include "node.h"
+
+#define POLL_CAPACITY 1024
+
+int main(int argc, char **argv)
+{
+    int ret;
+    NodeState state;
+
+    void*         poll_ctxs[POLL_CAPACITY];
+    struct pollfd poll_array[POLL_CAPACITY];
+    int poll_count;
+    int poll_timeout;
+
+    ret = node_init(
+        &state,
+        argc,
+        argv,
+        poll_ctxs,
+        poll_array,
+        POLL_CAPACITY,
+        &poll_count,
+        &poll_timeout
+    );
+    if (ret < 0)
+        return -1;
+
+    for (;;) {
+
+#ifdef _WIN32
+        WSAPoll(poll_array, poll_count, poll_timeout);
+#else
+        poll(poll_array, poll_count, poll_timeout);
+#endif
+
+        ret = node_tick(
+            &state,
+            poll_ctxs,
+            poll_array,
+            POLL_CAPACITY,
+            &poll_count,
+            &poll_timeout
+        );
+        if (ret < 0)
+            return -1;
+    }
+
+    node_free(&state);
+    return 0;
+}
+
+#endif // MAIN_SERVER

raft/node.h

@@ -0,0 +1,146 @@
+#ifndef NODE_INCLUDED
+#define NODE_INCLUDED
+
+#include <lib/tcp.h>
+#include <lib/basic.h>
+#include <lib/message.h>
+
+#include "wal.h"
+#include "config.h"
+
+enum {
+    MESSAGE_TYPE_REQUEST_VOTE,
+    MESSAGE_TYPE_VOTED,
+    MESSAGE_TYPE_APPEND_ENTRIES,
+    MESSAGE_TYPE_APPENDED,
+    MESSAGE_TYPE_REQUEST,
+    MESSAGE_TYPE_REPLY,
+    MESSAGE_TYPE_REDIRECT,
+};
+
+typedef struct {
+    MessageHeader base;
+    uint64_t term;
+    int      sender_idx;
+    int      last_log_index;
+    uint64_t last_log_term;
+} RequestVoteMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t term;
+    uint8_t  value;
+} VotedMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t term;
+    int      leader_idx;
+    int      prev_log_index;
+    uint64_t prev_log_term;
+    int      leader_commit;
+    int      entry_count;
+    // Followed by: LogEntry entries[entry_count]
+} AppendEntriesMessage;
+
+typedef struct {
+    MessageHeader base;
+    int      sender_idx;
+    uint64_t term;
+    uint8_t  success;
+    int      match_index;
+} AppendedMessage;
+
+typedef struct {
+    MessageHeader base;
+    Operation oper;
+    uint64_t client_id;
+    uint64_t request_id;
+} RequestMessage;
+
+typedef struct {
+    MessageHeader base;
+    OperationResult result;
+} ReplyMessage;
+
+typedef struct {
+    MessageHeader base;
+    int leader_idx;
+} RedirectMessage;
+
+typedef struct {
+    uint64_t        client_id;
+    uint64_t        last_request_id;
+    OperationResult last_result;
+    bool            pending;
+    int             conn_tag;
+} ClientTableEntry;
+
+typedef struct {
+    int count;
+    int capacity;
+    ClientTableEntry *entries;
+} ClientTable;
+
+typedef enum {
+    ROLE_FOLLOWER,
+    ROLE_CANDIDATE,
+    ROLE_LEADER,
+} Role;
+
+typedef struct {
+
+    TCP tcp;
+    WAL wal;
+
+    Address self_addr;
+    Address node_addrs[NODE_LIMIT];
+    int     num_nodes;
+
+    Role role;
+
+    uint64_t term;
+    int      voted_for;
+    Handle   term_and_vote_handle;
+
+    // Index of the current leader
+    int leader_idx;
+
+    int commit_index;
+    int last_applied;
+
+    // When CANDIDATE
+    int votes_received;
+
+    // When LEADER
+    int next_indices[NODE_LIMIT];
+    int match_indices[NODE_LIMIT];
+
+    // Relative timeout in nanoseconds
+    uint64_t election_timeout;
+
+    // Last heartbeat time
+    uint64_t watchdog;
+
+    // Keep track of client request order to enforce
+    // linearizability
+    ClientTable client_table;
+    int next_client_tag;
+
+    // The state machine to be replicated
+    StateMachine state_machine;
+
+} NodeState;
+
+struct pollfd;
+
+int node_init(void *state, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout);
+
+int node_tick(void *state, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+
+int node_free(void *state);
+
+#endif // NODE_INCLUDED

raft/wal.c

@@ -0,0 +1,127 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+#include <stdlib.h>
+
+#include "wal.h"
+
+int wal_init(WAL *wal, string file)
+{
+    Handle handle;
+    if (file_open(file, &handle) < 0)
+        return -1;
+    wal->handle = handle;
+
+    size_t size;
+    if (file_size(handle, &size) < 0) {
+        file_close(handle);
+        return -1;
+    }
+
+    if (size % sizeof(WALEntry) != 0) {
+        file_close(handle);
+        return -1;
+    }
+    int count = size / sizeof(WALEntry);
+
+    wal->count = count;
+    wal->capacity = count;
+    wal->entries = malloc(count * sizeof(WALEntry));
+    if (wal->entries == NULL) {
+        file_close(handle);
+        return -1;
+    }
+
+    if (file_read_exact(handle, (char*) wal->entries, count * sizeof(WALEntry)) < 0) {
+        file_close(handle);
+        return -1;
+    }
+
+    return 0;
+}
+
+void wal_free(WAL *wal)
+{
+    free(wal->entries);
+    file_close(wal->handle);
+}
+
+int wal_append(WAL *wal, WALEntry *entry)
+{
+    if (wal->count == wal->capacity) {
+        int n = 2 * wal->capacity;
+        if (n < 8) n = 8;
+        void *p = realloc(wal->entries, n * sizeof(WALEntry));
+        if (p == NULL)
+            return -1;
+        wal->capacity = n;
+        wal->entries = p;
+    }
+
+    // TODO: Should truncate file on partial writes
+    if (file_write_exact(wal->handle, (char*) entry, sizeof(*entry)) < 0)
+        return -1;
+
+    if (file_sync(wal->handle) < 0)
+        return -1;
+
+    wal->entries[wal->count++] = *entry;
+    return 0;
+}
+
+int wal_truncate(WAL *wal, int new_count)
+{
+    assert(new_count <= wal->count);
+    if (wal->count == new_count)
+        return 0;
+
+    if (file_truncate(wal->handle, new_count * sizeof(WALEntry)) < 0)
+        return -1;
+
+    if (file_set_offset(wal->handle, new_count * sizeof(WALEntry)) < 0)
+        return -1;
+
+    wal->count = new_count;
+    return 0;
+}
+
+uint64_t wal_last_term(WAL *wal)
+{
+    if (wal->count == 0)
+        return 0;
+    return wal->entries[wal->count-1].term;
+}
+
+int wal_entry_count(WAL *wal)
+{
+    return wal->count;
+}
+
+WALEntry *wal_peek_entry(WAL *wal, int idx)
+{
+    assert(idx > -1);
+    assert(idx < wal->count);
+    return &wal->entries[idx];
+}
+
+void wal_replay_init(WALReplay *wal_replay, WAL *wal)
+{
+    wal_replay->count = wal->count;
+    wal_replay->current = 0;
+    wal_replay->entries = wal->entries;
+}
+
+void wal_replay_free(WALReplay *wal_replay)
+{
+    (void) wal_replay;
+}
+
+WALEntry *wal_replay_next(WALReplay *wal_replay)
+{
+    if (wal_replay->current == wal_replay->count)
+        return NULL;
+    return &wal_replay->entries[wal_replay->current++];
+}

raft/wal.h

@@ -0,0 +1,38 @@
+#ifndef WAL_INCLUDED
+#define WAL_INCLUDED
+
+#include <lib/file_system.h>
+
+#include <state_machine/state_machine.h>
+
+typedef struct {
+    uint64_t  term;
+    uint64_t  client_id;
+    Operation oper;
+} WALEntry;
+
+typedef struct {
+    int       count;
+    int       capacity;
+    WALEntry *entries;
+    Handle    handle;
+} WAL;
+
+typedef struct {
+    int count;
+    int current;
+    WALEntry *entries;
+} WALReplay;
+
+int       wal_init(WAL *wal, string file);
+void      wal_free(WAL *wal);
+int       wal_append(WAL *wal, WALEntry *entry);
+int       wal_truncate(WAL *wal, int new_count);
+uint64_t  wal_last_term(WAL *wal);
+int       wal_entry_count(WAL *wal);
+WALEntry *wal_peek_entry(WAL *wal, int idx);
+void      wal_replay_init(WALReplay *wal_replay, WAL *wal);
+void      wal_replay_free(WALReplay *wal_replay);
+WALEntry *wal_replay_next(WALReplay *wal_replay);
+
+#endif // WAL_INCLUDED

state_machine/state_machine.c

@@ -0,0 +1,25 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+
+#include "state_machine.h"
+
+void state_machine_init(StateMachine *sm)
+{
+    (void)sm;
+}
+
+void state_machine_free(StateMachine *sm)
+{
+    (void)sm;
+}
+
+OperationResult state_machine_update(StateMachine *sm, Operation op)
+{
+    (void)sm;
+    (void)op;
+    return OPERATION_RESULT_X;
+}

state_machine/state_machine.h

@@ -0,0 +1,26 @@
+#ifndef STATE_MACHINE_INCLUDED
+#define STATE_MACHINE_INCLUDED
+
+typedef enum {
+    OPERATION_NOOP,
+    OPERATION_A,
+    OPERATION_B,
+    OPERATION_C,
+} Operation;
+
+typedef enum {
+    OPERATION_RESULT_X,
+    OPERATION_RESULT_Y,
+    OPERATION_RESULT_Z,
+} OperationResult;
+
+typedef struct {
+
+} StateMachine;
+
+void state_machine_init(StateMachine *sm);
+void state_machine_free(StateMachine *sm);
+
+OperationResult state_machine_update(StateMachine *sm, Operation op);
+
+#endif // STATE_MACHINE_INCLUDED

vsr/client.c

@@ -0,0 +1,273 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include <lib/file_system.h>
+
+#include "node.h"
+#include "client.h"
+
+//#define CLIENT_TRACE(fmt, ...) {}
+#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__);
+
+// Format time as seconds with 3 decimal places for trace output
+#define TIME_FMT "%.3fs"
+#define TIME_VAL(t) ((double)(t) / 1000000000.0)
+
+static int leader_idx(ClientState *state)
+{
+    return state->view_number % state->num_servers;
+}
+
+static int
+process_message(ClientState *state,
+    int conn_idx, uint8_t type, ByteView msg)
+{
+    (void) conn_idx;
+
+    if (!state->pending)
+        return -1;
+
+    if (type != MESSAGE_TYPE_REPLY)
+        return -1;
+
+    ReplyMessage reply_message;
+    if (msg.len != sizeof(ReplyMessage))
+        return -1;
+    memcpy(&reply_message, msg.ptr, sizeof(reply_message));
+
+    {
+        Time now = get_current_time();
+        CLIENT_TRACE("[" TIME_FMT "] received REPLY (rejected=%s)",
+            TIME_VAL(now),
+            reply_message.rejected ? "true" : "false");
+    }
+
+    state->pending = false;
+    return 0;
+}
+
+int client_init(void *state_, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout)
+{
+    ClientState *state = state_;
+
+    state->num_servers = 0;
+
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "--server")) {
+            i++;
+            if (i == argc) {
+                fprintf(stderr, "Option --server missing value. Usage is --server <addr>:<port>\n");
+                return -1;
+            }
+            if (state->num_servers == NODE_LIMIT) {
+                fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT);
+                return -1;
+            }
+            // TODO: Check address is not duplicated
+            if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) {
+                fprintf(stderr, "Malformed <addr>:<port> pair for --server option\n");
+                return -1;
+            }
+        } else {
+            printf("Ignoring option '%s'\n", argv[i]);
+        }
+    }
+
+    // Now sort the addresses
+    addr_sort(state->server_addrs, state->num_servers);
+
+    if (tcp_context_init(&state->tcp) < 0) {
+        fprintf(stderr, "Client :: Couldn't setup TCP context\n");
+        return -1;
+    }
+
+    state->pending = false;
+
+    state->view_number = 0;
+    state->request_id = 0;
+
+    // Load or generate a persistent client_id from disk.
+    // This ensures the client_id survives process restarts.
+    {
+        string path = S("client_id");
+        Handle fd;
+        bool loaded = false;
+        if (file_exists(path)) {
+            if (file_open(path, &fd) == 0) {
+                file_set_offset(fd, 0);
+                uint64_t saved_id;
+                if (file_read_exact(fd, (char*)&saved_id, sizeof(saved_id)) == (int)sizeof(saved_id)) {
+                    state->client_id = saved_id;
+                    loaded = true;
+                }
+                file_close(fd);
+            }
+        }
+        if (!loaded) {
+            Time now = get_current_time();
+            state->client_id = (uint64_t)now;
+            if (file_open(path, &fd) == 0) {
+                file_set_offset(fd, 0);
+                file_write_exact(fd, (char*)&state->client_id, sizeof(state->client_id));
+                file_sync(fd);
+                file_close(fd);
+            }
+        }
+    }
+
+    // Connect to all known servers
+    for (int i = 0; i < state->num_servers; i++) {
+        if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) {
+            fprintf(stderr, "Client :: Couldn't connect to server %d\n", i);
+            tcp_context_free(&state->tcp);
+            return -1;
+        }
+    }
+
+    {
+        Time now = get_current_time();
+        CLIENT_TRACE("[" TIME_FMT "] initialized: num_servers=%d, leader_idx=%d",
+            TIME_VAL(now), state->num_servers, leader_idx(state));
+    }
+
+    *timeout = 0;
+    if (pcap < TCP_POLL_CAPACITY) {
+        fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
+        return -1;
+    }
+    *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
+    return 0;
+}
+
+int client_tick(void *state_, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout)
+{
+    ClientState *state = state_;
+
+    Event events[TCP_EVENT_CAPACITY];
+    int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
+
+    for (int i = 0; i < num_events; i++) {
+        if (events[i].type == EVENT_DISCONNECT) {
+            int conn_idx = events[i].conn_idx;
+            int tag = tcp_get_tag(&state->tcp, conn_idx);
+            if (tag == leader_idx(state) && state->pending) {
+                Time now = get_current_time();
+                CLIENT_TRACE("[" TIME_FMT "] lost connection to leader (node %d), resetting pending request",
+                    TIME_VAL(now), leader_idx(state));
+                state->pending = false;
+            }
+            tcp_close(&state->tcp, conn_idx);
+            continue;
+        }
+        if (events[i].type != EVENT_MESSAGE)
+            continue;
+        int conn_idx = events[i].conn_idx;
+
+        for (;;) {
+
+            ByteView msg;
+            uint16_t msg_type;
+            int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type);
+            if (ret == 0)
+                break;
+            if (ret < 0) {
+                tcp_close(&state->tcp, conn_idx);
+                break;
+            }
+
+            ret = process_message(state, conn_idx, msg_type, msg);
+            if (ret < 0) {
+                tcp_close(&state->tcp, conn_idx);
+                break;
+            }
+
+            tcp_consume_message(&state->tcp, conn_idx);
+        }
+    }
+
+    Time now = get_current_time();
+
+    // If we've been waiting too long for a response, give up and
+    // try the next server (the current leader may have crashed and
+    // a view change may have happened)
+    if (state->pending) {
+        Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
+        if (request_deadline <= now) {
+            CLIENT_TRACE("[" TIME_FMT "] request to leader (node %d, view=%lu) timed out, trying next server",
+                TIME_VAL(now), leader_idx(state),
+                (unsigned long)state->view_number);
+            state->view_number++;
+            state->pending = false;
+        }
+    }
+
+    if (!state->pending) {
+
+        int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
+        if (conn_idx < 0) {
+            // Leader connection not available, try reconnecting
+            {
+                CLIENT_TRACE("[" TIME_FMT "] leader (node %d) not connected, reconnecting",
+                    TIME_VAL(now), leader_idx(state));
+            }
+            tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
+        } else {
+            // Now start a new operation
+            state->request_id++;
+
+            RequestMessage request_message = {
+                .base = {
+                    .version = MESSAGE_VERSION,
+                    .type    = MESSAGE_TYPE_REQUEST,
+                    .length  = sizeof(RequestMessage),
+                },
+                .oper = OPERATION_A,
+                .client_id = state->client_id,
+                .request_id = state->request_id,
+            };
+
+            ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
+            assert(output);
+
+            byte_queue_write(output, &request_message, request_message.base.length);
+
+            {
+                CLIENT_TRACE("[" TIME_FMT "] sent REQUEST to leader (node %d, view=%lu, client_id=%lu, req_id=%lu)",
+                    TIME_VAL(now), leader_idx(state),
+                    (unsigned long)state->view_number,
+                    (unsigned long)state->client_id,
+                    (unsigned long)state->request_id);
+            }
+
+            state->pending = true;
+            state->request_time = now;
+        }
+    }
+
+    // Set timeout based on pending request deadline
+    Time deadline = INVALID_TIME;
+    if (state->pending) {
+        nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
+    }
+    *timeout = deadline_to_timeout(deadline, now);
+    if (pcap < TCP_POLL_CAPACITY)
+        return -1;
+    *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
+    return 0;
+}
+
+int client_free(void *state_)
+{
+    ClientState *state = state_;
+
+    tcp_context_free(&state->tcp);
+    return 0;
+}

vsr/client.h

@@ -0,0 +1,38 @@
+#ifndef CLIENT_INCLUDED
+#define CLIENT_INCLUDED
+
+#include <lib/tcp.h>
+#include <lib/basic.h>
+
+#include "config.h"
+
+typedef struct {
+
+    TCP tcp;
+
+    // True if we are waiting for a response
+    bool pending;
+    Time request_time; // When the current request was sent
+
+    Address server_addrs[NODE_LIMIT];
+    int num_servers;
+
+    uint64_t view_number;
+
+    uint64_t client_id;
+    uint64_t request_id;
+
+} ClientState;
+
+struct pollfd;
+
+int client_init(void *state, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout);
+
+int client_tick(void *state, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+
+int client_free(void *state);
+
+#endif // CLIENT_INCLUDED

vsr/client_table.c

@@ -0,0 +1,70 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <stdbool.h>
+
+#include "client_table.h"
+
+void client_table_init(ClientTable *client_table)
+{
+    client_table->count = 0;
+    client_table->capacity = 0;
+    client_table->entries = NULL;
+}
+
+void client_table_free(ClientTable *client_table)
+{
+    free(client_table->entries);
+}
+
+ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id)
+{
+    for (int i = 0; i < client_table->count; i++) {
+        if (client_table->entries[i].client_id == client_id)
+            return &client_table->entries[i];
+    }
+    return NULL;
+}
+
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx)
+{
+    if (client_table->count == client_table->capacity) {
+        int n = 2 * client_table->capacity;
+        if (n == 0) n = 8;
+        void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
+        if (p == NULL)
+            return -1;
+        client_table->capacity = n;
+        client_table->entries = p;
+    }
+
+    client_table->entries[client_table->count++] = (ClientTableEntry) {
+        .client_id = client_id,
+        .last_request_id = request_id,
+        .pending = true,
+        .conn_idx = conn_idx,
+    };
+    return 0;
+}
+
+int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id)
+{
+    if (client_table->count == client_table->capacity) {
+        int n = 2 * client_table->capacity;
+        if (n == 0) n = 8;
+        void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
+        if (p == NULL)
+            return -1;
+        client_table->capacity = n;
+        client_table->entries = p;
+    }
+
+    client_table->entries[client_table->count++] = (ClientTableEntry) {
+        .client_id=client_id,
+        .last_request_id=request_id,
+        .pending=true,
+    };
+    return 0;
+}

vsr/client_table.h

@@ -0,0 +1,28 @@
+#ifndef CLIENT_TABLE_INCLUDED
+#define CLIENT_TABLE_INCLUDED
+
+#include <stdint.h>
+
+#include <state_machine/state_machine.h>
+
+typedef struct {
+    uint64_t        client_id;
+    uint64_t        last_request_id;
+    OperationResult last_result;
+    bool            pending;
+    int             conn_idx;
+} ClientTableEntry;
+
+typedef struct {
+    int count;
+    int capacity;
+    ClientTableEntry *entries;
+} ClientTable;
+
+void client_table_init(ClientTable *client_table);
+void client_table_free(ClientTable *client_table);
+ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_idx);
+int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
+
+#endif // CLIENT_TABLE_INCLUDED

vsr/config.h

@@ -0,0 +1,11 @@
+#ifndef CONFIG_INCLUDED
+#define CONFIG_INCLUDED
+
+#define NODE_LIMIT 32
+#define FUTURE_LIMIT 32
+#define HEARTBEAT_INTERVAL_SEC 1
+#define PRIMARY_DEATH_TIMEOUT_SEC 5
+#define RECOVERY_TIMEOUT_SEC 3
+#define RECOVERY_ATTEMPT_LIMIT 10
+
+#endif // CONFIG_INCLUDED

vsr/log.c

@@ -0,0 +1,37 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+
+#include "log.h"
+
+void log_init(Log *log)
+{
+    log->count = 0;
+    log->capacity = 0;
+    log->entries = NULL;
+}
+
+void log_free(Log *log)
+{
+    free(log->entries);
+}
+
+int log_append(Log *log, LogEntry entry)
+{
+    if (log->count == log->capacity) {
+        int n= 2 * log->capacity;
+        if (n < 8)
+            n = 8;
+        LogEntry *p = realloc(log->entries, n * sizeof(LogEntry));
+        if (p == NULL)
+            return -1;
+
+        log->entries = p;
+        log->capacity = n;
+    }
+
+    log->entries[log->count++] = entry;
+    return 0;
+}

vsr/log.h

@@ -0,0 +1,27 @@
+#ifndef LOG_INCLUDED
+#define LOG_INCLUDED
+
+#include <state_machine/state_machine.h>
+
+#include "config.h"
+
+typedef struct {
+    Operation oper;
+    uint32_t votes;
+    int view_number;
+    uint64_t client_id;
+} LogEntry;
+
+_Static_assert(NODE_LIMIT <= 32, "");
+
+typedef struct {
+    int count;
+    int capacity;
+    LogEntry *entries;
+} Log;
+
+void log_init(Log *log);
+void log_free(Log *log);
+int  log_append(Log *log, LogEntry entry);
+
+#endif // LOG_INCLUDED

vsr/main.c

@@ -0,0 +1,247 @@
+#ifdef MAIN_SIMULATION
+#define QUAKEY_ENABLE_MOCKS
+
+#include <signal.h>
+#include <stdint.h>
+#include <quakey.h>
+
+#include "node.h"
+#include "client.h"
+
+static volatile int simulation_running = 1;
+
+static void sigint_handler(int sig)
+{
+    (void)sig;
+    simulation_running = 0;
+}
+
+int main(void)
+{
+    signal(SIGINT, sigint_handler);
+
+    Quakey *quakey;
+    int ret = quakey_init(&quakey, 1);
+    if (ret < 0)
+        return -1;
+
+    QuakeyNode cli_1;
+    QuakeyNode cli_2;
+    QuakeyNode node_1;
+    QuakeyNode node_2;
+    QuakeyNode node_3;
+
+    // Client 1
+    {
+        QuakeySpawn config = {
+            .name       = "rndcli1",
+            .state_size = sizeof(ClientState),
+            .init_func  = client_init,
+            .tick_func  = client_tick,
+            .free_func  = client_free,
+            .addrs      = (char*[]) { "127.0.0.2" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        cli_1 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+    }
+
+    // Client 2
+    {
+        QuakeySpawn config = {
+            .name       = "rndcli2",
+            .state_size = sizeof(ClientState),
+            .init_func  = client_init,
+            .tick_func  = client_tick,
+            .free_func  = client_free,
+            .addrs      = (char*[]) { "127.0.0.3" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        cli_2 = quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
+    }
+
+    // Node 1
+    {
+        QuakeySpawn config = {
+            .name       = "node1",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.4" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
+    }
+
+    // Node 2
+    {
+        QuakeySpawn config = {
+            .name       = "node2",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.5" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
+    }
+
+    // Node 3
+    {
+        QuakeySpawn config = {
+            .name       = "node3",
+            .state_size = sizeof(NodeState),
+            .init_func  = node_init,
+            .tick_func  = node_tick,
+            .free_func  = node_free,
+            .addrs      = (char*[]) { "127.0.0.6" },
+            .num_addrs  = 1,
+            .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
+        };
+        node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
+    }
+
+    while (simulation_running) {
+        quakey_schedule_one(quakey);
+
+        NodeState *arr[] = {
+            quakey_node_state(node_1),
+            quakey_node_state(node_2),
+            quakey_node_state(node_3),
+        };
+        check_vsr_invariants(arr, sizeof(arr)/sizeof(arr[0]));
+    }
+
+    quakey_free(quakey);
+    return 0;
+}
+
+#endif // MAIN_SIMULATION
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+#ifdef MAIN_CLIENT
+
+#include <poll.h>
+
+#include "client.h"
+
+#define POLL_CAPACITY 1024
+
+int main(int argc, char **argv)
+{
+    int ret;
+    ClientState state;
+
+    void*         poll_ctxs[POLL_CAPACITY];
+    struct pollfd poll_array[POLL_CAPACITY];
+    int poll_count;
+    int poll_timeout;
+
+    ret = client_init(
+        &state,
+        argc,
+        argv,
+        poll_ctxs,
+        poll_array,
+        POLL_CAPACITY,
+        &poll_count,
+        &poll_timeout
+    );
+    if (ret < 0)
+        return -1;
+
+    for (;;) {
+
+#ifdef _WIN32
+        WSAPoll(poll_array, poll_count, poll_timeout);
+#else
+        poll(poll_array, poll_count, poll_timeout);
+#endif
+
+        ret = client_tick(
+            &state,
+            poll_ctxs,
+            poll_array,
+            POLL_CAPACITY,
+            &poll_count,
+            &poll_timeout
+        );
+        if (ret < 0)
+            return -1;
+    }
+
+    client_free(&state);
+    return 0;
+}
+
+#endif // MAIN_CLIENT
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////
+#ifdef MAIN_SERVER
+
+#include <poll.h>
+
+#include "node.h"
+
+#define POLL_CAPACITY 1024
+
+int main(int argc, char **argv)
+{
+    int ret;
+    NodeState state;
+
+    void*         poll_ctxs[POLL_CAPACITY];
+    struct pollfd poll_array[POLL_CAPACITY];
+    int poll_count;
+    int poll_timeout;
+
+    ret = node_init(
+        &state,
+        argc,
+        argv,
+        poll_ctxs,
+        poll_array,
+        POLL_CAPACITY,
+        &poll_count,
+        &poll_timeout
+    );
+    if (ret < 0)
+        return -1;
+
+    for (;;) {
+
+#ifdef _WIN32
+        WSAPoll(poll_array, poll_count, poll_timeout);
+#else
+        poll(poll_array, poll_count, poll_timeout);
+#endif
+
+        ret = node_tick(
+            &state,
+            poll_ctxs,
+            poll_array,
+            POLL_CAPACITY,
+            &poll_count,
+            &poll_timeout
+        );
+        if (ret < 0)
+            return -1;
+    }
+
+    node_free(&state);
+    return 0;
+}
+
+#endif // MAIN_SERVER

vsr/node.h

@@ -0,0 +1,176 @@
+#ifndef NODE_INCLUDED
+#define NODE_INCLUDED
+
+#include <lib/tcp.h>
+#include <lib/basic.h>
+#include <lib/message.h>
+
+#include <state_machine/state_machine.h>
+
+#include "log.h"
+#include "config.h"
+#include "client_table.h"
+
+enum {
+
+    // Normal Protocol
+    MESSAGE_TYPE_REQUEST,
+    MESSAGE_TYPE_REPLY,
+    MESSAGE_TYPE_PREPARE,
+    MESSAGE_TYPE_PREPARE_OK,
+    MESSAGE_TYPE_COMMIT,
+
+    // View Change Protocol
+    MESSAGE_TYPE_BEGIN_VIEW_CHANGE,
+    MESSAGE_TYPE_DO_VIEW_CHANGE,
+    MESSAGE_TYPE_BEGIN_VIEW,
+
+    // Recovery Protocol
+    MESSAGE_TYPE_RECOVERY,
+    MESSAGE_TYPE_RECOVERY_RESPONSE,
+};
+
+typedef struct {
+    MessageHeader base;
+    Operation oper;
+    uint64_t client_id;
+    uint64_t request_id;
+} RequestMessage;
+
+typedef struct {
+    MessageHeader base;
+    Operation oper;
+    int sender_idx;
+    int log_index;
+    int commit_index;
+    uint64_t view_number;
+} PrepareMessage;
+
+typedef struct {
+    MessageHeader base;
+    int sender_idx;
+    int log_index;
+    uint64_t view_number;
+} PrepareOKMessage;
+
+typedef struct {
+    MessageHeader base;
+    int commit_index;
+} CommitMessage;
+
+typedef struct {
+    MessageHeader base;
+    bool rejected;
+    OperationResult result;
+} ReplyMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+    int sender_idx;
+} BeginViewChangeMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;       // The new view number
+    uint64_t old_view_number;   // Last view number when replica was in normal status
+    int op_number;              // Number of entries in the log
+    int commit_index;
+    int sender_idx;
+    // Followed by: LogEntry log[op_number]
+} DoViewChangeMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+    int commit_index;
+    int op_number;  // Number of log entries that follow
+    // Followed by: LogEntry log[op_number]
+} BeginViewMessage;
+
+typedef struct {
+    MessageHeader base;
+    int sender_idx;
+    uint64_t nonce;
+} RecoveryMessage;
+
+typedef struct {
+    MessageHeader base;
+    uint64_t view_number;
+    int op_number;
+    uint64_t nonce;
+    int commit_index;
+    int sender_idx;
+} RecoveryResponseMessage;
+
+typedef enum {
+    STATUS_NORMAL,
+    STATUS_CHANGE_VIEW,
+    STATUS_RECOVERY,
+} Status;
+
+typedef struct {
+
+    TCP tcp;
+
+    Address  self_addr;
+    Address  node_addrs[NODE_LIMIT];
+    int      num_nodes;
+
+    Status status;
+
+    ClientTable client_table;
+
+    uint64_t view_number;
+
+    // These fields are used in recovery mode
+    uint32_t recovery_votes;
+    uint64_t recovery_nonce;
+    uint64_t max_view_seen;
+    int      latest_primary_idx;
+    bool     received_primary_state;
+    Log      potential_primary_log;
+    Time     recovery_start_time;
+    int      recovery_attempt_count;
+
+    ///////////////////////////////////////////////////////////
+    // VIEW CHANGE
+
+    uint32_t begin_view_change_votes;  // Bitmask of received BeginViewChange messages
+    uint32_t do_view_change_votes;     // Bitmask of received DoViewChange messages
+    uint64_t do_view_change_best_old_view;  // Best old_view_number seen in DoViewChange
+    int      do_view_change_best_commit;    // Best commit_index seen
+    Log      do_view_change_best_log;       // Best log seen
+
+    ///////////////////////////////////////////////////////////
+
+    // If this is the leader, commit_index is the index of the next uncommitted element
+    // of the log (if no uncommitted elements are present, it's equal to the total number
+    // of log elements)
+    int commit_index;
+
+    PrepareMessage future[FUTURE_LIMIT];
+    int num_future;
+
+    Log log;
+
+    Time last_heartbeat_time;
+
+    StateMachine state_machine;
+
+} NodeState;
+
+struct pollfd;
+
+int node_init(void *state, int argc, char **argv,
+    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
+    int *timeout);
+
+int node_tick(void *state, void **ctxs,
+    struct pollfd *pdata, int pcap, int *pnum, int *timeout);
+
+int node_free(void *state);
+
+void check_vsr_invariants(NodeState **nodes, int num_nodes);
+
+#endif // NODE_INCLUDED