Add TODO

Several bugs found via deterministic simulation testing (500 seeds, 30s each):

WAL format: Store full WALEntry on disk (including votes and checksum)
instead of a separate WALEntryDisk. This eliminates the need to
reconstruct transient vote bits after restart, and adds wal_update_entry()
for persisting in-place vote modifications.

Crash recovery: Restore 3-case startup logic (empty=normal, corrupt=recovery,
clean=replay). Fix last_normal_view initialization on clean restart. Call
adopt_view() after replay to reinitialize leader vote bits that may be stale
if a crash interrupted wal_replace before the atomic rename.

View change ordering: Move wal_replace before persist_state in
process_begin_view so the on-disk log is always at least as recent as the
persisted metadata. Reset votes on view_change_log entries before wal_replace
in complete_view_change_and_become_primary and complete_recovery.

View adoption: Add adopt_view() helper called from process_prepare and
process_prepare_ok when a NORMAL node learns of a higher view. Updates
last_normal_view and initializes leader vote bits.

Invariant checker: Track node liveness across ticks (prev_alive[]) so
min_commit monotonicity is correctly relaxed for nodes that just restarted.

https://claude.ai/code/session_01X2b5VRntA7LM32kdN9tPAM

.gitignore

@@ -1,11 +1,7 @@
-toasty_simulation
+toastyfs
-toasty_simulation.exe
+toastyfs_client
-toasty
+toastyfs_random_client
-toasty.exe
+toastyfs_simulation
-toasty_random_client
-toasty_random_client.exe
-toasty_proxy
-toasty_proxy.exe
 *.gcno
 *.gcda
 *.gcov
@@ -18,9 +14,6 @@ coverage_html/
 example
 example_large
 examples/example
+.claude/
 .cluster/
 vsr_boot_marker
-*.pem
-*.exe
-.claude/
-cluster-data/

LICENSE

@@ -1,7 +0,0 @@
-Copyright 2026 Francesco Cozzuto (cozis)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -0,0 +1,81 @@
+CC      ?= gcc
+CFLAGS  ?= -Wall -Wextra -ggdb -O0
+AR      ?= ar
+
+INCLUDES = -Iquakey/include -Iinclude -I.
+
+# ---- Library (client-side only) ----
+
+LIB_SRCS = src/client.c src/tcp.c src/byte_queue.c src/message.c src/basic.c
+LIB_OBJS = $(LIB_SRCS:.c=.o)
+
+STATIC_LIB = libtoastyfs.a
+SHARED_LIB = libtoastyfs.so
+
+# ---- Server binary ----
+
+SERVER_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
+              src/tcp.c src/server.c src/main.c src/wal.c src/client_table.c \
+              src/chunk_store.c src/metadata.c
+
+# ---- Client binary (random test client) ----
+
+CLIENT_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
+              src/tcp.c src/server.c src/client.c src/random_client.c src/main.c \
+              src/wal.c src/client_table.c src/chunk_store.c src/metadata.c
+
+# ---- Simulation binary ----
+
+SIM_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
+           src/tcp.c src/server.c src/client.c src/random_client.c src/main.c \
+           src/wal.c src/client_table.c src/invariant_checker.c src/chunk_store.c \
+           src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c
+
+# ---- Default target ----
+
+all: $(STATIC_LIB) $(SHARED_LIB) toastyfs toastyfs_client toastyfs_simulation
+
+# ---- Library targets ----
+
+lib: $(STATIC_LIB) $(SHARED_LIB)
+
+$(STATIC_LIB): $(LIB_OBJS)
+	$(AR) rcs $@ $^
+
+$(SHARED_LIB): $(LIB_SRCS)
+	$(CC) $(CFLAGS) -shared -fPIC $(INCLUDES) $^ -o $@
+
+src/%.o: src/%.c
+	$(CC) $(CFLAGS) $(INCLUDES) -fPIC -c $< -o $@
+
+# ---- Binary targets ----
+
+toastyfs: $(SERVER_SRCS)
+	$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_SERVER $^ -o $@
+
+toastyfs_client: $(CLIENT_SRCS)
+	$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_CLIENT $^ -o $@
+
+toastyfs_simulation: $(SIM_SRCS)
+	$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_SIMULATION -DFAULT_INJECTION $^ -o $@
+
+# ---- Install ----
+
+PREFIX    ?= /usr/local
+LIBDIR    ?= $(PREFIX)/lib
+INCLUDEDIR?= $(PREFIX)/include
+
+install: $(STATIC_LIB) $(SHARED_LIB)
+	install -d $(DESTDIR)$(LIBDIR)
+	install -d $(DESTDIR)$(INCLUDEDIR)
+	install -m 644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)/
+	install -m 755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR)/
+	install -m 644 include/toastyfs.h $(DESTDIR)$(INCLUDEDIR)/
+
+# ---- Clean ----
+
+clean:
+	rm -f $(LIB_OBJS) $(STATIC_LIB) $(SHARED_LIB)
+	rm -f toastyfs toastyfs_client toastyfs_simulation
+
+.PHONY: all lib clean install

README.md

@@ -1,165 +1,109 @@
 # ToastyFS
 
-ToastyFS is a simple, fault-tolerant, highly available object storage featuring:
+ToastyFS is a distributed and fault-tolerant object storage.
 
-* Deterministic Simulation Testing
+WARNING: I'm still finishing up :) Log compaction and persistence are missing.
-* Cross-Platform (Windows, Linux)
-* Minimal Dependencies (OpenSSL and SChannel)
-* Viewstamped Replication
-
-I initially started this project to learn about distributed systems. I asked myself what it would take to build my own Dropbox. A rabbit-hole later and here is my distributed storage system!
-
-Here is a cool pic of the hardware I'm building to run ToastyFS :D
-
-![photo of an 8 Raspberry PI cluster mouted in a small rack connected via a switch](pic.jpg)
-
-## Table of Contents
-
-* [ToastyFS](#toastyfs)
-  * [Project Status & Known Limitations](#project-status--known-limitations)
-* [Getting Started](#getting-started)
-  * [Building](#building)
-  * [Running a Cluster](#running-a-cluster)
-  * [Basic Usage](#basic-usage)
-* [Fault Tolerance & High Availability](#fault-tolerance--high-availability)
-  * [The Need for Replication](#the-need-for-replication)
-  * [Raft VS Viewstamped Replication](#raft-vs-viewstamped-replication)
-  * [Replication in ToastyFS](#replication-in-toastyfs)
-  * [Failure Scenarios](#failure-scenarios)
-* [Architecture](#architecture)
-  * [System Components](#system-components)
-  * [Request Lifecycle](#request-lifecycle)
-* [Testing](#testing)
-  * [Deterministic Simulation Testing](#deterministic-simulation-testing)
-  * [Testing in ToastyFS](#testing-in-toastyfs)
-
-### Project Status & Known Limitations
-
-This project should be considered a robust proof-of-concept at this time. Features that would be required for long-running instances are missing, such as:
-
-* Log compaction
-* Ability to shut down all nodes without resetting the system
 
 ## Getting Started
 
-### Building
+First, build ToastyFS by running the build script:
 
-ToastyFS supports Windows and Linux. Each platform has its own build script:
+```sh
-
-```
-# Windows
-.\build.bat
-
-# Linux
 ./build.sh
 ```
 
-The build script will produce the following executables:
+This will produce three executables: `toastyfs`, `toastyfs_random_client` and `toastyfs_simulation`.
 
-| Name | Description |
+The `toastyfs` is the one you need to run. The other executables are for testing purposes.
-| :--- | :---------- |
-| toasty_simulation | Runs a ToastyFS cluster in-memory, with sped-up time and serving a number of random operations. See [Testing](#testing) section for details. |
-| toasty | The actual ToastyFS program. This is what you need to run to use ToastyFS. |
-| toasty_random_client | An utility client which spams random requests towards a ToastyFS cluster. Useful for testing. |
-| toasty_proxy | An HTTP proxy that translates HTTP request to the ToastyFS-specific request protocol. |
 
-On Windows the executables will have the `.exe` extension.
+You can pick a cluster size based on how many faults you want it to handle. Let's start with a cluster size of 3 which will allow one node to crash at a given time.
 
-### Running a Cluster
+Run three instances of `toastyfs` while specifying to each one the addresses of the others:
 
-You can then start a ToastyFS cluster by running the cluster script. If you are on Windows, you will need a Linux-compatible shell to run it.
+```sh
-
+./toastyfs --addr 127.0.0.1:8081  --peer 127.0.0.1:8082 --peer 127.0.0.1:8083
-```
+./toastyfs --addr 127.0.0.1:8082  --peer 127.0.0.1:8081 --peer 127.0.0.1:8083
-./cluster.sh start
+./toastyfs --addr 127.0.0.1:8083  --peer 127.0.0.1:8081 --peer 127.0.0.1:8082
-./cluster.sh status
 ```
 
-The cluster is composed of 3 nodes and 1 HTTP proxy listening on `127.0.0.1:3000`.
+The cluster is now working.
 
-The cluster can be turned off by doing:
+To upload/download/delete objects, you need to compile a client program which needs the toasty client library.
 
-```
+To build the library, run the following, which will generate the `libtoasty.a` and `libtoasty.so` files:
-./cluster.sh stop
+
+```sh
+Makefile
 ```
 
-### Basic Usage
+You can use the following example program which will upload a simple object:
 
-The HTTP proxy makes it easy to manage ToastyFS instances. You can manage objects via regular HTTP verbs:
+```c
+#include <time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <toastyfs.h>
 
-```
+int main(void)
-$ curl -X PUT http://127.0.0.1:3000/first_object -d "I'm the first object"
+{
-$ curl -X GET http://127.0.0.1:3000/first_object
+    srand(time(NULL));
-I'm the first object
+
+    char addr1[] = "127.0.0.1:8081";
+    char addr2[] = "127.0.0.1:8082";
+    char addr3[] = "127.0.0.1:8083";
+    char *addrs[] = { addr1, addr2, addr3 };
+
+    printf("Connecting to cluster...\n");
+    ToastyFS *tfs = toastyfs_init(rand(), addrs, 3);
+    if (tfs == NULL) {
+        fprintf(stderr, "toastyfs_init failed\n");
+        return 1;
+    }
+
+    char key[]  = "hello";
+    char data[] = "world";
+
+    printf("PUT key=\"%s\" data=\"%s\" (%d bytes)\n", key, data, (int)strlen(data));
+
+    ToastyFS_Result res;
+    int ret = toastyfs_put(tfs, key, strlen(key), data, strlen(data), &res);
+    if (ret < 0) {
+        fprintf(stderr, "toastyfs_put returned %d\n", ret);
+        toastyfs_free(tfs);
+        return 1;
+    }
+
+    printf("Done.\n");
+    toastyfs_free(tfs);
+    return 0;
+}
 ```
 
-## Fault Tolerance & High Availability
+Save it as `example_client.c` and compile it by running:
 
-### The Need for Replication
+```
+gcc example_client.c libtoasty.a -o example
+```
 
-To build a fault-tolerant service, it's necessary for it to run on multiple machines in such a way that if a machine dies, the others can continue its work. This is referred to as replication. Designing and implementing a replication system that properly manages all edge cases is **incredibly hard**. Such an algorithm needs to account for any number of node crashes at any phase of the protocol, it needs to account for arbitrary network partitions (network failures that cause the system to be split in multiple groups of nodes) making it impossible for the state of each group to diverge. This is such a hard problem that not only few general algorithms have been designed, but also the number of their implementations is vanishingly low. The common replication algorithms are Raft and Paxos (*). This project uses a less known algorithm called Viewstamped Replication (VSR).
+By running the client while the cluster is running, the object will be created!
 
-(*) Strictly speaking, Paxos is a consensus algorithm. Replication can be built on top of it.
+## Fault Tolerance
 
-### Raft VS Viewstamped Replication
+ToastyFS allows any minority of nodes to be dead at any given time and continue running without issues. Generally speaking, if your system has 2f+1 nodes, it can work without f of them.
-
-For historical reasons, Paxos (1989) is the established algorithm to achieve replication. Paxos is notorious for being hard to understand and implement, so in 2014 a more understandable algorithm called Raft was introduced. Since then, Raft has been the go-to algorithm from new open source projects.
-
-There is also a lesser known replication algorithm called Viewstamped Replication (VSR), which never got much attention from the industry even though it was invented around the same time as Paxos (and arguably, before it). In 2012 the "Viewstamped Replication Revisited" paper was published which offered a modernized design of the protocol. The later paper is the reference for VSR used in this project.
-
-VSR's design is very similar to Raft (which came about 25 years later) but has some important differences. At a high level, Raft was optimized for understandability, while VSR for performance and robustness. VSR offers guarantees that neither Raft nor Paxos are able to offer by default.
-
-A key component of the Raft algorithm is the write-ahead log that each node uses to restore its state in case of a crash. If the disk fails, the log is lost. Raft only guarantees overall availability if no disks fail.
-
-Unlike Raft, VSR does not require a disk. All state necessary for replication can be stored in-memory. Disk storage can be used but it's merely an optional optimization. Even if a disk were used, implementations could fallback to the in-memory variant on the protocol in case of disk failure.
-
-### Replication in ToastyFS
-
-A ToastyFS system is made by a number of servers that are replicated according to VSR. One server acts as primary while the others as backups. Clients send read, update, delete operations to the primary which in turns forwards them to backups. When a majority has agreed to run that command, it is executed and the primary sends the result to the client. Reaching a consensus on operations to perform ensures that if any node crashes, another one can take its place with the same state.
-
-Since ToastyFS relies on reaching majority to update its state, it remains available as long as a majority of nodes stays online. For instance a cluster of 5 nodes will function as long as 3 nodes are live. Generally speaking, a cluster of `2f+1` nodes will tolerate `f` simultaneous failures.
-
-### Failure Scenarios
-
-The primary node periodically sends heartbeat messages to backups. When backups stop receiving messages from the primary, they assume it is dead. The specific timeout is configurable but the default value is 1-2 seconds. The live node then perform a "view change", which means they choose a new primary and ensure it holds the latest state of the system. Any interaction of clients with the original node is lost causing them to timeout. Dropped operations will need to be restarted. If the old primary comes back online, it will do so as a backup.
-
-If a backup node crashes, it simply will simply stop partecipating in consensus, and if a majority of nodes is dead no consensus will be reached at all causing the system to become unavailable.
-
-When a node (be it an old primary or backup) restarts, it enters a recovery state where it asks to rejoin the cluster and the current state of the system is sent back.
-
-A node in the recovery state can only rejoin the cluster if a majority of nodes is still alive, which means that if a majority of nodes dies, the system will become permanently unavailable. This is a consequence of the in-memory implementation of VSR. If it was extended to use a persistent log, the recovery state could be skipped entirely, allowing nodes to rejoin the cluster regardless of the number of nodes that are still alive.
-
-Network partitions are naturally handled by the VSR protocol. If the cluster nodes are partitioned in groups that can't talk with each other, a maximum of one group will be able to reach consensus and continue operation. All other minorities will become unavailable. If the partitions are resolved, stale nodes will request a state transfer and catch up.
-
-Disk corruption will not impact the replication protocol but it may cause object data to be lost on a node. Since such data is stored on a majority of nodes, as long as that data is available on at least one node, it will be possible to retrieve it.
-
-## Architecture
-
-### System Components
-
-ToastyFS is designed to be easy to deploy. A ToastyFS cluster is made of a number of server nodes and a proxy node.
-
-Server nodes store both object metadata and data and speak to each other via a custom protocol. An HTTP proxy node translates regular HTTP operations to the custom protocol.
-
-A cluster must have an odd number of servers that is greater than 1, like 3 or 5. Generally speaking if 2f+1 is the number of servers, the system will continue working if at least f+1 nodes are alive. This is a consequence of the quorum intersection property that VSR relies on.
-
-### Request Lifecycle
-
-When a client starts performs a PUT request to create a new object in the system, the object name and data is first sent to the HTTP proxty. The proxy, which acts as client relative to the ToastyFS servers, splits the data into chunks and for each chunk it picks the servers it should be replicated on. Each chunk must be replicated on a majority of servers. The proxy then uploads the chunks to the servers. If any single upload fails, the overall operation fails. If all uploads succeded, the proxy sends the list of chunks that were uploaded and their new locations to the primary node. The primary forwards the object's metadata to the backups and wait for them to acknowledge it. If a majority is reached, the metadata is committed and a success response is sent back to the proxy. The proxy then forwards the success response to the HTTP client.
-
-When a client performs a GET request, the HTTP proxy requests the list of chunks and their locations to the primary node. The primary node forwards the request to all other backups which acknowledge it. When the majority of servers acknowledge (primary included), the response is sent to the proxy. The proxy then downloads the chunks from their locations and reconstructs the object, which is then sent back to the client over HTTP.
-
-When a client performs a DELETE request, the HTTP proxy sends a DELETE operation to the primary, which forwards it to backups. When a majority of servers acknowledge it, the primary deletes the entry. 
 
 ## Testing
 
-### Deterministic Simulation Testing
+ToastyFS is tested with Deterministic Simulation Testing (DST).
 
-Testing one of the most important aspects of ToastyFS. Fault tolerant systems must be tested to ensure they are actually capable of handling failures, especially rare ones that is hard to prepare for. To ensure a high degree of certainty, ToastyFS uses **Deterministic Simulation Testing** (DST), the gold standard for distributed system validation.
+The entire system (server nodes and clients) is simulated in the memory of a single process deterministically. Faults (node crashes, network partitons) and latencies are pseudo-randonly injected in the simulation such that a variety of scenarios are tested but in a way that can be perfectly reproduced by using the same simulation seed from which all random events are chosen.
 
-Generally speaking, simulation testing (deterministic or not) is a way to run a system in an environment that creates a variety of different conditions to see how it reacts. This can be used to check for a system's correctness by aborting the system if incoherent states are reached via assert statements. This makes it possible to determine whether an incoherent state can be reached, but not **why** or **how** it was reached. This is where the deterministic parts comes in.
+After each node schedulation, the simulator inspects all node states at the same instant and verifies that none of the system's invariant were broken.
 
-If the entire simulation is deterministic, running it multiple times will cause the system to evolve in the exact same way, effectively allowing the replaying the causes of the error. Making a simulation deterministic is very hard as any interaction with the outside world needs to be simulated to make it controllable. Note that this includes scheduling: even if nodes in a system send the same exact sequence of messages, the evolution of the system may be completely different due to how each node is scheduled.
+You can build and run the simulation by doing:
 
-### Testing in ToastyFS
+```
+./build.sh
+./toastyfs_simulation
+```
 
-ToastyFS uses DST as main form of testing. Running the build script (see [Building](#building)) will produce the `toastyfs_simulation` executable which runs a simulation with 3 server nodes and 3 client nodes which issue random object operations. The entire cluster is simulated in a single process and deterministically. All I/O operations are mocked and a variety of faults are injected pseudo-randomly, such as disk bit flips, network partitions, network delays, node crashes. The cluster configuration and fault injection parameters can trivially be modified in `src/main.c`. The mocking of I/O operations is implemented at the system call level. Each node call mocked version of each system call which interacts with a simplified userspace kernel which routes operations to in-memory disks or in-memory network. Node parallelism is simulated via a non-preemptive userspace scheduler. 

TODO.txt

@@ -0,0 +1,4 @@
+[ ] Add log compaction
+[ ] Add log persistence
+[ ] Add chunk locations to metadata
+[ ] fsync parent directory when renaming log

build.bat

@@ -1,9 +0,0 @@
-set QKY_FILES=quakey/src/mockfs.c quakey/src/quakey.c
-set LIB_FILES=lib/basic.c lib/byte_queue.c lib/file_system.c lib/http_parse.c lib/http_server.c lib/message.c lib/tcp.c lib/tls_openssl.c lib/tls_schannel.c
-set SRC_FILES=src/chunk_store.c src/client.c src/client_table.c src/http_proxy.c src/invariant_checker.c src/log.c src/main.c src/metadata.c src/random_client.c src/server.c
-set FLAGS=-Wall -Wextra -ggdb -O0 -Iquakey/include -Iinclude -I.
-
-gcc -o toasty_simulation    %LIB_FILES% %SRC_FILES% %QKY_FILES% %FLAGS% -DMAIN_SIMULATION -DFAULT_INJECTION -lws2_32
-gcc -o toasty               %LIB_FILES% %SRC_FILES% %FLAGS% -DMAIN_SERVER -lws2_32
-gcc -o toasty_random_client %LIB_FILES% %SRC_FILES% %FLAGS% -DMAIN_CLIENT -lws2_32
-gcc -o toasty_proxy         %LIB_FILES% %SRC_FILES% %FLAGS% -DMAIN_HTTP_PROXY -DTLS_ENABLED -DTLS_SCHANNEL -lws2_32 -lsecur32 -lcrypt32 -lncrypt

build.sh

@@ -1,9 +1,3 @@
-QKY_FILES="quakey/src/mockfs.c quakey/src/quakey.c"
+gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/client.c src/random_client.c src/main.c src/wal.c src/client_table.c src/invariant_checker.c src/chunk_store.c src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c -o toastyfs_simulation -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
-LIB_FILES="lib/basic.c lib/byte_queue.c lib/file_system.c lib/http_parse.c lib/http_server.c lib/message.c lib/tcp.c lib/tls_openssl.c lib/tls_schannel.c"
+gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/main.c src/wal.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs        -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
-SRC_FILES="src/chunk_store.c src/client.c src/client_table.c src/http_proxy.c src/invariant_checker.c src/log.c src/main.c src/metadata.c src/random_client.c src/server.c"
+gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/client.c src/random_client.c src/main.c src/wal.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs_random_client -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
-FLAGS="-Wall -Wextra -ggdb -O0 -Iquakey/include -Iinclude -I."
-
-gcc -o toasty_simulation    $LIB_FILES $SRC_FILES $QKY_FILES $FLAGS -DMAIN_SIMULATION -DFAULT_INJECTION
-gcc -o toasty               $LIB_FILES $SRC_FILES $FLAGS -DMAIN_SERVER
-gcc -o toasty_random_client $LIB_FILES $SRC_FILES $FLAGS -DMAIN_CLIENT
-gcc -o toasty_proxy         $LIB_FILES $SRC_FILES $FLAGS -DMAIN_HTTP_PROXY -DTLS_ENABLED -DTLS_OPENSSL -lssl -lcrypto

build.zig

@@ -1,85 +0,0 @@
-const std = @import("std");
-
-const include_paths: []const []const u8 = &.{
-    ".",
-    "src",
-    "include",
-    "quakey/include",
-};
-
-const files: []const []const u8 = &.{
-
-    "src/chunk_store.c",
-    "src/client.c",
-    "src/client_table.c",
-    "src/http_proxy.c",
-    "src/invariant_checker.c",
-    "src/log.c",
-    "src/main.c",
-    "src/metadata.c",
-    "src/random_client.c",
-    "src/server.c",
-
-    "lib/basic.c",
-    "lib/byte_queue.c",
-    "lib/file_system.c",
-    "lib/http_parse.c",
-    "lib/http_server.c",
-    "lib/message.c",
-    "lib/tcp.c",
-    "lib/tls_openssl.c",
-    "lib/tls_schannel.c",
-
-    "quakey/src/mockfs.c",
-    "quakey/src/quakey.c",
-};
-
-fn buildTarget(b: *std.Build, name: []const u8, macros: []const []const u8, flags: []const []const u8) void {
-    const exe = b.addExecutable(.{
-        .name = name,
-        .root_module = b.createModule(.{
-            .target = b.graph.host,
-        }),
-    });
-    for (macros) |m| {
-        exe.root_module.addCMacro(m, "");
-    }
-    for (include_paths) |p| {
-        exe.root_module.addIncludePath(b.path(p));
-    }
-    exe.root_module.addCSourceFiles(.{ .files = files, .flags = flags });
-    exe.root_module.link_libc = true;
-
-    b.installArtifact(exe);
-}
-
-pub fn build(b: *std.Build) !void {
-
-    // Server
-    buildTarget(b,
-        "toasty",
-        &.{ "MAIN_SERVER" },
-        &.{ "-Wall", "-Wextra", "-ggdb", "-O0" },
-    );
-
-    // Client
-    buildTarget(b,
-        "toasty_random_client",
-        &.{ "MAIN_CLIENT" },
-        &.{ "-Wall", "-Wextra", "-ggdb", "-O0" },
-    );
-
-    // Proxy
-    buildTarget(b,
-        "toasty_proxy",
-        &.{ "MAIN_HTTP_PROXY" },
-        &.{ "-Wall", "-Wextra", "-ggdb", "-O0" },
-    );
-
-    // Simulation
-    buildTarget(b,
-        "toasty_simulation",
-        &.{ "MAIN_SIMULATION", "FAULT_INJECTION" },
-        &.{ "-Wall", "-Wextra", "-ggdb", "-O0" },
-    );
-}

cluster.sh

@@ -1,178 +1,176 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
-# Cluster management script for ToastyFS.
+# cluster.sh - Manage a local 3-node ToastyFS cluster.
-# Manages a 3-node server cluster with 1 HTTP proxy.
 #
 # Usage:
-#   ./cluster.sh start   - Build (if needed) and start all nodes + proxy
+#   ./cluster.sh up                     Build the server and start 3 nodes
-#   ./cluster.sh stop    - Stop all running nodes and the proxy
+#   ./cluster.sh down                   Stop all nodes
-#   ./cluster.sh status  - Show the status of each process
+#   ./cluster.sh status                 Show which nodes are running
-#   ./cluster.sh restart - Stop then start
+#   ./cluster.sh logs [1|2|3]           Tail logs (all nodes, or one)
-#   ./cluster.sh clean   - Stop everything and remove data/log directories
+#   ./cluster.sh run <source.c>         Build the library, compile <source.c>
+#                                       against it, start the cluster, run the
+#                                       binary, then tear down the cluster.
 #
-
+# The three server nodes listen on:
-set -e
+#   127.0.0.1:8081   127.0.0.1:8082   127.0.0.1:8083
+#
+set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-DATA_DIR="$SCRIPT_DIR/cluster-data"
+CLUSTER_DIR="$SCRIPT_DIR/.cluster"
-LOG_DIR="$DATA_DIR/logs"
+ADDRS=("127.0.0.1:8081" "127.0.0.1:8082" "127.0.0.1:8083")
-PID_DIR="$DATA_DIR/pids"
 
-NODE1_ADDR="127.0.0.1:8001"
+usage() {
-NODE2_ADDR="127.0.0.1:8002"
+    sed -n '3,14s/^# \?//p' "$0"
-NODE3_ADDR="127.0.0.1:8003"
-
-build() {
-    echo "Building ToastyFS..."
-    (cd "$SCRIPT_DIR" && bash build.sh)
-    echo "Build complete."
-}
-
-ensure_dirs() {
-    mkdir -p "$LOG_DIR" "$PID_DIR"
-    mkdir -p "$DATA_DIR/chunks-node1"
-    mkdir -p "$DATA_DIR/chunks-node2"
-    mkdir -p "$DATA_DIR/chunks-node3"
-}
-
-start_node() {
-    local name="$1"
-    local addr="$2"
-    local peer1="$3"
-    local peer2="$4"
-    local chunks="$5"
-
-    local pidfile="$PID_DIR/$name.pid"
-    local logfile="$LOG_DIR/$name.log"
-
-    if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
-        echo "  $name is already running (pid $(cat "$pidfile"))"
-        return
-    fi
-
-    "$SCRIPT_DIR/toasty" \
-        --addr "$addr" \
-        --peer "$peer1" \
-        --peer "$peer2" \
-        --chunks "$chunks" \
-        > "$logfile" 2>&1 &
-
-    local pid=$!
-    echo "$pid" > "$pidfile"
-    echo "  $name started (pid $pid) on $addr"
-}
-
-start_proxy() {
-    local pidfile="$PID_DIR/proxy.pid"
-    local logfile="$LOG_DIR/proxy.log"
-
-    if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
-        echo "  proxy is already running (pid $(cat "$pidfile"))"
-        return
-    fi
-
-    "$SCRIPT_DIR/toasty_proxy" \
-        --server "$NODE1_ADDR" \
-        --server "$NODE2_ADDR" \
-        --server "$NODE3_ADDR" \
-        > "$logfile" 2>&1 &
-
-    local pid=$!
-    echo "$pid" > "$pidfile"
-    echo "  proxy started (pid $pid) on 127.0.0.1:3000"
-}
-
-stop_process() {
-    local name="$1"
-    local pidfile="$PID_DIR/$name.pid"
-
-    if [ ! -f "$pidfile" ]; then
-        echo "  $name is not running (no pid file)"
-        return
-    fi
-
-    local pid
-    pid=$(cat "$pidfile")
-
-    if kill -0 "$pid" 2>/dev/null; then
-        kill "$pid"
-        # Wait briefly for graceful shutdown
-        local i=0
-        while kill -0 "$pid" 2>/dev/null && [ $i -lt 10 ]; do
-            sleep 0.1
-            i=$((i + 1))
-        done
-        if kill -0 "$pid" 2>/dev/null; then
-            kill -9 "$pid" 2>/dev/null || true
-        fi
-        echo "  $name stopped (was pid $pid)"
-    else
-        echo "  $name was not running (stale pid $pid)"
-    fi
-
-    rm -f "$pidfile"
-}
-
-do_start() {
-    if [ ! -f "$SCRIPT_DIR/toasty" ] || [ ! -f "$SCRIPT_DIR/toasty_proxy" ]; then
-        build
-    fi
-
-    ensure_dirs
-
-    echo "Starting cluster..."
-    start_node "node1" "$NODE1_ADDR" "$NODE2_ADDR" "$NODE3_ADDR" "$DATA_DIR/chunks-node1"
-    start_node "node2" "$NODE2_ADDR" "$NODE1_ADDR" "$NODE3_ADDR" "$DATA_DIR/chunks-node2"
-    start_node "node3" "$NODE3_ADDR" "$NODE1_ADDR" "$NODE2_ADDR" "$DATA_DIR/chunks-node3"
-
-    # Give servers a moment to start listening before launching the proxy
-    sleep 1
-
-    start_proxy
-    echo "Cluster is running. HTTP proxy at http://127.0.0.1:3000"
-}
-
-do_stop() {
-    echo "Stopping cluster..."
-    stop_process "proxy"
-    stop_process "node1"
-    stop_process "node2"
-    stop_process "node3"
-    rm -f "$SCRIPT_DIR/vsr_boot_marker"
-    echo "Cluster stopped."
-}
-
-do_status() {
-    local all_stopped=true
-    for name in node1 node2 node3 proxy; do
-        local pidfile="$PID_DIR/$name.pid"
-        if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
-            echo "  $name: running (pid $(cat "$pidfile"))"
-            all_stopped=false
-        else
-            echo "  $name: stopped"
-        fi
-    done
-    if $all_stopped; then
-        echo "Cluster is not running."
-    fi
-}
-
-do_clean() {
-    do_stop
-    echo "Removing cluster data..."
-    rm -rf "$DATA_DIR"
-    echo "Clean complete."
-}
-
-case "${1:-}" in
-    start)   do_start   ;;
-    stop)    do_stop    ;;
-    status)  do_status  ;;
-    restart) do_stop; do_start ;;
-    clean)   do_clean   ;;
-    *)
-        echo "Usage: $0 {start|stop|status|restart|clean}"
     exit 1
+}
+
+cluster_up() {
+    echo "==> Building server..."
+    make -C "$SCRIPT_DIR" toastyfs
+
+    mkdir -p "$CLUSTER_DIR"
+
+    for i in 1 2 3; do
+        pidfile="$CLUSTER_DIR/node${i}.pid"
+        if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
+            echo "    Node $i already running (pid $(cat "$pidfile"))"
+            continue
+        fi
+
+        node_dir="$CLUSTER_DIR/node${i}"
+        mkdir -p "$node_dir"
+
+        # Build argument list: --addr for self, --peer for the others.
+        args=()
+        for j in 1 2 3; do
+            idx=$((j - 1))
+            if [ "$j" -eq "$i" ]; then
+                args+=(--addr "${ADDRS[$idx]}")
+            else
+                args+=(--peer "${ADDRS[$idx]}")
+            fi
+        done
+
+        (
+            cd "$node_dir"
+            # ServerState is ~40 MB (MetaStore), which exceeds the
+            # default 8 MB stack.  Raise the limit for the server.
+            ulimit -s unlimited
+            "$SCRIPT_DIR/toastyfs" "${args[@]}" \
+                >> "$CLUSTER_DIR/node${i}.log" 2>&1 &
+            echo $! > "$pidfile"
+        )
+        echo "    Node $i started (pid $(cat "$pidfile")) on ${ADDRS[$((i-1))]}"
+    done
+    echo "==> Cluster is running."
+}
+
+cluster_down() {
+    if [ ! -d "$CLUSTER_DIR" ]; then
+        echo "No cluster state found."
+        return
+    fi
+
+    for i in 1 2 3; do
+        pidfile="$CLUSTER_DIR/node${i}.pid"
+        if [ -f "$pidfile" ]; then
+            pid="$(cat "$pidfile")"
+            if kill -0 "$pid" 2>/dev/null; then
+                kill "$pid" 2>/dev/null || true
+                echo "    Node $i stopped (pid $pid)"
+            else
+                echo "    Node $i was not running"
+            fi
+            rm -f "$pidfile"
+        fi
+    done
+
+    # Clean up node data directories so fresh starts don't trigger
+    # the crash-recovery path (vsr_boot_marker).
+    rm -rf "$CLUSTER_DIR"
+    echo "==> Cluster stopped."
+}
+
+cluster_status() {
+    if [ ! -d "$CLUSTER_DIR" ]; then
+        echo "No cluster state found. Run './cluster.sh up' first."
+        return
+    fi
+
+    printf "%-8s %-8s %-20s %s\n" "NODE" "PID" "ADDRESS" "STATUS"
+    for i in 1 2 3; do
+        pidfile="$CLUSTER_DIR/node${i}.pid"
+        addr="${ADDRS[$((i-1))]}"
+        if [ -f "$pidfile" ]; then
+            pid="$(cat "$pidfile")"
+            if kill -0 "$pid" 2>/dev/null; then
+                printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "running"
+            else
+                printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "dead"
+            fi
+        else
+            printf "%-8s %-8s %-20s %s\n" "node$i" "-" "$addr" "not started"
+        fi
+    done
+}
+
+cluster_logs() {
+    if [ $# -gt 0 ]; then
+        logfile="$CLUSTER_DIR/node${1}.log"
+        if [ ! -f "$logfile" ]; then
+            echo "No log file for node $1."
+            return 1
+        fi
+        tail -f "$logfile"
+    else
+        tail -f "$CLUSTER_DIR"/node*.log
+    fi
+}
+
+cluster_run() {
+    local src="$1"
+    local bin
+    bin="$(basename "${src%.c}")"
+
+    echo "==> Building libtoastyfs..."
+    make -C "$SCRIPT_DIR" lib
+
+    echo "==> Compiling $src..."
+    ${CC:-gcc} -Wall -Wextra -o "$bin" "$src" \
+        -I"$SCRIPT_DIR/include" \
+        -L"$SCRIPT_DIR" \
+        -ltoastyfs
+
+    echo "==> Starting cluster..."
+    cluster_up
+    # Give the servers a moment to elect a leader.
+    sleep 2
+
+    echo "==> Running ./$bin"
+    echo "---"
+    LD_LIBRARY_PATH="$SCRIPT_DIR:${LD_LIBRARY_PATH:-}" "./$bin" || true
+    echo "---"
+
+    echo "==> Stopping cluster..."
+    cluster_down
+}
+
+if [ $# -lt 1 ]; then
+    usage
+fi
+
+case "$1" in
+    up)     cluster_up ;;
+    down)   cluster_down ;;
+    status) cluster_status ;;
+    logs)   shift; cluster_logs "$@" ;;
+    run)
+        if [ $# -lt 2 ]; then
+            echo "Error: 'run' requires a source file argument." >&2
+            usage
+        fi
+        cluster_run "$2"
         ;;
+    *)      usage ;;
 esac

lib/http_parse.h

@@ -1,161 +0,0 @@
-#ifndef HTTP_PARSE_INCLUDED
-#define HTTP_PARSE_INCLUDED
-
-#include "basic.h"
-
-#define CHTTP_MAX_HEADERS 32
-
-typedef struct {
-	unsigned int data;
-} CHTTP_IPv4;
-
-typedef struct {
-	unsigned short data[8];
-} CHTTP_IPv6;
-
-typedef enum {
-	CHTTP_HOST_MODE_VOID = 0,
-	CHTTP_HOST_MODE_NAME,
-	CHTTP_HOST_MODE_IPV4,
-	CHTTP_HOST_MODE_IPV6,
-} CHTTP_HostMode;
-
-typedef struct {
-	CHTTP_HostMode mode;
-	string text;
-	union {
-		string name;
-		CHTTP_IPv4 ipv4;
-		CHTTP_IPv6 ipv6;
-	};
-} CHTTP_Host;
-
-typedef struct {
-	string userinfo;
-	CHTTP_Host host;
-	int        port;
-} CHTTP_Authority;
-
-// ZII
-typedef struct {
-	string scheme;
-	CHTTP_Authority authority;
-	string path;
-	string query;
-	string fragment;
-} CHTTP_URL;
-
-typedef enum {
-	CHTTP_METHOD_GET,
-	CHTTP_METHOD_HEAD,
-	CHTTP_METHOD_POST,
-	CHTTP_METHOD_PUT,
-	CHTTP_METHOD_DELETE,
-	CHTTP_METHOD_CONNECT,
-	CHTTP_METHOD_OPTIONS,
-	CHTTP_METHOD_TRACE,
-	CHTTP_METHOD_PATCH,
-} CHTTP_Method;
-
-typedef struct {
-	string name;
-	string value;
-} CHTTP_Header;
-
-typedef struct {
-    bool        secure;
-	CHTTP_Method method;
-	CHTTP_URL    url;
-	int         minor;
-	int         num_headers;
-	CHTTP_Header headers[CHTTP_MAX_HEADERS];
-	string body;
-} CHTTP_Request;
-
-typedef struct {
-    void*       context;
-	int         minor;
-	int         status;
-	string reason;
-	int         num_headers;
-	CHTTP_Header headers[CHTTP_MAX_HEADERS];
-	string body;
-} CHTTP_Response;
-
-int chttp_parse_ipv4(char *src, int len, CHTTP_IPv4     *ipv4);
-int chttp_parse_ipv6(char *src, int len, CHTTP_IPv6     *ipv6);
-int chttp_parse_url(char *src, int len, CHTTP_URL      *url);
-int chttp_parse_request(char *src, int len, CHTTP_Request  *req);
-int chttp_parse_response(char *src, int len, CHTTP_Response *res);
-
-int chttp_find_header(CHTTP_Header *headers, int num_headers, string name);
-
-string chttp_get_cookie(CHTTP_Request *req, string name);
-string chttp_get_param(string body, string str, char *mem, int cap);
-int    chttp_get_param_i(string body, string str);
-
-// Checks whether the request was meant for the host with the given
-// domain an port. If port is -1, the default value of 80 is assumed.
-bool chttp_match_host(CHTTP_Request *req, string domain, int port);
-
-// Date and cookie types for Set-Cookie header parsing
-typedef enum {
-    CHTTP_WEEKDAY_MON,
-    CHTTP_WEEKDAY_TUE,
-    CHTTP_WEEKDAY_WED,
-    CHTTP_WEEKDAY_THU,
-    CHTTP_WEEKDAY_FRI,
-    CHTTP_WEEKDAY_SAT,
-    CHTTP_WEEKDAY_SUN,
-} CHTTP_WeekDay;
-
-typedef enum {
-    CHTTP_MONTH_JAN,
-    CHTTP_MONTH_FEB,
-    CHTTP_MONTH_MAR,
-    CHTTP_MONTH_APR,
-    CHTTP_MONTH_MAY,
-    CHTTP_MONTH_JUN,
-    CHTTP_MONTH_JUL,
-    CHTTP_MONTH_AUG,
-    CHTTP_MONTH_SEP,
-    CHTTP_MONTH_OCT,
-    CHTTP_MONTH_NOV,
-    CHTTP_MONTH_DEC,
-} CHTTP_Month;
-
-typedef struct {
-    CHTTP_WeekDay week_day;
-    int          day;
-    CHTTP_Month   month;
-    int          year;
-    int          hour;
-    int          minute;
-    int          second;
-} CHTTP_Date;
-
-typedef struct {
-    string name;
-    string value;
-
-    bool secure;
-    bool chttp_only;
-
-    bool have_date;
-    CHTTP_Date date;
-
-    bool have_max_age;
-    uint32_t max_age;
-
-    bool have_domain;
-    string domain;
-
-    bool have_path;
-    string path;
-} CHTTP_SetCookie;
-
-// Parses a Set-Cookie header value
-// Returns 0 on success, -1 on error
-int chttp_parse_set_cookie(string str, CHTTP_SetCookie *out);
-
-#endif // HTTP_PARSE_INCLUDED

lib/http_server.c

@@ -1,392 +0,0 @@
-#include <quakey.h>
-#include <assert.h>
-
-#include "http_server.h"
-
-static string reason_phrase(int status)
-{
-	switch(status) {
-
-		case 100: return S("Continue");
-		case 101: return S("Switching Protocols");
-		case 102: return S("Processing");
-
-		case 200: return S("OK");
-		case 201: return S("Created");
-		case 202: return S("Accepted");
-		case 203: return S("Non-Authoritative Information");
-		case 204: return S("No Content");
-		case 205: return S("Reset Content");
-		case 206: return S("Partial Content");
-		case 207: return S("Multi-Status");
-		case 208: return S("Already Reported");
-
-		case 300: return S("Multiple Choices");
-		case 301: return S("Moved Permanently");
-		case 302: return S("Found");
-		case 303: return S("See Other");
-		case 304: return S("Not Modified");
-		case 305: return S("Use Proxy");
-		case 306: return S("Switch Proxy");
-		case 307: return S("Temporary Redirect");
-		case 308: return S("Permanent Redirect");
-
-		case 400: return S("Bad Request");
-		case 401: return S("Unauthorized");
-		case 402: return S("Payment Required");
-		case 403: return S("Forbidden");
-		case 404: return S("Not Found");
-		case 405: return S("Method Not Allowed");
-		case 406: return S("Not Acceptable");
-		case 407: return S("Proxy Authentication Required");
-		case 408: return S("Request Timeout");
-		case 409: return S("Conflict");
-		case 410: return S("Gone");
-		case 411: return S("Length Required");
-		case 412: return S("Precondition Failed");
-		case 413: return S("Request Entity Too Large");
-		case 414: return S("Request-URI Too Long");
-		case 415: return S("Unsupported Media Type");
-		case 416: return S("Requested Range Not Satisfiable");
-		case 417: return S("Expectation Failed");
-		case 418: return S("I'm a teapot");
-		case 420: return S("Enhance your calm");
-		case 422: return S("Unprocessable Entity");
-		case 426: return S("Upgrade Required");
-		case 429: return S("Too many requests");
-		case 431: return S("Request Header Fields Too Large");
-		case 449: return S("Retry With");
-		case 451: return S("Unavailable For Legal Reasons");
-
-		case 500: return S("Internal Server Error");
-		case 501: return S("Not Implemented");
-		case 502: return S("Bad Gateway");
-		case 503: return S("Service Unavailable");
-		case 504: return S("Gateway Timeout");
-		case 505: return S("HTTP Version Not Supported");
-		case 509: return S("Bandwidth Limit Exceeded");
-	}
-
-	return S("???");
-}
-
-int http_server_init(HTTP_Server *server, int max_conns)
-{
-    server->conns = malloc(max_conns * sizeof(HTTP_Conn));
-    if (server->conns == NULL)
-        return -1;
-
-    for (int i = 0; i < max_conns; i++) {
-        server->conns[i].state = HTTP_CONN_STATE_FREE;
-        server->conns[i].gen = 1;
-    }
-
-    server->max_conns = max_conns;
-    server->num_conns = 0;
-
-    server->tcp = tcp_init(max_conns);
-    if (server->tcp == NULL) {
-        free(server->conns);
-        return -1;
-    }
-
-    return 0;
-}
-
-void http_server_free(HTTP_Server *server)
-{
-    tcp_free(server->tcp);
-    free(server->conns);
-}
-
-int http_server_listen_tcp(HTTP_Server *server, Address addr)
-{
-    int ret = tcp_listen_tcp(server->tcp, addr);
-    if (ret < 0)
-        return -1;
-
-    return 0;
-}
-
-int http_server_listen_tls(HTTP_Server *server, Address addr,
-    string cert_file, string key_file)
-{
-#ifdef TLS_ENABLED
-    int ret = tcp_listen_tls(server->tcp, addr, cert_file, key_file);
-    if (ret < 0)
-        return -1;
-
-    return 0;
-#else
-    (void) server;
-    (void) addr;
-    (void) cert_file;
-    (void) key_file;
-    return -1;
-#endif
-}
-
-int http_server_add_cert(HTTP_Server *server, string domain, string cert_file, string key_file)
-{
-    int ret = tcp_add_cert(server->tcp, domain, cert_file, key_file);
-    if (ret < 0)
-        return -1;
-
-    return 0;
-}
-
-static void http_conn_init(HTTP_Conn *conn, TCP_Handle handle)
-{
-    assert(conn->state == HTTP_CONN_STATE_FREE);
-    conn->state = HTTP_CONN_STATE_IDLE;
-    conn->ready = false;
-    conn->num_served = 0;
-    conn->request_len = 0;
-    conn->keep_alive = true;
-    conn->handle = handle;
-}
-
-static void http_conn_free(HTTP_Conn *conn)
-{
-    assert(conn->state != HTTP_CONN_STATE_FREE);
-
-    conn->gen++;
-    if (conn->gen == 0)
-        conn->gen = 1;
-
-    tcp_close(conn->handle);
-
-    conn->state = HTTP_CONN_STATE_FREE;
-}
-
-void http_server_process_events(HTTP_Server *server,
-    void **ptrs, struct pollfd *arr, int num)
-{
-    tcp_process_events(server->tcp, ptrs, arr, num);
-
-    TCP_Event event;
-    while (tcp_next_event(server->tcp, &event)) {
-
-        if (event.flags & TCP_EVENT_NEW) {
-            // New connection. Find an HTTP_Conn struct.
-
-            int i = 0;
-            while (i < server->max_conns && server->conns[i].state != HTTP_CONN_STATE_FREE)
-                i++;
-            if (i == server->max_conns) {
-                tcp_close(event.handle);
-                continue;
-            }
-            HTTP_Conn *conn = &server->conns[i];
-
-            http_conn_init(conn, event.handle);
-            tcp_set_user_ptr(event.handle, conn);
-
-            server->num_conns++;
-        }
-
-        HTTP_Conn *conn = tcp_get_user_ptr(event.handle);
-        assert(conn);
-
-        bool defer_close = false;
-        if (event.flags & TCP_EVENT_DATA) {
-
-            string src = tcp_read_buf(event.handle);
-            int ret = chttp_parse_request((char*) src.ptr, src.len, &conn->request);
-            if (ret < 0) {
-                tcp_read_ack(event.handle, 0);
-                defer_close = true;
-            } else if (ret == 0) {
-                tcp_read_ack(event.handle, 0);
-            } else {
-                // Request was buffered
-
-                // Decide whether the connection should be closed
-                // after the next response
-
-                conn->keep_alive = true;
-
-                if (conn->num_served+1 >= 100)
-                    conn->keep_alive = false;
-
-                conn->response_offset = tcp_write_off(event.handle);
-                conn->request_len = ret;
-                conn->state = HTTP_CONN_STATE_STATUS;
-                conn->ready = true;
-            }
-        }
-
-        if (event.flags & TCP_EVENT_HUP) {
-            defer_close = true;
-        }
-
-        if (defer_close) {
-            http_conn_free(conn);
-            server->num_conns--;
-        }
-    }
-}
-
-int http_server_register_events(HTTP_Server *server,
-    void **ptrs, struct pollfd *arr, int cap)
-{
-    return tcp_register_events(server->tcp, ptrs, arr, cap);
-}
-
-bool http_server_next_request(HTTP_Server *server,
-    HTTP_Request **request, HTTP_ResponseBuilder *builder)
-{
-    for (int i = 0; i < server->max_conns; i++) {
-        if (server->conns[i].state == HTTP_CONN_STATE_FREE)
-            continue;
-
-        if (server->conns[i].ready) {
-            server->conns[i].ready = false;
-            *request = &server->conns[i].request;
-            *builder = (HTTP_ResponseBuilder) {
-                .server = server,
-                .idx = i,
-                .gen = server->conns[i].gen,
-            };
-            return true;
-        }
-    }
-
-    return false;
-}
-
-static HTTP_Conn*
-builder_to_conn(HTTP_ResponseBuilder builder)
-{
-    if (builder.server == NULL)
-        return NULL;
-    HTTP_Server *server = builder.server;
-
-    if (builder.idx < 0 || builder.idx >= server->max_conns)
-        return NULL;
-    HTTP_Conn *conn = &server->conns[builder.idx];
-
-    if (conn->state == HTTP_CONN_STATE_FREE || conn->gen != builder.gen)
-        return NULL;
-
-    return conn;
-}
-
-bool http_response_builder_is_valid(HTTP_ResponseBuilder builder)
-{
-    return builder_to_conn(builder) != NULL;
-}
-
-void http_response_builder_status(HTTP_ResponseBuilder builder, int status)
-{
-    HTTP_Conn *conn = builder_to_conn(builder);
-    if (conn == NULL)
-        return;
-
-    if (conn->state == HTTP_CONN_STATE_IDLE)
-        return;
-
-    if (conn->state != HTTP_CONN_STATE_STATUS) {
-        tcp_clear_from_offset(conn->handle, conn->response_offset);
-        conn->state = HTTP_CONN_STATE_STATUS;
-    }
-
-    assert(status > 99);
-    assert(status < 1000);
-    char tmp[3] = {
-        '0' + (status % 1000) / 100,
-        '0' + (status %  100) / 10,
-        '0' + (status %   10) / 1,
-    };
-
-    tcp_write(conn->handle, S("HTTP/1.1 "));
-    tcp_write(conn->handle, (string) { tmp, 3 });
-    tcp_write(conn->handle, S(" "));
-    tcp_write(conn->handle, reason_phrase(status));
-    tcp_write(conn->handle, S("\r\n"));
-
-    conn->state = HTTP_CONN_STATE_HEADER;
-}
-
-void http_response_builder_header(HTTP_ResponseBuilder builder, string header)
-{
-    HTTP_Conn *conn = builder_to_conn(builder);
-    if (conn == NULL)
-        return;
-
-    if (conn->state != HTTP_CONN_STATE_HEADER)
-        return;
-
-    tcp_write(conn->handle, header);
-    tcp_write(conn->handle, S("\r\n"));
-}
-
-static void append_special_headers(HTTP_Conn *conn)
-{
-    if (conn->keep_alive) {
-        tcp_write(conn->handle, S("Connection: Keep-Alive\r\n"));
-    } else {
-        tcp_write(conn->handle, S("Connection: Close\r\n"));
-    }
-    tcp_write(conn->handle, S("Content-Length: "));
-    conn->content_length_header_offset = tcp_write_off(conn->handle);
-    tcp_write(conn->handle, S("          "));
-    tcp_write(conn->handle, S("\r\n"));
-    tcp_write(conn->handle, S("\r\n"));
-    conn->content_offset = tcp_write_off(conn->handle);
-}
-
-void http_response_builder_content(HTTP_ResponseBuilder builder, string content)
-{
-    HTTP_Conn *conn = builder_to_conn(builder);
-    if (conn == NULL)
-        return;
-
-    if (conn->state == HTTP_CONN_STATE_STATUS)
-        return;
-
-    if (conn->state != HTTP_CONN_STATE_CONTENT) {
-        append_special_headers(conn);
-        conn->state = HTTP_CONN_STATE_CONTENT;
-    }
-
-    tcp_write(conn->handle, content);
-}
-
-void http_response_builder_submit(HTTP_ResponseBuilder builder)
-{
-    HTTP_Conn *conn = builder_to_conn(builder);
-    if (conn == NULL)
-        return;
-
-    if (conn->state == HTTP_CONN_STATE_STATUS)
-        return;
-
-    if (conn->state != HTTP_CONN_STATE_CONTENT) {
-        append_special_headers(conn);
-        conn->state = HTTP_CONN_STATE_CONTENT;
-    }
-
-    TCP_Offset current_offset = tcp_write_off(conn->handle);
-    int content_length = current_offset - conn->content_offset;
-
-    char buf[11];
-    int ret = snprintf(buf, sizeof(buf), "%d", content_length);
-    assert(ret > 0);
-    assert(ret < (int) sizeof(buf));
-    tcp_patch(conn->handle, conn->content_length_header_offset, (string) { buf, ret });
-
-    conn->num_served++;
-
-    tcp_read_ack(conn->handle, conn->request_len);
-    conn->request_len = 0;
-
-    if (conn->keep_alive) {
-        tcp_mark_ready(conn->handle);
-        conn->ready = false;
-        conn->state = HTTP_CONN_STATE_IDLE;
-    } else {
-        http_conn_free(conn);
-        builder.server->num_conns--;
-    }
-}

lib/http_server.h

@@ -1,69 +0,0 @@
-#ifndef HTTP_SERVER_INCLUDED
-#define HTTP_SERVER_INCLUDED
-
-#include "tcp.h"
-#include "http_parse.h"
-
-typedef CHTTP_Request HTTP_Request;
-
-typedef enum {
-    HTTP_CONN_STATE_FREE,
-    HTTP_CONN_STATE_IDLE,
-    HTTP_CONN_STATE_STATUS,
-    HTTP_CONN_STATE_HEADER,
-    HTTP_CONN_STATE_CONTENT,
-} HTTP_ConnState;
-
-typedef struct {
-    HTTP_ConnState state;
-    uint16_t gen;
-    bool ready;
-    int  num_served;
-    int  request_len;
-    bool keep_alive;
-    TCP_Handle handle;
-    HTTP_Request request;
-    TCP_Offset content_offset;
-    TCP_Offset content_length_header_offset;
-    TCP_Offset response_offset;
-} HTTP_Conn;
-
-typedef struct {
-    TCP *tcp;
-    int num_conns;
-    int max_conns;
-    HTTP_Conn *conns;
-} HTTP_Server;
-
-int  http_server_init(HTTP_Server *server, int max_conns);
-void http_server_free(HTTP_Server *server);
-
-int http_server_listen_tcp(HTTP_Server *server, Address addr);
-
-int http_server_listen_tls(HTTP_Server *server, Address addr,
-    string cert_file, string key_file);
-
-int http_server_add_cert(HTTP_Server *server, string domain, string cert_file, string key_file);
-
-void http_server_process_events(HTTP_Server *server,
-    void **ptrs, struct pollfd *arr, int cap);
-
-int http_server_register_events(HTTP_Server *server,
-    void **ptrs, struct pollfd *arr, int cap);
-
-typedef struct {
-    HTTP_Server *server;
-    int idx;
-    int gen;
-} HTTP_ResponseBuilder;
-
-bool http_server_next_request(HTTP_Server *server,
-    HTTP_Request **request, HTTP_ResponseBuilder *builder);
-
-bool http_response_builder_is_valid(HTTP_ResponseBuilder builder);
-void http_response_builder_status(HTTP_ResponseBuilder builder, int status);
-void http_response_builder_header(HTTP_ResponseBuilder builder, string header);
-void http_response_builder_content(HTTP_ResponseBuilder builder, string content);
-void http_response_builder_submit(HTTP_ResponseBuilder builder);
-
-#endif // HTTP_SERVER_INCLUDED

lib/message.c

@@ -1,294 +0,0 @@
-#ifdef MAIN_SIMULATION
-#define QUAKEY_ENABLE_MOCKS
-#endif
-
-#include <quakey.h>
-#include <assert.h>
-
-#include "tcp.h"
-#include "message.h"
-
-#define MESSAGE_SYSTEM_VERSION 1
-#define MESSAGE_SYSTEM_NODE_LIMIT 8
-
-typedef struct {
-    bool       used;
-    TCP_Handle handle;
-    int        senders[MESSAGE_SYSTEM_NODE_LIMIT];
-    int        num_senders;
-    void*      message;
-} Channel;
-
-struct MessageSystem {
-
-    TCP *tcp;
-
-    Address addrs[MESSAGE_SYSTEM_NODE_LIMIT];
-    int num_addrs;
-
-    int max_channels;
-    Channel channels[];
-};
-
-MessageSystem *message_system_init(Address *addrs, int num_addrs)
-{
-    int max_channels = 2 * num_addrs + 1;
-    MessageSystem *msys = malloc(sizeof(MessageSystem) + max_channels * sizeof(Channel));
-    if (msys == NULL)
-        return NULL;
-
-    if (num_addrs > MESSAGE_SYSTEM_NODE_LIMIT) {
-        free(msys);
-        return NULL;
-    }
-    for (int i = 0; i < num_addrs; i++)
-        msys->addrs[i] = addrs[i];
-    msys->num_addrs = num_addrs;
-    addr_sort(msys->addrs, msys->num_addrs);
-
-    msys->max_channels = max_channels;
-    for (int i = 0; i < max_channels; i++)
-        msys->channels[i].used = 0;
-
-    msys->tcp = tcp_init(max_channels);
-    if (msys->tcp == NULL) {
-        free(msys);
-        return NULL;
-    }
-
-    return msys;
-}
-
-void message_system_free(MessageSystem *msys)
-{
-    tcp_free(msys->tcp);
-    free(msys);
-}
-
-int message_system_listen_tcp(MessageSystem *msys, Address addr)
-{
-    int ret = tcp_listen_tcp(msys->tcp, addr);
-    if (ret < 0)
-        return -1;
-    return 0;
-}
-
-void message_system_process_events(MessageSystem *msys,
-    void **ptrs, struct pollfd *pfds, int num)
-{
-    tcp_process_events(msys->tcp, ptrs, pfds, num);
-}
-
-int message_system_register_events(MessageSystem *msys,
-    void **ptrs, struct pollfd *pfds, int cap)
-{
-    return tcp_register_events(msys->tcp, ptrs, pfds, cap);
-}
-
-static int
-find_free_channel_struct(MessageSystem *msys)
-{
-    int i = 0;
-    while (i < msys->max_channels && msys->channels[i].used)
-        i++;
-
-    if (i == msys->max_channels)
-        return -1; // No free space
-
-    return i;
-}
-
-static bool has_sender(Channel *channel, int sender_idx)
-{
-    for (int i = 0; i < channel->num_senders; i++)
-        if (channel->senders[i] == sender_idx)
-            return true;
-    return false;
-}
-
-static void add_sender(Channel *channel, int sender_idx)
-{
-    if (has_sender(channel, sender_idx))
-        return;
-    channel->senders[channel->num_senders++] = sender_idx;
-}
-
-void *get_next_message(MessageSystem *msys)
-{
-    TCP_Event event;
-    while (tcp_next_event(msys->tcp, &event)) {
-
-        if (event.flags & TCP_EVENT_NEW) {
-
-            int channel_idx = find_free_channel_struct(msys);
-            if (channel_idx < 0) {
-                tcp_close(event.handle);
-                continue;
-            }
-            Channel *channel = &msys->channels[channel_idx];
-
-            channel->used = true;
-            channel->num_senders = 0;
-            channel->message = NULL;
-            channel->handle = event.handle;
-
-            tcp_set_user_ptr(event.handle, channel);
-        }
-
-        Channel *channel = tcp_get_user_ptr(event.handle);
-
-        if (event.flags & TCP_EVENT_HUP) {
-            channel->used = false;
-            tcp_close(channel->handle); // TODO: What if the user is still hanging on the message pointer?
-        }
-
-        if (!(event.flags & TCP_EVENT_DATA))
-            continue;
-
-        string buf = tcp_read_buf(event.handle);
-
-        Message message;
-        if (buf.len < (int) sizeof(message)) {
-            tcp_read_ack(event.handle, 0);
-            continue;
-        }
-        memcpy(&message, buf.ptr, sizeof(message));
-        // TODO: endianess?
-
-        if (message.version != MESSAGE_SYSTEM_VERSION) {
-            assert(0); // TODO
-        }
-
-        if (message.length > (uint64_t) buf.len) {
-            tcp_read_ack(event.handle, 0);
-            continue; // Still buffering
-        }
-
-        add_sender(channel, message.sender);
-
-        channel->message = buf.ptr;
-        return buf.ptr;
-    }
-
-    return NULL;
-}
-
-static int find_channel_by_message(MessageSystem *msys, void *raw_message)
-{
-    for (int i = 0; i < msys->max_channels; i++) {
-        Channel *channel = &msys->channels[i];
-        if (!channel->used)
-            continue;
-        if (channel->message == raw_message)
-            return i;
-    }
-    return -1;
-}
-
-int message_length(void *raw_message)
-{
-    Message message;
-    memcpy(&message, raw_message, sizeof(message));
-    return message.length;
-}
-
-void consume_message(MessageSystem *msys, void *raw_message)
-{
-    int channel_idx = find_channel_by_message(msys, raw_message);
-    if (channel_idx < 0)
-        return;
-    Channel *channel = &msys->channels[channel_idx];
-
-    channel->message = NULL;
-    tcp_read_ack(channel->handle, message_length(raw_message));
-    tcp_mark_ready(channel->handle);
-}
-
-static int find_channel_by_target_idx(MessageSystem *msys, int target_idx)
-{
-    for (int i = 0; i < msys->max_channels; i++) {
-
-        Channel *channel = &msys->channels[i];
-        if (!channel->used)
-            continue;
-
-        for (int j = 0; j < channel->num_senders; j++) {
-            if (channel->senders[j] == target_idx)
-                return i;
-        }
-    }
-
-    return -1;
-}
-
-void send_message_ex(MessageSystem *msys, int target_idx,
-    Message *message, void *extra, int extra_len)
-{
-    int channel_idx = find_channel_by_target_idx(msys, target_idx);
-    if (channel_idx < 0) {
-
-        // Find an unused channel struct
-        channel_idx = find_free_channel_struct(msys);
-        if (channel_idx < 0)
-            return; // No free space
-
-        // Establish a new connection
-        TCP_Handle handle;
-        if (tcp_connect(msys->tcp, false, &msys->addrs[target_idx], 1, &handle) < 0)
-            return;
-
-        Channel *channel = &msys->channels[channel_idx];
-        channel->used = true;
-        channel->handle = handle;
-        channel->senders[0] = target_idx;
-        channel->num_senders = 1;
-        channel->message = NULL;
-
-        tcp_set_user_ptr(handle, channel);
-    }
-    Channel *channel = &msys->channels[channel_idx];
-
-    tcp_write(channel->handle, (string) { (void*) message, message->length - extra_len });
-    if (extra_len > 0)
-        tcp_write(channel->handle, (string) { extra, extra_len });
-}
-
-void send_message(MessageSystem *msys, int target_idx,
-    Message *message)
-{
-    send_message_ex(msys, target_idx, message, NULL, 0);
-}
-
-void reply_to_message_ex(MessageSystem *msys, void *incoming_message,
-    Message *outgoing_message, void *extra, int extra_len)
-{
-    int channel_idx = find_channel_by_message(msys, incoming_message);
-    if (channel_idx < 0)
-        return;
-    Channel *channel = &msys->channels[channel_idx];
-
-    int header_size = outgoing_message->length - extra_len;
-    tcp_write(channel->handle, (string) { (char*) outgoing_message, header_size });
-    if (extra_len > 0)
-        tcp_write(channel->handle, (string) { (char*) extra, extra_len });
-}
-
-void reply_to_message(MessageSystem *msys, void *incoming_message,
-    Message *outgoing_message)
-{
-    reply_to_message_ex(msys, incoming_message, outgoing_message, NULL, 0);
-}
-
-void broadcast_message_ex(MessageSystem *msys, int self_idx,
-    Message *message, void *extra, int extra_len)
-{
-    for (int i = 0; i < msys->num_addrs; i++) {
-        if (i != self_idx)
-            send_message_ex(msys, i, message, extra, extra_len);
-    }
-}
-
-void broadcast_message(MessageSystem *msys, int self_idx, Message *message)
-{
-    broadcast_message_ex(msys, self_idx, message, NULL, 0);
-}

lib/message.h

@@ -1,53 +0,0 @@
-#ifndef MESSAGE_INCLUDED
-#define MESSAGE_INCLUDED
-
-#include <stdint.h>
-
-#include "basic.h"
-
-typedef struct MessageSystem MessageSystem;
-
-typedef struct {
-    uint16_t version;
-    uint16_t type;
-    uint16_t sender;
-    uint64_t length;
-} Message;
-
-MessageSystem *message_system_init(Address *addrs, int num_addrs);
-
-void message_system_free(MessageSystem *msys);
-
-int message_system_listen_tcp(MessageSystem *msys, Address addr);
-
-struct pollfd;
-
-void message_system_process_events(MessageSystem *msys,
-    void **ptrs, struct pollfd *pfds, int num);
-
-int message_system_register_events(MessageSystem *msys,
-    void **ptrs, struct pollfd *pfds, int cap);
-
-void *get_next_message(MessageSystem *msys);
-
-int message_length(void *raw_message);
-
-void consume_message(MessageSystem *msys, void *ptr);
-
-void send_message(MessageSystem *msys, int target, Message *message);
-
-void send_message_ex(MessageSystem *msys, int target,
-    Message *header, void *extra, int extra_len);
-
-void reply_to_message(MessageSystem *msys, void *incoming_message,
-    Message *outgoing_message);
-
-void reply_to_message_ex(MessageSystem *msys, void *incoming_message,
-    Message *outgoing_message, void *extra, int extra_len);
-
-void broadcast_message(MessageSystem *msys, int self_idx, Message *message);
-
-void broadcast_message_ex(MessageSystem *msys, int self_idx,
-    Message *header, void *extra, int extra_len);
-
-#endif // MESSAGE_INCLUDED

lib/tcp.h

@@ -1,211 +0,0 @@
-#ifndef TCP_INCLUDED
-#define TCP_INCLUDED
-
-#include "byte_queue.h"
-
-// Abstraction over TCP and TLS sockets.
-//
-// It works by creating a pool of TCP connections. Connections can be added
-// to the pool by connecting to other processes via the tcp_connect() function,
-// or by adding them automatically as they arrive from other peers, if the
-// pool is configured in listening mode. This allows the same abstraction to
-// work for servers, clients, and nodes in a larger network that behave both
-// as clients and servers.
-//
-// It features:
-//   - Cross-platform (Windows and Linux)
-//   - All I/O is multiplexed, which means slow connections will not stall faster ones.
-//   - Input and output buffering
-//   - Encryption via TLS (OpenSSL on Linux and SChannel on Windows)
-
-// The TCP structure holds the state of a single instance. It is dynamically
-// allocated internally so the caller doesn't need to read its contents.
-typedef struct TCP TCP;
-
-// Create an instance of the TCP subsystem. The max_conns argument is the
-// maximum number of TCP connection this instance will be able to manage.
-TCP *tcp_init(int max_conns);
-
-// Free a TCP subsystem instance. Any resources provided by the subsystem
-// will be forcefully released too.
-void tcp_free(TCP *tcp);
-
-// Enable a listening interface for this TCP pool. Connections accepted via
-// this interface will be plaintext.
-int tcp_listen_tcp(TCP *tcp, Address addr);
-
-// Enable a listening interface for this TCP pool. Connections accepted via
-// this interface will be encrypted. A single TCP pool may be configured for
-// plaintext and encrypted connections at the same time. From the user's
-// perspective, the interface from which a connections was accepted is totally
-// transparent.
-// The cert_file and key_file parameters refer to the certificate file and
-// associated private key file to use for encryption, both in PEM format.
-int tcp_listen_tls(TCP *tcp, Address addr, string cert_file, string key_file);
-
-// If the TCP pool is configured in TLS mode (tcp_listen_tls was called), this
-// function can be used to add an additional certificate. Connecting sockets
-// will be able to pick the right certificate by expressing the domain name they
-// are expecting to talk to.
-int tcp_add_cert(TCP *tcp, string domain, string cert_file, string key_file);
-
-// Handle structure representing a TCP connection of the TCP pool. The contents
-// should not be interpreted by users.
-typedef struct {
-    TCP *tcp;
-    int  idx;
-    int  gen;
-} TCP_Handle;
-
-// Add a connection to the TCP pool by establishing one towards the specified
-// peer. The addrs array (of size num_addrs) contains the list of IP addresses
-// for the host. The TCP pool will try each address one by one until a connection
-// is established. If the secure argument is true, the connection will be
-// encrypted.
-int tcp_connect(TCP *tcp, bool secure, Address *addrs, int num_addrs, TCP_Handle *handle);
-
-// Forward-declare poll item type. The user must include poll.h (Linux) or
-// winsock2.h (Windows) to get this definition (and the definition of poll()
-// and WSAPoll()).
-struct pollfd;
-
-// Initialize an array of pollfd structures with all the descriptor the pool
-// needs to monitor with the associated events. The array is such that the caller
-// can then call poll() on it to block execution of the process while the TCP
-// pool has no work to be done. The number of items written to the array is
-// returned.
-// The ptrs array is some state set by the TCP pool to associate metadata to
-// each descriptor for internal book-keping.
-int tcp_register_events(TCP *tcp, void **ptrs, struct pollfd *pfds, int cap);
-
-// After poll() is called and revents flags are set on the array initialized by
-// tcp_register_events, this function can be called to go over the triggered
-// events and update the internal state of the TCP pool. The ptrs array should
-// be passed in as it was initialized by the tcp_register_events as-is.
-void tcp_process_events(TCP *tcp, void **ptrs, struct pollfd *pfds, int num);
-
-// Flags for the "flags" field in TCP_Event.
-enum {
-    TCP_EVENT_NEW  = 1<<0,
-    TCP_EVENT_HUP  = 1<<1,
-    TCP_EVENT_DATA = 1<<2,
-};
-
-// See tcp_next_event.
-typedef struct {
-    int flags;
-    TCP_Handle handle;
-} TCP_Event;
-
-// After tcp_process_events is called, some new events may be available for the
-// user. This function returns the next event in the TCP pool.
-//
-// If an event is available, true is returned and the event structure is
-// initialized with the handle to the connection and flags that identify the
-// events that triggered associated to that handle. The events are:
-//   TCP_EVENT_NEW:  This connection was just established. It's the first time the
-//                   user's code sees it.
-//   TCP_EVENT_HUP:  The peer disconnected and therefore the user should close
-//                   the connection associated to it.
-//   TCP_EVENT_DATA: Some bytes were buffered for this connection.
-//                   (It's possible that this event to triggered with 0 new bytes,
-//                   for instance if the user called tcp_mark_ready)
-// Any of these events may happen at the same time. They are not exclusive.
-//
-// If no event is available, false is returned.
-//
-// The general way one would use is function is by doing:
-//   tcp_process_events(...)
-//   for (TCP_Event event; tcp_next_event(tcp, &event); ) {
-//     if (event.flags & TCP_EVENT_NEW) {
-//       // ...
-//     }
-//
-//     if (event.flags & TCP_EVENT_DATA) {
-//       // ...
-//     }
-//
-//     if (event.flags & TCP_EVENT_HUP) {
-//       tcp_close(event.handle);
-//     }
-//   }
-//
-// Note that the handle returned by the TCP_EVENT_NEW event
-// (and all subsequent events) will be valid until the user
-// calls tcp_close() on it.
-bool tcp_next_event(TCP *tcp, TCP_Event *event);
-
-// Start a read operation into the TCP connection's input buffer.
-//
-// This function returns a slice of the input buffer. The user
-// may inspect the contents and decide to consume some bytes from
-// the buffer by calling tcp_read_ack(handle, num) with the number
-// of bytes. Reading the input buffer with this function locks the
-// buffer not allowing new bytes to be buffered. For this reason
-// tcp_read_ack(handle, 0) must be called even if no bytes were
-// consumed.
-//
-// Note that returned bytes are plaintext regardless of whether
-// the connection was accepted via the plaintext or encrypted
-// listening interface.
-string tcp_read_buf(TCP_Handle handle);
-
-// Complete a read operation into the TCP connection's input buffer.
-void tcp_read_ack(TCP_Handle handle, int num);
-
-// Start a write operation into the TCP connection's output buffer.
-//
-// This function is specular to tcp_read_buf except the user must
-// write into the returned slice instead of reading from it.
-string tcp_write_buf(TCP_Handle handle);
-
-// Complete a write operation into the TCP connection's output buffer.
-// The num argument is the number of bytes written into the slice by
-// the user.
-void tcp_write_ack(TCP_Handle handle, int num);
-
-// See tcp_write_off
-typedef ByteQueueOffset TCP_Offset;
-
-// Returns the offset of the next byte that would be written into the
-// output buffer.
-//
-// This offset is such that removing previous data from the output
-// buffer will not invalidate such offset. It's useful to calcuate
-// the number of bytes between to offsets of apply operations on
-// bytes since a given offset on the buffer.
-TCP_Offset tcp_write_off(TCP_Handle handle);
-
-// Writes bytes into the TCP connections' output buffer. It's just
-// a shorthand for tcp_write_buf/tcp_write_ack.
-void tcp_write(TCP_Handle handle, string data);
-
-// Writes bytes at the specified offset of the output buffer. Note
-// that this only overwrites bytes in the buffer and does not grow
-// its size, therefore the user must have already inserted some values
-// after that offset. Also, the region referred by the offset must
-// still be into the buffer and not be read out.
-void tcp_patch(TCP_Handle handle, TCP_Offset offset, string data);
-
-// Removes all bytes in the TCP connection's output buffer from the
-// specified offset onwards.
-void tcp_clear_from_offset(TCP_Handle handle, TCP_Offset offset);
-
-// Close a TCP connection. Previously buffered output bytes will be
-// sent out asynchronously.
-void tcp_close(TCP_Handle handle);
-
-// Associate an opaque pointer value to this connection. The tcp_get_user_ptr
-// can be used to retrieve the pointer at any time.
-void tcp_set_user_ptr(TCP_Handle handle, void *user_ptr);
-
-// Retrieve the user pointer associated to a TCP connection. If no user
-// pointer was previously set, NULL is returned.
-void *tcp_get_user_ptr(TCP_Handle handle);
-
-// Mark the TCP connection as "ready" causing it to be returned once more
-// by the tcp_next_event() function with the TCP_EVENT_DATA flag set, even
-// if no more data was buffered.
-void tcp_mark_ready(TCP_Handle handle);
-
-#endif // TCP_INCLUDED

lib/tls.h

@@ -1,92 +0,0 @@
-#ifdef TLS_ENABLED
-#ifndef TLS_INCLUDED
-#define TLS_INCLUDED
-
-#include <stdbool.h>
-#include "basic.h"
-
-#define TLS_CERT_LIMIT 8
-
-#ifdef TLS_SCHANNEL
-
-#define SECURITY_WIN32
-#include <windows.h>
-#include <security.h>
-#include <schannel.h>
-
-#include "byte_queue.h"
-
-typedef struct {
-    char       domain[128];
-    CredHandle cred;
-} TLS_Cert;
-
-typedef struct {
-    CredHandle     cred;
-    PCCERT_CONTEXT cert_ctx;
-    int            num_certs;
-    TLS_Cert       certs[TLS_CERT_LIMIT];
-} TLS_Server;
-
-typedef struct {
-    CtxtHandle                ctx;
-    bool                      ctx_valid;
-    bool                      handshake;
-    CredHandle               *cred;
-    ByteQueue                 in_buf;
-    ByteQueue                 out_buf;
-    SecPkgContext_StreamSizes sizes;
-    char                     *pending;
-    int                       pending_off;
-    int                       pending_len;
-} TLS_Conn;
-
-#else // OpenSSL
-
-typedef struct ssl_ctx_st SSL_CTX;
-typedef struct ssl_st     SSL;
-typedef struct bio_st     BIO;
-
-typedef struct {
-    char domain[128];
-    SSL_CTX *ctx;
-} TLS_Cert;
-
-typedef struct {
-    SSL_CTX *ctx;
-    int num_certs;
-    TLS_Cert certs[TLS_CERT_LIMIT];
-} TLS_Server;
-
-typedef struct {
-    SSL  *ssl;
-    BIO  *network_bio;
-    bool  handshake;
-} TLS_Conn;
-
-#endif // TLS_SCHANNEL
-
-void tls_global_init(void);
-void tls_global_free(void);
-
-int   tls_server_init(TLS_Server *server, string cert_file, string key_file);
-void  tls_server_free(TLS_Server *server);
-int   tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file);
-
-int   tls_conn_init(TLS_Conn *conn, TLS_Server *server);
-void  tls_conn_free(TLS_Conn *conn);
-
-int   tls_conn_handshake(TLS_Conn *conn);
-
-char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap);
-void  tls_conn_net_write_ack(TLS_Conn *conn, int num);
-
-char *tls_conn_net_read_buf(TLS_Conn *conn, int *num);
-void  tls_conn_net_read_ack(TLS_Conn *conn, int num);
-
-int   tls_conn_app_write(TLS_Conn *conn, char *dst, int num);
-int   tls_conn_app_read(TLS_Conn *conn, char *src, int cap);
-int   tls_conn_needs_flushing(TLS_Conn *conn);
-
-#endif // TLS_INCLUDED
-#endif // TLS_ENABLED

lib/tls_openssl.c

@@ -1,293 +0,0 @@
-#ifdef TLS_ENABLED
-#ifdef TLS_OPENSSL
-
-#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
-#define QUAKEY_ENABLE_MOCKS
-#endif
-
-#include <quakey.h>
-#include <assert.h>
-
-// Avoid name collision between basic.h's SHA256 typedef
-// and OpenSSL's SHA256 function
-#define SHA256 openssl_SHA256
-#include <openssl/ssl.h>
-#include <openssl/err.h>
-#undef SHA256
-
-#include "tls.h"
-
-void tls_global_init(void)
-{
-    SSL_library_init();
-    SSL_load_error_strings();
-    OpenSSL_add_all_algorithms();
-}
-
-void tls_global_free(void)
-{
-    EVP_cleanup();
-}
-
-static int servername_callback(SSL *ssl, int *ad, void *arg)
-{
-    TLS_Server *server = arg;
-
-    // The 'ad' parameter is used to set the alert description when returning
-    // SSL_TLSEXT_ERR_ALERT_FATAL. Since we only return OK or NOACK, it's unused.
-    (void) ad;
-
-    const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
-    if (servername == NULL)
-        return SSL_TLSEXT_ERR_NOACK;
-
-    for (int i = 0; i < server->num_certs; i++) {
-        TLS_Cert *cert = &server->certs[i];
-        if (!strcmp(cert->domain, servername)) {
-            SSL_set_SSL_CTX(ssl, cert->ctx);
-            return SSL_TLSEXT_ERR_OK;
-        }
-    }
-
-    return SSL_TLSEXT_ERR_NOACK;
-}
-
-int tls_server_init(TLS_Server *server, string cert_file, string key_file)
-{
-    SSL_CTX *ctx = SSL_CTX_new(TLS_server_method());
-    if (ctx == NULL)
-        return -1;
-
-    SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
-
-    char cert_buf[1024];
-    if (cert_file.len >= (int) sizeof(cert_buf)) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-    memcpy(cert_buf, cert_file.ptr, cert_file.len);
-    cert_buf[cert_file.len] = '\0';
-
-    char key_buf[1024];
-    if (key_file.len >= (int) sizeof(key_buf)) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-    memcpy(key_buf, key_file.ptr, key_file.len);
-    key_buf[key_file.len] = '\0';
-
-    // Load certificate and private key
-    if (SSL_CTX_use_certificate_chain_file(ctx, cert_buf) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    if (SSL_CTX_use_PrivateKey_file(ctx, key_buf, SSL_FILETYPE_PEM) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    if (SSL_CTX_check_private_key(ctx) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    SSL_CTX_set_tlsext_servername_callback(ctx, servername_callback);
-    SSL_CTX_set_tlsext_servername_arg(ctx, server);
-
-    server->ctx = ctx;
-    server->num_certs = 0;
-    return 0;
-}
-
-void tls_server_free(TLS_Server *server)
-{
-    for (int i = 0; i < server->num_certs; i++)
-        SSL_CTX_free(server->certs[i].ctx);
-    SSL_CTX_free(server->ctx);
-}
-
-// TODO: Can the domain be inferred from the cert?
-int tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file)
-{
-    if (server->num_certs == TLS_CERT_LIMIT)
-        return -1;
-
-    SSL_CTX *ctx = SSL_CTX_new(TLS_server_method());
-    if (!ctx)
-        return -1;
-
-    SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
-
-    char cert_buf[1024];
-    if (cert_file.len >= (int) sizeof(cert_buf)) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-    memcpy(cert_buf, cert_file.ptr, cert_file.len);
-    cert_buf[cert_file.len] = '\0';
-
-    char key_buf[1024];
-    if (key_file.len >= (int) sizeof(key_buf)) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-    memcpy(key_buf, key_file.ptr, key_file.len);
-    key_buf[key_file.len] = '\0';
-
-    if (SSL_CTX_use_certificate_chain_file(ctx, cert_buf) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    if (SSL_CTX_use_PrivateKey_file(ctx, key_buf, SSL_FILETYPE_PEM) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    if (SSL_CTX_check_private_key(ctx) != 1) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-
-    TLS_Cert *cert = &server->certs[server->num_certs];
-    if (domain.len >= (int) sizeof(cert->domain)) {
-        SSL_CTX_free(ctx);
-        return -1;
-    }
-    memcpy(cert->domain, domain.ptr, domain.len);
-    cert->domain[domain.len] = '\0';
-    cert->ctx = ctx;
-    server->num_certs++;
-    return 0;
-}
-
-int tls_conn_init(TLS_Conn *conn, TLS_Server *server)
-{
-    SSL *ssl = SSL_new(server->ctx);
-    if (ssl == NULL)
-        return -1;
-
-    // Create a BIO pair:
-    //   internal_bio  — attached to the SSL object (OpenSSL uses it internally)
-    //   network_bio   — you read/write encrypted data from/to this
-    BIO *internal_bio = NULL;
-    BIO *network_bio = NULL;
-    if (!BIO_new_bio_pair(&internal_bio, 0, &network_bio, 0)) {
-        SSL_free(ssl);
-        return -1;
-    }
-
-    // Bind the internal side to the SSL object
-    SSL_set_bio(ssl, internal_bio, internal_bio);
-
-    // We're the server side
-    SSL_set_accept_state(ssl);
-
-    conn->ssl = ssl;
-    conn->network_bio = network_bio;
-    conn->handshake = true;
-    return 0;
-}
-
-void tls_conn_free(TLS_Conn *conn)
-{
-    SSL_free(conn->ssl);
-    BIO_free(conn->network_bio);
-}
-
-// Write ciphertext from the connection object to the network
-char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap)
-{
-    char *buf;
-    int ret = BIO_nwrite0(conn->network_bio, &buf);
-    if (ret <= 0)
-        return NULL;
-    *cap = ret;
-    return buf;
-}
-
-// Complete the write from the connection object
-void tls_conn_net_write_ack(TLS_Conn *conn, int num)
-{
-    char *dummy;
-    BIO_nwrite(conn->network_bio, &dummy, num);
-}
-
-// Read ciphertext from the network into the connection object
-char *tls_conn_net_read_buf(TLS_Conn *conn, int *num)
-{
-    char *buf;
-    int ret = BIO_nread0(conn->network_bio, &buf);
-    if (ret <= 0)
-        return NULL;
-    *num = ret;
-    return buf;
-}
-
-// Complete the read from the network
-void tls_conn_net_read_ack(TLS_Conn *conn, int num)
-{
-    char *dummy;
-    BIO_nread(conn->network_bio, &dummy, num);
-}
-
-// Write plaintext from the application to the connection object
-int tls_conn_app_write(TLS_Conn *conn, char *dst, int num)
-{
-    assert(!conn->handshake);
-
-    int n = SSL_write(conn->ssl, dst, num);
-    if (n > 0)
-        return n;
-
-    int err = SSL_get_error(conn->ssl, n);
-    if (err == SSL_ERROR_WANT_READ ||
-        err == SSL_ERROR_WANT_WRITE)
-        return 0;
-
-    return -1;
-}
-
-// Read plaintext from the connection object into the application
-int tls_conn_app_read(TLS_Conn *conn, char *src, int cap)
-{
-    assert(!conn->handshake);
-
-    int n = SSL_read(conn->ssl, src, cap);
-    if (n > 0)
-        return n;
-
-    int err = SSL_get_error(conn->ssl, n);
-    if (err == SSL_ERROR_WANT_READ ||
-        err == SSL_ERROR_WANT_WRITE)
-        return 0;
-
-    return -1;
-}
-
-int tls_conn_handshake(TLS_Conn *conn)
-{
-    assert(conn->handshake);
-
-    int n = SSL_do_handshake(conn->ssl);
-    if (n == 1) {
-        conn->handshake = false;
-        return 1;
-    }
-
-    int err = SSL_get_error(conn->ssl, n);
-    if (err == SSL_ERROR_WANT_READ ||
-        err == SSL_ERROR_WANT_WRITE)
-        return 0;
-
-    return -1;
-}
-
-int tls_conn_needs_flushing(TLS_Conn *conn)
-{
-    return BIO_ctrl_pending(conn->network_bio) > 0;
-}
-
-#endif // TLS_OPENSSL
-#endif // TLS_ENABLED

lib/tls_schannel.c

@@ -1,695 +0,0 @@
-#ifdef TLS_ENABLED
-#ifdef TLS_SCHANNEL
-
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-#define SECURITY_WIN32
-#include <windows.h>
-#include <wincrypt.h>
-#include <security.h>
-#include <schannel.h>
-
-#include "tls.h"
-
-#pragma comment(lib, "secur32.lib")
-#pragma comment(lib, "crypt32.lib")
-#pragma comment(lib, "ncrypt.lib")
-
-#define TLS_BUF_LIMIT (32 * 1024)
-
-// ============================================================
-// Certificate loading
-// ============================================================
-
-// Read entire file into malloc'd buffer. Caller must free.
-static char *read_file(const char *path, int *out_len)
-{
-    FILE *f = fopen(path, "rb");
-    if (!f) return NULL;
-
-    fseek(f, 0, SEEK_END);
-    long len = ftell(f);
-    fseek(f, 0, SEEK_SET);
-
-    if (len <= 0 || len > 1024 * 1024) {
-        fclose(f);
-        return NULL;
-    }
-
-    char *buf = malloc((size_t) len + 1);
-    if (!buf) {
-        fclose(f);
-        return NULL;
-    }
-
-    size_t nread = fread(buf, 1, (size_t) len, f);
-    fclose(f);
-
-    buf[nread] = '\0';
-    *out_len = (int) nread;
-    return buf;
-}
-
-// Decode PEM base64 content to DER binary. Returns malloc'd buffer.
-static BYTE *pem_to_der(const char *pem, int pem_len, DWORD *out_len)
-{
-    DWORD len = 0;
-    if (!CryptStringToBinaryA(pem, pem_len, CRYPT_STRING_BASE64HEADER,
-                              NULL, &len, NULL, NULL))
-        return NULL;
-
-    BYTE *der = malloc(len);
-    if (!der) return NULL;
-
-    if (!CryptStringToBinaryA(pem, pem_len, CRYPT_STRING_BASE64HEADER,
-                              der, &len, NULL, NULL)) {
-        free(der);
-        return NULL;
-    }
-
-    *out_len = len;
-    return der;
-}
-
-// Import an in-memory PFX blob, acquire SChannel credential.
-static int load_pfx_blob(const BYTE *pfx_data, DWORD pfx_len,
-                          CredHandle *cred_out, PCCERT_CONTEXT *cert_ctx_out)
-{
-    CRYPT_DATA_BLOB blob;
-    blob.pbData = (BYTE *) pfx_data;
-    blob.cbData = pfx_len;
-
-    HCERTSTORE store = PFXImportCertStore(&blob, L"", CRYPT_EXPORTABLE);
-    if (!store) return -1;
-
-    // Find first cert with a private key
-    PCCERT_CONTEXT cert_ctx = NULL;
-    while ((cert_ctx = CertEnumCertificatesInStore(store, cert_ctx)) != NULL) {
-        HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
-        DWORD key_spec = 0;
-        BOOL caller_free = FALSE;
-        if (CryptAcquireCertificatePrivateKey(cert_ctx,
-                CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG | CRYPT_ACQUIRE_SILENT_FLAG,
-                NULL, &hkey, &key_spec, &caller_free)) {
-            if (caller_free && hkey) {
-                if (key_spec == CERT_NCRYPT_KEY_SPEC)
-                    NCryptFreeObject(hkey);
-                else
-                    CryptReleaseContext(hkey, 0);
-            }
-            break;
-        }
-    }
-
-    if (!cert_ctx) {
-        CertCloseStore(store, 0);
-        return -1;
-    }
-
-    PCCERT_CONTEXT dup_ctx = CertDuplicateCertificateContext(cert_ctx);
-
-    SCHANNEL_CRED sc_cred = {0};
-    sc_cred.dwVersion = SCHANNEL_CRED_VERSION;
-    sc_cred.cCreds = 1;
-    sc_cred.paCred = &dup_ctx;
-    sc_cred.grbitEnabledProtocols = SP_PROT_TLS1_2_SERVER;
-
-    TimeStamp expiry;
-    SECURITY_STATUS ss = AcquireCredentialsHandleA(
-        NULL, UNISP_NAME_A, SECPKG_CRED_INBOUND,
-        NULL, &sc_cred, NULL, NULL,
-        cred_out, &expiry);
-
-    if (ss != SEC_E_OK) {
-        CertFreeCertificateContext(dup_ctx);
-        CertCloseStore(store, 0);
-        return -1;
-    }
-
-    *cert_ctx_out = dup_ctx;
-    // Keep store open — SChannel needs access to the cert+key
-    return 0;
-}
-
-// Load PEM cert+key using pure CryptoAPI (no external tools).
-// Decodes PEM, imports key into a temporary CAPI keyset, exports as
-// in-memory PFX, then imports via PFXImportCertStore.
-static int load_pem_credential(string cert_file, string key_file,
-                               CredHandle *cred_out, PCCERT_CONTEXT *cert_ctx_out)
-{
-    int result = -1;
-    char cert_z[1024], key_z[1024], container[64];
-    char *cert_pem = NULL, *key_pem = NULL;
-    BYTE *cert_der = NULL, *key_der = NULL;
-    BYTE *rsa_blob = NULL, *pkcs8_buf = NULL;
-    PCCERT_CONTEXT cert_ctx = NULL;
-    HCRYPTPROV hprov = 0;
-    HCRYPTKEY hkey = 0;
-    HCERTSTORE mem_store = NULL;
-    CRYPT_DATA_BLOB pfx = {0};
-    int cert_pem_len, key_pem_len;
-    DWORD cert_der_len, key_der_len, rsa_blob_len, pkcs8_len;
-
-    snprintf(container, sizeof(container), "tls_tmp_%lu",
-             (unsigned long) GetCurrentProcessId());
-
-    // Null-terminate file paths
-    if (cert_file.len >= (int) sizeof(cert_z)) goto done;
-    memcpy(cert_z, cert_file.ptr, cert_file.len);
-    cert_z[cert_file.len] = '\0';
-
-    if (key_file.len >= (int) sizeof(key_z)) goto done;
-    memcpy(key_z, key_file.ptr, key_file.len);
-    key_z[key_file.len] = '\0';
-
-    // --- Certificate: PEM -> DER -> CERT_CONTEXT ---
-    cert_pem = read_file(cert_z, &cert_pem_len);
-    if (!cert_pem) goto done;
-
-    cert_der = pem_to_der(cert_pem, cert_pem_len, &cert_der_len);
-    if (!cert_der) goto done;
-
-    cert_ctx = CertCreateCertificateContext(
-        X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, cert_der, cert_der_len);
-    if (!cert_ctx) goto done;
-
-    // --- Private key: PEM -> DER -> CAPI RSA blob ---
-    key_pem = read_file(key_z, &key_pem_len);
-    if (!key_pem) goto done;
-
-    key_der = pem_to_der(key_pem, key_pem_len, &key_der_len);
-    if (!key_der) goto done;
-
-    if (strstr(key_pem, "-----BEGIN PRIVATE KEY-----")) {
-        // PKCS#8: unwrap to get inner RSA key
-        if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_PRIVATE_KEY_INFO,
-                key_der, key_der_len, CRYPT_DECODE_ALLOC_FLAG,
-                NULL, &pkcs8_buf, &pkcs8_len))
-            goto done;
-
-        CRYPT_PRIVATE_KEY_INFO *pki = (CRYPT_PRIVATE_KEY_INFO *) pkcs8_buf;
-
-        if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY,
-                pki->PrivateKey.pbData, pki->PrivateKey.cbData,
-                CRYPT_DECODE_ALLOC_FLAG, NULL, &rsa_blob, &rsa_blob_len))
-            goto done;
-    } else {
-        // Traditional RSA (BEGIN RSA PRIVATE KEY)
-        if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY,
-                key_der, key_der_len, CRYPT_DECODE_ALLOC_FLAG,
-                NULL, &rsa_blob, &rsa_blob_len))
-            goto done;
-    }
-
-    // --- Import key into temporary CAPI keyset ---
-    CryptAcquireContextA(&hprov, container, MS_ENH_RSA_AES_PROV_A,
-                         PROV_RSA_AES, CRYPT_DELETEKEYSET);
-    hprov = 0;
-
-    if (!CryptAcquireContextA(&hprov, container, MS_ENH_RSA_AES_PROV_A,
-                              PROV_RSA_AES, CRYPT_NEWKEYSET))
-        goto done;
-
-    if (!CryptImportKey(hprov, rsa_blob, rsa_blob_len, 0, CRYPT_EXPORTABLE, &hkey))
-        goto done;
-
-    // --- Bind key to cert, export PFX in memory ---
-    {
-        wchar_t containerW[64];
-        MultiByteToWideChar(CP_ACP, 0, container, -1, containerW, 64);
-
-        CRYPT_KEY_PROV_INFO prov_info = {0};
-        prov_info.pwszContainerName = containerW;
-        prov_info.pwszProvName      = (LPWSTR) L"Microsoft Enhanced RSA and AES Cryptographic Provider";
-        prov_info.dwProvType        = PROV_RSA_AES;
-        prov_info.dwKeySpec         = AT_KEYEXCHANGE;
-
-        if (!CertSetCertificateContextProperty(cert_ctx,
-                CERT_KEY_PROV_INFO_PROP_ID, 0, &prov_info))
-            goto done;
-    }
-
-    mem_store = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, 0, 0, NULL);
-    if (!mem_store) goto done;
-
-    if (!CertAddCertificateContextToStore(mem_store, cert_ctx,
-            CERT_STORE_ADD_ALWAYS, NULL))
-        goto done;
-
-    // Two-pass PFX export: get size, then export
-    pfx.cbData = 0;
-    pfx.pbData = NULL;
-    if (!PFXExportCertStoreEx(mem_store, &pfx, L"", NULL, EXPORT_PRIVATE_KEYS))
-        goto done;
-
-    pfx.pbData = malloc(pfx.cbData);
-    if (!pfx.pbData) goto done;
-
-    if (!PFXExportCertStoreEx(mem_store, &pfx, L"", NULL, EXPORT_PRIVATE_KEYS))
-        goto done;
-
-    // --- Import PFX blob and acquire SChannel credential ---
-    result = load_pfx_blob(pfx.pbData, pfx.cbData, cred_out, cert_ctx_out);
-
-done:
-    free(pfx.pbData);
-    if (mem_store) CertCloseStore(mem_store, 0);
-    if (hkey) CryptDestroyKey(hkey);
-    if (hprov) CryptReleaseContext(hprov, 0);
-    {   HCRYPTPROV tmp = 0;
-        CryptAcquireContextA(&tmp, container, MS_ENH_RSA_AES_PROV_A,
-                             PROV_RSA_AES, CRYPT_DELETEKEYSET); }
-    if (pkcs8_buf) LocalFree(pkcs8_buf);
-    if (rsa_blob) LocalFree(rsa_blob);
-    free(key_der);
-    free(key_pem);
-    free(cert_der);
-    free(cert_pem);
-    if (cert_ctx) CertFreeCertificateContext(cert_ctx);
-    return result;
-}
-
-// ============================================================
-// Global init/free (no-ops for SChannel)
-// ============================================================
-
-void tls_global_init(void)
-{
-}
-
-void tls_global_free(void)
-{
-}
-
-// ============================================================
-// Server init/free
-// ============================================================
-
-int tls_server_init(TLS_Server *server, string cert_file, string key_file)
-{
-    memset(server, 0, sizeof(*server));
-    server->num_certs = 0;
-
-    int ret = load_pem_credential(cert_file, key_file, &server->cred, &server->cert_ctx);
-    if (ret < 0)
-        return -1;
-
-    return 0;
-}
-
-void tls_server_free(TLS_Server *server)
-{
-    FreeCredentialsHandle(&server->cred);
-    if (server->cert_ctx) {
-        HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
-        DWORD key_spec = 0;
-        BOOL caller_free = FALSE;
-        if (CryptAcquireCertificatePrivateKey(server->cert_ctx,
-                CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG, NULL, &hkey, &key_spec, &caller_free)) {
-            if (caller_free && hkey)
-                NCryptFreeObject(hkey);
-        }
-        CertFreeCertificateContext(server->cert_ctx);
-    }
-
-    for (int i = 0; i < server->num_certs; i++)
-        FreeCredentialsHandle(&server->certs[i].cred);
-}
-
-int tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file)
-{
-    if (server->num_certs >= TLS_CERT_LIMIT)
-        return -1;
-
-    TLS_Cert *cert = &server->certs[server->num_certs];
-    if (domain.len >= (int) sizeof(cert->domain))
-        return -1;
-
-    PCCERT_CONTEXT cert_ctx = NULL;
-    int ret = load_pem_credential(cert_file, key_file, &cert->cred, &cert_ctx);
-    if (ret < 0)
-        return -1;
-
-    if (cert_ctx) {
-        HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
-        DWORD key_spec = 0;
-        BOOL caller_free = FALSE;
-        if (CryptAcquireCertificatePrivateKey(cert_ctx,
-                CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG, NULL, &hkey, &key_spec, &caller_free)) {
-            if (caller_free && hkey)
-                NCryptFreeObject(hkey);
-        }
-        CertFreeCertificateContext(cert_ctx);
-    }
-
-    memcpy(cert->domain, domain.ptr, domain.len);
-    cert->domain[domain.len] = '\0';
-    server->num_certs++;
-    return 0;
-}
-
-// ============================================================
-// Connection init/free
-// ============================================================
-
-int tls_conn_init(TLS_Conn *conn, TLS_Server *server)
-{
-    memset(conn, 0, sizeof(*conn));
-    conn->ctx_valid = false;
-    conn->handshake = true;
-    conn->cred = &server->cred;
-    byte_queue_init(&conn->in_buf, TLS_BUF_LIMIT);
-    byte_queue_init(&conn->out_buf, TLS_BUF_LIMIT);
-    conn->pending = NULL;
-    conn->pending_off = 0;
-    conn->pending_len = 0;
-    SecInvalidateHandle(&conn->ctx);
-    return 0;
-}
-
-void tls_conn_free(TLS_Conn *conn)
-{
-    if (conn->ctx_valid)
-        DeleteSecurityContext(&conn->ctx);
-    byte_queue_free(&conn->in_buf);
-    byte_queue_free(&conn->out_buf);
-    free(conn->pending);
-}
-
-// ============================================================
-// Handshake
-// ============================================================
-
-int tls_conn_handshake(TLS_Conn *conn)
-{
-    assert(conn->handshake);
-
-    // Read available ciphertext from in_buf
-    string in = byte_queue_read_buf(&conn->in_buf);
-    if (!in.ptr || in.len == 0) {
-        byte_queue_read_ack(&conn->in_buf, 0);
-        return 0;
-    }
-
-    int in_avail = (int) in.len;
-
-    // Input buffers
-    SecBuffer in_bufs[2];
-    in_bufs[0].BufferType = SECBUFFER_TOKEN;
-    in_bufs[0].pvBuffer   = in.ptr;
-    in_bufs[0].cbBuffer   = (unsigned long) in_avail;
-    in_bufs[1].BufferType = SECBUFFER_EMPTY;
-    in_bufs[1].pvBuffer   = NULL;
-    in_bufs[1].cbBuffer   = 0;
-
-    SecBufferDesc in_desc;
-    in_desc.ulVersion = SECBUFFER_VERSION;
-    in_desc.cBuffers  = 2;
-    in_desc.pBuffers  = in_bufs;
-
-    // Output buffers
-    SecBuffer out_bufs[1];
-    out_bufs[0].BufferType = SECBUFFER_TOKEN;
-    out_bufs[0].pvBuffer   = NULL;
-    out_bufs[0].cbBuffer   = 0;
-
-    SecBufferDesc out_desc;
-    out_desc.ulVersion = SECBUFFER_VERSION;
-    out_desc.cBuffers  = 1;
-    out_desc.pBuffers  = out_bufs;
-
-    DWORD flags = ASC_REQ_STREAM
-                | ASC_REQ_SEQUENCE_DETECT
-                | ASC_REQ_REPLAY_DETECT
-                | ASC_REQ_CONFIDENTIALITY
-                | ASC_REQ_ALLOCATE_MEMORY;
-
-    DWORD out_flags = 0;
-    TimeStamp expiry;
-
-    SECURITY_STATUS ss = AcceptSecurityContext(
-        conn->cred,
-        conn->ctx_valid ? &conn->ctx : NULL,
-        &in_desc,
-        flags,
-        0,
-        conn->ctx_valid ? NULL : &conn->ctx,
-        &out_desc,
-        &out_flags,
-        &expiry);
-
-    if (ss == SEC_E_OK || ss == SEC_I_CONTINUE_NEEDED)
-        conn->ctx_valid = true;
-
-    // Copy output token to out_buf
-    if (out_bufs[0].pvBuffer && out_bufs[0].cbBuffer > 0) {
-        byte_queue_write_setmincap(&conn->out_buf, out_bufs[0].cbBuffer);
-        byte_queue_write(&conn->out_buf, out_bufs[0].pvBuffer, out_bufs[0].cbBuffer);
-        FreeContextBuffer(out_bufs[0].pvBuffer);
-    }
-
-    // Calculate how much input was consumed
-    int consumed = in_avail;
-    if (in_bufs[1].BufferType == SECBUFFER_EXTRA && in_bufs[1].cbBuffer > 0)
-        consumed = in_avail - (int) in_bufs[1].cbBuffer;
-
-    if (ss == SEC_E_INCOMPLETE_MESSAGE) {
-        // SChannel didn't consume anything
-        byte_queue_read_ack(&conn->in_buf, 0);
-        return 0;
-    }
-
-    byte_queue_read_ack(&conn->in_buf, consumed);
-
-    if (ss == SEC_I_CONTINUE_NEEDED)
-        return 0;
-
-    if (ss == SEC_E_OK) {
-        conn->handshake = false;
-        ss = QueryContextAttributes(&conn->ctx, SECPKG_ATTR_STREAM_SIZES, &conn->sizes);
-        if (ss != SEC_E_OK)
-            return -1;
-        return 1;
-    }
-
-    return -1;
-}
-
-// ============================================================
-// Network I/O (ciphertext ↔ socket)
-// ============================================================
-
-char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap)
-{
-    byte_queue_write_setmincap(&conn->in_buf, 4096);
-    string bv = byte_queue_write_buf(&conn->in_buf);
-    if (!bv.ptr || bv.len == 0) {
-        byte_queue_write_ack(&conn->in_buf, 0);
-        return NULL;
-    }
-    *cap = (int) bv.len;
-    return (char *) bv.ptr;
-}
-
-void tls_conn_net_write_ack(TLS_Conn *conn, int num)
-{
-    byte_queue_write_ack(&conn->in_buf, num);
-}
-
-char *tls_conn_net_read_buf(TLS_Conn *conn, int *num)
-{
-    string bv = byte_queue_read_buf(&conn->out_buf);
-    if (!bv.ptr || bv.len == 0) {
-        byte_queue_read_ack(&conn->out_buf, 0);
-        return NULL;
-    }
-    *num = (int) bv.len;
-    return (char *) bv.ptr;
-}
-
-void tls_conn_net_read_ack(TLS_Conn *conn, int num)
-{
-    byte_queue_read_ack(&conn->out_buf, num);
-}
-
-// ============================================================
-// Application I/O (encrypt/decrypt)
-// ============================================================
-
-int tls_conn_app_write(TLS_Conn *conn, char *src, int num)
-{
-    assert(!conn->handshake);
-
-    if (num <= 0) return 0;
-
-    int max_msg = (int) conn->sizes.cbMaximumMessage;
-    if (num > max_msg)
-        num = max_msg;
-
-    int header_size  = (int) conn->sizes.cbHeader;
-    int trailer_size = (int) conn->sizes.cbTrailer;
-    int total = header_size + num + trailer_size;
-
-    // Ensure output buffer has enough space
-    byte_queue_write_setmincap(&conn->out_buf, total);
-    string bv = byte_queue_write_buf(&conn->out_buf);
-    if (!bv.ptr || (int) bv.len < total) {
-        // Try with less data
-        if (!bv.ptr || (int) bv.len < header_size + trailer_size + 1) {
-            byte_queue_write_ack(&conn->out_buf, 0);
-            return 0;
-        }
-        num = (int) bv.len - header_size - trailer_size;
-        total = header_size + num + trailer_size;
-    }
-
-    char *out_ptr = (char *) bv.ptr;
-
-    // Copy plaintext into the data portion
-    memcpy(out_ptr + header_size, src, num);
-
-    // Set up SecBuffers for in-place encryption
-    SecBuffer bufs[4];
-    bufs[0].BufferType = SECBUFFER_STREAM_HEADER;
-    bufs[0].pvBuffer   = out_ptr;
-    bufs[0].cbBuffer   = (unsigned long) header_size;
-
-    bufs[1].BufferType = SECBUFFER_DATA;
-    bufs[1].pvBuffer   = out_ptr + header_size;
-    bufs[1].cbBuffer   = (unsigned long) num;
-
-    bufs[2].BufferType = SECBUFFER_STREAM_TRAILER;
-    bufs[2].pvBuffer   = out_ptr + header_size + num;
-    bufs[2].cbBuffer   = (unsigned long) trailer_size;
-
-    bufs[3].BufferType = SECBUFFER_EMPTY;
-    bufs[3].pvBuffer   = NULL;
-    bufs[3].cbBuffer   = 0;
-
-    SecBufferDesc desc;
-    desc.ulVersion = SECBUFFER_VERSION;
-    desc.cBuffers  = 4;
-    desc.pBuffers  = bufs;
-
-    SECURITY_STATUS ss = EncryptMessage(&conn->ctx, 0, &desc, 0);
-    if (ss != SEC_E_OK) {
-        byte_queue_write_ack(&conn->out_buf, 0);
-        return -1;
-    }
-
-    int written = (int)(bufs[0].cbBuffer + bufs[1].cbBuffer + bufs[2].cbBuffer);
-    byte_queue_write_ack(&conn->out_buf, written);
-
-    return num;
-}
-
-int tls_conn_app_read(TLS_Conn *conn, char *dst, int cap)
-{
-    assert(!conn->handshake);
-
-    // Drain any pending plaintext from a previous partial read
-    if (conn->pending_len > 0) {
-        int n = conn->pending_len;
-        if (n > cap) n = cap;
-        memcpy(dst, conn->pending + conn->pending_off, n);
-        conn->pending_off += n;
-        conn->pending_len -= n;
-        if (conn->pending_len == 0) {
-            free(conn->pending);
-            conn->pending = NULL;
-            conn->pending_off = 0;
-        }
-        return n;
-    }
-
-    string in = byte_queue_read_buf(&conn->in_buf);
-    if (!in.ptr || in.len == 0) {
-        byte_queue_read_ack(&conn->in_buf, 0);
-        return 0;
-    }
-
-    int in_avail = (int) in.len;
-
-    // DecryptMessage operates in-place
-    SecBuffer bufs[4];
-    bufs[0].BufferType = SECBUFFER_DATA;
-    bufs[0].pvBuffer   = in.ptr;
-    bufs[0].cbBuffer   = (unsigned long) in_avail;
-    bufs[1].BufferType = SECBUFFER_EMPTY;
-    bufs[1].pvBuffer   = NULL;
-    bufs[1].cbBuffer   = 0;
-    bufs[2].BufferType = SECBUFFER_EMPTY;
-    bufs[2].pvBuffer   = NULL;
-    bufs[2].cbBuffer   = 0;
-    bufs[3].BufferType = SECBUFFER_EMPTY;
-    bufs[3].pvBuffer   = NULL;
-    bufs[3].cbBuffer   = 0;
-
-    SecBufferDesc desc;
-    desc.ulVersion = SECBUFFER_VERSION;
-    desc.cBuffers  = 4;
-    desc.pBuffers  = bufs;
-
-    SECURITY_STATUS ss = DecryptMessage(&conn->ctx, &desc, 0, NULL);
-
-    if (ss == SEC_E_INCOMPLETE_MESSAGE) {
-        byte_queue_read_ack(&conn->in_buf, 0);
-        return 0;
-    }
-
-    if (ss != SEC_E_OK && ss != SEC_I_RENEGOTIATE) {
-        byte_queue_read_ack(&conn->in_buf, 0);
-        return -1;
-    }
-
-    // Find decrypted data and extra ciphertext buffers
-    SecBuffer *data_buf = NULL;
-    SecBuffer *extra_buf = NULL;
-    for (int i = 0; i < 4; i++) {
-        if (bufs[i].BufferType == SECBUFFER_DATA)
-            data_buf = &bufs[i];
-        else if (bufs[i].BufferType == SECBUFFER_EXTRA)
-            extra_buf = &bufs[i];
-    }
-
-    int result = 0;
-    if (data_buf && data_buf->cbBuffer > 0) {
-        int total = (int) data_buf->cbBuffer;
-        int n = total;
-        if (n > cap) n = cap;
-        memcpy(dst, data_buf->pvBuffer, n);
-        result = n;
-
-        // Save excess plaintext for next call
-        int leftover = total - n;
-        if (leftover > 0) {
-            conn->pending = malloc(leftover);
-            if (conn->pending) {
-                memcpy(conn->pending, (char *) data_buf->pvBuffer + n, leftover);
-                conn->pending_off = 0;
-                conn->pending_len = leftover;
-            }
-        }
-    }
-
-    // Consume processed input, keeping any extra ciphertext
-    int consumed = in_avail;
-    if (extra_buf && extra_buf->cbBuffer > 0)
-        consumed = in_avail - (int) extra_buf->cbBuffer;
-    byte_queue_read_ack(&conn->in_buf, consumed);
-
-    if (ss == SEC_I_RENEGOTIATE)
-        return result > 0 ? result : 0;
-
-    return result;
-}
-
-int tls_conn_needs_flushing(TLS_Conn *conn)
-{
-    return !byte_queue_empty(&conn->out_buf);
-}
-
-#endif // TLS_SCHANNEL
-#endif // TLS_ENABLED

quakey/include/quakey.h

@@ -4,7 +4,6 @@
 #define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdbool.h>
 #ifdef _WIN32
 #include <winsock2.h>
 #include <ws2tcpip.h>
@@ -27,6 +26,8 @@
 #include <pthread.h>
 #endif
 
+#include <stdbool.h>
+
 typedef struct {} Quakey;
 
 // Function pointers to a simulated program's code
@@ -34,6 +35,11 @@ typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, s
 typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
 typedef int (*QuakeyFreeFunc)(void *state);
 
+typedef enum {
+    QUAKEY_LINUX,
+    QUAKEY_WINDOWS,
+} QuakeyPlatform;
+
 typedef struct {
 
     // Label associated to the process for debugging
@@ -55,6 +61,9 @@ typedef struct {
     // Disk size for the process
     int disk_size;
 
+    // Platform used by this program
+    QuakeyPlatform platform;
+
 } QuakeySpawn;
 
 typedef unsigned long long QuakeyUInt64;
@@ -75,6 +84,9 @@ void *quakey_node_state(QuakeyNode node);
 // Schedule and executes one program until it would block, then returns
 int quakey_schedule_one(Quakey *quakey);
 
+// Get the current simulated time in nanoseconds
+QuakeyUInt64 quakey_current_time(Quakey *quakey);
+
 // Generate a random u64
 QuakeyUInt64 quakey_random(void);
 
@@ -86,50 +98,36 @@ typedef struct {
 void quakey_signal(char *name);
 int  quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
 
+// Limit the number of hosts that can be crashed at the same time.
+// Set to 0 for no limit (default).
+void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
+
+void quakey_network_partitioning(Quakey *quakey, bool enable);
+
 // Access spawned host information
 int         quakey_num_hosts(Quakey *quakey);
 void       *quakey_host_state(Quakey *quakey, int idx);  // Returns NULL if host is dead
 int         quakey_host_is_dead(Quakey *quakey, int idx);
 const char *quakey_host_name(Quakey *quakey, int idx);
 
-// Fault injection control
+// Enter/leave a host's context so that mock_xxx functions (file I/O,
-void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
+// etc.) operate on that host's resources. Use from code that runs
-void quakey_network_partitioning(Quakey *quakey, bool enabled);
+// outside of the normal init/tick/free callbacks (e.g. invariant
-
+// checkers).
-// Simulation time
-QuakeyUInt64 quakey_current_time(Quakey *quakey);
-
-// Host context (for accessing mock filesystem from outside scheduled context)
 void quakey_enter_host(QuakeyNode node);
 void quakey_leave_host(void);
 
 int *mock_errno_ptr(void);
 
-// Network mocks (POSIX-style, available on all platforms)
-int   mock_socket(int domain, int type, int protocol);
-int   mock_bind(int fd, void *addr, unsigned long addr_len);
-int   mock_connect(int fd, void *addr, unsigned long addr_len);
-int   mock_getsockopt(int fd, int level, int optname, void *optval, unsigned int *optlen);
-int   mock_listen(int fd, int backlog);
-int   mock_accept(int fd, void *addr, unsigned int *addr_len);
-int   mock_pipe(int *fds);
-int   mock_recv(int fd, char *dst, int len, int flags);
-int   mock_send(int fd, char *src, int len, int flags);
-
 #ifdef _WIN32
-// Windows-specific socket functions (use SOCKET type)
-int   mock_closesocket(SOCKET fd);
-int   mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp);
 BOOL   mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
 BOOL   mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
 HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
-DWORD  mock_GetFileAttributesA(char *lpFileName);
 BOOL   mock_CloseHandle(HANDLE handle);
 BOOL   mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
 BOOL   mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
 BOOL   mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
 DWORD  mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
-BOOL   mock_SetEndOfFile(HANDLE hFile);
 BOOL   mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
 BOOL   mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
 BOOL   mock_FlushFileBuffers(HANDLE handle);
@@ -139,18 +137,23 @@ HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
 BOOL   mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
 BOOL   mock_FindClose(HANDLE hFindFile);
 int    mock__mkdir(char *path);
-int   mock_open(char *path, int flags, int mode);
-int   mock_close(int fd);
-int   mock_read(int fd, char *dst, int len);
-int   mock_write(int fd, char *src, int len);
-int   mock_remove(char *path);
-int   mock_rename(char *oldpath, char *newpath);
-int   mock_mkdir(char *path, int mode);
 #else
+int   mock_socket(int domain, int type, int protocol);
+int   mock_closesocket(int fd);
+int   mock_ioctlsocket(int fd, long cmd, unsigned long *argp);
+int   mock_bind(int fd, void *addr, unsigned long addr_len);
+int   mock_connect(int fd, void *addr, unsigned long addr_len);
+int   mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
+int   mock_listen(int fd, int backlog);
+int   mock_accept(int fd, void *addr, socklen_t *addr_len);
+int   mock_pipe(int *fds);
+int   mock_recv(int fd, char *dst, int len, int flags);
+int   mock_send(int fd, char *src, int len, int flags);
 int   mock_clock_gettime(clockid_t clockid, struct timespec *tp);
 int   mock_open(char *path, int flags, int mode);
 int   mock_fcntl(int fd, int cmd, int flags);
 int   mock_close(int fd);
+int   mock_access(const char *path, int mode);
 int   mock_ftruncate(int fd, size_t new_size);
 int   mock_fstat(int fd, struct stat *buf);
 int   mock_read(int fd, char *dst, int len);
@@ -199,9 +202,9 @@ void  mock_free(void *ptr);
 #define open             mock_open
 #define fcntl            mock_fcntl
 #define close            mock_close
+#define access           mock_access
 #define ftruncate        mock_ftruncate
 #define CreateFileW      mock_CreateFileW
-#define GetFileAttributesA mock_GetFileAttributesA
 #define CloseHandle      mock_CloseHandle
 #define ReadFile         mock_ReadFile
 #define WriteFile        mock_WriteFile
@@ -211,7 +214,6 @@ void  mock_free(void *ptr);
 #define GetFileSizeEx    mock_GetFileSizeEx
 #define lseek            mock_lseek
 #define SetFilePointer   mock_SetFilePointer
-#define SetEndOfFile     mock_SetEndOfFile
 #define flock            mock_flock
 #define LockFile         mock_LockFile
 #define UnlockFile       mock_UnlockFile

quakey/src/quakey.c

@@ -5,7 +5,6 @@
 #include <quakey.h>
 #include <stdint.h>
 #include <assert.h>
-#include <fcntl.h>
 
 /////////////////////////////////////////////////////////////////
 // Utilities
@@ -245,6 +244,9 @@ struct Host {
 
     char *name;
 
+    // Platform used by this host
+    QuakeyPlatform platform;
+
     // State of the ephimeral port allocation
     u16 next_ephemeral_port;
 
@@ -400,6 +402,11 @@ struct Sim {
     // reached, a new random target is generated.
     HostPairList partition;
     HostPairList target_partition;
+
+    // Maximum number of hosts that can be dead at the same time.
+    int max_crashes;
+
+    bool network_partitioning;
 };
 
 static void time_event_wakeup(Sim *sim, Nanos time, Host *host);
@@ -738,7 +745,6 @@ static b32 socket_queue_empty(SocketQueue *queue)
     return queue->used == 0;
 }
 
-
 /////////////////////////////////////////////////////////////////
 // Accept Queue Code
 
@@ -934,6 +940,7 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg)
 {
     host->sim = sim;
     host->name = config.name;
+    host->platform = config.platform;
     host->next_ephemeral_port = FIRST_EPHEMERAL_PORT;
 
     // Save spawn config for restart after crash
@@ -1055,8 +1062,11 @@ static void host_free(Host *host)
 }
 
 // Remove all events associated with a host (WAKEUP events and events
-// whose source or destination descriptors belong to this host)
+// whose source or destination descriptors belong to this host).
-static void remove_events_for_host(Sim *sim, Host *host)
+// If skip_restart is true, RESTART events are kept (used during
+// restart failure to avoid removing the event currently being
+// processed by process_events_at_current_time).
+static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
 {
     int i = 0;
     while (i < sim->num_events) {
@@ -1065,7 +1075,7 @@ static void remove_events_for_host(Sim *sim, Host *host)
 
         if (event->type == EVENT_TYPE_WAKEUP && event->host == host)
             remove = true;
-        if (event->type == EVENT_TYPE_RESTART && event->host == host)
+        if (!skip_restart && event->type == EVENT_TYPE_RESTART && event->host == host)
             remove = true;
         if (event->src_desc && event->src_desc->host == host)
             remove = true;
@@ -1081,6 +1091,11 @@ static void remove_events_for_host(Sim *sim, Host *host)
     }
 }
 
+static void remove_events_for_host(Sim *sim, Host *host)
+{
+    remove_events_for_host_ex(sim, host, false);
+}
+
 static int count_dead(Sim *sim)
 {
     int n = 0;
@@ -1108,6 +1123,11 @@ static void host_crash(Host *host)
     // Remove all pending events for this host
     remove_events_for_host(sim, host);
 
+    // Let the application clean up (e.g. log the crash)
+    host___ = host;
+    host->free_func(host->state);
+    host___ = NULL;
+
     // Free application state
     free(host->state);
     host->state = NULL;
@@ -1118,7 +1138,6 @@ static void host_crash(Host *host)
     host->poll_count = 0;
     host->poll_timeout = -1;
 
-    printf("Quakey: Host crash (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
 }
 
 // Restart a crashed host: re-initialize from saved config,
@@ -1189,10 +1208,22 @@ static void host_restart(Host *host)
                 desc_free(sim, &host->desc[i], true);
         }
         host->num_desc = 0;
-        remove_events_for_host(sim, host);
+        // Skip removing RESTART events: this function is called from
+        // time_event_process (EVENT_TYPE_RESTART), so the current
+        // restart event is still in the array being iterated by
+        // process_events_at_current_time. Removing it here would cause
+        // the outer loop's swap-removal to discard an unrelated event.
+        remove_events_for_host_ex(sim, host, true);
         free(host->state);
         host->state = NULL;
         host->dead = true;
+
+        // Schedule another restart attempt. Init can fail due to
+        // transient I/O errors from fault injection; permanently
+        // killing the host would be unrealistic since a real
+        // process supervisor would keep retrying.
+        Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL);
+        time_event_restart(sim, sim->current_time + restart_delay, host);
         return;
     }
 
@@ -1202,9 +1233,17 @@ static void host_restart(Host *host)
     } else if (host->poll_timeout == 0)
         host->timedout = true;
 
-    printf("Quakey: Host restart (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
 }
 
+static b32 host_is_linux(Host *host)
+{
+    return host->platform == QUAKEY_LINUX;
+}
+
+static b32 host_is_windows(Host *host)
+{
+    return host->platform == QUAKEY_WINDOWS;
+}
 
 static Nanos host_time(Host *host)
 {
@@ -1363,7 +1402,10 @@ static void host_update(Host *host)
     );
     host___ = NULL;
     if (ret < 0) {
-        TODO;
+        host_crash(host);
+        Nanos restart_delay = 1000000000ULL + (sim_random(host->sim) % 9000000000ULL);
+        time_event_restart(host->sim, host->sim->current_time + restart_delay, host);
+        return;
     }
 
     if (host->poll_timeout > 0) {
@@ -1943,7 +1985,7 @@ static int host_read(Host *host, int desc_idx, char *dst, int len)
         desc->type == DESC_PIPE) {
         num = recv_inner(desc, dst, len);
     } else if (desc->type == DESC_FILE) {
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
         uint64_t roll = sim_random(host->sim) % 1000;
         if (roll == 0) return HOST_ERROR_IO;
 #endif
@@ -1987,7 +2029,7 @@ static int host_write(Host *host, int desc_idx, char *src, int len)
         desc->type == DESC_PIPE) {
         num = send_inner(desc, src, len);
     } else if (desc->type == DESC_FILE) {
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
         uint64_t roll = sim_random(host->sim) % 1000;
         if (roll == 0) return HOST_ERROR_IO;
         if (roll == 1) return HOST_ERROR_NOSPC;
@@ -2163,7 +2205,7 @@ static int host_fsync(Host *host, int desc_idx)
     if (desc->type != DESC_FILE)
         return HOST_ERROR_BADIDX;
 
-#ifdef FAULT_INJECTION
+#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
     uint64_t roll = sim_random(host->sim) % 100;
     if (roll == 0)
         return HOST_ERROR_IO;
@@ -2388,7 +2430,7 @@ static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b);
 
 static Nanos pick_partition_delay(Sim *sim, bool longer)
 {
-    Nanos min_delay = 10000000; // 10ms
+    Nanos min_delay = 1000000000; // 1s
     Nanos max_delay = 10000000000; // 10s
     if (longer) {
         min_delay = 10000000000;
@@ -2405,10 +2447,13 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
         {
             // Try to move one step toward the target partition.
             // If current already matches target, pick a new target.
-            if (!break_or_repair_link(sim))
+            if (!break_or_repair_link(sim)) {
+                if (sim->network_partitioning) {
                     if (change_target_partition(sim) < 0) {
                         TODO;
                     }
+                }
+            }
 
             // Reschedule for the next partition step
             event->time = sim->current_time + pick_partition_delay(sim, true);
@@ -2428,8 +2473,8 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
             Host *peer_host = sim->hosts[idx];
 
             if (hosts_are_partitioned(sim, src_desc->host, peer_host)) {
-                SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
+                //SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
-                    src_desc->host->name, peer_host->name);
+                //    src_desc->host->name, peer_host->name);
                 src_desc->connect_status = CONNECT_STATUS_NOHOST;
                 break;
             }
@@ -2746,11 +2791,22 @@ static void sim_init(Sim *sim, uint64_t seed)
     sim->events = NULL;
     sim->num_signals = 0;
     sim->head_signal = 0;
+    sim->max_crashes = 0;
+    sim->network_partitioning = true;
     host_pair_list_init(&sim->partition);
     host_pair_list_init(&sim->target_partition);
     time_event_partition(sim, pick_partition_delay(sim, false));
 }
 
+static void sim_network_partitioning(Sim *sim, bool enable)
+{
+    sim->network_partitioning = enable;
+    if (!enable) {
+        host_pair_list_free(&sim->target_partition);
+        host_pair_list_init(&sim->target_partition);
+    }
+}
+
 static void sim_free(Sim *sim)
 {
     host_pair_list_free(&sim->target_partition);
@@ -2834,8 +2890,8 @@ static void process_events_at_current_time(Sim *sim)
         }
     }
 
-    if (sim->num_events > 10000)
+    //if (sim->num_events > 10000)
-        SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
+    //    SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
 }
 
 static int find_first_ready_host(Sim *sim)
@@ -2898,14 +2954,14 @@ static b32 sim_update(Sim *sim)
     if (sim->num_hosts > 1) {
         uint64_t rng = sim_random(sim);
         if ((rng % 10000) == 0) {
-            // Pick a random live host to crash
+
             int live_count = 0;
             for (int i = 0; i < sim->num_hosts; i++)
                 if (!sim->hosts[i]->dead)
                     live_count++;
+            int dead_count = sim->num_hosts - live_count;
 
-            // Only crash if at least 2 hosts are alive (keep a majority)
+            if (dead_count < sim->max_crashes) {
-            if (live_count >= 2) {
                 // Pick one of the live hosts at random
                 int target = sim_random(sim) % live_count;
                 int j = 0;
@@ -2997,6 +3053,18 @@ void quakey_free(Quakey *quakey)
     }
 }
 
+void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
+{
+    Sim *sim = (Sim*) quakey;
+    sim->max_crashes = max_crashes;
+}
+
+void quakey_network_partitioning(Quakey *quakey, bool enable)
+{
+    Sim *sim = (Sim*) quakey;
+    sim_network_partitioning(sim, enable);
+}
+
 QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg)
 {
     return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg);
@@ -3008,11 +3076,28 @@ void *quakey_node_state(QuakeyNode node)
     return host->state;
 }
 
+void quakey_enter_host(QuakeyNode node)
+{
+    Host *host = (void*) node;
+    host___ = host;
+}
+
+void quakey_leave_host(void)
+{
+    host___ = NULL;
+}
+
 int quakey_schedule_one(Quakey *quakey)
 {
     return sim_update((Sim*) quakey);
 }
 
+QuakeyUInt64 quakey_current_time(Quakey *quakey)
+{
+    Sim *sim = (Sim*) quakey;
+    return sim->current_time;
+}
+
 QuakeyUInt64 quakey_random(void)
 {
     Host *host = host___;
@@ -3072,39 +3157,6 @@ const char *quakey_host_name(Quakey *quakey, int idx)
     return sim->hosts[idx]->name;
 }
 
-void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
-{
-    // TODO: Implement crash limiting. For now this is a no-op
-    // that allows the simulation to run without the feature.
-    (void) quakey;
-    (void) max_crashes;
-}
-
-void quakey_network_partitioning(Quakey *quakey, bool enabled)
-{
-    // TODO: Implement network partitioning toggle. For now this is
-    // a no-op that allows the simulation to run without the feature.
-    (void) quakey;
-    (void) enabled;
-}
-
-QuakeyUInt64 quakey_current_time(Quakey *quakey)
-{
-    Sim *sim = (Sim*) quakey;
-    return (QuakeyUInt64) sim->current_time;
-}
-
-void quakey_enter_host(QuakeyNode node)
-{
-    Host *host = (void*) node;
-    host___ = host;
-}
-
-void quakey_leave_host(void)
-{
-    host___ = NULL;
-}
-
 /////////////////////////////////////////////////////////////////
 // Mock System Calls
 
@@ -3117,7 +3169,7 @@ int *mock_errno_ptr(void)
     return host_errno_ptr(host);
 }
 
-// Platform-independent mock functions (used by lib/ on all platforms)
+#ifndef _WIN32
 
 int mock_socket(int domain, int type, int protocol)
 {
@@ -3169,6 +3221,8 @@ int mock_close(int fd)
     if (host == NULL)
         abort_("Call to mock_close() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_close() not from Linux\n");
 
     int desc_idx = fd;
     int ret = host_close(host, desc_idx, false);
@@ -3187,6 +3241,39 @@ int mock_close(int fd)
     return 0;
 }
 
+// NOTE: mock_access must be implemented for simulation to work correctly.
+// chunk_store_exists() uses file_exists() which calls access(). Without a
+// working mock_access, the simulation cannot check chunk existence and
+// must fall back to file_open (O_CREAT), which creates empty files as a
+// side effect and leads to data corruption (servers return uninitialized
+// data for chunks they don't actually have).
+int mock_access(const char *path, int mode)
+{
+    (void) mode; // No permission bits in mockfs; all checks reduce to existence
+
+    Host *host = host___;
+    if (host == NULL)
+        abort_("Call to mock_access() with no node scheduled\n");
+
+    if (!host_is_linux(host))
+        abort_("Call to mock_access() not from Linux\n");
+
+    MockFS_OpenFile open_file;
+    int ret = mockfs_open(host->mfs, (char *) path, strlen(path), MOCKFS_O_RDONLY, &open_file);
+    if (ret == MOCKFS_ERRNO_ISDIR) {
+        // Path exists and is a directory
+        return 0;
+    }
+    if (ret < 0) {
+        *host_errno_ptr(host) = ENOENT;
+        return -1;
+    }
+
+    // File exists; close it immediately
+    mockfs_close_file(&open_file);
+    return 0;
+}
+
 static int convert_addr(void *addr, size_t addr_len,
     Addr *converted_addr, uint16_t *converted_port)
 {
@@ -3374,11 +3461,12 @@ static int convert_linux_open_flags_to_mockfs(int flags)
 
 int mock_open(char *path, int flags, int mode)
 {
-    (void)mode;
     Host *host = host___;
     if (host == NULL)
         abort_("Call to mock_open() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_open() not from Linux\n");
 
     int converted_flags = convert_linux_open_flags_to_mockfs(flags);
 
@@ -3408,6 +3496,8 @@ int mock_read(int fd, char *dst, int len)
     if (host == NULL)
         abort_("Call to mock_read() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_read() not from Linux\n");
 
     int ret = host_read(host, fd, dst, len);
     if (ret < 0) {
@@ -3438,6 +3528,8 @@ int mock_write(int fd, char *src, int len)
     if (host == NULL)
         abort_("Call to mock_write() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_write() not from Linux\n");
 
     int ret = host_write(host, fd, src, len);
     if (ret < 0) {
@@ -3533,7 +3625,7 @@ int mock_send(int fd, char *src, int len, int flags)
     return ret;
 }
 
-int mock_accept(int fd, void *addr, unsigned int *addr_len)
+int mock_accept(int fd, void *addr, socklen_t *addr_len)
 {
     Host *host = host___;
     if (host == NULL)
@@ -3591,7 +3683,7 @@ int mock_accept(int fd, void *addr, unsigned int *addr_len)
     return new_fd;
 }
 
-int mock_getsockopt(int fd, int level, int optname, void *optval, unsigned int *optlen)
+int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
 {
     if (level != SOL_SOCKET)
         abort_("Call to mock_getsockopt() with level other than SOL_SOCKET\n");
@@ -3646,6 +3738,8 @@ int mock_remove(char *path)
     if (host == NULL)
         abort_("Call to mock_remove() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_remove() not from Linux\n");
 
     int ret = host_remove(host, path);
     if (ret < 0) {
@@ -3672,6 +3766,8 @@ int mock_rename(char *oldpath, char *newpath)
     if (host == NULL)
         abort_("Call to mock_rename() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_rename() not from Linux\n");
 
     int ret = host_rename(host, oldpath, newpath);
     if (ret < 0) {
@@ -3698,14 +3794,14 @@ int mock_rename(char *oldpath, char *newpath)
     return 0;
 }
 
-#ifndef _WIN32
-
 int mock_clock_gettime(clockid_t clockid, struct timespec *tp)
 {
     Host *host = host___;
     if (host == NULL)
         abort_("Call to mock_clock_gettime() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_clock_gettime() not from Linux\n");
 
     if (tp == NULL) {
         *host_errno_ptr(host) = EINVAL;
@@ -3734,8 +3830,6 @@ int mock_clock_gettime(clockid_t clockid, struct timespec *tp)
 int mock_flock(int fd, int op)
 {
     // TODO
-    (void)fd;
-    (void)op;
     return 0;
 }
 
@@ -3745,6 +3839,8 @@ int mock_fsync(int fd)
     if (host == NULL)
         abort_("Call to mock_fsync() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_fsync() not from Linux\n");
 
     int ret = host_fsync(host, fd);
     if (ret < 0) {
@@ -3764,6 +3860,8 @@ int mock_ftruncate(int fd, size_t new_size)
     if (host == NULL)
         abort_("Call to mock_ftruncate() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_ftruncate() not from Linux\n");
 
     int ret = host_ftruncate(host, fd, (int) new_size);
     if (ret < 0) {
@@ -3785,6 +3883,8 @@ off_t mock_lseek(int fd, off_t offset, int whence)
     if (host == NULL)
         abort_("Call to mock_lseek() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_lseek() not from Linux\n");
 
     // Convert POSIX whence to HOST whence
     int host_whence;
@@ -3821,6 +3921,8 @@ int mock_fstat(int fd, struct stat *buf)
     if (host == NULL)
         abort_("Call to mock_fstat() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_fstat() not from Linux\n");
 
     if (buf == NULL) {
         *host_errno_ptr(host) = EINVAL;
@@ -3853,7 +3955,6 @@ int mock_fstat(int fd, struct stat *buf)
 
 int mock_mkstemp(char *path)
 {
-    (void)path;
     TODO;
 }
 
@@ -3863,6 +3964,8 @@ char *mock_realpath(char *path, char *dst)
     if (host == NULL)
         abort_("Call to mock_realpath() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_realpath() not from Linux\n");
 
     if (path == NULL) {
         *host_errno_ptr(host) = EINVAL;
@@ -3976,6 +4079,8 @@ int mock_mkdir(char *path, mode_t mode)
     if (host == NULL)
         abort_("Call to mock_mkdir() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_mkdir() not from Linux\n");
 
     // LittleFS doesn't use mode, but we accept it for API compatibility
     (void) mode;
@@ -4005,6 +4110,8 @@ int mock_fcntl(int fd, int cmd, int flags)
     if (host == NULL)
         abort_("Call to mock_fcntl() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_fcntl() not from Linux\n");
 
     switch (cmd) {
 
@@ -4057,6 +4164,8 @@ DIR *mock_opendir(char *name)
     if (host == NULL)
         abort_("Call to mock_opendir() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_opendir() not from Linux\n");
 
     int ret = host_open_dir(host, name);
     if (ret < 0) {
@@ -4093,6 +4202,8 @@ struct dirent* mock_readdir(DIR *dirp)
     if (host == NULL)
         abort_("Call to mock_readdir() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_readdir() not from Linux\n");
 
     DIR_ *dirp_ = (DIR_*) dirp;
 
@@ -4141,6 +4252,8 @@ int mock_closedir(DIR *dirp)
     if (host == NULL)
         abort_("Call to mock_closedir() with no node scheduled\n");
 
+    if (!host_is_linux(host))
+        abort_("Call to mock_closedir() not from Linux\n");
 
     DIR_ *dirp_ = (DIR_*) dirp;
 
@@ -4175,6 +4288,8 @@ int mock_GetLastError(void)
     if (host == NULL)
         abort_("Call to mock_GetLastError() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_GetLastError() not from Windows\n");
 
     // Note that technically on windows errno and GetLastError
     // are different things. Here we use errno_ to store the
@@ -4194,6 +4309,8 @@ void mock_SetLastError(int err)
     if (host == NULL)
         abort_("Call to mock_SetLastError() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_SetLastError() not from Windows\n");
 
     *host_errno_ptr(host) = err;
 }
@@ -4209,6 +4326,8 @@ int mock_closesocket(SOCKET fd)
     if (host == NULL)
         abort_("Call to mock_closesocket() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_closesocket() not from Windows\n");
 
     int desc_idx = fd;
     int ret = host_close(host, desc_idx, true);  // expect_socket = true
@@ -4229,19 +4348,7 @@ int mock_closesocket(SOCKET fd)
 
 int mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp)
 {
-    Host *host = host___;
+    TODO;
-    if (host == NULL)
-        abort_("Call to mock_ioctlsocket() with no node scheduled\n");
-
-    if (cmd == FIONBIO) {
-        int host_flags = (*argp != 0) ? HOST_FLAG_NONBLOCK : 0;
-        int ret = host_setdescflags(host, (int) fd, host_flags);
-        if (ret < 0)
-            return SOCKET_ERROR;
-        return 0;
-    }
-
-    return SOCKET_ERROR;
 }
 
 // Helper function to convert wide string to narrow string (ASCII subset)
@@ -4268,11 +4375,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
 
     // Convert access mode
     if ((dwDesiredAccess & GENERIC_READ) && (dwDesiredAccess & GENERIC_WRITE))
-        lfs_flags = MOCKFS_O_RDWR;
+        lfs_flags = LFS_O_RDWR;
     else if (dwDesiredAccess & GENERIC_WRITE)
-        lfs_flags = MOCKFS_O_WRONLY;
+        lfs_flags = LFS_O_WRONLY;
     else
-        lfs_flags = MOCKFS_O_RDONLY;
+        lfs_flags = LFS_O_RDONLY;
 
     *truncate = false;
 
@@ -4280,11 +4387,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
     switch (dwCreationDisposition) {
     case CREATE_NEW:
         // Creates a new file, fails if file exists
-        lfs_flags |= MOCKFS_O_CREAT | MOCKFS_O_EXCL;
+        lfs_flags |= LFS_O_CREAT | LFS_O_EXCL;
         break;
     case CREATE_ALWAYS:
         // Creates a new file, always (truncates if exists)
-        lfs_flags |= MOCKFS_O_CREAT | MOCKFS_O_TRUNC;
+        lfs_flags |= LFS_O_CREAT | LFS_O_TRUNC;
         *truncate = true;
         break;
     case OPEN_EXISTING:
@@ -4293,11 +4400,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
         break;
     case OPEN_ALWAYS:
         // Opens file if it exists, creates if it doesn't
-        lfs_flags |= MOCKFS_O_CREAT;
+        lfs_flags |= LFS_O_CREAT;
         break;
     case TRUNCATE_EXISTING:
         // Opens and truncates, fails if file doesn't exist
-        lfs_flags |= MOCKFS_O_TRUNC;
+        lfs_flags |= LFS_O_TRUNC;
         *truncate = true;
         break;
     default:
@@ -4317,6 +4424,8 @@ HANDLE mock_CreateFileW(WCHAR *lpFileName,
     if (host == NULL)
         abort_("Call to mock_CreateFileW() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_CreateFileW() not from Windows\n");
 
     // lpSecurityAttributes and hTemplateFile are typically NULL
     (void) lpSecurityAttributes;
@@ -4371,31 +4480,14 @@ HANDLE mock_CreateFileW(WCHAR *lpFileName,
     return (HANDLE)(long long)desc_idx;
 }
 
-DWORD mock_GetFileAttributesA(char *lpFileName)
-{
-    Host *host = host___;
-    if (host == NULL)
-        abort_("Call to mock_GetFileAttributesA() with no node scheduled\n");
-
-    // Try opening the file read-only to check existence
-    int ret = host_open_file(host, lpFileName, MOCKFS_O_RDONLY);
-    if (ret < 0) {
-        *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND;
-        return INVALID_FILE_ATTRIBUTES;
-    }
-
-    // File exists — close it and return normal attributes
-    host_close(host, ret, false);
-    *host_errno_ptr(host) = ERROR_SUCCESS;
-    return FILE_ATTRIBUTE_NORMAL;
-}
-
 BOOL mock_CloseHandle(HANDLE handle)
 {
     Host *host = host___;
     if (host == NULL)
         abort_("Call to mock_CloseHandle() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_CloseHandle() not from Windows\n");
 
     if (handle == INVALID_HANDLE_VALUE || handle == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4443,6 +4535,8 @@ BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *
     if (host == NULL)
         abort_("Call to mock_ReadFile() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_ReadFile() not from Windows\n");
 
     // We don't support overlapped (async) I/O
     if (ov != NULL)
@@ -4490,6 +4584,8 @@ BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED
     if (host == NULL)
         abort_("Call to mock_WriteFile() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_WriteFile() not from Windows\n");
 
     // We don't support overlapped (async) I/O
     if (ov != NULL)
@@ -4533,6 +4629,8 @@ DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceTo
     if (host == NULL)
         abort_("Call to mock_SetFilePointer() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_SetFilePointer() not from Windows\n");
 
     if (hFile == INVALID_HANDLE_VALUE || hFile == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4593,51 +4691,14 @@ DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceTo
     return (DWORD)(new_pos & 0xFFFFFFFF);
 }
 
-BOOL mock_SetEndOfFile(HANDLE hFile)
-{
-    Host *host = host___;
-    if (host == NULL)
-        abort_("Call to mock_SetEndOfFile() with no node scheduled\n");
-
-    if (hFile == INVALID_HANDLE_VALUE || hFile == NULL) {
-        *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
-        return 0;  // FALSE
-    }
-
-    int desc_idx = (int)(long long)hFile;
-
-    // Get current file position to use as the new size
-    int cur_pos = host_lseek(host, desc_idx, 0, HOST_SEEK_CUR);
-    if (cur_pos < 0) {
-        *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
-        return 0;  // FALSE
-    }
-
-    int ret = host_ftruncate(host, desc_idx, cur_pos);
-    if (ret < 0) {
-        switch (ret) {
-        case HOST_ERROR_BADIDX:
-            *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
-            return 0;
-        case HOST_ERROR_NOSPC:
-            *host_errno_ptr(host) = ERROR_DISK_FULL;
-            return 0;
-        default:
-            *host_errno_ptr(host) = ERROR_INVALID_PARAMETER;
-            return 0;
-        }
-    }
-
-    *host_errno_ptr(host) = ERROR_SUCCESS;
-    return 1;  // TRUE
-}
-
 BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf)
 {
     Host *host = host___;
     if (host == NULL)
         abort_("Call to mock_GetFileSizeEx() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_GetFileSizeEx() not from Windows\n");
 
     if (handle == INVALID_HANDLE_VALUE || handle == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4677,6 +4738,8 @@ BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount)
     if (host == NULL)
         abort_("Call to mock_QueryPerformanceCounter() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_QueryPerformanceCounter() not from Windows\n");
 
     if (lpPerformanceCount == NULL)
         return 0;  // FALSE
@@ -4695,6 +4758,8 @@ BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency)
     if (host == NULL)
         abort_("Call to mock_QueryPerformanceFrequency() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_QueryPerformanceFrequency() not from Windows\n");
 
     if (lpFrequency == NULL)
         return 0;  // FALSE
@@ -4712,6 +4777,8 @@ char *mock__fullpath(char *path, char *dst, int cap)
     if (host == NULL)
         abort_("Call to mock__fullpath() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock__fullpath() not from Windows\n");
 
     if (path == NULL) {
         *host_errno_ptr(host) = EINVAL;
@@ -4819,6 +4886,8 @@ int mock__mkdir(char *path)
     if (host == NULL)
         abort_("Call to mock__mkdir() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock__mkdir() not from Windows\n");
 
     int ret = host_mkdir(host, path);
     if (ret < 0) {
@@ -4869,6 +4938,8 @@ HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData)
     if (host == NULL)
         abort_("Call to mock_FindFirstFileA() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_FindFirstFileA() not from Windows\n");
 
     if (lpFileName == NULL || lpFindFileData == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_PARAMETER;
@@ -4967,6 +5038,8 @@ BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData)
     if (host == NULL)
         abort_("Call to mock_FindNextFileA() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_FindNextFileA() not from Windows\n");
 
     if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL || lpFindFileData == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -5009,6 +5082,8 @@ BOOL mock_FindClose(HANDLE hFindFile)
     if (host == NULL)
         abort_("Call to mock_FindClose() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_FindClose() not from Windows\n");
 
     if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL) {
         *host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -5041,6 +5116,8 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
     if (host == NULL)
         abort_("Call to mock_MoveFileExW() with no node scheduled\n");
 
+    if (!host_is_windows(host))
+        abort_("Call to mock_MoveFileExW() not from Windows\n");
 
     // Validate parameters
     if (lpExistingFileName == NULL) {
@@ -5078,7 +5155,7 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
     // We need to check this before calling host_rename
     if (!(dwFlags & MOVEFILE_REPLACE_EXISTING)) {
         // Try to check if destination exists by attempting to open it
-        int check = host_open_file(host, newpath, MOCKFS_O_RDONLY);
+        int check = host_open_file(host, newpath, LFS_O_RDONLY);
         if (check >= 0) {
             // File exists, close it and return error
             host_close(host, check, false);
@@ -5113,31 +5190,6 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
     return 1;  // TRUE
 }
 
-int mock_mkdir(char *path, int mode)
-{
-    (void) mode;
-    Host *host = host___;
-    if (host == NULL)
-        abort_("Call to mock_mkdir() with no node scheduled\n");
-
-    int ret = host_mkdir(host, path);
-    if (ret < 0) {
-        switch (ret) {
-        case HOST_ERROR_EXIST:
-            *host_errno_ptr(host) = EEXIST;
-            return -1;
-        case HOST_ERROR_NOENT:
-            *host_errno_ptr(host) = ENOENT;
-            return -1;
-        default:
-            *host_errno_ptr(host) = EIO;
-            return -1;
-        }
-    }
-
-    return 0;
-}
-
 #endif
 
 void *mock_malloc(size_t size)

src/basic.c

@@ -71,7 +71,10 @@ int deadline_to_timeout(Time deadline, Time current_time)
 {
     if (deadline == INVALID_TIME)
         return -1;
-    return (deadline - current_time) / 1000000;
+    if (deadline <= current_time)
+        return 0;
+    // Round up so sub-millisecond deadlines become 1ms, not 0
+    return CEIL(deadline - current_time, 1000000);
 }
 
 bool getargb(int argc, char **argv, char *name)
@@ -184,31 +187,27 @@ bool addr_eql(Address a, Address b)
     return true;
 }
 
-int parse_addr_arg(string arg, Address *out)
+int parse_addr_arg(char *arg, Address *out)
 {
-    char buf[1<<7];
+    int len = strlen(arg);
-    if (arg.len >= (int) sizeof(buf))
-        return -1;
-    memcpy(buf, arg.ptr, arg.len);
-    arg.ptr = buf;
 
     int i = 0;
-    while (i < arg.len && arg.ptr[i] != ':')
+    while (i < len && arg[i] != ':')
         i++;
 
-    if (i == arg.len)
+    if (i == len)
         return -1; // No ':' character.
-    arg.ptr[i] = '\0';
+    arg[i] = '\0';
 
     IPv4 ipv4;
-    int ret = inet_pton(AF_INET, arg.ptr, &ipv4);
+    int ret = inet_pton(AF_INET, arg, &ipv4);
-    arg.ptr[i] = ':';
+    arg[i] = ':';
 
     if (ret != 1)
         return -1;
 
     errno = 0;
-    ret = atoi(arg.ptr + i + 1);
+    ret = atoi(arg + i + 1);
     if (ret == 0 && errno != 0)
         return -1;
 

src/basic.h

@@ -37,11 +37,10 @@ typedef uint64_t Time;
 
 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
 #define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+#define CEIL(X, Y) (((X) + (Y) - 1) / (Y))
 
 #define UNREACHABLE __builtin_trap();
 
-#define STATIC_ASSERT(X) _Static_assert((X), "")
-
 bool streq(string s1, string s2);
 
 Time get_current_time(void);
@@ -57,7 +56,7 @@ void append_hex_as_str(char *out, SHA256 hash);
 bool addr_eql(Address a, Address b);
 bool addr_lower(Address a, Address b);
 
-int parse_addr_arg(string arg, Address *out);
+int parse_addr_arg(char *arg, Address *out);
 void addr_sort(Address *addrs, int count);
 
 #endif // BASIC_INCLUDED

src/byte_queue.c

@@ -68,10 +68,10 @@ int byte_queue_full(ByteQueue *queue)
 //
 // Note:
 //   - You can't have more than one pending read.
-string byte_queue_read_buf(ByteQueue *queue)
+ByteView byte_queue_read_buf(ByteQueue *queue)
 {
     if (queue->flags & BYTE_QUEUE_ERROR)
-        return (string) {NULL, 0};
+        return (ByteView) {NULL, 0};
 
     assert((queue->flags & BYTE_QUEUE_READ) == 0);
     queue->flags |= BYTE_QUEUE_READ;
@@ -79,9 +79,9 @@ string byte_queue_read_buf(ByteQueue *queue)
     queue->read_target_size = queue->size;
 
     if (queue->data == NULL)
-        return (string) {NULL, 0};
+        return (ByteView) {NULL, 0};
 
-    return (string) { queue->data + queue->head, queue->used };
+    return (ByteView) { queue->data + queue->head, queue->used };
 }
 
 // Complete a previously started operation on the queue.
@@ -108,20 +108,15 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
     }
 }
 
-bool byte_queue_reading(ByteQueue *queue)
+ByteView byte_queue_write_buf(ByteQueue *queue)
-{
-    return (queue->flags & BYTE_QUEUE_READ) != 0;
-}
-
-string byte_queue_write_buf(ByteQueue *queue)
 {
     if ((queue->flags & BYTE_QUEUE_ERROR) || queue->data == NULL)
-        return (string) {NULL, 0};
+        return (ByteView) {NULL, 0};
 
     assert((queue->flags & BYTE_QUEUE_WRITE) == 0);
     queue->flags |= BYTE_QUEUE_WRITE;
 
-    return (string) {
+    return (ByteView) {
         queue->data + (queue->head + queue->used),
         queue->size - (queue->head + queue->used),
     };
@@ -232,7 +227,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
             if (size > queue->limit)
                 size = queue->limit;
 
-            char *data = malloc(size);
+            uint8_t *data = malloc(size);
             if (!data) {
                 queue->flags |= BYTE_QUEUE_ERROR;
                 return 0;
@@ -263,7 +258,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
 void byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len)
 {
     byte_queue_write_setmincap(queue, len);
-    string dst = byte_queue_write_buf(queue);
+    ByteView dst = byte_queue_write_buf(queue);
     if (dst.ptr) {
         memcpy(dst.ptr, ptr, len);
         byte_queue_write_ack(queue, len);
@@ -290,7 +285,7 @@ void byte_queue_patch(ByteQueue *queue, ByteQueueOffset off,
     assert(len <= queue->used - (off - queue->curs));
 
     // Perform the patch
-    char *dst = queue->data + queue->head + (off - queue->curs);
+    uint8_t *dst = queue->data + queue->head + (off - queue->curs);
     memcpy(dst, src, len);
 }
 

src/byte_queue.h

@@ -5,14 +5,19 @@
 
 #include "basic.h"
 
+typedef struct {
+    uint8_t *ptr;
+    size_t   len;
+} ByteView;
+
 typedef struct {
     uint64_t curs;
-    char*    data;
+    uint8_t* data;
     uint32_t head;
     uint32_t size;
     uint32_t used;
     uint32_t limit;
-    char*    read_target;
+    uint8_t* read_target;
     uint32_t read_target_size;
     int flags;
 } ByteQueue;
@@ -32,11 +37,10 @@ int byte_queue_error(ByteQueue *queue);
 int byte_queue_empty(ByteQueue *queue);
 int byte_queue_full(ByteQueue *queue);
 
-string byte_queue_read_buf(ByteQueue *queue);
+ByteView byte_queue_read_buf(ByteQueue *queue);
 void     byte_queue_read_ack(ByteQueue *queue, uint32_t num);
-bool     byte_queue_reading(ByteQueue *queue);
 
-string byte_queue_write_buf(ByteQueue *queue);
+ByteView byte_queue_write_buf(ByteQueue *queue);
 void     byte_queue_write_ack(ByteQueue *queue, uint32_t num);
 int      byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap);
 void     byte_queue_write(ByteQueue *queue, void *ptr, uint32_t len);

src/chunk_store.c

@@ -6,7 +6,7 @@
 #include <stdio.h>
 
 #include "chunk_store.h"
-#include <lib/file_system.h>
+#include "file_system.h"
 
 // Build the full path for a chunk: "base_path/HEX_HASH"
 // SHA256 hex = 64 chars. Returns string pointing into buf.

src/chunk_store.h

@@ -4,7 +4,7 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-#include <lib/basic.h>
+#include "basic.h"
 
 typedef struct {
     char base_path[256];

src/client.c

@@ -6,17 +6,16 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <assert.h>
+#include <poll.h>
 
-#include <lib/basic.h>
+#include "tcp.h"
-
+#include "basic.h"
 #include "config.h"
 #include "metadata.h"
 #include "server.h"
 #include <toastyfs.h>
 #include <stdio.h>
 
-#define POLL_CAPACITY (NODE_LIMIT * 2 + 4)
-
 typedef enum {
     STEP_IDLE,
     STEP_STORE_CHUNK,
@@ -46,7 +45,7 @@ typedef struct {
 
 struct ToastyFS {
 
-    MessageSystem *msys;
+    TCP tcp;
 
     Address server_addrs[NODE_LIMIT];
     int num_servers;
@@ -69,14 +68,14 @@ struct ToastyFS {
 
     SHA256 chunks[META_CHUNKS_MAX];
     int    chunk_sizes[META_CHUNKS_MAX];
-    uint16_t chunk_num_holders[META_CHUNKS_MAX];
-    uint16_t chunk_holders[META_CHUNKS_MAX][REPLICATION_FACTOR];
     int    num_chunks;
 
     char *put_data;
     int   put_data_len;
 };
 
+// ---- Client logging infrastructure (mirrors server node_log) ----
+
 #define TIME_FMT "%7.3fs"
 #define TIME_VAL(t) ((double)(t) / 1000000000.0)
 
@@ -98,7 +97,7 @@ static const char *step_name(Step step)
 
 static void client_log_impl(ToastyFS *tfs, const char *event, const char *detail)
 {
-    Time now = get_current_time(); // TODO: check error
+    Time now = get_current_time();
     printf("[" TIME_FMT "] CLIENT %lu %-12s V%-3lu | %-20s %s\n",
         TIME_VAL(now),
         tfs->client_id,
@@ -140,7 +139,7 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs)
     tfs->request_id  = 0;
 
     for (int i = 0; i < num_addrs; i++) {
-        if (parse_addr_arg((string) { addrs[i], strlen(addrs[i]) }, &tfs->server_addrs[i]) < 0) {
+        if (parse_addr_arg(addrs[i], &tfs->server_addrs[i]) < 0) {
             free(tfs);
             return NULL;
         }
@@ -148,8 +147,7 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs)
     tfs->num_servers = num_addrs;
     addr_sort(tfs->server_addrs, tfs->num_servers);
 
-    tfs->msys = message_system_init(tfs->server_addrs, num_addrs);
+    if (tcp_context_init(&tfs->tcp) < 0) {
-    if (tfs->msys == NULL) {
         free(tfs);
         return NULL;
     }
@@ -163,13 +161,12 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs)
 
 void toastyfs_free(ToastyFS *tfs)
 {
-    message_system_free(tfs->msys);
+    tcp_context_free(&tfs->tcp);
     free(tfs->put_data);
 }
 
 static int find_completed_transfer_for_hash(ToastyFS *tfs, SHA256 hash);
 static bool all_chunk_transfers_completed(ToastyFS *tfs);
-static bool all_transfers_finished(ToastyFS *tfs);
 
 static bool
 transfer_for_hash_already_started(ToastyFS *tfs, SHA256 hash)
@@ -206,6 +203,41 @@ static int leader_idx(ToastyFS *tfs)
     return tfs->view_number % tfs->num_servers;
 }
 
+static void send_message_to_server(ToastyFS *tfs, int server_idx, MessageHeader *msg)
+{
+    ByteQueue *output;
+    int conn_idx = tcp_index_from_tag(&tfs->tcp, server_idx);
+    if (conn_idx < 0) {
+        if (tcp_connect(&tfs->tcp, tfs->server_addrs[server_idx], server_idx, &output) < 0)
+            return;
+    } else {
+        output = tcp_output_buffer(&tfs->tcp, conn_idx);
+        if (output == NULL)
+            return;
+    }
+
+    byte_queue_write(output, msg, msg->length);
+}
+
+static void
+send_message_to_server_ex(ToastyFS *tfs, int server_idx,
+    MessageHeader *msg, void *extra, int extra_len)
+{
+    ByteQueue *output;
+    int conn_idx = tcp_index_from_tag(&tfs->tcp, server_idx);
+    if (conn_idx < 0) {
+        if (tcp_connect(&tfs->tcp, tfs->server_addrs[server_idx], server_idx, &output) < 0)
+            return;
+    } else {
+        output = tcp_output_buffer(&tfs->tcp, conn_idx);
+        if (output == NULL)
+            return;
+    }
+
+    byte_queue_write(output, msg, msg->length - extra_len);
+    byte_queue_write(output, extra, extra_len);
+}
+
 static int begin_transfers(ToastyFS *tfs)
 {
     // Count started transfers
@@ -233,7 +265,7 @@ static int begin_transfers(ToastyFS *tfs)
                     .hash = tfs->transfers[i].hash,
                     .size = tfs->transfers[i].size,
                 };
-                send_message_ex(tfs->msys, tfs->transfers[i].location,
+                send_message_to_server_ex(tfs, tfs->transfers[i].location,
                     &msg.base, tfs->transfers[i].data, tfs->transfers[i].size);
             } else {
                 FetchChunkMessage msg = {
@@ -245,7 +277,7 @@ static int begin_transfers(ToastyFS *tfs)
                     .hash = tfs->transfers[i].hash,
                     .sender_idx = -1, // Client (not a peer server)
                 };
-                send_message(tfs->msys, tfs->transfers[i].location, &msg.base);
+                send_message_to_server(tfs, tfs->transfers[i].location, &msg.base);
             }
             tfs->transfers[i].state = TRANSFER_STARTED;
 
@@ -308,11 +340,8 @@ static void replay_request(ToastyFS *tfs)
             for (int i = 0; i < tfs->num_chunks; i++) {
                 msg.oper.chunks[i].hash = tfs->chunks[i];
                 msg.oper.chunks[i].size = tfs->chunk_sizes[i];
-                msg.oper.chunks[i].num_holders = tfs->chunk_num_holders[i];
-                for (int j = 0; j < tfs->chunk_num_holders[i]; j++)
-                    msg.oper.chunks[i].holders[j] = tfs->chunk_holders[i][j];
             }
-            send_message(tfs->msys, leader_idx(tfs), &msg.base);
+            send_message_to_server(tfs, leader_idx(tfs), &msg.base);
         }
         break;
     case STEP_DELETE:
@@ -331,7 +360,7 @@ static void replay_request(ToastyFS *tfs)
             };
             memcpy(msg.oper.bucket, tfs->bucket, META_BUCKET_MAX);
             memcpy(msg.oper.key,    tfs->key,    META_KEY_MAX);
-            send_message(tfs->msys, leader_idx(tfs), &msg.base);
+            send_message_to_server(tfs, leader_idx(tfs), &msg.base);
         }
         break;
     case STEP_GET:
@@ -345,7 +374,7 @@ static void replay_request(ToastyFS *tfs)
             };
             memcpy(msg.bucket, tfs->bucket, META_BUCKET_MAX);
             memcpy(msg.key, tfs->key, META_KEY_MAX);
-            send_message(tfs->msys, leader_idx(tfs), &msg.base);
+            send_message_to_server(tfs, leader_idx(tfs), &msg.base);
         }
         break;
     default:
@@ -353,17 +382,10 @@ static void replay_request(ToastyFS *tfs)
     }
 }
 
-static int process_message(ToastyFS *tfs, uint16_t type, string msg)
+static int process_message(ToastyFS *tfs, int conn_idx,
+    uint8_t type, ByteView msg)
 {
-    // Discard messages that arrive after the operation has completed or
+    (void) conn_idx;
-    // before a new one has started.  This handles late/stale responses
-    // from previous operations (e.g. a STORE_CHUNK_ACK arriving after the
-    // client already moved on to a GET).
-    if (tfs->step == STEP_IDLE
-        || tfs->step == STEP_PUT_DONE
-        || tfs->step == STEP_GET_DONE
-        || tfs->step == STEP_DELETE_DONE)
-        return 0;
 
     switch (tfs->step) {
     case STEP_FETCH_CHUNK:
@@ -371,7 +393,7 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
             if (type == MESSAGE_TYPE_FETCH_CHUNK_RESPONSE) {
 
                 FetchChunkResponseMessage resp;
-                if (msg.len < (int) sizeof(FetchChunkResponseMessage))
+                if (msg.len < sizeof(FetchChunkResponseMessage))
                     return -1;
                 memcpy(&resp, msg.ptr, sizeof(resp));
                 char *data = (char *)msg.ptr + sizeof(resp);
@@ -429,27 +451,10 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
                     assert(transfer_idx > -1);
 
                     tfs->transfers[transfer_idx].state = TRANSFER_COMPLETED;
-                    // NOTE: Do NOT abort pending transfers for this hash.
+                    mark_waiting_transfers_for_hash_as_aborted(tfs, ack.hash);
-                    // We need all REPLICATION_FACTOR copies to be stored.
 
                     begin_transfers(tfs);
-                    if (all_transfers_finished(tfs)) {
+                    if (all_chunk_transfers_completed(tfs)) {
-
-                        // Extract holder info from completed transfers
-                        for (int i = 0; i < tfs->num_chunks; i++) {
-                            tfs->chunk_num_holders[i] = 0;
-                            for (int t = 0; t < tfs->num_transfers; t++) {
-                                if (!memcmp(&tfs->transfers[t].hash, &tfs->chunks[i], sizeof(SHA256))
-                                    && tfs->transfers[t].state == TRANSFER_COMPLETED)
-                                {
-                                    int h = tfs->chunk_num_holders[i];
-                                    if (h < REPLICATION_FACTOR) {
-                                        tfs->chunk_holders[i][h] = (uint16_t)tfs->transfers[t].location;
-                                        tfs->chunk_num_holders[i]++;
-                                    }
-                                }
-                            }
-                        }
 
                         tfs->request_id++;
                         CommitPutMessage msg = {
@@ -472,11 +477,8 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
                         for (int i = 0; i < tfs->num_chunks; i++) {
                             msg.oper.chunks[i].hash = tfs->chunks[i];
                             msg.oper.chunks[i].size = tfs->chunk_sizes[i];
-                            msg.oper.chunks[i].num_holders = tfs->chunk_num_holders[i];
-                            for (int j = 0; j < tfs->chunk_num_holders[i]; j++)
-                                msg.oper.chunks[i].holders[j] = tfs->chunk_holders[i][j];
                         }
-                        send_message(tfs->msys, leader_idx(tfs), &msg.base);
+                        send_message_to_server(tfs, leader_idx(tfs), &msg.base);
                         tfs->step = STEP_COMMIT;
                         client_log(tfs, "SEND COMMIT_PUT", "key=%s chunks=%d req=%lu",
                             tfs->key, tfs->num_chunks, tfs->request_id);
@@ -512,8 +514,8 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
 
             } else if (type == MESSAGE_TYPE_REPLY) {
 
-                VsrReplyMessage reply;
+                ReplyMessage reply;
-                if (msg.len != sizeof(VsrReplyMessage))
+                if (msg.len != sizeof(ReplyMessage))
                     return -1;
                 memcpy(&reply, msg.ptr, sizeof(reply));
 
@@ -563,8 +565,8 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
 
             } else if (type == MESSAGE_TYPE_REPLY) {
 
-                VsrReplyMessage reply;
+                ReplyMessage reply;
-                if (msg.len != sizeof(VsrReplyMessage))
+                if (msg.len != sizeof(ReplyMessage))
                     return -1;
                 memcpy(&reply, msg.ptr, sizeof(reply));
 
@@ -633,9 +635,12 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
 
                     tfs->num_transfers = 0;
                     for (int i = 0; i < (int)resp.num_chunks; i++) {
-                        for (int j = 0; j < resp.chunks[i].num_holders; j++) {
+                        // TODO: The server selection formula is a temporary
+                        //       solution. Figure out a proper strategy for
+                        //       picking which servers to fetch chunks from.
+                        for (int j = 0; j < REPLICATION_FACTOR; j++) {
                             add_transfer(tfs, resp.chunks[i].hash,
-                                resp.chunks[i].holders[j], NULL, 0);
+                                (i + j) % tfs->num_servers, NULL, 0);
                         }
                         tfs->chunks[i] = resp.chunks[i].hash;
                         tfs->chunk_sizes[i] = resp.chunks[i].size;
@@ -669,19 +674,47 @@ static int process_message(ToastyFS *tfs, uint16_t type, string msg)
     return 0;
 }
 
-void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pnum)
+void toastyfs_process_events(ToastyFS *tfs, void **ctxs,
+    struct pollfd *pdata, int pnum)
 {
-    message_system_process_events(tfs->msys, ctxs, pdata, pnum);
+    Event events[TCP_EVENT_CAPACITY];
+    int num_events = tcp_translate_events(&tfs->tcp, events, ctxs, pdata, pnum);
 
-    void *raw_message;
+    for (int i = 0; i < num_events; i++) {
-    while ((raw_message = get_next_message(tfs->msys)) != NULL) {
+
-        Message *header = (Message *)raw_message;
+        if (events[i].type == EVENT_DISCONNECT) {
-        string msg_view = { .ptr = raw_message, .len = header->length };
+            int conn_idx = events[i].conn_idx;
-        process_message(tfs, header->type, msg_view);
+            client_log(tfs, "DISCONNECT", "conn=%d", conn_idx);
-        consume_message(tfs->msys, raw_message);
+            tcp_close(&tfs->tcp, conn_idx);
+            continue;
         }
 
-    // Check for operation timeout -- retry the current operation if the
+        if (events[i].type != EVENT_MESSAGE)
+            continue;
+
+        int conn_idx = events[i].conn_idx;
+        for (;;) {
+            ByteView msg;
+            uint16_t msg_type;
+            int ret = tcp_next_message(&tfs->tcp, conn_idx, &msg, &msg_type);
+            if (ret == 0)
+                break;
+            if (ret < 0) {
+                tcp_close(&tfs->tcp, conn_idx);
+                break;
+            }
+
+            ret = process_message(tfs, conn_idx, msg_type, msg);
+            if (ret < 0) {
+                tcp_close(&tfs->tcp, conn_idx);
+                break;
+            }
+
+            tcp_consume_message(&tfs->tcp, conn_idx);
+        }
+    }
+
+    // Check for operation timeout — retry the current operation if the
     // deadline has passed (handles initial sends that were dropped because
     // the TCP connection wasn't established yet, and unresponsive servers).
     if (tfs->step != STEP_IDLE
@@ -717,7 +750,8 @@ void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i
 // TODO: The toastyfs client needs to determine a timeout based on the
 //       pending operation status, not just use PRIMARY_DEATH_TIMEOUT_SEC
 //       for everything.
-int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pcap, int *timeout)
+int toastyfs_register_events(ToastyFS *tfs, void **ctxs,
+    struct pollfd *pdata, int pcap, int *timeout)
 {
     Time now = get_current_time(); // TODO: Handle INVALID_TIME error
     Time deadline = INVALID_TIME;
@@ -727,9 +761,9 @@ int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i
     }
 
     *timeout = deadline_to_timeout(deadline, now);
-    if (pcap < POLL_CAPACITY)
+    if (pcap < TCP_POLL_CAPACITY)
         return -1;
-    return message_system_register_events(tfs->msys, ctxs, pdata, pcap);
+    return tcp_register_events(&tfs->tcp, ctxs, pdata);
 }
 
 static void
@@ -835,7 +869,7 @@ int toastyfs_async_get(ToastyFS *tfs, char *key, int key_len)
     tfs->step_time = get_current_time(); // TODO: Handle INVALID_TIME error
     tfs->step = STEP_GET;
     client_log(tfs, "ASYNC GET", "key=%s leader=%d", tfs->key, leader_idx(tfs));
-    send_message(tfs->msys, leader_idx(tfs), &msg.base);
+    send_message_to_server(tfs, leader_idx(tfs), &msg.base);
     return 0;
 }
 
@@ -869,7 +903,7 @@ int toastyfs_async_delete(ToastyFS *tfs, char *key, int key_len)
     tfs->step_time = get_current_time(); // TODO: Handle INVALID_TIME error
     tfs->step = STEP_DELETE;
     client_log(tfs, "ASYNC DELETE", "key=%s req=%lu leader=%d", tfs->key, tfs->request_id, leader_idx(tfs));
-    send_message(tfs->msys, leader_idx(tfs), &msg.base);
+    send_message_to_server(tfs, leader_idx(tfs), &msg.base);
     return 0;
 }
 
@@ -959,17 +993,6 @@ static void get_result(ToastyFS *tfs, ToastyFS_Result *result)
     result->size  = blob_size;
 }
 
-static bool
-all_transfers_finished(ToastyFS *tfs)
-{
-    for (int i = 0; i < tfs->num_transfers; i++) {
-        if (tfs->transfers[i].state == TRANSFER_PENDING
-            || tfs->transfers[i].state == TRANSFER_STARTED)
-            return false;
-    }
-    return true;
-}
-
 static bool
 all_chunk_transfers_completed(ToastyFS *tfs)
 {
@@ -1053,18 +1076,14 @@ ToastyFS_Result toastyfs_get_result(ToastyFS *tfs)
 static int wait_until_result(ToastyFS *tfs, ToastyFS_Result *res)
 {
     for (;;) {
-        void *ctxs[POLL_CAPACITY];
+        void *ctxs[TCP_POLL_CAPACITY];
-        struct pollfd arr[POLL_CAPACITY];
+        struct pollfd arr[TCP_POLL_CAPACITY];
         int poll_timeout;
-        int num = toastyfs_register_events(tfs, ctxs, arr, POLL_CAPACITY, &poll_timeout);
+        int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY, &poll_timeout);
         if (num < 0)
             return num;
 
-#ifdef _WIN32
-        WSAPoll(arr, num, poll_timeout);
-#else
         poll(arr, num, poll_timeout);
-#endif
 
         toastyfs_process_events(tfs, ctxs, arr, num);
         *res = toastyfs_get_result(tfs);

src/client_table.c

@@ -28,7 +28,7 @@ ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_i
     return NULL;
 }
 
-int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, void *pending_message)
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
 {
     if (client_table->count == client_table->capacity) {
         int n = 2 * client_table->capacity;
@@ -44,7 +44,7 @@ int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t req
         .client_id = client_id,
         .last_request_id = request_id,
         .pending = true,
-        .pending_message = pending_message,
+        .conn_tag = conn_tag,
     };
     return 0;
 }

src/client_table.h

@@ -13,7 +13,7 @@ typedef struct {
                              // the REQUEST. After a view change, a new leader
                              // may find stale entries with pending=false from
                              // a previous view when it was leader before.
-    void           *pending_message; // Raw message pointer for deferred reply
+    int             conn_tag;
 } ClientTableEntry;
 
 typedef struct {
@@ -25,7 +25,7 @@ typedef struct {
 void client_table_init(ClientTable *client_table);
 void client_table_free(ClientTable *client_table);
 ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
-int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, void *pending_message);
+int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
 int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
 
 #endif // CLIENT_TABLE_INCLUDED

src/config.h

@@ -1,8 +1,6 @@
 #ifndef CONFIG_INCLUDED
 #define CONFIG_INCLUDED
 
-#define CEIL(a, b) (((a) + (b) - 1) / (b))
-
 #define NODE_LIMIT 32
 #define FUTURE_LIMIT 32
 #define HEARTBEAT_INTERVAL_SEC 1

src/file_system.c

@@ -36,7 +36,7 @@ int file_open(string path, Handle *fd)
     memcpy(zt, path.ptr, path.len);
     zt[path.len] = '\0';
 
-    int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
+    int ret = open(zt, O_RDWR | O_CREAT, 0644);
     if (ret < 0)
         return -1;
 
@@ -86,12 +86,7 @@ int file_truncate(Handle fd, size_t new_size)
 #endif
 
 #ifdef _WIN32
-    DWORD result = SetFilePointer((HANDLE) fd.data, (LONG) new_size, NULL, FILE_BEGIN);
+    return -1; // TODO: Not implemented
-    if (result == INVALID_SET_FILE_POINTER)
-        return -1;
-    if (!SetEndOfFile((HANDLE) fd.data))
-        return -1;
-    return 0;
 #endif
 }
 
@@ -298,7 +293,7 @@ int get_full_path(string path, char *dst)
 #endif
 
 #ifdef _WIN32
-    if (_fullpath(dst, path_zt, PATH_MAX) == NULL)
+    if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
         return -1;
 #endif
 

src/http_proxy.c

@@ -1,336 +0,0 @@
-#include "http_proxy.h"
-#include "config.h"
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#ifdef _WIN32
-#include <winsock2.h>
-#else
-#include <poll.h>
-#endif
-
-int http_proxy_init(void *state, int argc, char **argv,
-    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
-    int *timeout)
-{
-    HTTPProxy *proxy = state;
-
-    char *addrs[NODE_LIMIT];
-    int num_addrs = 0;
-
-    int      max_opers = 128;
-    string   http_addr = S("127.0.0.1:3000");
-    uint64_t client_id = 999;
-
-    for (int i = 1; i < argc; i++) {
-        if (!strcmp(argv[i], "--server")) {
-            i++;
-            if (i == argc) {
-                fprintf(stderr, "Option --server missing value\n");
-                return -1;
-            }
-            if (num_addrs == NODE_LIMIT) {
-                fprintf(stderr, "Node limit reached\n");
-                return -1;
-            }
-            addrs[num_addrs] = argv[i];
-            num_addrs++;
-        } else if (!strcmp(argv[i], "--addr")) {
-            i++;
-            if (i == argc) {
-                fprintf(stderr, "Option --addr missing value\n");
-                return -1;
-            }
-            http_addr = (string) { argv[i], strlen(argv[i]) };
-        } else if (!strcmp(argv[i], "--max-ops")) {
-            i++;
-            if (i == argc) {
-                fprintf(stderr, "Option --max-ops missing value\n");
-                return -1;
-            }
-            max_opers = atoi(argv[i]); // TODO: don't use atoi
-            if (max_opers == 0) {
-                fprintf(stderr, "Invalid value for option --max-ops\n");
-                return -1;
-            }
-        } else if (!strcmp(argv[i], "--client-id")) {
-            i++;
-            if (i == argc) {
-                fprintf(stderr, "Invalid value for option --client-id\n");
-                return -1;
-            }
-            client_id = atoi(argv[i]);
-            if (client_id == 0) {
-                fprintf(stderr, "Invalid value for option --client-id\n");
-                return -1;
-            }
-        } else {
-            // Ignore unknown options
-        }
-    }
-
-    proxy->max_opers = max_opers;
-    proxy->opers = malloc(max_opers * sizeof(ProxyOper));
-    if (proxy->opers == NULL)
-        return -1;
-
-    for (int i = 0; i < max_opers; i++)
-        proxy->opers[i].state = PROXY_OPER_FREE;
-
-    if (http_server_init(&proxy->http_server, max_opers) < 0) {
-        free(proxy->opers);
-        return -1;
-    }
-
-    Address http_addr_2;
-    if (parse_addr_arg(http_addr, &http_addr_2) < 0) {
-        http_server_free(&proxy->http_server);
-        free(proxy->opers);
-        return -1;
-    }
-    if (http_server_listen_tcp(&proxy->http_server, http_addr_2) < 0) {
-        http_server_free(&proxy->http_server);
-        free(proxy->opers);
-        return -1;
-    }
-
-    proxy->toastyfs = toastyfs_init(client_id, addrs, num_addrs);
-    if (proxy->toastyfs == NULL) {
-        http_server_free(&proxy->http_server);
-        free(proxy->opers);
-        return -1;
-    }
-
-    // Register events that need to be monitored
-    {
-        int ret;
-        *pnum = 0;
-        *timeout = -1;
-
-        ret = toastyfs_register_events(proxy->toastyfs, ctxs, pdata, pcap, timeout);
-        if (ret < 0)
-            return -1;
-        *pnum += ret;
-        proxy->num_polled_by_toasty = ret;
-
-        ret = http_server_register_events(&proxy->http_server,
-            ctxs + proxy->num_polled_by_toasty,
-            pdata + proxy->num_polled_by_toasty,
-            pcap - proxy->num_polled_by_toasty);
-        if (ret < 0)
-            return -1;
-        *pnum += ret;
-    }
-    return 0;
-}
-
-int http_proxy_tick(void *state, void **ctxs,
-    struct pollfd *pdata, int pcap, int *pnum, int *timeout)
-{
-    HTTPProxy *proxy = state;
-
-    http_server_process_events(&proxy->http_server,
-        ctxs  + proxy->num_polled_by_toasty,
-        pdata + proxy->num_polled_by_toasty,
-        *pnum - proxy->num_polled_by_toasty);
-    toastyfs_process_events(proxy->toastyfs,
-        ctxs, pdata, proxy->num_polled_by_toasty);
-
-    // Process operation resolutions
-    for (;;) {
-        ToastyFS_Result result = toastyfs_get_result(proxy->toastyfs);
-        if (result.type == TOASTYFS_RESULT_VOID)
-            break;
-
-        // Find the started operation
-        int i = 0;
-        while (i < proxy->max_opers && proxy->opers[i].state != PROXY_OPER_STARTED)
-            i++;
-        assert(i < proxy->max_opers); // Wasn't expecting this result
-
-        ProxyOper *oper = &proxy->opers[i];
-        assert(oper->state == PROXY_OPER_STARTED);
-
-        HTTP_ResponseBuilder builder = oper->builder;
-
-        switch (result.type) {
-        case TOASTYFS_RESULT_PUT:
-            if (result.error == TOASTYFS_ERROR_VOID) {
-                http_response_builder_status(builder, 201);
-                http_response_builder_submit(builder);
-            } else if (result.error == TOASTYFS_ERROR_FULL) {
-                http_response_builder_status(builder, 507);
-                http_response_builder_submit(builder);
-            } else {
-                http_response_builder_status(builder, 500);
-                http_response_builder_submit(builder);
-            }
-            break;
-        case TOASTYFS_RESULT_GET:
-            if (result.error == TOASTYFS_ERROR_VOID) {
-                http_response_builder_status(builder, 200);
-                http_response_builder_content(builder, (string) { result.data, result.size });
-                http_response_builder_submit(builder);
-                free(result.data);
-            } else if (result.error == TOASTYFS_ERROR_NOT_FOUND) {
-                http_response_builder_status(builder, 404);
-                http_response_builder_submit(builder);
-            } else {
-                http_response_builder_status(builder, 500);
-                http_response_builder_submit(builder);
-            }
-            break;
-        case TOASTYFS_RESULT_DELETE:
-            if (result.error == TOASTYFS_ERROR_VOID) {
-                http_response_builder_status(builder, 204);
-                http_response_builder_submit(builder);
-            } else if (result.error == TOASTYFS_ERROR_NOT_FOUND) {
-                http_response_builder_status(builder, 404);
-                http_response_builder_submit(builder);
-            } else {
-                http_response_builder_status(builder, 500);
-                http_response_builder_submit(builder);
-            }
-            break;
-        default:
-            UNREACHABLE;
-        }
-
-        oper->state = PROXY_OPER_FREE;
-    }
-
-    // Discard pending operations whose connection has been closed.
-    // When a client disconnects, the HTTP_Conn is freed and the builder
-    // becomes invalid.  The request pointers (url, body) reference the
-    // now-freed TCP read buffer, so we must not dereference them.
-    for (int i = 0; i < proxy->max_opers; i++) {
-        if (proxy->opers[i].state == PROXY_OPER_PENDING
-            && !http_response_builder_is_valid(proxy->opers[i].builder)) {
-            proxy->opers[i].state = PROXY_OPER_FREE;
-        }
-    }
-
-    // Buffer operation requests
-    for (;;) {
-        HTTP_Request *request;
-        HTTP_ResponseBuilder builder;
-        if (!http_server_next_request(&proxy->http_server, &request, &builder))
-            break;
-
-        // Only allow GET, PUT, DELETE requests
-        if (request->method != CHTTP_METHOD_GET &&
-            request->method != CHTTP_METHOD_PUT &&
-            request->method != CHTTP_METHOD_DELETE) {
-            http_response_builder_status(builder, 405);
-            http_response_builder_submit(builder);
-            continue;
-        }
-
-        // Look for a free operation slot
-        int i = 0;
-        while (i < proxy->max_opers && proxy->opers[i].state != PROXY_OPER_FREE)
-            i++;
-
-        if (i == proxy->max_opers) {
-            // Queue is full
-            http_response_builder_status(builder, 503);
-            http_response_builder_submit(builder);
-            continue;
-        }
-
-        ProxyOper *oper = &proxy->opers[i];
-        assert(oper->state == PROXY_OPER_FREE);
-
-        oper->state = PROXY_OPER_PENDING;
-        oper->request = request;
-        oper->builder = builder;
-    }
-
-    // Start operations
-    {
-        // Look for a started operation
-        bool started = false;
-        for (int i = 0; i < proxy->max_opers; i++) {
-            if (proxy->opers[i].state == PROXY_OPER_STARTED) {
-                started = true;
-                break;
-            }
-        }
-
-        // Start an operation if necessary
-        if (!started) {
-            // Look for a pending operation
-            int i = 0;
-            while (i < proxy->max_opers && proxy->opers[i].state != PROXY_OPER_PENDING)
-                i++;
-
-            if (i < proxy->max_opers) {
-                // Found pending operation
-                ProxyOper *oper = &proxy->opers[i];
-                HTTP_Request *request = oper->request;
-                int ret = -1;
-
-                switch (request->method) {
-                case CHTTP_METHOD_GET:
-                    ret = toastyfs_async_get(proxy->toastyfs,
-                        request->url.path.ptr, request->url.path.len);
-                    break;
-                case CHTTP_METHOD_PUT:
-                    ret = toastyfs_async_put(proxy->toastyfs,
-                        request->url.path.ptr, request->url.path.len,
-                        request->body.ptr, request->body.len);
-                    break;
-                case CHTTP_METHOD_DELETE:
-                    ret = toastyfs_async_delete(proxy->toastyfs,
-                        request->url.path.ptr, request->url.path.len);
-                    break;
-                default:
-                    UNREACHABLE;
-                }
-
-                if (ret < 0) {
-                    // Async operation failed to start -- respond with error
-                    // and free the slot so the proxy doesn't get stuck.
-                    http_response_builder_status(oper->builder, 500);
-                    http_response_builder_submit(oper->builder);
-                    oper->state = PROXY_OPER_FREE;
-                } else {
-                    oper->state = PROXY_OPER_STARTED;
-                }
-            }
-        }
-    }
-
-    // Register events that need to be monitored
-    {
-        int ret;
-        *pnum = 0;
-        *timeout = -1;
-
-        ret = toastyfs_register_events(proxy->toastyfs, ctxs, pdata, pcap, timeout);
-        if (ret < 0)
-            return -1;
-        *pnum += ret;
-        proxy->num_polled_by_toasty = ret;
-
-        ret = http_server_register_events(&proxy->http_server,
-            ctxs + proxy->num_polled_by_toasty,
-            pdata + proxy->num_polled_by_toasty,
-            pcap - proxy->num_polled_by_toasty);
-        if (ret < 0)
-            return -1;
-        *pnum += ret;
-    }
-    return 0;
-}
-
-int http_proxy_free(void *state)
-{
-    HTTPProxy *proxy = state;
-
-    toastyfs_free(proxy->toastyfs);
-    http_server_free(&proxy->http_server);
-    free(proxy->opers);
-    return 0;
-}

src/http_proxy.h

@@ -1,38 +0,0 @@
-#ifndef HTTP_PROXY_INCLUDED
-#define HTTP_PROXY_INCLUDED
-
-#include <toastyfs.h>
-#include <lib/http_server.h>
-
-typedef enum {
-    PROXY_OPER_FREE,
-    PROXY_OPER_PENDING,
-    PROXY_OPER_STARTED,
-} ProxyOperState;
-
-typedef struct {
-    ProxyOperState       state;
-    HTTP_Request*        request;
-    HTTP_ResponseBuilder builder;
-} ProxyOper;
-
-typedef struct {
-    HTTP_Server http_server;
-    ToastyFS *toastyfs;
-    ProxyOper *opers;
-    int max_opers;
-    int num_polled_by_toasty;
-} HTTPProxy;
-
-struct pollfd;
-
-int http_proxy_init(void *state, int argc, char **argv,
-    void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
-    int *timeout);
-
-int http_proxy_tick(void *state, void **ctxs,
-    struct pollfd *pdata, int pcap, int *pnum, int *timeout);
-
-int http_proxy_free(void *state);
-
-#endif // HTTP_PROXY_INCLUDED

src/invariant_checker.c

@@ -1,5 +1,3 @@
-#ifdef MAIN_SIMULATION
-
 // VSR invariant checker with external shadow log tracking.
 //
 // This file runs in the main simulation loop outside Quakey-scheduled
@@ -11,23 +9,16 @@
 #include "chunk_store.h"
 
 #include <assert.h>
-#include <stdio.h>
 
 // Restore real allocators (see checker/linearizability.c for precedent).
 #undef malloc
 #undef realloc
 #undef free
 #include <stdlib.h>
-#include <string.h>
-
-// Forward declarations for quakey host context functions
-// (we can't include quakey.h because it would re-mock malloc/realloc/free)
-void quakey_enter_host(unsigned long long node);
-void quakey_leave_host(void);
 
 // These helpers are static in node.c; duplicated here for the checker.
 
-static int self_idx(Server *state)
+static int self_idx(ServerState *state)
 {
     for (int i = 0; i < state->num_nodes; i++)
         if (addr_eql(state->node_addrs[i], state->self_addr))
@@ -35,12 +26,12 @@ static int self_idx(Server *state)
     UNREACHABLE;
 }
 
-static int leader_idx(Server *state)
+static int leader_idx(ServerState *state)
 {
     return state->view_number % state->num_nodes;
 }
 
-static bool is_leader(Server *state)
+static bool is_leader(ServerState *state)
 {
     if (state->status == STATUS_RECOVERY)
         return false;
@@ -67,8 +58,10 @@ void invariant_checker_init(InvariantChecker *ic)
 {
     ic->last_min_commit = -1;
     ic->last_max_commit = -1;
-    for (int i = 0; i < NODE_LIMIT; i++)
+    for (int i = 0; i < NODE_LIMIT; i++) {
         ic->prev_status[i] = STATUS_NORMAL;
+        ic->prev_alive[i] = false;
+    }
     ic->shadow_log = NULL;
     ic->shadow_count = 0;
     ic->shadow_capacity = 0;
@@ -81,7 +74,7 @@ void invariant_checker_free(InvariantChecker *ic)
     free(ic->shadow_log);
 }
 
-void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
+void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
     unsigned long long *node_handles)
 {
     int min_commit = -1;
@@ -99,13 +92,17 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
     }
 
     for (int i = 0; i < num_nodes; i++) {
-        Server *n = nodes[i];
+        ServerState *n = nodes[i];
         if (n == NULL || n->status == STATUS_RECOVERY)
             continue;
 
         if (min_commit < 0 || min_commit > n->commit_index) {
             min_commit = n->commit_index;
-            min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY);
+            // A node that just recovered or just restarted after a crash
+            // may have a stale commit_index (persisted state can lag behind
+            // the in-memory value if the crash interrupted file_sync).
+            min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY)
+                                     || !ic->prev_alive[i];
         }
 
         if (max_commit < 0 || max_commit < n->commit_index) {
@@ -162,20 +159,21 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
     ic->last_min_commit = min_commit;
     ic->last_max_commit = max_commit;
     for (int i = 0; i < num_nodes; i++) {
+        ic->prev_alive[i] = (nodes[i] != NULL);
         if (nodes[i])
             ic->prev_status[i] = nodes[i]->status;
     }
 
     for (int i = 0; i < num_nodes; i++) {
-        Server *s = nodes[i];
+        ServerState *s = nodes[i];
         if (s == NULL)
             continue;
 
         // 1. commit_index <= log.count
         //    A node cannot have committed more entries than it has in its log.
-        if (s->commit_index > s->log.count) {
+        if (s->commit_index > s->wal.count) {
             fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n",
-                i, s->commit_index, s->log.count);
+                i, s->commit_index, s->wal.count);
             __builtin_trap();
         }
 
@@ -215,11 +213,11 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
         // 9. Log entry view numbers must not exceed the node's current view.
         //    Entries are created in the view they were proposed. No entry
         //    should carry a view number from the future.
-        for (int k = 0; k < s->log.count; k++) {
+        for (int k = 0; k < s->wal.count; k++) {
-            if ((uint64_t)s->log.entries[k].view_number > s->view_number) {
+            if ((uint64_t)s->wal.entries[k].view_number > s->view_number) {
                 fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) "
                     "> view_number (%lu)\n",
-                    i, k, s->log.entries[k].view_number,
+                    i, k, s->wal.entries[k].view_number,
                     (unsigned long)s->view_number);
                 __builtin_trap();
             }
@@ -230,8 +228,8 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
         //     votes for its own entries.
         if (s->status == STATUS_NORMAL && is_leader(s)) {
             int idx = self_idx(s);
-            for (int k = s->commit_index; k < s->log.count; k++) {
+            for (int k = s->commit_index; k < s->wal.count; k++) {
-                if (!(s->log.entries[k].votes & (1 << idx))) {
+                if (!(s->wal.entries[k].votes & (1 << idx))) {
                     fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): "
                         "uncommitted log[%d] missing leader's own vote bit\n",
                         i, k);
@@ -275,7 +273,7 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
                 mc = nodes[j]->commit_index;
 
             for (int k = 0; k < mc; k++) {
-                if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(MetaOper)) != 0) {
+                if (memcmp(&nodes[i]->wal.entries[k].oper, &nodes[j]->wal.entries[k].oper, sizeof(MetaOper)) != 0) {
                     fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d "
                         "between node %d and node %d\n", k, i, j);
                     __builtin_trap();
@@ -306,12 +304,12 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
     // Phase 2: Append newly committed entries to the shadow log.
     if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
 
-        Server *source = nodes[source_node_idx];
+        ServerState *source = nodes[source_node_idx];
-        assert(source->log.count >= observed_max_commit);
+        assert(source->wal.count >= observed_max_commit);
 
         for (int k = ic->shadow_count; k < observed_max_commit; k++) {
 
-            MetaOper *source_oper = &source->log.entries[k].oper;
+            MetaOper *source_oper = &source->wal.entries[k].oper;
 
             // Cross-validate against other live non-recovering nodes
             // that have also committed this entry.
@@ -324,10 +322,10 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
                     continue;
                 if (nodes[j]->commit_index <= k)
                     continue;
-                if (nodes[j]->log.count <= k)
+                if (nodes[j]->wal.count <= k)
                     continue;
 
-                if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(MetaOper)) != 0) {
+                if (memcmp(&nodes[j]->wal.entries[k].oper, source_oper, sizeof(MetaOper)) != 0) {
                     fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
                         "between source node %d and node %d during shadow log append\n",
                         k, source_node_idx, j);
@@ -349,15 +347,15 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
         for (int i = 0; i < num_nodes; i++) {
             if (nodes[i] == NULL)
                 continue;
-            if (nodes[i]->log.count <= k)
+            if (nodes[i]->wal.count <= k)
                 continue;
             if (nodes[i]->commit_index <= k)
                 continue;
 
-            if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) != 0) {
+            if (memcmp(&nodes[i]->wal.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) != 0) {
                 char shadow_buf[128], node_buf[128];
                 meta_snprint_oper(shadow_buf, sizeof(shadow_buf), &ic->shadow_log[k]);
-                meta_snprint_oper(node_buf, sizeof(node_buf), &nodes[i]->log.entries[k].oper);
+                meta_snprint_oper(node_buf, sizeof(node_buf), &nodes[i]->wal.entries[k].oper);
                 fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
                     "  shadow: %s\n"
                     "  node:   %s\n",
@@ -386,9 +384,9 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
                     num_dead++;
                     continue;
                 }
-                if (nodes[i]->log.count <= k)
+                if (nodes[i]->wal.count <= k)
                     continue;
-                if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) == 0)
+                if (memcmp(&nodes[i]->wal.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) == 0)
                     holders++;
             }
 
@@ -452,5 +450,3 @@ void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
         }
     }
 }
-
-#endif // MAIN_SIMULATION

src/log.c

@@ -1,3 +1,4 @@
+#if 0
 #if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
 #define QUAKEY_ENABLE_MOCKS
 #endif
@@ -53,3 +54,5 @@ int log_append(Log *log, LogEntry entry)
     log->entries[log->count++] = entry;
     return 0;
 }
+
+#endif

src/log.h

@@ -1,3 +1,4 @@
+#if 0
 #ifndef LOG_INCLUDED
 #define LOG_INCLUDED
 
@@ -28,3 +29,4 @@ void log_move(Log *dst, Log *src);
 int  log_append(Log *log, LogEntry entry);
 
 #endif // LOG_INCLUDED
+#endif

src/main.c

@@ -52,6 +52,7 @@ int main(int argc, char **argv)
             .addrs      = (char*[]) { "127.0.0.2" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         (void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
     }
@@ -67,6 +68,7 @@ int main(int argc, char **argv)
             .addrs      = (char*[]) { "127.0.0.3" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         (void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
     }
@@ -82,6 +84,7 @@ int main(int argc, char **argv)
             .addrs      = (char*[]) { "127.0.0.7" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         (void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
     }
@@ -90,13 +93,14 @@ int main(int argc, char **argv)
     {
         QuakeySpawn config = {
             .name       = "server1",
-            .state_size = sizeof(Server),
+            .state_size = sizeof(ServerState),
             .init_func  = server_init,
             .tick_func  = server_tick,
             .free_func  = server_free,
             .addrs      = (char*[]) { "127.0.0.4" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
     }
@@ -105,13 +109,14 @@ int main(int argc, char **argv)
     {
         QuakeySpawn config = {
             .name       = "server2",
-            .state_size = sizeof(Server),
+            .state_size = sizeof(ServerState),
             .init_func  = server_init,
             .tick_func  = server_tick,
             .free_func  = server_free,
             .addrs      = (char*[]) { "127.0.0.5" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
     }
@@ -120,13 +125,14 @@ int main(int argc, char **argv)
     {
         QuakeySpawn config = {
             .name       = "server3",
-            .state_size = sizeof(Server),
+            .state_size = sizeof(ServerState),
             .init_func  = server_init,
             .tick_func  = server_tick,
             .free_func  = server_free,
             .addrs      = (char*[]) { "127.0.0.6" },
             .num_addrs  = 1,
             .disk_size  = 10<<20,
+            .platform   = QUAKEY_LINUX,
         };
         node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
     }
@@ -145,7 +151,7 @@ int main(int argc, char **argv)
 
         quakey_schedule_one(quakey);
 
-        Server *arr[] = {
+        ServerState *arr[] = {
             quakey_node_state(node_1),
             quakey_node_state(node_2),
             quakey_node_state(node_3),
@@ -176,11 +182,7 @@ int main(int argc, char **argv)
 ////////////////////////////////////////////////////////////////////
 #ifdef MAIN_CLIENT
 
-#ifdef _WIN32
-#include <winsock2.h>
-#else
 #include <poll.h>
-#endif
 #include <stdlib.h>
 #include <time.h>
 #include <unistd.h>
@@ -244,14 +246,8 @@ int main(int argc, char **argv)
 ////////////////////////////////////////////////////////////////////
 #ifdef MAIN_SERVER
 
-#ifdef _WIN32
-#include <winsock2.h>
-#else
 #include <poll.h>
-#endif
 
-#include <stdio.h>
-#include <stdlib.h>
 #include "server.h"
 
 #define POLL_CAPACITY 1024
@@ -260,11 +256,11 @@ int main(int argc, char **argv)
 {
     int ret;
 
-    // Server is ~40 MB (MetaStore holds 4096 ObjectMeta entries),
+    // ServerState is ~40 MB (MetaStore holds 4096 ObjectMeta entries),
     // which exceeds the default stack size.  Heap-allocate it.
-    Server *state = malloc(sizeof(Server));
+    ServerState *state = malloc(sizeof(ServerState));
     if (state == NULL) {
-        fprintf(stderr, "Failed to allocate Server\n");
+        fprintf(stderr, "Failed to allocate ServerState\n");
         return -1;
     }
 
@@ -312,80 +308,3 @@ int main(int argc, char **argv)
 }
 
 #endif // MAIN_SERVER
-////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////
-#ifdef MAIN_HTTP_PROXY
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <stdlib.h>
-#include <stdio.h>
-static void invalid_param_handler(const wchar_t *expr, const wchar_t *func,
-    const wchar_t *file, unsigned int line, uintptr_t reserved)
-{
-    (void)expr; (void)func; (void)file; (void)line; (void)reserved;
-    fprintf(stderr, "[CRT] Invalid parameter in %ls (%ls:%u)\n",
-        func ? func : L"?", file ? file : L"?", line);
-    fflush(stderr);
-}
-#else
-#include <poll.h>
-#endif
-
-#include "http_proxy.h"
-
-#define POLL_CAPACITY 1024
-
-int main(int argc, char **argv)
-{
-#ifdef _WIN32
-    _set_invalid_parameter_handler(invalid_param_handler);
-#endif
-    int ret;
-
-    HTTPProxy state;
-
-    void*         poll_ctxs[POLL_CAPACITY];
-    struct pollfd poll_array[POLL_CAPACITY];
-    int poll_count;
-    int poll_timeout;
-
-    ret = http_proxy_init(
-        &state,
-        argc,
-        argv,
-        poll_ctxs,
-        poll_array,
-        POLL_CAPACITY,
-        &poll_count,
-        &poll_timeout
-    );
-    if (ret < 0)
-        return -1;
-
-    for (;;) {
-
-#ifdef _WIN32
-        WSAPoll(poll_array, poll_count, poll_timeout);
-#else
-        poll(poll_array, poll_count, poll_timeout);
-#endif
-
-        ret = http_proxy_tick(
-            &state,
-            poll_ctxs,
-            poll_array,
-            POLL_CAPACITY,
-            &poll_count,
-            &poll_timeout
-        );
-        if (ret < 0)
-            return -1;
-    }
-
-    http_proxy_free(&state);
-    return 0;
-}
-
-#endif // MAIN_HTTP_PROXY

src/message.c

@@ -0,0 +1,81 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "message.h"
+
+bool binary_read(BinaryReader *reader, void *dst, int len)
+{
+    if (reader->len - reader->cur < len)
+        return false;
+    if (dst)
+        memcpy(dst, reader->src + reader->cur, len);
+    reader->cur += len;
+    return true;
+}
+
+void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type)
+{
+    uint16_t version = MESSAGE_VERSION;
+    uint32_t dummy = 0; // Dummy value
+    writer->output = output;
+    writer->start  = byte_queue_offset(output);
+    byte_queue_write(output, &version, sizeof(version));
+    byte_queue_write(output, &type, sizeof(type));
+    writer->patch = byte_queue_offset(output);
+    byte_queue_write(output, &dummy, sizeof(dummy));
+}
+
+bool message_writer_free(MessageWriter *writer)
+{
+    uint32_t length = byte_queue_size_from_offset(writer->output, writer->start);
+    byte_queue_patch(writer->output, writer->patch, &length, sizeof(length));
+    if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure?
+        return false;
+    return true;
+}
+
+void message_write(MessageWriter *writer, void *mem, int len)
+{
+    byte_queue_write(writer->output, mem, len);
+}
+
+void message_write_u8(MessageWriter *writer, uint8_t value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+void message_write_u32(MessageWriter *writer, uint32_t value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+void message_write_hash(MessageWriter *writer, SHA256 value)
+{
+    message_write(writer, &value, (int) sizeof(value));
+}
+
+int message_peek(ByteView msg, uint16_t *type, uint32_t *len)
+{
+    if (msg.len < (int) sizeof(MessageHeader))
+        return 0;
+
+    MessageHeader header;
+    memcpy(&header, msg.ptr, sizeof(header));
+
+    // (We ignore endianess for now)
+
+    if (header.version != MESSAGE_VERSION)
+        return -1;
+
+    if (header.length > msg.len)
+        return 0;
+
+    if (type) *type = header.type;
+    if (len) *len = header.length;
+
+    return 1;
+}

src/message.h

@@ -0,0 +1,46 @@
+#ifndef MESSAGE_INCLUDED
+#define MESSAGE_INCLUDED
+
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <stdint.h>
+#include <quakey.h>
+
+#include "basic.h"
+#include "byte_queue.h"
+
+#define MESSAGE_VERSION 1
+
+typedef struct {
+    uint8_t *src;
+    int      len;
+    int      cur;
+} BinaryReader;
+
+typedef struct {
+    uint16_t version;
+    uint16_t type;
+    uint32_t length;
+} MessageHeader;
+
+typedef struct {
+    ByteQueue *output;
+    ByteQueueOffset start;
+    ByteQueueOffset patch;
+} MessageWriter;
+
+bool binary_read(BinaryReader *reader, void *dst, int len);
+
+void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type);
+bool message_writer_free(MessageWriter *writer);
+void message_write(MessageWriter *writer, void *mem, int len);
+void message_write_u8(MessageWriter *writer, uint8_t value);
+void message_write_u32(MessageWriter *writer, uint32_t value);
+void message_write_hash(MessageWriter *writer, SHA256 value);
+
+int  message_peek(ByteView msg, uint16_t *type, uint32_t *len);
+void message_dump(FILE *stream, ByteView msg);
+
+#endif // MESSAGE_INCLUDED

src/metadata.h

@@ -4,14 +4,11 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-#include <lib/basic.h>
+#include "basic.h"
-#include "config.h"
 
 typedef struct {
     SHA256   hash;
     uint32_t size;
-    uint16_t num_holders;
-    uint16_t holders[REPLICATION_FACTOR];
 } ChunkRef;
 
 #define META_BUCKET_MAX  64

src/random_client.c

@@ -82,7 +82,10 @@ int random_client_init(void *state_, int argc, char **argv,
         return -1;
     state->started = false;
 
-    (void) pcap;
+    if (pcap < TCP_POLL_CAPACITY) {
+        fprintf(stderr, "Random client :: Not enough poll capacity\n");
+        return -1;
+    }
     *pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout);
     *timeout = 0; // Ensure first tick fires immediately to start an operation
     return 0;

src/random_client.h

@@ -3,8 +3,8 @@
 
 #include <toastyfs.h>
 
-#include <lib/tcp.h>
+#include "tcp.h"
-#include <lib/basic.h>
+#include "basic.h"
 
 #include "config.h"
 #include "metadata.h"

src/server.h

@@ -1,16 +1,15 @@
-#ifndef SERVER_INCLUDED
+#ifndef NODE_INCLUDED
-#define SERVER_INCLUDED
+#define NODE_INCLUDED
 
-#include <lib/message.h>
+#include "tcp.h"
-
+#include "basic.h"
-#include "log.h"
+#include "message.h"
+#include "wal.h"
 #include "config.h"
 #include "metadata.h"
 #include "chunk_store.h"
 #include "client_table.h"
 
-#define MESSAGE_VERSION 1
-
 enum {
 
     // Normal Protocol
@@ -49,14 +48,14 @@ enum {
 };
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     MetaOper oper;
     uint64_t client_id;
     uint64_t request_id;
 } RequestMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     MetaOper oper;
     int sender_idx;
     int log_index;
@@ -67,58 +66,58 @@ typedef struct {
 } PrepareMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     int sender_idx;
     int log_index;
     uint64_t view_number;
 } PrepareOKMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int sender_idx;
     int commit_index;
 } CommitMessage;
 
 typedef struct {
-    Message    base;
+    MessageHeader base;
     bool rejected;
     MetaResult result;
     uint64_t request_id;
-} VsrReplyMessage;
+} ReplyMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int sender_idx;
 } BeginViewChangeMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;       // The new view number
     uint64_t old_view_number;   // Last view number when replica was in normal status
     int op_number;              // Number of entries in the log
     int commit_index;
     int sender_idx;
-    // Followed by: LogEntry log[op_number]
+    // Followed by: WALEntry log[op_number]
 } DoViewChangeMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int commit_index;
-    int      op_number;  // Number of log entries that follow
+    int op_number;  // Number of WAL entries that follow
-    // Followed by: LogEntry log[op_number]
+    // Followed by: WALEntry log[op_number]
 } BeginViewMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     int sender_idx;
     uint64_t nonce;
 } RecoveryMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int op_number;
     uint64_t nonce;
@@ -127,29 +126,29 @@ typedef struct {
 } RecoveryResponseMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int op_number;      // Requester's current log count
     int sender_idx;
 } GetStateMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
     int op_number;      // Number of log entries that follow
     int commit_index;
     int start_index;    // Global log index of the first entry in the suffix
-    // Followed by: LogEntry log[op_number]
+    // Followed by: WALEntry log[op_number]
 } NewStateMessage;
 
 typedef struct {
-    Message  base;
+    MessageHeader base;
     uint64_t view_number;
 } RedirectMessage;
 
 // StoreChunk: client -> any server. Carries chunk data as trailing bytes.
 typedef struct {
-    Message  base;
+    MessageHeader base;
     SHA256   hash;
     uint32_t size;
     // Followed by: uint8_t data[size]
@@ -157,21 +156,21 @@ typedef struct {
 
 // StoreChunkAck: server -> client.
 typedef struct {
-    Message base;
+    MessageHeader base;
     SHA256   hash;
     bool     success;
 } StoreChunkAckMessage;
 
 // FetchChunk: client/server -> server. Request a chunk by hash.
 typedef struct {
-    Message base;
+    MessageHeader base;
     SHA256   hash;
     int      sender_idx; // -1 if from a client
 } FetchChunkMessage;
 
 // FetchChunkResponse: server -> client/server. Chunk data as trailing bytes.
 typedef struct {
-    Message  base;
+    MessageHeader base;
     SHA256   hash;
     uint32_t size; // 0 if chunk not found
     // Followed by: uint8_t data[size]
@@ -180,7 +179,7 @@ typedef struct {
 // CommitPut: blob client -> leader. Commits metadata after chunks uploaded.
 // Processed like a REQUEST (goes through VSR log).
 typedef struct {
-    Message  base;
+    MessageHeader base;
     MetaOper oper;
     uint64_t client_id;
     uint64_t request_id;
@@ -188,14 +187,14 @@ typedef struct {
 
 // GetBlob: client -> any server. Request metadata for a blob.
 typedef struct {
-    Message base;
+    MessageHeader base;
     char bucket[META_BUCKET_MAX];
     char key[META_KEY_MAX];
 } GetBlobMessage;
 
 // GetBlobResponse: server -> client. Returns blob metadata.
 typedef struct {
-    Message  base;
+    MessageHeader base;
     bool     found;
     uint64_t size;
     SHA256   content_hash;
@@ -211,7 +210,7 @@ typedef enum {
 
 typedef struct {
 
-    MessageSystem *msys;
+    TCP tcp;
 
     Address  self_addr;
     Address  node_addrs[NODE_LIMIT];
@@ -220,6 +219,7 @@ typedef struct {
     Status status;
 
     ClientTable client_table;
+    int next_client_tag;
 
     uint64_t view_number;
     uint64_t last_normal_view;  // Latest view where status was NORMAL
@@ -228,7 +228,7 @@ typedef struct {
     uint32_t recovery_votes;
     uint64_t recovery_nonce;
     uint64_t recovery_view;
-    Log      recovery_log;
+    WAL      recovery_log;
     uint64_t recovery_log_view;
     Time     recovery_time;
     int      recovery_commit;
@@ -238,7 +238,7 @@ typedef struct {
 
     uint32_t view_change_begin_votes;
     uint32_t view_change_apply_votes;
-    Log      view_change_log; // Best log seen
+    WAL      view_change_log; // Best log seen
     uint64_t view_change_old_view;  // Best old_view_number seen in DoViewChange
     int      view_change_commit;    // Best commit_index seen
 
@@ -255,7 +255,8 @@ typedef struct {
     bool state_transfer_pending;
     Time state_transfer_time;
 
-    Log log;
+    WAL wal;
+    ViewAndCommit vc;
 
     Time heartbeat;
 
@@ -265,7 +266,7 @@ typedef struct {
     // Set at each wakeup
     Time now;
 
-} Server;
+} ServerState;
 
 struct pollfd;
 
@@ -282,6 +283,7 @@ typedef struct {
     int last_min_commit;
     int last_max_commit;
     Status prev_status[NODE_LIMIT];
+    bool   prev_alive[NODE_LIMIT];
 
     // External shadow log of committed operations (unbounded, dynamically allocated)
     MetaOper *shadow_log;
@@ -294,7 +296,7 @@ void invariant_checker_free(InvariantChecker *ic);
 
 // node_handles: opaque QuakeyNode handles for entering host context.
 // Pass NULL when running outside the simulation (real mode).
-void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
+void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
     unsigned long long *node_handles);
 
-#endif // SERVER_INCLUDED
+#endif // NODE_INCLUDED

src/tcp.c

@@ -0,0 +1,504 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+
+#include "tcp.h"
+#include "message.h"
+
+static int set_socket_blocking(SOCKET sock, bool value)
+{
+#ifdef _WIN32
+    u_long mode = !value;
+    if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
+        return -1;
+#else
+    int flags = fcntl(sock, F_GETFL, 0);
+    if (flags < 0)
+        return -1;
+    if (value) flags &= ~O_NONBLOCK;
+    else       flags |= O_NONBLOCK;
+    if (fcntl(sock, F_SETFL, flags) < 0)
+        return -1;
+#endif
+
+    return 0;
+}
+
+static SOCKET create_listen_socket(Address addr)
+{
+    SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd == INVALID_SOCKET)
+        return INVALID_SOCKET;
+
+    if (set_socket_blocking(fd, false) < 0) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+#ifndef MAIN_SIMULATION
+    int reuse = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
+#endif
+
+    if (!addr.is_ipv4) {
+        assert(0); // TODO
+    }
+
+    struct sockaddr_in bind_buf;
+    bind_buf.sin_family = AF_INET;
+    bind_buf.sin_port   = htons(addr.port);
+    memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4));
+    if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+    int backlog = 32;
+    if (listen(fd, backlog) < 0) {
+        CLOSE_SOCKET(fd);
+        return INVALID_SOCKET;
+    }
+
+    return fd;
+}
+
+static int create_socket_pair(SOCKET *a, SOCKET *b)
+{
+#ifdef _WIN32
+    SOCKET sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    if (sock == INVALID_SOCKET)
+        return -1;
+
+    // Bind to loopback address with port 0 (dynamic port assignment)
+    struct sockaddr_in addr;
+    int addr_len = sizeof(addr);
+    memset(&addr, 0, sizeof(addr));
+    addr.sin_family = AF_INET;
+    addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1
+    addr.sin_port = 0; // Let system choose port
+
+    if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    if (getsockname(sock, (struct sockaddr*)&addr, &addr_len) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    if (connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
+        closesocket(sock);
+        return -1;
+    }
+
+    *a = sock;
+    *b = sock;
+
+    // Optional: Set socket to non-blocking mode
+    // This prevents send() from blocking if the receive buffer is full
+    u_long mode = 1;
+    ioctlsocket(sock, FIONBIO, &mode); // TODO: does this fail?
+    return 0;
+#else
+    int fds[2];
+    if (pipe(fds) < 0)
+        return -1;
+    *a = fds[0];
+    *b = fds[1];
+    return 0;
+#endif
+}
+
+static void close_socket_pair(SOCKET a, SOCKET b)
+{
+#ifdef _WIN32
+    closesocket(a);
+    (void) b;
+#else
+    close(a);
+    close(b);
+#endif
+}
+
+static void conn_init(Connection *conn, SOCKET fd, bool connecting)
+{
+    conn->fd = fd;
+    conn->tag = -1;
+    conn->connecting = connecting;
+    conn->closing = false;
+    conn->msglen = 0;
+    byte_queue_init(&conn->input, 1<<30);
+    byte_queue_init(&conn->output, 1<<30);
+}
+
+static void conn_free(Connection *conn)
+{
+    CLOSE_SOCKET(conn->fd);
+    byte_queue_free(&conn->input);
+    byte_queue_free(&conn->output);
+}
+
+static int conn_events(Connection *conn)
+{
+    int events = 0;
+
+    if (conn->connecting)
+        events |= POLLOUT;
+    else {
+
+        assert(!byte_queue_full(&conn->input));
+        if (!conn->closing)
+            events |= POLLIN;
+
+        if (!byte_queue_empty(&conn->output))
+            events |= POLLOUT;
+    }
+    return events;
+}
+
+int tcp_context_init(TCP *tcp)
+{
+    tcp->listen_fd = INVALID_SOCKET;
+    tcp->num_conns = 0;
+
+    if (create_socket_pair(&tcp->wait_fd, &tcp->signal_fd) < 0)
+        return -1;
+
+    return 0;
+}
+
+void tcp_context_free(TCP *tcp)
+{
+    // Free all connection byte queues without closing sockets
+    // (sockets are managed by the simulation and will be cleaned up separately)
+    for (int i = 0; i < tcp->num_conns; i++) {
+        byte_queue_free(&tcp->conns[i].input);
+        byte_queue_free(&tcp->conns[i].output);
+    }
+    tcp->num_conns = 0;
+
+    if (tcp->listen_fd != INVALID_SOCKET)
+        CLOSE_SOCKET(tcp->listen_fd);
+
+    close_socket_pair(tcp->wait_fd, tcp->signal_fd);
+}
+
+int tcp_wakeup(TCP *tcp)
+{
+    send(tcp->signal_fd, "0", 1, 0); // TODO: Handle error
+    return 0;
+}
+
+int tcp_index_from_tag(TCP *tcp, int tag)
+{
+    for (int i = 0; i < tcp->num_conns; i++)
+        if (tcp->conns[i].tag == tag)
+            return i;
+    return -1;
+}
+
+int tcp_listen(TCP *tcp, Address addr)
+{
+    SOCKET listen_fd = create_listen_socket(addr);
+    if (listen_fd == INVALID_SOCKET)
+        return -1;
+
+    tcp->listen_fd = listen_fd;
+    return 0;
+}
+
+int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type)
+{
+    *msg = byte_queue_read_buf(&tcp->conns[conn_idx].input);
+
+    uint32_t len;
+    int ret = message_peek(*msg, type, &len);
+
+    // Invalid message?
+    if (ret < 0) {
+        byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
+        return -1;
+    }
+
+    // Still buffering header?
+    if (ret == 0) {
+        byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
+        if (byte_queue_full(&tcp->conns[conn_idx].input))
+            return -1;
+        return 0;
+    }
+
+    // Message received
+    assert(ret > 0);
+    msg->len = len;
+    tcp->conns[conn_idx].msglen = len;
+
+    return 1;
+}
+
+void tcp_consume_message(TCP *tcp, int conn_idx)
+{
+    byte_queue_read_ack(&tcp->conns[conn_idx].input, tcp->conns[conn_idx].msglen);
+    tcp->conns[conn_idx].msglen = 0;
+}
+
+int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled)
+{
+    int num_polled = 0;
+
+    polled[num_polled].fd = tcp->wait_fd;
+    polled[num_polled].events = POLLIN;
+    polled[num_polled].revents = 0;
+    contexts[num_polled] = NULL;
+    num_polled++;
+
+    if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < TCP_CONNECTION_LIMIT) {
+        polled[num_polled].fd = tcp->listen_fd;
+        polled[num_polled].events = POLLIN;
+        polled[num_polled].revents = 0;
+        contexts[num_polled] = NULL;
+        num_polled++;
+    }
+
+    for (int i = 0; i < tcp->num_conns; i++) {
+        int events = conn_events(&tcp->conns[i]);
+        if (events) {
+            polled[num_polled].fd = tcp->conns[i].fd;
+            polled[num_polled].events = events;
+            polled[num_polled].revents = 0;
+            contexts[num_polled] = &tcp->conns[i];
+            num_polled++;
+        }
+    }
+
+    return num_polled;
+}
+
+// The "events" array must be an array of capacity TCP_EVENT_CAPACITY,
+// while "contexts" and "polled" must have capacity TCP_POLL_CAPACITY.
+int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled)
+{
+    bool removed[TCP_POLL_CAPACITY];
+    for (int i = 0; i < TCP_POLL_CAPACITY; i++)
+        removed[i] = false;
+
+    int num_events = 0;
+    for (int i = 1; i < num_polled; i++) {
+
+        if (polled[i].fd == tcp->wait_fd) {
+
+            if (polled[i].revents & POLLIN) {
+                char buf[100];
+                recv(tcp->wait_fd, buf, sizeof(buf), 0); // TODO: Make sure all bytes are consumed
+                events[num_events++] = (Event) { EVENT_WAKEUP, -1, -1 };
+            }
+
+        } else if (polled[i].fd == tcp->listen_fd) {
+
+            assert(contexts[i] == NULL);
+
+            if (polled[i].revents & POLLIN) {
+                SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
+                if (new_fd != INVALID_SOCKET) {
+
+                    if (set_socket_blocking(new_fd, false) < 0)
+                        CLOSE_SOCKET(new_fd);
+                    else {
+                        conn_init(&tcp->conns[tcp->num_conns++], new_fd, false);
+                        events[num_events++] = (Event) { EVENT_CONNECT, tcp->num_conns-1, tcp->conns[tcp->num_conns-1].tag };
+                    }
+                }
+            }
+            removed[i] = false;
+
+        } else {
+
+            Connection *conn = contexts[i];
+            bool defer_close = false;
+            bool defer_ready = false;
+
+            if (conn->connecting) {
+
+                // Check for error conditions on the socket
+                if (polled[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+                    defer_close = true;
+                } else if (polled[i].revents & POLLOUT) {
+
+                    int err = 0;
+                    socklen_t len = sizeof(err);
+                    if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
+                        defer_close = true;
+                    else {
+                        conn->connecting = false;
+                        events[num_events++] = (Event) { EVENT_CONNECT, conn - tcp->conns, conn->tag };
+                    }
+                }
+
+            } else {
+
+                if (polled[i].revents & POLLIN) {
+                    byte_queue_write_setmincap(&conn->input, 1<<9);
+                    ByteView buf = byte_queue_write_buf(&conn->input);
+                    int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
+                    if (num == 0)
+                        defer_close = true;
+                    else if (num < 0) {
+                        if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) // TODO: does Windows return these error codes or not?
+                            defer_close = true;
+                        num = 0;
+                    }
+                    byte_queue_write_ack(&conn->input, num);
+                    ByteView msg = byte_queue_read_buf(&conn->input);
+                    int ret = message_peek(msg, NULL, NULL);
+                    byte_queue_read_ack(&conn->input, 0);
+                    if (ret < 0) {
+                        // Invalid message
+                        defer_close = true;
+                    } else if (ret == 0) {
+                        // Still buffering
+                        if (byte_queue_full(&conn->input))
+                            defer_close = true;
+                    } else {
+                        // Message received
+                        assert(ret > 0);
+                        defer_ready = true;
+                    }
+                }
+
+                if (polled[i].revents & POLLOUT) {
+                    ByteView buf = byte_queue_read_buf(&conn->output);
+                    int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
+                    if (num < 0) {
+                        if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
+                            defer_close = true;
+                        num = 0;
+                    }
+                    byte_queue_read_ack(&conn->output, num);
+                    if (conn->closing && byte_queue_empty(&conn->output))
+                        defer_close = true;
+                }
+            }
+
+            // TODO: byte_queue_error here?
+
+            removed[i] = defer_close;
+            if (0) {}
+            else if (defer_close) events[num_events++] = (Event) { EVENT_DISCONNECT, conn - tcp->conns, conn->tag };
+            else if (defer_ready) events[num_events++] = (Event) { EVENT_MESSAGE,    conn - tcp->conns, conn->tag };
+        }
+    }
+
+    for (int i = 1; i < num_polled; i++) {
+        if (removed[i]) {
+            Connection *conn = contexts[i];
+            assert(conn);
+            int removed_idx = conn - tcp->conns;
+            conn_free(conn);
+            int last_idx = --tcp->num_conns;
+            if (removed_idx != last_idx) {
+                *conn = tcp->conns[last_idx];
+                // Update event conn_idx values to reflect the swap
+                for (int j = 0; j < num_events; j++) {
+                    if (events[j].conn_idx == last_idx)
+                        events[j].conn_idx = removed_idx;
+                }
+                // Update contexts pointers for remaining iterations
+                for (int j = i + 1; j < num_polled; j++) {
+                    if (contexts[j] == &tcp->conns[last_idx])
+                        contexts[j] = conn;
+                }
+            }
+        }
+    }
+
+    return num_events;
+}
+
+ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx)
+{
+    return &tcp->conns[conn_idx].output;
+}
+
+int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
+{
+    if (tcp->num_conns == TCP_CONNECTION_LIMIT)
+        return -1;
+    int conn_idx = tcp->num_conns;
+
+    SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd == INVALID_SOCKET)
+        return -1;
+
+    if (set_socket_blocking(fd, false) < 0) {
+        CLOSE_SOCKET(fd);
+        return -1;
+    }
+
+    int ret;
+    if (addr.is_ipv4) {
+        struct sockaddr_in buf;
+        buf.sin_family = AF_INET;
+        buf.sin_port = htons(addr.port);
+        memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
+        ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
+    } else {
+        struct sockaddr_in6 buf;
+        buf.sin6_family = AF_INET6;
+        buf.sin6_port = htons(addr.port);
+        memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
+        ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
+    }
+
+    bool connecting;
+    if (ret == 0) {
+        connecting = false;
+    } else {
+        if (errno != EINPROGRESS) {
+            CLOSE_SOCKET(fd);
+            return -1;
+        }
+        connecting = true;
+    }
+
+    // Check that this tag wasn't already used
+    for (int i = 0; i < tcp->num_conns; i++)
+        assert(tcp->conns[i].tag != tag);
+
+    conn_init(&tcp->conns[conn_idx], fd, connecting);
+    tcp->conns[conn_idx].tag = tag;
+
+    if (output)
+        *output = &tcp->conns[conn_idx].output;
+
+    tcp->num_conns++;
+    return 0;
+}
+
+void tcp_close(TCP *tcp, int conn_idx)
+{
+    tcp->conns[conn_idx].closing = true;
+    tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection
+    // TODO: if no event will be triggered, the connection will not be closed
+    //       if the output buffer is empty, the connection should be closed here.
+}
+
+void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique)
+{
+    assert(tag != -1);
+
+    if (unique) {
+        for (int i = 0; i < tcp->num_conns; i++)
+            assert(tcp->conns[i].tag != tag);
+    }
+
+    tcp->conns[conn_idx].tag = tag;
+}
+
+int tcp_get_tag(TCP *tcp, int conn_idx)
+{
+    return tcp->conns[conn_idx].tag;
+}

src/tcp.h

@@ -0,0 +1,87 @@
+#ifndef TCP_INCLUDED
+#define TCP_INCLUDED
+
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#   define QUAKEY_ENABLE_MOCKS
+#   include <quakey.h>
+#else
+#   ifdef _WIN32
+#       include <winsock2.h>
+#   endif
+#endif
+
+#include "byte_queue.h"
+
+#ifdef _WIN32
+#define CLOSE_SOCKET closesocket
+#else
+#define SOCKET int
+#define INVALID_SOCKET -1
+#define CLOSE_SOCKET close
+#endif
+
+#ifndef TCP_CONNECTION_LIMIT
+// Maximum number of connections that can be managed
+// simultaneously.
+#define TCP_CONNECTION_LIMIT 512
+#endif
+
+// This is the maximum number of descriptors that the
+// TCP system will want to wait at any given time.
+// One descriptor per connection plus a listener socket
+// and a self-pipe handle for wakeup.
+#define TCP_POLL_CAPACITY (TCP_CONNECTION_LIMIT+2)
+
+// Number of TCP events that can be returned at a given
+// time by "tcp_translate_events". There may be a single
+// event per connection (MESSAGE, CONNECT, DISCONNECT)
+// plus a general WAKEUP event.
+#define TCP_EVENT_CAPACITY (TCP_CONNECTION_LIMIT+1)
+
+typedef enum {
+    EVENT_WAKEUP,
+    EVENT_MESSAGE,
+    EVENT_CONNECT,
+    EVENT_DISCONNECT,
+} EventType;
+
+typedef struct {
+    EventType type;
+    int conn_idx;
+    int tag;
+} Event;
+
+typedef struct {
+    SOCKET    fd;
+    int       tag;
+    bool      connecting;
+    bool      closing;
+    uint32_t  msglen;
+    ByteQueue input;
+    ByteQueue output;
+} Connection;
+
+typedef struct {
+    SOCKET listen_fd;
+    SOCKET wait_fd;
+    SOCKET signal_fd;
+    int    num_conns;
+    Connection conns[TCP_CONNECTION_LIMIT];
+} TCP;
+
+int  tcp_context_init(TCP *tcp);
+void tcp_context_free(TCP *tcp);
+int  tcp_wakeup(TCP *tcp);
+int  tcp_index_from_tag(TCP *tcp, int tag);
+int  tcp_listen(TCP *tcp, Address addr);
+int  tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type);
+void tcp_consume_message(TCP *tcp, int conn_idx);
+int  tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled);
+int  tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled);
+ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx);
+int  tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output);
+void tcp_close(TCP *tcp, int conn_idx);
+void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique);
+int  tcp_get_tag(TCP *tcp, int conn_idx);
+
+#endif // TCP_INCLUDED

src/wal.c

@@ -0,0 +1,422 @@
+#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
+#define QUAKEY_ENABLE_MOCKS
+#endif
+
+#include <quakey.h>
+#include <assert.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "wal.h"
+
+// FNV-1a checksum over all WALEntry fields except the checksum itself.
+static uint32_t wal_entry_checksum(WALEntry *entry)
+{
+    uint32_t h = 2166136261u;
+    const unsigned char *p = (const unsigned char *)entry;
+    size_t len = offsetof(WALEntry, checksum);
+    for (size_t i = 0; i < len; i++) {
+        h ^= p[i];
+        h *= 16777619u;
+    }
+    return h;
+}
+
+static void wal_entry_set_checksum(WALEntry *entry)
+{
+    entry->checksum = wal_entry_checksum(entry);
+}
+
+static bool wal_is_file_backed(WAL *wal)
+{
+    return wal->handle.data != 0;
+}
+
+static int wal_grow(WAL *wal)
+{
+    int n = 2 * wal->capacity;
+    if (n < 8) n = 8;
+    WALEntry *p = realloc(wal->entries, n * sizeof(WALEntry));
+    if (p == NULL)
+        return -1;
+    wal->entries = p;
+    wal->capacity = n;
+    return 0;
+}
+
+void wal_init(WAL *wal)
+{
+    wal->count = 0;
+    wal->capacity = 0;
+    wal->entries = NULL;
+    wal->handle = (Handle) { 0 };
+}
+
+int wal_init_from_file(WAL *wal, string file, bool *was_truncated)
+{
+    if (was_truncated)
+        *was_truncated = false;
+
+    Handle handle;
+    if (file_open(file, &handle) < 0)
+        return -1;
+
+    size_t size;
+    if (file_size(handle, &size) < 0) {
+        file_close(handle);
+        return -1;
+    }
+
+    // Discard any partial trailing entry (crash during write).
+    int raw_count = size / sizeof(WALEntry);
+    size_t valid_size = raw_count * sizeof(WALEntry);
+    if (valid_size < size)
+        file_truncate(handle, valid_size);
+
+    WALEntry *entries = malloc(raw_count * sizeof(WALEntry));
+    if (entries == NULL && raw_count > 0) {
+        file_close(handle);
+        return -1;
+    }
+
+    if (file_set_offset(handle, 0) < 0) {
+        file_close(handle);
+        free(entries);
+        return -1;
+    }
+
+    if (raw_count > 0 && file_read_exact(handle, (char *)entries, raw_count * sizeof(WALEntry)) < 0) {
+        file_close(handle);
+        free(entries);
+        return -1;
+    }
+
+    // Verify checksums: truncate at the first corrupted entry.
+    int count = raw_count;
+    for (int i = 0; i < raw_count; i++) {
+        if (entries[i].checksum != wal_entry_checksum(&entries[i])) {
+            count = i;
+            file_truncate(handle, count * sizeof(WALEntry));
+            if (was_truncated)
+                *was_truncated = true;
+            break;
+        }
+    }
+
+    // Position file offset at end for future appends.
+    if (file_set_offset(handle, count * sizeof(WALEntry)) < 0) {
+        file_close(handle);
+        free(entries);
+        return -1;
+    }
+
+    wal->count = count;
+    wal->capacity = count;
+    wal->entries = entries;
+    wal->handle = handle;
+    return 0;
+}
+
+int wal_init_from_network(WAL *wal, void *src, int num)
+{
+    wal->count = num;
+    wal->capacity = num;
+    wal->entries = NULL;
+    if (num > 0) {
+        wal->entries = malloc(num * sizeof(WALEntry));
+        if (wal->entries == NULL)
+            return -1;
+        memcpy(wal->entries, src, num * sizeof(WALEntry));
+    }
+    wal->handle = (Handle) { 0 };
+    return 0;
+}
+
+void wal_free(WAL *wal)
+{
+    free(wal->entries);
+    if (wal_is_file_backed(wal))
+        file_close(wal->handle);
+}
+
+void wal_move(WAL *dst, WAL *src)
+{
+    free(dst->entries);
+    dst->count = src->count;
+    dst->capacity = src->capacity;
+    dst->entries = src->entries;
+    // Do NOT touch dst->handle — caller manages file backing.
+    wal_init(src);
+}
+
+int wal_append(WAL *wal, WALEntry entry)
+{
+    if (wal->count == wal->capacity) {
+        if (wal_grow(wal) < 0)
+            return -1;
+    }
+
+    if (wal_is_file_backed(wal)) {
+        wal_entry_set_checksum(&entry);
+
+        if (file_write_exact(wal->handle, (char *)&entry, sizeof(entry)) < 0) {
+            // Partial write may have advanced file offset. Truncate and
+            // rewind so the file stays consistent with in-memory count.
+            file_truncate(wal->handle, wal->count * sizeof(WALEntry));
+            file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
+            return -1;
+        }
+
+        if (file_sync(wal->handle) < 0) {
+            file_truncate(wal->handle, wal->count * sizeof(WALEntry));
+            file_set_offset(wal->handle, wal->count * sizeof(WALEntry));
+            return -1;
+        }
+    }
+
+    wal->entries[wal->count++] = entry;
+    return 0;
+}
+
+int wal_update_entry(WAL *wal, int idx)
+{
+    assert(idx >= 0 && idx < wal->count);
+
+    if (wal_is_file_backed(wal)) {
+        WALEntry entry = wal->entries[idx];
+        wal_entry_set_checksum(&entry);
+
+        size_t offset = idx * sizeof(WALEntry);
+        size_t end_offset = wal->count * sizeof(WALEntry);
+
+        if (file_set_offset(wal->handle, offset) < 0)
+            return -1;
+        if (file_write_exact(wal->handle, (char *)&entry, sizeof(entry)) < 0)
+            return -1;
+        if (file_sync(wal->handle) < 0)
+            return -1;
+
+        // Restore file offset to end for future appends.
+        if (file_set_offset(wal->handle, end_offset) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int wal_truncate(WAL *wal, int new_count)
+{
+    assert(new_count <= wal->count);
+    if (wal->count == new_count)
+        return 0;
+
+    if (wal_is_file_backed(wal)) {
+        if (file_truncate(wal->handle, new_count * sizeof(WALEntry)) < 0)
+            return -1;
+        if (file_set_offset(wal->handle, new_count * sizeof(WALEntry)) < 0)
+            return -1;
+    }
+
+    wal->count = new_count;
+    return 0;
+}
+
+int wal_replace(WAL *wal, WALEntry *entries, int count)
+{
+    assert(wal_is_file_backed(wal));
+
+    string tmp_path = S("tmp.log");
+    string wal_path = S("vsr.log");
+
+    // Open tmp.log (file_open creates if not exists, opens if exists).
+    Handle tmp;
+    if (file_open(tmp_path, &tmp) < 0)
+        return -1;
+
+    // Truncate in case tmp.log already exists from a previous crash.
+    if (file_truncate(tmp, 0) < 0) {
+        file_close(tmp);
+        return -1;
+    }
+    if (file_set_offset(tmp, 0) < 0) {
+        file_close(tmp);
+        return -1;
+    }
+
+    // Write all entries to tmp.log.
+    for (int i = 0; i < count; i++) {
+        wal_entry_set_checksum(&entries[i]);
+        if (file_write_exact(tmp, (char *)&entries[i], sizeof(entries[i])) < 0) {
+            file_close(tmp);
+            return -1;
+        }
+    }
+
+    // Fsync tmp.log to ensure all data is on disk.
+    if (file_sync(tmp) < 0) {
+        file_close(tmp);
+        return -1;
+    }
+    file_close(tmp);
+
+    // Close the current WAL handle before rename.
+    file_close(wal->handle);
+    wal->handle = (Handle) { 0 };
+
+    // Atomically replace the WAL file.
+    if (rename_file_or_dir(tmp_path, wal_path) < 0)
+        return -1;
+
+    // TODO: fsync on parent directory
+
+    // Reopen the WAL file and seek to end for future appends.
+    Handle new_handle;
+    if (file_open(wal_path, &new_handle) < 0)
+        return -1;
+    if (file_set_offset(new_handle, count * sizeof(WALEntry)) < 0) {
+        file_close(new_handle);
+        return -1;
+    }
+    wal->handle = new_handle;
+
+    // Update in-memory state.
+    free(wal->entries);
+    wal->entries = NULL;
+    wal->count = 0;
+    wal->capacity = 0;
+
+    if (count > 0) {
+        wal->entries = malloc(count * sizeof(WALEntry));
+        if (wal->entries == NULL)
+            return -1;
+        memcpy(wal->entries, entries, count * sizeof(WALEntry));
+    }
+    wal->count = count;
+    wal->capacity = count;
+
+    return 0;
+}
+
+int wal_entry_count(WAL *wal)
+{
+    return wal->count;
+}
+
+WALEntry *wal_peek_entry(WAL *wal, int idx)
+{
+    assert(idx >= 0);
+    assert(idx < wal->count);
+    return &wal->entries[idx];
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// ViewAndCommit — persistent view_number, last_normal_view and commit_index
+///////////////////////////////////////////////////////////////////////////////
+
+// On-disk layout (24 bytes):
+//   [view_number: 8] [last_normal_view: 8] [commit_index: 4] [checksum: 4]
+typedef struct {
+    uint64_t view_number;
+    uint64_t last_normal_view;
+    int      commit_index;
+    uint32_t checksum;
+} ViewAndCommitDisk;
+
+static uint32_t vc_checksum(uint64_t view_number,
+    uint64_t last_normal_view, int commit_index)
+{
+    uint32_t h = 2166136261u;
+
+    const unsigned char *p = (const unsigned char *)&view_number;
+    for (int i = 0; i < (int)sizeof(view_number); i++) {
+        h ^= p[i];
+        h *= 16777619u;
+    }
+
+    p = (const unsigned char *)&last_normal_view;
+    for (int i = 0; i < (int)sizeof(last_normal_view); i++) {
+        h ^= p[i];
+        h *= 16777619u;
+    }
+
+    p = (const unsigned char *)&commit_index;
+    for (int i = 0; i < (int)sizeof(commit_index); i++) {
+        h ^= p[i];
+        h *= 16777619u;
+    }
+
+    return h;
+}
+
+int view_and_commit_init(ViewAndCommit *vc, string file)
+{
+    Handle handle;
+    if (file_open(file, &handle) < 0)
+        return -1;
+
+    vc->handle = handle;
+    vc->view_number = 0;
+    vc->last_normal_view = 0;
+    vc->commit_index = 0;
+
+    size_t size;
+    if (file_size(handle, &size) < 0) {
+        file_close(handle);
+        return -1;
+    }
+
+    if (size >= sizeof(ViewAndCommitDisk)) {
+
+        if (file_set_offset(handle, 0) < 0) {
+            file_close(handle);
+            return -1;
+        }
+
+        ViewAndCommitDisk disk;
+        if (file_read_exact(handle, (char *)&disk, sizeof(disk)) < 0) {
+            file_close(handle);
+            return -1;
+        }
+
+        uint32_t expected_checksum = vc_checksum(disk.view_number,
+            disk.last_normal_view, disk.commit_index);
+
+        if (disk.checksum == expected_checksum) {
+            vc->view_number = disk.view_number;
+            vc->last_normal_view = disk.last_normal_view;
+            vc->commit_index = disk.commit_index;
+        }
+
+        // If checksum doesn't match, start from defaults (0, 0, 0)
+    }
+
+    return 0;
+}
+
+void view_and_commit_free(ViewAndCommit *vc)
+{
+    if (vc->handle.data != 0)
+        file_close(vc->handle);
+}
+
+int set_view_and_commit(ViewAndCommit *vc, uint64_t view_number,
+    uint64_t last_normal_view, int commit_index)
+{
+    ViewAndCommitDisk disk = {
+        .view_number = view_number,
+        .last_normal_view = last_normal_view,
+        .commit_index = commit_index,
+    };
+    disk.checksum = vc_checksum(view_number, last_normal_view, commit_index);
+
+    if (file_set_offset(vc->handle, 0) < 0)
+        return -1;
+    if (file_write_exact(vc->handle, (char *)&disk, sizeof(disk)) < 0)
+        return -1;
+    if (file_sync(vc->handle) < 0)
+        return -1;
+
+    vc->view_number = view_number;
+    vc->last_normal_view = last_normal_view;
+    vc->commit_index = commit_index;
+    return 0;
+}

src/wal.h

@@ -0,0 +1,77 @@
+#ifndef WAL_INCLUDED
+#define WAL_INCLUDED
+
+#include "metadata.h"
+#include "file_system.h"
+#include "config.h"
+
+typedef struct {
+    MetaOper oper;
+    uint32_t votes;
+    int      view_number;
+    uint64_t client_id;
+    uint64_t request_id;
+    uint32_t checksum;    // FNV-1a over all preceding fields
+} WALEntry;
+
+_Static_assert(NODE_LIMIT <= 32, "");
+
+typedef struct {
+    int       count;
+    int       capacity;
+    WALEntry *entries;
+    Handle    handle;     // file handle to the WAL (0 = memory-only)
+} WAL;
+
+// Initialize an empty, memory-only WAL (no file backing).
+void wal_init(WAL *wal);
+
+// Initialize a WAL from an on-disk file. Recovers valid entries,
+// discards partial/corrupted trailing entries. If was_truncated is
+// non-NULL, *was_truncated is set to true when entries were discarded
+// due to corruption (not just a partial trailing write).
+int  wal_init_from_file(WAL *wal, string file, bool *was_truncated);
+
+// Initialize a WAL from network data (memory-only, no file).
+int  wal_init_from_network(WAL *wal, void *src, int num);
+
+void wal_free(WAL *wal);
+
+// Move ownership from src to dst. If dst is file-backed, the file
+// is NOT affected; use wal_replace for atomic file replacement.
+void wal_move(WAL *dst, WAL *src);
+
+// Append an entry. If the WAL is file-backed, writes to disk and
+// fsyncs before updating the in-memory buffer.
+int  wal_append(WAL *wal, WALEntry entry);
+
+// Write a modified in-memory entry back to disk (recomputes checksum).
+int  wal_update_entry(WAL *wal, int idx);
+
+// Truncate the WAL to new_count entries.
+int  wal_truncate(WAL *wal, int new_count);
+
+// Atomically replace the WAL file contents with the given entries.
+// Writes to a temporary file, fsyncs, then renames over the WAL.
+// Updates the in-memory buffer and reopens the file handle.
+int  wal_replace(WAL *wal, WALEntry *entries, int count);
+
+int       wal_entry_count(WAL *wal);
+WALEntry *wal_peek_entry(WAL *wal, int idx);
+
+// Persistent view_number, last_normal_view and commit_index,
+// analogous to raft's TermAndVote. Backed by a small file
+// ("vsr.state") with a checksum for integrity.
+typedef struct {
+    uint64_t view_number;
+    uint64_t last_normal_view;
+    int      commit_index;
+    Handle   handle;
+} ViewAndCommit;
+
+int  view_and_commit_init(ViewAndCommit *vc, string file);
+void view_and_commit_free(ViewAndCommit *vc);
+int  set_view_and_commit(ViewAndCommit *vc, uint64_t view_number,
+                         uint64_t last_normal_view, int commit_index);
+
+#endif // WAL_INCLUDED