cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Major refactoring

Browse Source
This commit is contained in:
cozis
2026-02-25 10:44:43 +01:00
parent bac3477991
commit ef1d65ad2b
45 changed files with 5404 additions and 2120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -15,3 +15,6 @@ example_large
examples/example
.cluster/
vsr_boot_marker
*.pem
*.exe
.claude/
Makefile
-81
View File
@@ -1,81 +0,0 @@
CC ?= gcc
CFLAGS ?= -Wall -Wextra -ggdb -O0
AR ?= ar
INCLUDES = -Iquakey/include -Iinclude -I.
# ---- Library (client-side only) ----
LIB_SRCS = src/client.c src/tcp.c src/byte_queue.c src/message.c src/basic.c
LIB_OBJS = $(LIB_SRCS:.c=.o)
STATIC_LIB = libtoastyfs.a
SHARED_LIB = libtoastyfs.so
# ---- Server binary ----
SERVER_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
src/tcp.c src/server.c src/main.c src/log.c src/client_table.c \
src/chunk_store.c src/metadata.c
# ---- Client binary (random test client) ----
CLIENT_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
src/tcp.c src/server.c src/client.c src/random_client.c src/main.c \
src/log.c src/client_table.c src/chunk_store.c src/metadata.c
# ---- Simulation binary ----
SIM_SRCS = src/basic.c src/file_system.c src/byte_queue.c src/message.c \
src/tcp.c src/server.c src/client.c src/random_client.c src/main.c \
src/log.c src/client_table.c src/invariant_checker.c src/chunk_store.c \
src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c
# ---- Default target ----
all: $(STATIC_LIB) $(SHARED_LIB) toastyfs toastyfs_client toastyfs_simulation
# ---- Library targets ----
lib: $(STATIC_LIB) $(SHARED_LIB)
$(STATIC_LIB): $(LIB_OBJS)
$(AR) rcs $@ $^
$(SHARED_LIB): $(LIB_SRCS)
$(CC) $(CFLAGS) -shared -fPIC $(INCLUDES) $^ -o $@
src/%.o: src/%.c
$(CC) $(CFLAGS) $(INCLUDES) -fPIC -c $< -o $@
# ---- Binary targets ----
toastyfs: $(SERVER_SRCS)
$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_SERVER $^ -o $@
toastyfs_client: $(CLIENT_SRCS)
$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_CLIENT $^ -o $@
toastyfs_simulation: $(SIM_SRCS)
$(CC) $(CFLAGS) $(INCLUDES) -DMAIN_SIMULATION -DFAULT_INJECTION $^ -o $@
# ---- Install ----
PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR?= $(PREFIX)/include
install: $(STATIC_LIB) $(SHARED_LIB)
install -d $(DESTDIR)$(LIBDIR)
install -d $(DESTDIR)$(INCLUDEDIR)
install -m 644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)/
install -m 755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR)/
install -m 644 include/toastyfs.h $(DESTDIR)$(INCLUDEDIR)/
# ---- Clean ----
clean:
rm -f $(LIB_OBJS) $(STATIC_LIB) $(SHARED_LIB)
rm -f toastyfs toastyfs_client toastyfs_simulation
.PHONY: all lib clean install
README.md
-109
View File
@@ -1,109 +0,0 @@
# ToastyFS
ToastyFS is a distributed and fault-tolerant object storage.
WARNING: I'm still finishing up :) Log compaction and persistence are missing.
## Getting Started
First, build ToastyFS by running the build script:
```sh
./build.sh
```
This will produce three executables: `toastyfs`, `toastyfs_random_client` and `toastyfs_simulation`.
The `toastyfs` is the one you need to run. The other executables are for testing purposes.
You can pick a cluster size based on how many faults you want it to handle. Let's start with a cluster size of 3 which will allow one node to crash at a given time.
Run three instances of `toastyfs` while specifying to each one the addresses of the others:
```sh
./toastyfs --addr 127.0.0.1:8081 --peer 127.0.0.1:8082 --peer 127.0.0.1:8083
./toastyfs --addr 127.0.0.1:8082 --peer 127.0.0.1:8081 --peer 127.0.0.1:8083
./toastyfs --addr 127.0.0.1:8083 --peer 127.0.0.1:8081 --peer 127.0.0.1:8082
```
The cluster is now working.
To upload/download/delete objects, you need to compile a client program which needs the toasty client library.
To build the library, run the following, which will generate the `libtoasty.a` and `libtoasty.so` files:
```sh
Makefile
```
You can use the following example program which will upload a simple object:
```c
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <toastyfs.h>
int main(void)
{
srand(time(NULL));
char addr1[] = "127.0.0.1:8081";
char addr2[] = "127.0.0.1:8082";
char addr3[] = "127.0.0.1:8083";
char *addrs[] = { addr1, addr2, addr3 };
printf("Connecting to cluster...\n");
ToastyFS *tfs = toastyfs_init(rand(), addrs, 3);
if (tfs == NULL) {
fprintf(stderr, "toastyfs_init failed\n");
return 1;
}
char key[] = "hello";
char data[] = "world";
printf("PUT key=\"%s\" data=\"%s\" (%d bytes)\n", key, data, (int)strlen(data));
ToastyFS_Result res;
int ret = toastyfs_put(tfs, key, strlen(key), data, strlen(data), &res);
if (ret < 0) {
fprintf(stderr, "toastyfs_put returned %d\n", ret);
toastyfs_free(tfs);
return 1;
}
printf("Done.\n");
toastyfs_free(tfs);
return 0;
}
```
Save it as `example_client.c` and compile it by running:
```
gcc example_client.c libtoasty.a -o example
```
By running the client while the cluster is running, the object will be created!
## Fault Tolerance
ToastyFS allows any minority of nodes to be dead at any given time and continue running without issues. Generally speaking, if your system has 2f+1 nodes, it can work without f of them.
## Testing
ToastyFS is tested with Deterministic Simulation Testing (DST).
The entire system (server nodes and clients) is simulated in the memory of a single process deterministically. Faults (node crashes, network partitons) and latencies are pseudo-randonly injected in the simulation such that a variety of scenarios are tested but in a way that can be perfectly reproduced by using the same simulation seed from which all random events are chosen.
After each node schedulation, the simulator inspects all node states at the same instant and verifies that none of the system's invariant were broken.
You can build and run the simulation by doing:
```
./build.sh
./toastyfs_simulation
```
TODO.txt
+2 -1
View File
@@ -1,3 +1,4 @@
[ ] Add log compaction
[ ] Add log persistence
[ ] Add chunk locations to metadata
[ ] Add chunk locations to metadata
[ ] Remove sender_idx fields in messages since it's provided by the message header
build.bat
+41
View File
@@ -0,0 +1,41 @@
@echo off
setlocal
set INCLUDES=-Iquakey/include -Iinclude -I.
set CFLAGS=-Wall -Wextra -ggdb -O0
set LIB_CORE=lib/basic.c lib/byte_queue.c lib/tcp.c
set LIB_FULL=%LIB_CORE% lib/file_system.c lib/message.c
set LIB_TLS=%LIB_FULL% lib/tls_schannel.c
set LIB_HTTP=%LIB_TLS% lib/http_parse.c lib/http_server.c
set SRC_SERVER=src/server.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c
set QUAKEY=quakey/src/mockfs.c quakey/src/quakey.c
rem 1. Simulation
echo Building simulation...
gcc %LIB_FULL% %SRC_SERVER% src/client.c src/random_client.c src/main.c src/invariant_checker.c %QUAKEY% -o toastyfs_simulation.exe %INCLUDES% %CFLAGS% -DMAIN_SIMULATION -DFAULT_INJECTION -lws2_32
if errorlevel 1 goto :error
rem 2. Server
echo Building server...
gcc %LIB_TLS% %SRC_SERVER% src/main.c -o toastyfs_server.exe %INCLUDES% %CFLAGS% -DMAIN_SERVER -DTLS_ENABLED -DTLS_SCHANNEL -lws2_32 -lsecur32 -lcrypt32 -lncrypt
if errorlevel 1 goto :error
rem 3. Random client
echo Building random client...
gcc %LIB_CORE% src/client.c src/random_client.c src/main.c -o toastyfs_random_client.exe %INCLUDES% %CFLAGS% -DMAIN_CLIENT -lws2_32
if errorlevel 1 goto :error
rem 4. HTTP proxy
echo Building http proxy...
gcc %LIB_HTTP% src/client.c src/http_proxy.c src/main.c -o toastyfs_http_proxy.exe %INCLUDES% %CFLAGS% -DMAIN_HTTP_PROXY -DTLS_ENABLED -DTLS_SCHANNEL -lws2_32 -lsecur32 -lcrypt32 -lncrypt
if errorlevel 1 goto :error
echo Done.
goto :eof
:error
echo Build failed.
exit /b 1
build.sh
+45 -3
View File
@@ -1,3 +1,45 @@
gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/client.c src/random_client.c src/main.c src/log.c src/client_table.c src/invariant_checker.c src/chunk_store.c src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c -o toastyfs_simulation -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc src/basic.c src/file_system.c src/byte_queue.c src/message.c src/tcp.c src/server.c src/client.c src/random_client.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs_random_client -Iquakey/include -Iinclude -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
#!/bin/bash
set -e
INCLUDES="-Iquakey/include -Iinclude -I."
CFLAGS="-Wall -Wextra -ggdb -O0"
LIB_CORE="lib/basic.c lib/byte_queue.c lib/tcp.c"
LIB_FULL="$LIB_CORE lib/file_system.c lib/message.c"
LIB_TLS="$LIB_FULL lib/tls_openssl.c"
LIB_HTTP="$LIB_TLS lib/http_parse.c lib/http_server.c"
SRC_SERVER="src/server.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c"
QUAKEY="quakey/src/mockfs.c quakey/src/quakey.c"
# 1. Simulation
echo "Building simulation..."
gcc $LIB_FULL \
$SRC_SERVER src/client.c src/random_client.c src/main.c src/invariant_checker.c \
$QUAKEY \
-o toastyfs_simulation \
$INCLUDES $CFLAGS -DMAIN_SIMULATION -DFAULT_INJECTION
# 2. Server
echo "Building server..."
gcc $LIB_TLS \
$SRC_SERVER src/main.c \
-o toastyfs_server \
$INCLUDES $CFLAGS -DMAIN_SERVER -DTLS_ENABLED -DTLS_OPENSSL -lssl -lcrypto
# 3. Random client
echo "Building random client..."
gcc $LIB_CORE lib/message.c \
src/client.c src/random_client.c src/main.c \
-o toastyfs_random_client \
$INCLUDES $CFLAGS -DMAIN_CLIENT
# 4. HTTP proxy
echo "Building http proxy..."
gcc $LIB_HTTP \
src/client.c src/http_proxy.c src/main.c \
-o toastyfs_http_proxy \
$INCLUDES $CFLAGS -DMAIN_HTTP_PROXY -DTLS_ENABLED -DTLS_OPENSSL -lssl -lcrypto
echo "Done."
cluster.sh
-176
View File
@@ -1,176 +0,0 @@
#!/usr/bin/env bash
#
# cluster.sh - Manage a local 3-node ToastyFS cluster.
#
# Usage:
# ./cluster.sh up Build the server and start 3 nodes
# ./cluster.sh down Stop all nodes
# ./cluster.sh status Show which nodes are running
# ./cluster.sh logs [1|2|3] Tail logs (all nodes, or one)
# ./cluster.sh run <source.c> Build the library, compile <source.c>
# against it, start the cluster, run the
# binary, then tear down the cluster.
#
# The three server nodes listen on:
# 127.0.0.1:8081 127.0.0.1:8082 127.0.0.1:8083
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CLUSTER_DIR="$SCRIPT_DIR/.cluster"
ADDRS=("127.0.0.1:8081" "127.0.0.1:8082" "127.0.0.1:8083")
usage() {
sed -n '3,14s/^# \?//p' "$0"
exit 1
}
cluster_up() {
echo "==> Building server..."
make -C "$SCRIPT_DIR" toastyfs
mkdir -p "$CLUSTER_DIR"
for i in 1 2 3; do
pidfile="$CLUSTER_DIR/node${i}.pid"
if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
echo " Node $i already running (pid $(cat "$pidfile"))"
continue
fi
node_dir="$CLUSTER_DIR/node${i}"
mkdir -p "$node_dir"
# Build argument list: --addr for self, --peer for the others.
args=()
for j in 1 2 3; do
idx=$((j - 1))
if [ "$j" -eq "$i" ]; then
args+=(--addr "${ADDRS[$idx]}")
else
args+=(--peer "${ADDRS[$idx]}")
fi
done
(
cd "$node_dir"
# ServerState is ~40 MB (MetaStore), which exceeds the
# default 8 MB stack. Raise the limit for the server.
ulimit -s unlimited
"$SCRIPT_DIR/toastyfs" "${args[@]}" \
>> "$CLUSTER_DIR/node${i}.log" 2>&1 &
echo $! > "$pidfile"
)
echo " Node $i started (pid $(cat "$pidfile")) on ${ADDRS[$((i-1))]}"
done
echo "==> Cluster is running."
}
cluster_down() {
if [ ! -d "$CLUSTER_DIR" ]; then
echo "No cluster state found."
return
fi
for i in 1 2 3; do
pidfile="$CLUSTER_DIR/node${i}.pid"
if [ -f "$pidfile" ]; then
pid="$(cat "$pidfile")"
if kill -0 "$pid" 2>/dev/null; then
kill "$pid" 2>/dev/null || true
echo " Node $i stopped (pid $pid)"
else
echo " Node $i was not running"
fi
rm -f "$pidfile"
fi
done
# Clean up node data directories so fresh starts don't trigger
# the crash-recovery path (vsr_boot_marker).
rm -rf "$CLUSTER_DIR"
echo "==> Cluster stopped."
}
cluster_status() {
if [ ! -d "$CLUSTER_DIR" ]; then
echo "No cluster state found. Run './cluster.sh up' first."
return
fi
printf "%-8s %-8s %-20s %s\n" "NODE" "PID" "ADDRESS" "STATUS"
for i in 1 2 3; do
pidfile="$CLUSTER_DIR/node${i}.pid"
addr="${ADDRS[$((i-1))]}"
if [ -f "$pidfile" ]; then
pid="$(cat "$pidfile")"
if kill -0 "$pid" 2>/dev/null; then
printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "running"
else
printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "dead"
fi
else
printf "%-8s %-8s %-20s %s\n" "node$i" "-" "$addr" "not started"
fi
done
}
cluster_logs() {
if [ $# -gt 0 ]; then
logfile="$CLUSTER_DIR/node${1}.log"
if [ ! -f "$logfile" ]; then
echo "No log file for node $1."
return 1
fi
tail -f "$logfile"
else
tail -f "$CLUSTER_DIR"/node*.log
fi
}
cluster_run() {
local src="$1"
local bin
bin="$(basename "${src%.c}")"
echo "==> Building libtoastyfs..."
make -C "$SCRIPT_DIR" lib
echo "==> Compiling $src..."
${CC:-gcc} -Wall -Wextra -o "$bin" "$src" \
-I"$SCRIPT_DIR/include" \
-L"$SCRIPT_DIR" \
-ltoastyfs
echo "==> Starting cluster..."
cluster_up
# Give the servers a moment to elect a leader.
sleep 2
echo "==> Running ./$bin"
echo "---"
LD_LIBRARY_PATH="$SCRIPT_DIR:${LD_LIBRARY_PATH:-}" "./$bin" || true
echo "---"
echo "==> Stopping cluster..."
cluster_down
}
if [ $# -lt 1 ]; then
usage
fi
case "$1" in
up) cluster_up ;;
down) cluster_down ;;
status) cluster_status ;;
logs) shift; cluster_logs "$@" ;;
run)
if [ $# -lt 2 ]; then
echo "Error: 'run' requires a source file argument." >&2
usage
fi
cluster_run "$2"
;;
*) usage ;;
esac
src/basic.c → lib/basic.c
+1 -4
View File
@@ -71,10 +71,7 @@ int deadline_to_timeout(Time deadline, Time current_time)
{
if (deadline == INVALID_TIME)
return -1;
if (deadline <= current_time)
return 0;
// Round up so sub-millisecond deadlines become 1ms, not 0
return CEIL(deadline - current_time, 1000000);
return (deadline - current_time) / 1000000;
}
bool getargb(int argc, char **argv, char *name)
src/basic.h → lib/basic.h
-1
View File
@@ -37,7 +37,6 @@ typedef uint64_t Time;
#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
#define CEIL(X, Y) (((X) + (Y) - 1) / (Y))
#define UNREACHABLE __builtin_trap();
src/byte_queue.c → lib/byte_queue.c
+5
View File
@@ -108,6 +108,11 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
}
}
bool byte_queue_reading(ByteQueue *queue)
{
return (queue->flags & BYTE_QUEUE_READ) != 0;
}
ByteView byte_queue_write_buf(ByteQueue *queue)
{
if ((queue->flags & BYTE_QUEUE_ERROR) || queue->data == NULL)
src/byte_queue.h → lib/byte_queue.h
+1
View File
@@ -39,6 +39,7 @@ int byte_queue_full(ByteQueue *queue);
ByteView byte_queue_read_buf(ByteQueue *queue);
void byte_queue_read_ack(ByteQueue *queue, uint32_t num);
bool byte_queue_reading(ByteQueue *queue);
ByteView byte_queue_write_buf(ByteQueue *queue);
void byte_queue_write_ack(ByteQueue *queue, uint32_t num);
src/file_system.c → lib/file_system.c
+2 -2
View File
@@ -36,7 +36,7 @@ int file_open(string path, Handle *fd)
memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0';
int ret = open(zt, O_RDWR | O_CREAT, 0644);
int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
if (ret < 0)
return -1;
@@ -293,7 +293,7 @@ int get_full_path(string path, char *dst)
#endif
#ifdef _WIN32
if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
if (_fullpath(dst, path_zt, PATH_MAX) == NULL)
return -1;
#endif
src/file_system.h → lib/file_system.h
View File
lib/http_parse.c
+1633
View File
File diff suppressed because it is too large Load Diff
lib/http_parse.h
+161
View File
@@ -0,0 +1,161 @@
#ifndef HTTP_PARSE_INCLUDED
#define HTTP_PARSE_INCLUDED
#include "basic.h"
#define CHTTP_MAX_HEADERS 32
typedef struct {
unsigned int data;
} CHTTP_IPv4;
typedef struct {
unsigned short data[8];
} CHTTP_IPv6;
typedef enum {
CHTTP_HOST_MODE_VOID = 0,
CHTTP_HOST_MODE_NAME,
CHTTP_HOST_MODE_IPV4,
CHTTP_HOST_MODE_IPV6,
} CHTTP_HostMode;
typedef struct {
CHTTP_HostMode mode;
string text;
union {
string name;
CHTTP_IPv4 ipv4;
CHTTP_IPv6 ipv6;
};
} CHTTP_Host;
typedef struct {
string userinfo;
CHTTP_Host host;
int port;
} CHTTP_Authority;
// ZII
typedef struct {
string scheme;
CHTTP_Authority authority;
string path;
string query;
string fragment;
} CHTTP_URL;
typedef enum {
CHTTP_METHOD_GET,
CHTTP_METHOD_HEAD,
CHTTP_METHOD_POST,
CHTTP_METHOD_PUT,
CHTTP_METHOD_DELETE,
CHTTP_METHOD_CONNECT,
CHTTP_METHOD_OPTIONS,
CHTTP_METHOD_TRACE,
CHTTP_METHOD_PATCH,
} CHTTP_Method;
typedef struct {
string name;
string value;
} CHTTP_Header;
typedef struct {
bool secure;
CHTTP_Method method;
CHTTP_URL url;
int minor;
int num_headers;
CHTTP_Header headers[CHTTP_MAX_HEADERS];
string body;
} CHTTP_Request;
typedef struct {
void* context;
int minor;
int status;
string reason;
int num_headers;
CHTTP_Header headers[CHTTP_MAX_HEADERS];
string body;
} CHTTP_Response;
int chttp_parse_ipv4(char *src, int len, CHTTP_IPv4 *ipv4);
int chttp_parse_ipv6(char *src, int len, CHTTP_IPv6 *ipv6);
int chttp_parse_url(char *src, int len, CHTTP_URL *url);
int chttp_parse_request(char *src, int len, CHTTP_Request *req);
int chttp_parse_response(char *src, int len, CHTTP_Response *res);
int chttp_find_header(CHTTP_Header *headers, int num_headers, string name);
string chttp_get_cookie(CHTTP_Request *req, string name);
string chttp_get_param(string body, string str, char *mem, int cap);
int chttp_get_param_i(string body, string str);
// Checks whether the request was meant for the host with the given
// domain an port. If port is -1, the default value of 80 is assumed.
bool chttp_match_host(CHTTP_Request *req, string domain, int port);
// Date and cookie types for Set-Cookie header parsing
typedef enum {
CHTTP_WEEKDAY_MON,
CHTTP_WEEKDAY_TUE,
CHTTP_WEEKDAY_WED,
CHTTP_WEEKDAY_THU,
CHTTP_WEEKDAY_FRI,
CHTTP_WEEKDAY_SAT,
CHTTP_WEEKDAY_SUN,
} CHTTP_WeekDay;
typedef enum {
CHTTP_MONTH_JAN,
CHTTP_MONTH_FEB,
CHTTP_MONTH_MAR,
CHTTP_MONTH_APR,
CHTTP_MONTH_MAY,
CHTTP_MONTH_JUN,
CHTTP_MONTH_JUL,
CHTTP_MONTH_AUG,
CHTTP_MONTH_SEP,
CHTTP_MONTH_OCT,
CHTTP_MONTH_NOV,
CHTTP_MONTH_DEC,
} CHTTP_Month;
typedef struct {
CHTTP_WeekDay week_day;
int day;
CHTTP_Month month;
int year;
int hour;
int minute;
int second;
} CHTTP_Date;
typedef struct {
string name;
string value;
bool secure;
bool chttp_only;
bool have_date;
CHTTP_Date date;
bool have_max_age;
uint32_t max_age;
bool have_domain;
string domain;
bool have_path;
string path;
} CHTTP_SetCookie;
// Parses a Set-Cookie header value
// Returns 0 on success, -1 on error
int chttp_parse_set_cookie(string str, CHTTP_SetCookie *out);
#endif // HTTP_PARSE_INCLUDED
lib/http_server.c
+390
View File
@@ -0,0 +1,390 @@
#include <quakey.h>
#include <assert.h>
#include "http_server.h"
static string reason_phrase(int status)
{
switch(status) {
case 100: return S("Continue");
case 101: return S("Switching Protocols");
case 102: return S("Processing");
case 200: return S("OK");
case 201: return S("Created");
case 202: return S("Accepted");
case 203: return S("Non-Authoritative Information");
case 204: return S("No Content");
case 205: return S("Reset Content");
case 206: return S("Partial Content");
case 207: return S("Multi-Status");
case 208: return S("Already Reported");
case 300: return S("Multiple Choices");
case 301: return S("Moved Permanently");
case 302: return S("Found");
case 303: return S("See Other");
case 304: return S("Not Modified");
case 305: return S("Use Proxy");
case 306: return S("Switch Proxy");
case 307: return S("Temporary Redirect");
case 308: return S("Permanent Redirect");
case 400: return S("Bad Request");
case 401: return S("Unauthorized");
case 402: return S("Payment Required");
case 403: return S("Forbidden");
case 404: return S("Not Found");
case 405: return S("Method Not Allowed");
case 406: return S("Not Acceptable");
case 407: return S("Proxy Authentication Required");
case 408: return S("Request Timeout");
case 409: return S("Conflict");
case 410: return S("Gone");
case 411: return S("Length Required");
case 412: return S("Precondition Failed");
case 413: return S("Request Entity Too Large");
case 414: return S("Request-URI Too Long");
case 415: return S("Unsupported Media Type");
case 416: return S("Requested Range Not Satisfiable");
case 417: return S("Expectation Failed");
case 418: return S("I'm a teapot");
case 420: return S("Enhance your calm");
case 422: return S("Unprocessable Entity");
case 426: return S("Upgrade Required");
case 429: return S("Too many requests");
case 431: return S("Request Header Fields Too Large");
case 449: return S("Retry With");
case 451: return S("Unavailable For Legal Reasons");
case 500: return S("Internal Server Error");
case 501: return S("Not Implemented");
case 502: return S("Bad Gateway");
case 503: return S("Service Unavailable");
case 504: return S("Gateway Timeout");
case 505: return S("HTTP Version Not Supported");
case 509: return S("Bandwidth Limit Exceeded");
}
return S("???");
}
int http_server_init(HTTP_Server *server, int max_conns)
{
server->conns = malloc(max_conns * sizeof(HTTP_Conn));
if (server->conns == NULL)
return -1;
for (int i = 0; i < max_conns; i++) {
server->conns[i].state = HTTP_CONN_STATE_FREE;
server->conns[i].gen = 1;
}
server->max_conns = max_conns;
server->num_conns = 0;
int ret = tcp_init(&server->tcp, max_conns);
if (ret < 0) {
free(server->conns);
return -1;
}
return 0;
}
void http_server_free(HTTP_Server *server)
{
tcp_free(&server->tcp);
free(server->conns);
}
int http_server_listen_tcp(HTTP_Server *server, string addr, uint16_t port)
{
int ret = tcp_listen_tcp(&server->tcp, addr, port, true, 128);
if (ret < 0)
return -1;
return 0;
}
int http_server_listen_tls(HTTP_Server *server, string addr, uint16_t port,
string cert_file, string key_file)
{
#ifdef TLS_ENABLED
int ret = tls_server_init(&server->tcp.tls, cert_file, key_file);
if (ret < 0)
return -1;
ret = tcp_listen_tls(&server->tcp, addr, port, true, 128);
if (ret < 0)
return -1;
return 0;
#else
(void)server; (void)addr; (void)port; (void)cert_file; (void)key_file;
return -1;
#endif
}
int http_server_add_cert(HTTP_Server *server, string cert_file, string key_file)
{
int ret = tcp_add_cert(&server->tcp, cert_file, key_file);
if (ret < 0)
return -1;
return 0;
}
static void http_conn_init(HTTP_Conn *conn, TCP_Handle handle)
{
assert(conn->state == HTTP_CONN_STATE_FREE);
conn->state = HTTP_CONN_STATE_IDLE;
conn->ready = false;
conn->num_served = 0;
conn->request_len = 0;
conn->keep_alive = true;
conn->handle = handle;
}
static void http_conn_free(HTTP_Conn *conn)
{
assert(conn->state != HTTP_CONN_STATE_FREE);
conn->gen++;
if (conn->gen == 0)
conn->gen = 1;
tcp_close(conn->handle);
conn->state = HTTP_CONN_STATE_FREE;
}
void http_server_process_events(HTTP_Server *server,
void **ptrs, struct pollfd *arr, int num)
{
tcp_process_events(&server->tcp, ptrs, arr, num);
TCP_Event event;
while (tcp_next_event(&server->tcp, &event)) {
if (event.flags & TCP_EVENT_NEW) {
// New connection. Find an HTTP_Conn struct.
int i = 0;
while (i < server->max_conns && server->conns[i].state != HTTP_CONN_STATE_FREE)
i++;
if (i == server->max_conns) {
tcp_close(event.handle);
continue;
}
HTTP_Conn *conn = &server->conns[i];
http_conn_init(conn, event.handle);
tcp_set_user_ptr(event.handle, conn);
server->num_conns++;
}
HTTP_Conn *conn = tcp_get_user_ptr(event.handle);
assert(conn);
bool defer_close = false;
if (event.flags & TCP_EVENT_DATA) {
// Only idle connections can buffer bytes
assert(conn->state == HTTP_CONN_STATE_IDLE);
ByteView src = tcp_read_buf(event.handle);
int ret = chttp_parse_request((char*) src.ptr, src.len, &conn->request);
if (ret < 0) {
tcp_read_ack(event.handle, 0);
defer_close = true;
} else if (ret == 0) {
tcp_read_ack(event.handle, 0);
} else {
// Request was buffered
// Decide whether the connection should be closed
// after the next response
conn->keep_alive = true;
if (conn->num_served+1 >= 100)
conn->keep_alive = false;
conn->response_offset = tcp_write_off(event.handle);
conn->request_len = ret;
conn->state = HTTP_CONN_STATE_STATUS;
conn->ready = true;
}
}
if (event.flags & TCP_EVENT_HUP) {
defer_close = true;
}
if (defer_close) {
http_conn_free(conn);
server->num_conns--;
}
}
}
int http_server_register_events(HTTP_Server *server,
void **ptrs, struct pollfd *arr, int cap)
{
return tcp_register_events(&server->tcp, ptrs, arr, cap);
}
bool http_server_next_request(HTTP_Server *server,
HTTP_Request **request, HTTP_ResponseBuilder *builder)
{
for (int i = 0; i < server->max_conns; i++) {
if (server->conns[i].state == HTTP_CONN_STATE_FREE)
continue;
if (server->conns[i].ready) {
*request = &server->conns[i].request;
*builder = (HTTP_ResponseBuilder) {
.server = server,
.idx = i,
.gen = server->conns[i].gen,
};
return true;
}
}
return false;
}
static HTTP_Conn*
builder_to_conn(HTTP_ResponseBuilder builder)
{
if (builder.server == NULL)
return NULL;
HTTP_Server *server = builder.server;
if (builder.idx < 0 || builder.idx >= server->max_conns)
return NULL;
HTTP_Conn *conn = &server->conns[builder.idx];
if (conn->state == HTTP_CONN_STATE_FREE || conn->gen != builder.gen)
return NULL;
return conn;
}
void http_response_builder_status(HTTP_ResponseBuilder builder, int status)
{
HTTP_Conn *conn = builder_to_conn(builder);
if (conn == NULL)
return;
if (conn->state == HTTP_CONN_STATE_IDLE)
return;
if (conn->state != HTTP_CONN_STATE_STATUS) {
tcp_clear_from_offset(conn->handle, conn->response_offset);
conn->state = HTTP_CONN_STATE_STATUS;
}
assert(status > 99);
assert(status < 1000);
char tmp[3] = {
'0' + (status % 1000) / 100,
'0' + (status % 100) / 10,
'0' + (status % 10) / 1,
};
tcp_write(conn->handle, S("HTTP/1.1 "));
tcp_write(conn->handle, (string) { tmp, 3 });
tcp_write(conn->handle, S(" "));
tcp_write(conn->handle, reason_phrase(status));
tcp_write(conn->handle, S("\r\n"));
conn->state = HTTP_CONN_STATE_HEADER;
}
void http_response_builder_header(HTTP_ResponseBuilder builder, string header)
{
HTTP_Conn *conn = builder_to_conn(builder);
if (conn == NULL)
return;
if (conn->state != HTTP_CONN_STATE_HEADER)
return;
tcp_write(conn->handle, header);
tcp_write(conn->handle, S("\r\n"));
}
static void append_special_headers(HTTP_Conn *conn)
{
if (conn->keep_alive) {
tcp_write(conn->handle, S("Connection: Keep-Alive\r\n"));
} else {
tcp_write(conn->handle, S("Connection: Close\r\n"));
}
tcp_write(conn->handle, S("Content-Length: "));
conn->content_length_header_offset = tcp_write_off(conn->handle);
tcp_write(conn->handle, S(" "));
tcp_write(conn->handle, S("\r\n"));
tcp_write(conn->handle, S("\r\n"));
conn->content_offset = tcp_write_off(conn->handle);
}
void http_response_builder_content(HTTP_ResponseBuilder builder, string content)
{
HTTP_Conn *conn = builder_to_conn(builder);
if (conn == NULL)
return;
if (conn->state == HTTP_CONN_STATE_STATUS)
return;
if (conn->state != HTTP_CONN_STATE_CONTENT) {
append_special_headers(conn);
conn->state = HTTP_CONN_STATE_CONTENT;
}
tcp_write(conn->handle, content);
}
void http_response_builder_submit(HTTP_ResponseBuilder builder)
{
HTTP_Conn *conn = builder_to_conn(builder);
if (conn == NULL)
return;
if (conn->state == HTTP_CONN_STATE_STATUS)
return;
if (conn->state != HTTP_CONN_STATE_CONTENT) {
append_special_headers(conn);
conn->state = HTTP_CONN_STATE_CONTENT;
}
TCP_Offset current_offset = tcp_write_off(conn->handle);
int content_length = current_offset - conn->content_offset;
char buf[11];
int ret = snprintf(buf, sizeof(buf), "%d", content_length);
assert(ret > 0);
assert(ret < (int) sizeof(buf));
tcp_patch(conn->handle, conn->content_length_header_offset, buf, ret);
conn->num_served++;
tcp_read_ack(conn->handle, conn->request_len);
conn->request_len = 0;
if (conn->keep_alive) {
tcp_mark_ready(conn->handle);
conn->ready = false;
conn->state = HTTP_CONN_STATE_IDLE;
} else {
http_conn_free(conn);
builder.server->num_conns--;
}
}
lib/http_server.h
+68
View File
@@ -0,0 +1,68 @@
#ifndef HTTP_SERVER_INCLUDED
#define HTTP_SERVER_INCLUDED
#include "tcp.h"
#include "http_parse.h"
typedef CHTTP_Request HTTP_Request;
typedef enum {
HTTP_CONN_STATE_FREE,
HTTP_CONN_STATE_IDLE,
HTTP_CONN_STATE_STATUS,
HTTP_CONN_STATE_HEADER,
HTTP_CONN_STATE_CONTENT,
} HTTP_ConnState;
typedef struct {
HTTP_ConnState state;
uint16_t gen;
bool ready;
int num_served;
int request_len;
bool keep_alive;
TCP_Handle handle;
HTTP_Request request;
TCP_Offset content_offset;
TCP_Offset content_length_header_offset;
TCP_Offset response_offset;
} HTTP_Conn;
typedef struct {
TCP tcp;
int num_conns;
int max_conns;
HTTP_Conn *conns;
} HTTP_Server;
int http_server_init(HTTP_Server *server, int max_conns);
void http_server_free(HTTP_Server *server);
int http_server_listen_tcp(HTTP_Server *server, string addr, uint16_t port);
int http_server_listen_tls(HTTP_Server *server, string addr, uint16_t port,
string cert_file, string key_file);
int http_server_add_cert(HTTP_Server *server, string cert_file, string key_file);
void http_server_process_events(HTTP_Server *server,
void **ptrs, struct pollfd *arr, int cap);
int http_server_register_events(HTTP_Server *server,
void **ptrs, struct pollfd *arr, int cap);
typedef struct {
HTTP_Server *server;
int idx;
int gen;
} HTTP_ResponseBuilder;
bool http_server_next_request(HTTP_Server *server,
HTTP_Request **request, HTTP_ResponseBuilder *builder);
void http_response_builder_status(HTTP_ResponseBuilder builder, int status);
void http_response_builder_header(HTTP_ResponseBuilder builder, string header);
void http_response_builder_content(HTTP_ResponseBuilder builder, string content);
void http_response_builder_submit(HTTP_ResponseBuilder builder);
#endif // HTTP_SERVER_INCLUDED
lib/message.c
+289
View File
@@ -0,0 +1,289 @@
#include <quakey.h>
#include <assert.h>
#include "message.h"
#define MESSAGE_SYSTEM_VERSION 1
int message_system_init(MessageSystem *msys,
Address *addrs, int num_addrs)
{
if (num_addrs > MESSAGE_SYSTEM_NODE_LIMIT)
return -1;
for (int i = 0; i < num_addrs; i++)
msys->addrs[i] = addrs[i];
msys->num_addrs = num_addrs;
int max_conns = 2 * num_addrs + 1;
msys->conns = malloc(max_conns * sizeof(ConnMetadata));
if (msys->conns == NULL)
return -1;
msys->max_conns = max_conns;
for (int i = 0; i < max_conns; i++) {
msys->conns[i].used = false;
msys->conns[i].gen = 0;
}
int ret = tcp_init(&msys->tcp, max_conns);
if (ret < 0) {
free(msys->conns);
return -1;
}
return 0;
}
int message_system_free(MessageSystem *msys)
{
tcp_free(&msys->tcp);
free(msys->conns);
return 0;
}
int message_system_listen_tcp(MessageSystem *msys, Address addr)
{
int ret = tcp_listen_tcp(&msys->tcp, S(""), addr.port, true, 128);
if (ret < 0)
return -1;
return 0;
}
int message_system_listen_tls(MessageSystem *msys, Address addr)
{
int ret = tcp_listen_tls(&msys->tcp, S(""), addr.port, true, 128);
if (ret < 0)
return -1;
return 0;
}
void message_system_process_events(MessageSystem *msys,
void **ptrs, struct pollfd *arr, int num)
{
tcp_process_events(&msys->tcp, ptrs, arr, num);
}
int message_system_register_events(MessageSystem *msys,
void **ptrs, struct pollfd *arr, int cap)
{
return tcp_register_events(&msys->tcp, ptrs, arr, cap);
}
void *get_next_message(MessageSystem *msys)
{
TCP_Event event;
while (tcp_next_event(&msys->tcp, &event)) {
if (event.flags & TCP_EVENT_NEW) {
// Skip if already set up by ensure_conn (outbound connection)
if (tcp_get_user_ptr(event.handle) == NULL) {
int i = 0;
while (i < msys->max_conns && msys->conns[i].used)
i++;
if (i == msys->max_conns) {
tcp_close(event.handle);
continue;
}
ConnMetadata *meta = &msys->conns[i];
meta->used = true;
meta->num_senders = 0;
meta->message = NULL;
tcp_set_user_ptr(event.handle, meta);
}
}
ConnMetadata *meta = tcp_get_user_ptr(event.handle);
assert(meta);
if (event.flags & TCP_EVENT_HUP) {
meta->used = false;
tcp_close(event.handle);
continue;
}
if (!(event.flags & TCP_EVENT_DATA))
continue;
if (meta->message)
continue; // Already processing message
ByteView buf = tcp_read_buf(event.handle);
Message message;
if (buf.len < sizeof(message)) {
tcp_read_ack(event.handle, 0);
continue;
}
memcpy(&message, buf.ptr, sizeof(message));
// TODO: endianess?
if (message.version != MESSAGE_SYSTEM_VERSION) {
assert(0); // TODO
}
if (message.length > (uint64_t)buf.len) {
tcp_read_ack(event.handle, 0);
continue; // Still buffering
}
// Associate this sender with the TCP connection
if (message.sender < msys->num_addrs) {
bool found = false;
for (int i = 0; i < meta->num_senders; i++) {
if (meta->senders[i] == message.sender) {
found = true;
break;
}
}
if (!found) {
meta->senders[meta->num_senders++] = message.sender;
}
}
meta->message = buf.ptr;
return buf.ptr;
}
return NULL;
}
static TCP_Handle find_conn_by_message(MessageSystem *msys, void *message)
{
for (int i = 0; i < msys->max_conns; i++) {
ConnMetadata *meta = &msys->conns[i];
if (meta->message == message)
return (TCP_Handle) { &msys->tcp, meta->gen, i };
}
return (TCP_Handle) {0};
}
static TCP_Handle find_conn_by_target(MessageSystem *msys, int target)
{
for (int i = 0; i < msys->max_conns; i++) {
ConnMetadata *meta = &msys->conns[i];
for (int j = 0; j < meta->num_senders; j++) {
if (meta->senders[j] == target) {
return (TCP_Handle) { &msys->tcp, meta->gen, i };
}
}
}
return (TCP_Handle) {0};
}
static TCP_Handle ensure_conn(MessageSystem *msys, int target)
{
TCP_Handle handle = find_conn_by_target(msys, target);
if (handle.tcp)
return handle;
if (target < 0 || target >= msys->num_addrs)
return (TCP_Handle) {0};
int ret = tcp_connect(&msys->tcp, false, &msys->addrs[target], 1);
if (ret < 0)
return (TCP_Handle) {0};
// Find the newly created connection slot and pre-associate with target
for (int i = 0; i < msys->max_conns; i++) {
TCP_Conn *conn = &msys->tcp.conns[i];
if (conn->state == TCP_CONN_STATE_FREE)
continue;
if (conn->user_ptr != NULL)
continue;
if (conn->state != TCP_CONN_STATE_CONNECTING
&& conn->state != TCP_CONN_STATE_ESTABLISHED)
continue;
ConnMetadata *meta = &msys->conns[i];
meta->used = true;
meta->gen = conn->gen;
meta->num_senders = 1;
meta->senders[0] = target;
meta->message = NULL;
TCP_Handle h = { &msys->tcp, conn->gen, i };
tcp_set_user_ptr(h, meta);
return h;
}
return (TCP_Handle) {0};
}
void consume_message(MessageSystem *msys, void *ptr)
{
int i = 0;
while (i < msys->max_conns && msys->conns[i].message != ptr)
i++;
if (i == msys->max_conns)
return; // Not found
Message message;
memcpy(&message, ptr, sizeof(message));
TCP_Handle handle = { &msys->tcp, msys->conns[i].gen, i };
tcp_read_ack(handle, message.length);
tcp_mark_ready(handle);
msys->conns[i].message = NULL;
}
void send_message(MessageSystem *msys, int target, Message *message)
{
TCP_Handle handle = ensure_conn(msys, target);
if (!handle.tcp) return;
tcp_write(handle, (string) { (char*) message, message->length });
}
void send_message_ex(MessageSystem *msys, int target,
Message *header, void *extra, int extra_len)
{
TCP_Handle handle = ensure_conn(msys, target);
if (!handle.tcp) return;
int header_size = header->length - extra_len;
tcp_write(handle, (string) { (char*) header, header_size });
tcp_write(handle, (string) { (char*) extra, extra_len });
}
void reply_to_message(MessageSystem *msys, void *incoming_message,
Message *outgoing_message)
{
TCP_Handle handle = find_conn_by_message(msys, incoming_message);
if (!handle.tcp) return;
tcp_write(handle, (string) { (char*) outgoing_message, outgoing_message->length });
}
void reply_to_message_ex(MessageSystem *msys, void *incoming_message,
Message *outgoing_message, void *extra, int extra_len)
{
TCP_Handle handle = find_conn_by_message(msys, incoming_message);
if (!handle.tcp) return;
int header_size = outgoing_message->length - extra_len;
tcp_write(handle, (string) { (char*) outgoing_message, header_size });
tcp_write(handle, (string) { (char*) extra, extra_len });
}
void broadcast_message(MessageSystem *msys, int self_idx, Message *message)
{
for (int i = 0; i < msys->num_addrs; i++) {
if (i != self_idx)
send_message(msys, i, message);
}
}
void broadcast_message_ex(MessageSystem *msys, int self_idx,
Message *header, void *extra, int extra_len)
{
for (int i = 0; i < msys->num_addrs; i++) {
if (i != self_idx)
send_message_ex(msys, i, header, extra, extra_len);
}
}
lib/message.h
+68
View File
@@ -0,0 +1,68 @@
#ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED
#include "tcp.h"
#define MESSAGE_SYSTEM_NODE_LIMIT 8
typedef struct {
bool used;
uint16_t gen;
int senders[MESSAGE_SYSTEM_NODE_LIMIT];
int num_senders;
void* message;
} ConnMetadata;
typedef struct {
TCP tcp;
Address addrs[MESSAGE_SYSTEM_NODE_LIMIT];
int num_addrs;
int max_conns;
ConnMetadata *conns;
} MessageSystem;
typedef struct {
uint16_t version;
uint16_t type;
uint16_t sender;
uint64_t length;
} Message;
int message_system_init(MessageSystem *msys,
Address *addrs, int num_addrs);
int message_system_free(MessageSystem *msys);
int message_system_listen_tcp(MessageSystem *msys, Address addr);
int message_system_listen_tls(MessageSystem *msys, Address addr);
void message_system_process_events(MessageSystem *msys,
void **ptrs, struct pollfd *arr, int num);
int message_system_register_events(MessageSystem *msys,
void **ptrs, struct pollfd *arr, int cap);
void *get_next_message(MessageSystem *msys);
void consume_message(MessageSystem *msys, void *ptr);
void send_message(MessageSystem *msys, int target, Message *message);
void send_message_ex(MessageSystem *msys, int target,
Message *header, void *extra, int extra_len);
void reply_to_message(MessageSystem *msys, void *incoming_message,
Message *outgoing_message);
void reply_to_message_ex(MessageSystem *msys, void *incoming_message,
Message *outgoing_message, void *extra, int extra_len);
void broadcast_message(MessageSystem *msys, int self_idx, Message *message);
void broadcast_message_ex(MessageSystem *msys, int self_idx,
Message *header, void *extra, int extra_len);
#endif // MESSAGE_INCLUDED
lib/tcp.c
+917
View File
@@ -0,0 +1,917 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include "tcp.h"
#define MIN_RECV 4096
#ifdef _WIN32
typedef SOCKET NATIVE_SOCKET;
#else
typedef int NATIVE_SOCKET;
#endif
static void tcp_conn_free(TCP_Conn *conn);
static bool tcp_conn_free_maybe(TCP_Conn *conn);
static int set_socket_blocking(NATIVE_SOCKET sock, bool value)
{
#ifdef _WIN32
u_long mode = !value;
if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
return -1;
return 0;
#else
int flags = fcntl(sock, F_GETFL, 0);
if (flags < 0)
return -1;
if (value)
flags &= ~O_NONBLOCK;
else
flags |= O_NONBLOCK;
if (fcntl(sock, F_SETFL, flags) < 0)
return -1;
return 0;
#endif
}
static int create_listen_socket(string addr, uint16_t port,
bool reuse_addr, int backlog)
{
#ifdef _WIN32
// TODO: Only do this if socket creation fails due to
// winsock not being initialized, then try again
// with the socket
WSADATA wsa;
WSAStartup(MAKEWORD(2, 2), &wsa);
#endif
int fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == -1)
return -1;
if (set_socket_blocking(fd, false) < 0) {
close(fd);
return -1;
}
if (reuse_addr) {
int one = 1;
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void*) &one, sizeof(one));
}
struct in_addr addr_buf;
if (addr.len == 0)
addr_buf.s_addr = htonl(INADDR_ANY);
else {
char copy[100];
if (addr.len >= (int) sizeof(copy)) {
close(fd);
return -1;
}
memcpy(copy, addr.ptr, addr.len);
copy[addr.len] = '\0';
if (inet_pton(AF_INET, copy, &addr_buf) < 0) {
close(fd);
return -1;
}
}
struct sockaddr_in bind_buf;
bind_buf.sin_family = AF_INET;
bind_buf.sin_addr = addr_buf;
bind_buf.sin_port = htons(port);
if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf)) < 0) {
close(fd);
return -1;
}
if (listen(fd, backlog) < 0) {
close(fd);
return -1;
}
return fd;
}
int tcp_init(TCP *tcp, int max_conns)
{
TCP_Conn *conns = malloc(max_conns * sizeof(TCP_Conn));
if (conns == NULL)
return -1;
tcp->tls_listen_fd = -1;
tcp->tcp_listen_fd = -1;
tcp->num_conns = 0;
tcp->max_conns = max_conns;
tcp->conns = conns;
for (int i = 0; i < tcp->max_conns; i++) {
tcp->conns[i].state = TCP_CONN_STATE_FREE;
tcp->conns[i].gen = 0;
}
return 0;
}
void tcp_free(TCP *tcp)
{
for (int i = 0; i < tcp->max_conns; i++) {
if (tcp->conns[i].state != TCP_CONN_STATE_FREE)
tcp_conn_free(&tcp->conns[i]);
}
free(tcp->conns);
if (tcp->tcp_listen_fd != -1)
close(tcp->tcp_listen_fd);
#ifdef TLS_ENABLED
if (tcp->tls_listen_fd != -1) {
close(tcp->tls_listen_fd);
tls_server_free(&tcp->tls);
}
#endif
}
int tcp_listen_tcp(TCP *tcp, string addr, uint16_t port, bool reuse_addr, int backlog)
{
if (tcp->tcp_listen_fd != -1)
return -1;
int fd = create_listen_socket(addr, port, reuse_addr, backlog);
if (fd == -1)
return -1;
tcp->tcp_listen_fd = fd;
return 0;
}
int tcp_listen_tls(TCP *tcp, string addr, uint16_t port, bool reuse_addr, int backlog)
{
#ifdef TLS_ENABLED
if (tcp->tls_listen_fd != -1)
return -1;
int fd = create_listen_socket(addr, port, reuse_addr, backlog);
if (fd == -1)
return -1;
tcp->tls_listen_fd = fd;
return 0;
#else
(void)tcp; (void)addr; (void)port; (void)reuse_addr; (void)backlog;
return -1;
#endif
}
int tcp_add_cert(TCP *tcp, string cert_file, string key_file)
{
#ifdef TLS_ENABLED
return tls_server_add_cert(&tcp->tls, S(""), cert_file, key_file);
#else
(void)tcp; (void)cert_file; (void)key_file;
return -1;
#endif
}
static void tcp_conn_init(TCP *tcp, TCP_Conn *conn, bool secure, TCP_ConnState state, int fd)
{
conn->state = state;
conn->flags = 0;
conn->events = 0;
conn->handled = false;
conn->closing = false;
conn->fd = fd;
conn->num_addrs = 0;
conn->addr_idx = 0;
conn->user_ptr = NULL;
byte_queue_init(&conn->input, 1<<20);
byte_queue_init(&conn->output, 1<<20);
#ifdef TLS_ENABLED
if (secure) {
conn->flags |= TCP_CONN_FLAG_SECURE;
tls_conn_init(&conn->tls, &tcp->tls);
}
#else
(void)tcp;
(void)secure;
#endif
}
static void tcp_conn_free(TCP_Conn *conn)
{
if (conn->fd >= 0) {
close(conn->fd);
conn->fd = -1;
}
byte_queue_free(&conn->input);
byte_queue_free(&conn->output);
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
tls_conn_free(&conn->tls);
}
#endif
conn->state = TCP_CONN_STATE_FREE;
}
static void tcp_conn_set_addrs(TCP_Conn *conn,
Address *addrs, int num_addrs)
{
assert(num_addrs <= TCP_CONNECT_ADDR_LIMIT);
for (int i = 0; i < num_addrs; i++)
conn->addrs[i] = addrs[i];
conn->num_addrs = num_addrs;
}
static ByteView tcp_conn_write_buf(TCP_Conn *conn)
{
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
int cap;
char *ptr = tls_conn_net_write_buf(&conn->tls, &cap);
if (ptr == NULL)
return (ByteView) {0};
return (ByteView) { (uint8_t*) ptr, cap };
}
#endif
byte_queue_write_setmincap(&conn->input, MIN_RECV);
return byte_queue_write_buf(&conn->input);
}
static int tcp_conn_write_ack(TCP_Conn *conn, int num)
{
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
int ret = 0;
tls_conn_net_write_ack(&conn->tls, num);
for (bool done = false; !done; ) {
byte_queue_write_setmincap(&conn->input, MIN_RECV);
ByteView buf = byte_queue_write_buf(&conn->input);
int n = tls_conn_app_read(&conn->tls, (char*) buf.ptr, buf.len);
if (n <= 0) {
if (n < 0) {
ret = -1;
n = 0;
}
done = true;
}
byte_queue_write_ack(&conn->input, n);
}
return ret;
}
#endif
byte_queue_write_ack(&conn->input, num);
return 0;
}
#ifdef TLS_ENABLED
// Encrypt plaintext from the output queue through SSL_write into the BIO.
static void tcp_conn_tls_encrypt_output(TCP_Conn *conn)
{
while (!byte_queue_empty(&conn->output)) {
ByteView src = byte_queue_read_buf(&conn->output);
if (!src.ptr || src.len == 0) {
byte_queue_read_ack(&conn->output, 0);
break;
}
int n = tls_conn_app_write(&conn->tls, (char*) src.ptr, src.len);
if (n <= 0) {
byte_queue_read_ack(&conn->output, 0);
break;
}
byte_queue_read_ack(&conn->output, n);
}
}
#endif
static ByteView tcp_conn_read_buf(TCP_Conn *conn)
{
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
tcp_conn_tls_encrypt_output(conn);
int n;
char *ptr = tls_conn_net_read_buf(&conn->tls, &n);
if (ptr == NULL)
return (ByteView) {0};
return (ByteView) { (uint8_t*) ptr, n };
}
#endif
return byte_queue_read_buf(&conn->output);
}
static void tcp_conn_read_ack(TCP_Conn *conn, int num)
{
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
tls_conn_net_read_ack(&conn->tls, num);
return;
}
#endif
byte_queue_read_ack(&conn->output, num);
}
static bool tcp_conn_needs_flushing(TCP_Conn *conn)
{
#ifdef TLS_ENABLED
if (conn->flags & TCP_CONN_FLAG_SECURE) {
return !byte_queue_empty(&conn->output)
|| tls_conn_needs_flushing(&conn->tls);
}
#endif
return !byte_queue_empty(&conn->output);
}
static bool tcp_conn_is_buffering(TCP_Conn *conn)
{
if (conn->closing)
return false;
if (conn->state == TCP_CONN_STATE_HANDSHAKE ||
conn->state == TCP_CONN_STATE_ACCEPTING)
return true;
return !byte_queue_reading(&conn->input);
}
static bool tcp_conn_free_maybe(TCP_Conn *conn)
{
if (!conn->handled && conn->fd < 0) {
tcp_conn_free(conn);
return true;
} else {
return false;
}
}
static void tcp_conn_invalidate_handles(TCP_Conn *conn)
{
conn->gen++;
if (conn->gen == 0)
conn->gen = 1;
}
static int build_sockaddr(Address *addr, struct sockaddr_in *out)
{
memset(out, 0, sizeof(*out));
if (!addr->is_ipv4)
return -1; // Only IPv4 supported for now
out->sin_family = AF_INET;
out->sin_port = htons(addr->port);
memcpy(&out->sin_addr, &addr->ipv4, sizeof(addr->ipv4));
return 0;
}
int tcp_connect(TCP *tcp, bool secure, Address *addrs, int num_addrs)
{
if (tcp->num_conns == tcp->max_conns)
return -1;
int i = 0;
while (i < tcp->max_conns && tcp->conns[i].state != TCP_CONN_STATE_FREE)
i++;
assert(i < tcp->max_conns);
int fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) {
return -1;
}
if (set_socket_blocking(fd, false) < 0) {
close(fd);
return -1;
}
struct sockaddr_in sa;
if (build_sockaddr(&addrs[0], &sa) < 0) {
close(fd);
return -1;
}
TCP_ConnState state;
int ret = connect(fd, (struct sockaddr*) &sa, sizeof(sa));
if (ret == 0) {
if (secure) {
state = TCP_CONN_STATE_HANDSHAKE;
} else {
state = TCP_CONN_STATE_ESTABLISHED;
}
} else {
assert(ret < 0);
if (errno == EINPROGRESS) {
state = TCP_CONN_STATE_CONNECTING;
} else {
close(fd);
return -1;
}
}
tcp_conn_init(tcp, &tcp->conns[i], secure, state, fd);
tcp_conn_set_addrs(&tcp->conns[i], addrs, num_addrs);
tcp->num_conns++;
return 0;
}
static int restart_connect(TCP_Conn *conn)
{
close(conn->fd);
conn->fd = -1;
conn->addr_idx++;
if (conn->addr_idx == conn->num_addrs) {
return -1;
}
int fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) {
return -1;
}
if (set_socket_blocking(fd, false) < 0) {
close(fd);
return -1;
}
struct sockaddr_in sa;
if (build_sockaddr(&conn->addrs[conn->addr_idx], &sa) < 0) {
close(fd);
return -1;
}
TCP_ConnState state;
int ret = connect(fd, (struct sockaddr*) &sa, sizeof(sa));
if (ret == 0) {
if (conn->flags & TCP_CONN_FLAG_SECURE) {
state = TCP_CONN_STATE_HANDSHAKE;
} else {
state = TCP_CONN_STATE_ESTABLISHED;
}
} else {
assert(ret < 0);
if (errno == EINPROGRESS) {
state = TCP_CONN_STATE_CONNECTING;
} else {
close(fd);
return -1;
}
}
conn->fd = fd;
conn->state = state;
return 0;
}
void tcp_process_events(TCP *tcp, void **ptrs, struct pollfd *arr, int num)
{
for (int i = 0; i < num; i++) {
if (arr[i].fd == tcp->tcp_listen_fd ||
arr[i].fd == tcp->tls_listen_fd) {
assert(ptrs[i] == NULL);
if (arr[i].revents & POLLIN) {
if (tcp->num_conns == tcp->max_conns)
continue;
bool is_tls = false;
if (arr[i].fd == tcp->tls_listen_fd)
is_tls = true;
int new_fd = accept(arr[i].fd, NULL, NULL);
if (new_fd == -1)
continue;
if (set_socket_blocking(new_fd, false) < 0) {
close(new_fd);
continue;
}
// Find a free connection slot
int slot = 0;
while (slot < tcp->max_conns && tcp->conns[slot].state != TCP_CONN_STATE_FREE)
slot++;
if (slot == tcp->max_conns) {
close(new_fd);
continue;
}
TCP_ConnState state;
if (is_tls) {
state = TCP_CONN_STATE_ACCEPTING;
} else {
state = TCP_CONN_STATE_ESTABLISHED;
}
TCP_Conn *conn = &tcp->conns[slot];
tcp_conn_init(tcp, conn, is_tls, state, new_fd);
if (!is_tls)
conn->events |= TCP_EVENT_NEW;
tcp->num_conns++;
}
} else {
TCP_Conn *conn = ptrs[i];
bool defer_ready = false;
bool defer_close = false;
bool defer_connect = false;
switch (conn->state) {
case TCP_CONN_STATE_CONNECTING:
{
int err = 0;
socklen_t len = sizeof(err);
if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0) {
defer_connect = true;
break;
}
if (err) {
defer_connect = true;
break;
}
if (conn->flags & TCP_CONN_FLAG_SECURE) {
conn->state = TCP_CONN_STATE_HANDSHAKE;
} else {
conn->state = TCP_CONN_STATE_ESTABLISHED;
conn->events |= TCP_EVENT_NEW;
}
}
break;
case TCP_CONN_STATE_HANDSHAKE:
case TCP_CONN_STATE_ACCEPTING:
{
#ifdef TLS_ENABLED
if (arr[i].revents & POLLIN) {
int cap;
char *buf = tls_conn_net_write_buf(&conn->tls, &cap);
if (buf) {
int n = recv(conn->fd, buf, cap, 0);
if (n == 0) { defer_close = true; break; }
if (n < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
{ defer_close = true; break; }
n = 0;
}
tls_conn_net_write_ack(&conn->tls, n);
}
}
int ret = tls_conn_handshake(&conn->tls);
if (ret == -1) {
defer_close = true;
break;
}
if (arr[i].revents & POLLOUT) {
int num;
char *buf = tls_conn_net_read_buf(&conn->tls, &num);
if (buf) {
int n = send(conn->fd, buf, num, 0);
if (n < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
{ defer_close = true; break; }
n = 0;
}
tls_conn_net_read_ack(&conn->tls, n);
}
}
if (ret == 1) {
conn->state = TCP_CONN_STATE_ESTABLISHED;
conn->events |= TCP_EVENT_NEW;
// Decrypt any application data already in the BIO
for (;;) {
byte_queue_write_setmincap(&conn->input, MIN_RECV);
ByteView buf = byte_queue_write_buf(&conn->input);
if (!buf.ptr) break;
int n = tls_conn_app_read(&conn->tls, (char*) buf.ptr, buf.len);
if (n <= 0) { byte_queue_write_ack(&conn->input, 0); break; }
byte_queue_write_ack(&conn->input, n);
conn->events |= TCP_EVENT_DATA;
}
}
#else
defer_close = true;
#endif
}
break;
case TCP_CONN_STATE_ESTABLISHED:
{
if (arr[i].revents & POLLIN) {
ByteView buf = tcp_conn_write_buf(conn);
int n = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
if (n == 0) {
defer_close = true;
} else {
if (n < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
defer_close = true;
n = 0;
}
}
int ret = tcp_conn_write_ack(conn, n);
if (ret < 0)
defer_close = true;
defer_ready = true;
}
if (arr[i].revents & POLLOUT) {
ByteView buf = tcp_conn_read_buf(conn);
int n = send(conn->fd, (char*) buf.ptr, buf.len, 0);
if (n < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
defer_close = true;
n = 0;
}
tcp_conn_read_ack(conn, n);
if (conn->closing && !tcp_conn_needs_flushing(conn))
defer_close = true;
}
}
break;
case TCP_CONN_STATE_SHUTDOWN:
{
// TLS shutdown — just close for now
defer_close = true;
}
break;
default:
break;
}
if (defer_connect) {
int ret = restart_connect(conn);
if (ret < 0) {
defer_close = true;
}
}
if (defer_ready) {
conn->events |= TCP_EVENT_DATA;
}
if (defer_close) {
close(conn->fd);
conn->fd = -1;
conn->events |= TCP_EVENT_HUP;
if (tcp_conn_free_maybe(conn)) {
tcp->num_conns--;
}
}
}
}
}
int tcp_register_events(TCP *tcp, void **ptrs, struct pollfd *arr, int cap)
{
if (cap < tcp->num_conns+2)
return -1;
int ret = 0;
if (tcp->tcp_listen_fd > -1) {
if (tcp->num_conns < tcp->max_conns) {
arr[ret].fd = tcp->tcp_listen_fd;
arr[ret].events = POLLIN;
arr[ret].revents = 0;
ptrs[ret] = NULL;
ret++;
}
}
if (tcp->tls_listen_fd > -1) {
if (tcp->num_conns < tcp->max_conns) {
arr[ret].fd = tcp->tls_listen_fd;
arr[ret].events = POLLIN;
arr[ret].revents = 0;
ptrs[ret] = NULL;
ret++;
}
}
for (int i=0, j=0; j < tcp->num_conns; i++) {
TCP_Conn *conn = &tcp->conns[i];
if (conn->state == TCP_CONN_STATE_FREE)
continue;
j++;
int events = 0;
if (conn->state == TCP_CONN_STATE_CONNECTING)
events |= POLLOUT;
if (tcp_conn_is_buffering(conn))
events |= POLLIN;
if (tcp_conn_needs_flushing(conn))
events |= POLLOUT;
if (events) {
arr[ret].fd = conn->fd;
arr[ret].events = events;
arr[ret].revents = 0;
ptrs[ret] = conn;
ret++;
}
}
return ret;
}
static TCP_Handle conn_to_handle(TCP *tcp, TCP_Conn *conn)
{
TCP_Handle handle = {
.tcp=tcp,
.gen=conn->gen,
.idx=conn - tcp->conns,
};
return handle;
}
static TCP_Conn *handle_to_conn(TCP_Handle handle)
{
if (handle.tcp == NULL)
return NULL;
TCP *tcp = handle.tcp;
if (handle.idx < 0 || handle.idx >= tcp->max_conns)
return NULL;
TCP_Conn *conn = &tcp->conns[handle.idx];
if (conn->state == TCP_CONN_STATE_FREE || conn->gen != handle.gen)
return NULL;
return conn;
}
static bool conn_to_event(TCP *tcp, TCP_Conn *conn, TCP_Event *event)
{
if (!conn->events)
return false;
*event = (TCP_Event) {
.flags = conn->events,
.handle = conn_to_handle(tcp, conn),
};
conn->events = 0;
return true;
}
bool tcp_next_event(TCP *tcp, TCP_Event *event)
{
for (int i = 0, j = 0; j < tcp->num_conns; i++) {
TCP_Conn *conn = &tcp->conns[i];
if (conn->state == TCP_CONN_STATE_FREE)
continue;
j++;
if (conn->flags & TCP_CONN_FLAG_CLOSED)
continue; // User isn't interested in this connection anymore
if (conn_to_event(tcp, conn, event))
return true;
}
return false;
}
ByteView tcp_read_buf(TCP_Handle handle)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return (ByteView) {0};
return byte_queue_read_buf(&conn->input);
}
void tcp_read_ack(TCP_Handle handle, int num)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
byte_queue_read_ack(&conn->input, num);
}
ByteView tcp_write_buf(TCP_Handle handle)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return (ByteView) {0};
return byte_queue_write_buf(&conn->output);
}
void tcp_write_ack(TCP_Handle handle, int num)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
byte_queue_write_ack(&conn->output, num);
}
TCP_Offset tcp_write_off(TCP_Handle handle)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return 0;
return byte_queue_offset(&conn->output);
}
void tcp_write(TCP_Handle handle, string str)
{
while (str.len > 0) {
byte_queue_write_setmincap(&handle_to_conn(handle)->output, str.len);
ByteView buf = tcp_write_buf(handle);
int num = MIN(buf.len, str.len);
memcpy(buf.ptr, str.ptr, num);
tcp_write_ack(handle, num);
str.ptr += num;
str.len -= num;
}
}
void tcp_patch(TCP_Handle handle, TCP_Offset offset, void *src, int len)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
byte_queue_patch(&conn->output, offset, src, len);
}
void tcp_clear_from_offset(TCP_Handle handle, TCP_Offset offset)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
byte_queue_remove_from_offset(&conn->output, offset);
}
void tcp_close(TCP_Handle handle)
{
TCP *tcp = handle.tcp;
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
conn->flags |= TCP_CONN_FLAG_CLOSED;
conn->handled = false;
tcp_conn_invalidate_handles(conn);
if (tcp_conn_free_maybe(conn)) {
tcp->num_conns--;
}
}
void tcp_set_user_ptr(TCP_Handle handle, void *ptr)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
conn->user_ptr = ptr;
}
void *tcp_get_user_ptr(TCP_Handle handle)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return NULL;
return conn->user_ptr;
}
void tcp_mark_ready(TCP_Handle handle)
{
TCP_Conn *conn = handle_to_conn(handle);
if (conn == NULL)
return;
conn->events |= TCP_EVENT_DATA;
}
lib/tcp.h
+99
View File
@@ -0,0 +1,99 @@
#ifndef TCP_INCLUDED
#define TCP_INCLUDED
#include "basic.h"
#include "byte_queue.h"
#include "tls.h"
typedef struct TCP TCP;
typedef struct {
TCP* tcp;
uint16_t gen;
int idx;
} TCP_Handle;
typedef enum {
TCP_CONN_STATE_FREE,
TCP_CONN_STATE_HANDSHAKE,
TCP_CONN_STATE_ESTABLISHED,
TCP_CONN_STATE_CONNECTING,
TCP_CONN_STATE_ACCEPTING,
TCP_CONN_STATE_SHUTDOWN,
} TCP_ConnState;
#define TCP_CONNECT_ADDR_LIMIT 8
enum {
TCP_EVENT_NEW = 1<<0,
TCP_EVENT_HUP = 1<<1,
TCP_EVENT_DATA = 1<<2,
};
enum {
TCP_CONN_FLAG_CLOSED = 1<<0,
TCP_CONN_FLAG_SECURE = 1<<1,
};
typedef struct {
TCP_ConnState state;
int flags;
int events;
uint16_t gen;
int fd;
bool handled;
bool closing;
void *user_ptr;
ByteQueue input;
ByteQueue output;
#ifdef TLS_ENABLED
TLS_Conn tls;
#endif
Address addrs[TCP_CONNECT_ADDR_LIMIT];
int num_addrs;
int addr_idx;
} TCP_Conn;
struct TCP {
int tls_listen_fd;
int tcp_listen_fd;
int num_conns;
int max_conns;
TCP_Conn *conns;
#ifdef TLS_ENABLED
TLS_Server tls;
#endif
};
typedef struct {
int flags;
TCP_Handle handle;
} TCP_Event;
typedef ByteQueueOffset TCP_Offset;
struct pollfd;
int tcp_init(TCP *tcp, int max_conns);
void tcp_free(TCP *tcp);
int tcp_listen_tcp(TCP *tcp, string addr, uint16_t port, bool reuse_addr, int backlog);
int tcp_listen_tls(TCP *tcp, string addr, uint16_t port, bool reuse_addr, int backlog);
int tcp_add_cert(TCP *tcp, string cert_file, string key_file);
int tcp_connect(TCP *tcp, bool secure, Address *addrs, int num_addrs);
void tcp_process_events(TCP *tcp, void **ptrs, struct pollfd *arr, int num);
int tcp_register_events(TCP *tcp, void **ptrs, struct pollfd *arr, int cap);
bool tcp_next_event(TCP *tcp, TCP_Event *event);
ByteView tcp_read_buf(TCP_Handle handle);
void tcp_read_ack(TCP_Handle handle, int num);
ByteView tcp_write_buf(TCP_Handle handle);
void tcp_write_ack(TCP_Handle handle, int num);
TCP_Offset tcp_write_off(TCP_Handle handle);
void tcp_write(TCP_Handle handle, string str);
void tcp_patch(TCP_Handle handle, TCP_Offset offset, void *src, int len);
void tcp_clear_from_offset(TCP_Handle handle, TCP_Offset offset);
void tcp_close(TCP_Handle handle);
void tcp_set_user_ptr(TCP_Handle handle, void *ptr);
void *tcp_get_user_ptr(TCP_Handle handle);
void tcp_mark_ready(TCP_Handle handle);
#endif // TCP_INCLUDED
lib/tls.h
+92
View File
@@ -0,0 +1,92 @@
#ifdef TLS_ENABLED
#ifndef TLS_INCLUDED
#define TLS_INCLUDED
#include <stdbool.h>
#include "basic.h"
#define TLS_CERT_LIMIT 8
#ifdef TLS_SCHANNEL
#define SECURITY_WIN32
#include <windows.h>
#include <security.h>
#include <schannel.h>
#include "byte_queue.h"
typedef struct {
char domain[128];
CredHandle cred;
} TLS_Cert;
typedef struct {
CredHandle cred;
PCCERT_CONTEXT cert_ctx;
int num_certs;
TLS_Cert certs[TLS_CERT_LIMIT];
} TLS_Server;
typedef struct {
CtxtHandle ctx;
bool ctx_valid;
bool handshake;
CredHandle *cred;
ByteQueue in_buf;
ByteQueue out_buf;
SecPkgContext_StreamSizes sizes;
char *pending;
int pending_off;
int pending_len;
} TLS_Conn;
#else // OpenSSL
typedef struct ssl_ctx_st SSL_CTX;
typedef struct ssl_st SSL;
typedef struct bio_st BIO;
typedef struct {
char domain[128];
SSL_CTX *ctx;
} TLS_Cert;
typedef struct {
SSL_CTX *ctx;
int num_certs;
TLS_Cert certs[TLS_CERT_LIMIT];
} TLS_Server;
typedef struct {
SSL *ssl;
BIO *network_bio;
bool handshake;
} TLS_Conn;
#endif // TLS_SCHANNEL
void tls_global_init(void);
void tls_global_free(void);
int tls_server_init(TLS_Server *server, string cert_file, string key_file);
void tls_server_free(TLS_Server *server);
int tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file);
int tls_conn_init(TLS_Conn *conn, TLS_Server *server);
void tls_conn_free(TLS_Conn *conn);
int tls_conn_handshake(TLS_Conn *conn);
char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap);
void tls_conn_net_write_ack(TLS_Conn *conn, int num);
char *tls_conn_net_read_buf(TLS_Conn *conn, int *num);
void tls_conn_net_read_ack(TLS_Conn *conn, int num);
int tls_conn_app_write(TLS_Conn *conn, char *dst, int num);
int tls_conn_app_read(TLS_Conn *conn, char *src, int cap);
int tls_conn_needs_flushing(TLS_Conn *conn);
#endif // TLS_INCLUDED
#endif // TLS_ENABLED
lib/tls_openssl.c
+293
View File
@@ -0,0 +1,293 @@
#ifdef TLS_ENABLED
#ifdef TLS_OPENSSL
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
// Avoid name collision between basic.h's SHA256 typedef
// and OpenSSL's SHA256 function
#define SHA256 openssl_SHA256
#include <openssl/ssl.h>
#include <openssl/err.h>
#undef SHA256
#include "tls.h"
void tls_global_init(void)
{
SSL_library_init();
SSL_load_error_strings();
OpenSSL_add_all_algorithms();
}
void tls_global_free(void)
{
EVP_cleanup();
}
static int servername_callback(SSL *ssl, int *ad, void *arg)
{
TLS_Server *server = arg;
// The 'ad' parameter is used to set the alert description when returning
// SSL_TLSEXT_ERR_ALERT_FATAL. Since we only return OK or NOACK, it's unused.
(void) ad;
const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
if (servername == NULL)
return SSL_TLSEXT_ERR_NOACK;
for (int i = 0; i < server->num_certs; i++) {
TLS_Cert *cert = &server->certs[i];
if (!strcmp(cert->domain, servername)) {
SSL_set_SSL_CTX(ssl, cert->ctx);
return SSL_TLSEXT_ERR_OK;
}
}
return SSL_TLSEXT_ERR_NOACK;
}
int tls_server_init(TLS_Server *server, string cert_file, string key_file)
{
SSL_CTX *ctx = SSL_CTX_new(TLS_server_method());
if (ctx == NULL)
return -1;
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
char cert_buf[1024];
if (cert_file.len >= (int) sizeof(cert_buf)) {
SSL_CTX_free(ctx);
return -1;
}
memcpy(cert_buf, cert_file.ptr, cert_file.len);
cert_buf[cert_file.len] = '\0';
char key_buf[1024];
if (key_file.len >= (int) sizeof(key_buf)) {
SSL_CTX_free(ctx);
return -1;
}
memcpy(key_buf, key_file.ptr, key_file.len);
key_buf[key_file.len] = '\0';
// Load certificate and private key
if (SSL_CTX_use_certificate_chain_file(ctx, cert_buf) != 1) {
SSL_CTX_free(ctx);
return -1;
}
if (SSL_CTX_use_PrivateKey_file(ctx, key_buf, SSL_FILETYPE_PEM) != 1) {
SSL_CTX_free(ctx);
return -1;
}
if (SSL_CTX_check_private_key(ctx) != 1) {
SSL_CTX_free(ctx);
return -1;
}
SSL_CTX_set_tlsext_servername_callback(ctx, servername_callback);
SSL_CTX_set_tlsext_servername_arg(ctx, server);
server->ctx = ctx;
server->num_certs = 0;
return 0;
}
void tls_server_free(TLS_Server *server)
{
for (int i = 0; i < server->num_certs; i++)
SSL_CTX_free(server->certs[i].ctx);
SSL_CTX_free(server->ctx);
}
// TODO: Can the domain be inferred from the cert?
int tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file)
{
if (server->num_certs == TLS_CERT_LIMIT)
return -1;
SSL_CTX *ctx = SSL_CTX_new(TLS_server_method());
if (!ctx)
return -1;
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
char cert_buf[1024];
if (cert_file.len >= (int) sizeof(cert_buf)) {
SSL_CTX_free(ctx);
return -1;
}
memcpy(cert_buf, cert_file.ptr, cert_file.len);
cert_buf[cert_file.len] = '\0';
char key_buf[1024];
if (key_file.len >= (int) sizeof(key_buf)) {
SSL_CTX_free(ctx);
return -1;
}
memcpy(key_buf, key_file.ptr, key_file.len);
key_buf[key_file.len] = '\0';
if (SSL_CTX_use_certificate_chain_file(ctx, cert_buf) != 1) {
SSL_CTX_free(ctx);
return -1;
}
if (SSL_CTX_use_PrivateKey_file(ctx, key_buf, SSL_FILETYPE_PEM) != 1) {
SSL_CTX_free(ctx);
return -1;
}
if (SSL_CTX_check_private_key(ctx) != 1) {
SSL_CTX_free(ctx);
return -1;
}
TLS_Cert *cert = &server->certs[server->num_certs];
if (domain.len >= (int) sizeof(cert->domain)) {
SSL_CTX_free(ctx);
return -1;
}
memcpy(cert->domain, domain.ptr, domain.len);
cert->domain[domain.len] = '\0';
cert->ctx = ctx;
server->num_certs++;
return 0;
}
int tls_conn_init(TLS_Conn *conn, TLS_Server *server)
{
SSL *ssl = SSL_new(server->ctx);
if (ssl == NULL)
return -1;
// Create a BIO pair:
// internal_bio — attached to the SSL object (OpenSSL uses it internally)
// network_bio — you read/write encrypted data from/to this
BIO *internal_bio = NULL;
BIO *network_bio = NULL;
if (!BIO_new_bio_pair(&internal_bio, 0, &network_bio, 0)) {
SSL_free(ssl);
return -1;
}
// Bind the internal side to the SSL object
SSL_set_bio(ssl, internal_bio, internal_bio);
// We're the server side
SSL_set_accept_state(ssl);
conn->ssl = ssl;
conn->network_bio = network_bio;
conn->handshake = true;
return 0;
}
void tls_conn_free(TLS_Conn *conn)
{
SSL_free(conn->ssl);
BIO_free(conn->network_bio);
}
// Write ciphertext from the connection object to the network
char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap)
{
char *buf;
int ret = BIO_nwrite0(conn->network_bio, &buf);
if (ret <= 0)
return NULL;
*cap = ret;
return buf;
}
// Complete the write from the connection object
void tls_conn_net_write_ack(TLS_Conn *conn, int num)
{
char *dummy;
BIO_nwrite(conn->network_bio, &dummy, num);
}
// Read ciphertext from the network into the connection object
char *tls_conn_net_read_buf(TLS_Conn *conn, int *num)
{
char *buf;
int ret = BIO_nread0(conn->network_bio, &buf);
if (ret <= 0)
return NULL;
*num = ret;
return buf;
}
// Complete the read from the network
void tls_conn_net_read_ack(TLS_Conn *conn, int num)
{
char *dummy;
BIO_nread(conn->network_bio, &dummy, num);
}
// Write plaintext from the application to the connection object
int tls_conn_app_write(TLS_Conn *conn, char *dst, int num)
{
assert(!conn->handshake);
int n = SSL_write(conn->ssl, dst, num);
if (n > 0)
return n;
int err = SSL_get_error(conn->ssl, n);
if (err == SSL_ERROR_WANT_READ ||
err == SSL_ERROR_WANT_WRITE)
return 0;
return -1;
}
// Read plaintext from the connection object into the application
int tls_conn_app_read(TLS_Conn *conn, char *src, int cap)
{
assert(!conn->handshake);
int n = SSL_read(conn->ssl, src, cap);
if (n > 0)
return n;
int err = SSL_get_error(conn->ssl, n);
if (err == SSL_ERROR_WANT_READ ||
err == SSL_ERROR_WANT_WRITE)
return 0;
return -1;
}
int tls_conn_handshake(TLS_Conn *conn)
{
assert(conn->handshake);
int n = SSL_do_handshake(conn->ssl);
if (n == 1) {
conn->handshake = false;
return 1;
}
int err = SSL_get_error(conn->ssl, n);
if (err == SSL_ERROR_WANT_READ ||
err == SSL_ERROR_WANT_WRITE)
return 0;
return -1;
}
int tls_conn_needs_flushing(TLS_Conn *conn)
{
return BIO_ctrl_pending(conn->network_bio) > 0;
}
#endif // TLS_OPENSSL
#endif // TLS_ENABLED
lib/tls_schannel.c
+695
View File
@@ -0,0 +1,695 @@
#ifdef TLS_ENABLED
#ifdef TLS_SCHANNEL
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define SECURITY_WIN32
#include <windows.h>
#include <wincrypt.h>
#include <security.h>
#include <schannel.h>
#include "tls.h"
#pragma comment(lib, "secur32.lib")
#pragma comment(lib, "crypt32.lib")
#pragma comment(lib, "ncrypt.lib")
#define TLS_BUF_LIMIT (32 * 1024)
// ============================================================
// Certificate loading
// ============================================================
// Read entire file into malloc'd buffer. Caller must free.
static char *read_file(const char *path, int *out_len)
{
FILE *f = fopen(path, "rb");
if (!f) return NULL;
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
if (len <= 0 || len > 1024 * 1024) {
fclose(f);
return NULL;
}
char *buf = malloc((size_t) len + 1);
if (!buf) {
fclose(f);
return NULL;
}
size_t nread = fread(buf, 1, (size_t) len, f);
fclose(f);
buf[nread] = '\0';
*out_len = (int) nread;
return buf;
}
// Decode PEM base64 content to DER binary. Returns malloc'd buffer.
static BYTE *pem_to_der(const char *pem, int pem_len, DWORD *out_len)
{
DWORD len = 0;
if (!CryptStringToBinaryA(pem, pem_len, CRYPT_STRING_BASE64HEADER,
NULL, &len, NULL, NULL))
return NULL;
BYTE *der = malloc(len);
if (!der) return NULL;
if (!CryptStringToBinaryA(pem, pem_len, CRYPT_STRING_BASE64HEADER,
der, &len, NULL, NULL)) {
free(der);
return NULL;
}
*out_len = len;
return der;
}
// Import an in-memory PFX blob, acquire SChannel credential.
static int load_pfx_blob(const BYTE *pfx_data, DWORD pfx_len,
CredHandle *cred_out, PCCERT_CONTEXT *cert_ctx_out)
{
CRYPT_DATA_BLOB blob;
blob.pbData = (BYTE *) pfx_data;
blob.cbData = pfx_len;
HCERTSTORE store = PFXImportCertStore(&blob, L"", CRYPT_EXPORTABLE);
if (!store) return -1;
// Find first cert with a private key
PCCERT_CONTEXT cert_ctx = NULL;
while ((cert_ctx = CertEnumCertificatesInStore(store, cert_ctx)) != NULL) {
HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
DWORD key_spec = 0;
BOOL caller_free = FALSE;
if (CryptAcquireCertificatePrivateKey(cert_ctx,
CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG | CRYPT_ACQUIRE_SILENT_FLAG,
NULL, &hkey, &key_spec, &caller_free)) {
if (caller_free && hkey) {
if (key_spec == CERT_NCRYPT_KEY_SPEC)
NCryptFreeObject(hkey);
else
CryptReleaseContext(hkey, 0);
}
break;
}
}
if (!cert_ctx) {
CertCloseStore(store, 0);
return -1;
}
PCCERT_CONTEXT dup_ctx = CertDuplicateCertificateContext(cert_ctx);
SCHANNEL_CRED sc_cred = {0};
sc_cred.dwVersion = SCHANNEL_CRED_VERSION;
sc_cred.cCreds = 1;
sc_cred.paCred = &dup_ctx;
sc_cred.grbitEnabledProtocols = SP_PROT_TLS1_2_SERVER;
TimeStamp expiry;
SECURITY_STATUS ss = AcquireCredentialsHandleA(
NULL, UNISP_NAME_A, SECPKG_CRED_INBOUND,
NULL, &sc_cred, NULL, NULL,
cred_out, &expiry);
if (ss != SEC_E_OK) {
CertFreeCertificateContext(dup_ctx);
CertCloseStore(store, 0);
return -1;
}
*cert_ctx_out = dup_ctx;
// Keep store open — SChannel needs access to the cert+key
return 0;
}
// Load PEM cert+key using pure CryptoAPI (no external tools).
// Decodes PEM, imports key into a temporary CAPI keyset, exports as
// in-memory PFX, then imports via PFXImportCertStore.
static int load_pem_credential(string cert_file, string key_file,
CredHandle *cred_out, PCCERT_CONTEXT *cert_ctx_out)
{
int result = -1;
char cert_z[1024], key_z[1024], container[64];
char *cert_pem = NULL, *key_pem = NULL;
BYTE *cert_der = NULL, *key_der = NULL;
BYTE *rsa_blob = NULL, *pkcs8_buf = NULL;
PCCERT_CONTEXT cert_ctx = NULL;
HCRYPTPROV hprov = 0;
HCRYPTKEY hkey = 0;
HCERTSTORE mem_store = NULL;
CRYPT_DATA_BLOB pfx = {0};
int cert_pem_len, key_pem_len;
DWORD cert_der_len, key_der_len, rsa_blob_len, pkcs8_len;
snprintf(container, sizeof(container), "tls_tmp_%lu",
(unsigned long) GetCurrentProcessId());
// Null-terminate file paths
if (cert_file.len >= (int) sizeof(cert_z)) goto done;
memcpy(cert_z, cert_file.ptr, cert_file.len);
cert_z[cert_file.len] = '\0';
if (key_file.len >= (int) sizeof(key_z)) goto done;
memcpy(key_z, key_file.ptr, key_file.len);
key_z[key_file.len] = '\0';
// --- Certificate: PEM -> DER -> CERT_CONTEXT ---
cert_pem = read_file(cert_z, &cert_pem_len);
if (!cert_pem) goto done;
cert_der = pem_to_der(cert_pem, cert_pem_len, &cert_der_len);
if (!cert_der) goto done;
cert_ctx = CertCreateCertificateContext(
X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, cert_der, cert_der_len);
if (!cert_ctx) goto done;
// --- Private key: PEM -> DER -> CAPI RSA blob ---
key_pem = read_file(key_z, &key_pem_len);
if (!key_pem) goto done;
key_der = pem_to_der(key_pem, key_pem_len, &key_der_len);
if (!key_der) goto done;
if (strstr(key_pem, "-----BEGIN PRIVATE KEY-----")) {
// PKCS#8: unwrap to get inner RSA key
if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_PRIVATE_KEY_INFO,
key_der, key_der_len, CRYPT_DECODE_ALLOC_FLAG,
NULL, &pkcs8_buf, &pkcs8_len))
goto done;
CRYPT_PRIVATE_KEY_INFO *pki = (CRYPT_PRIVATE_KEY_INFO *) pkcs8_buf;
if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY,
pki->PrivateKey.pbData, pki->PrivateKey.cbData,
CRYPT_DECODE_ALLOC_FLAG, NULL, &rsa_blob, &rsa_blob_len))
goto done;
} else {
// Traditional RSA (BEGIN RSA PRIVATE KEY)
if (!CryptDecodeObjectEx(X509_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY,
key_der, key_der_len, CRYPT_DECODE_ALLOC_FLAG,
NULL, &rsa_blob, &rsa_blob_len))
goto done;
}
// --- Import key into temporary CAPI keyset ---
CryptAcquireContextA(&hprov, container, MS_ENH_RSA_AES_PROV_A,
PROV_RSA_AES, CRYPT_DELETEKEYSET);
hprov = 0;
if (!CryptAcquireContextA(&hprov, container, MS_ENH_RSA_AES_PROV_A,
PROV_RSA_AES, CRYPT_NEWKEYSET))
goto done;
if (!CryptImportKey(hprov, rsa_blob, rsa_blob_len, 0, CRYPT_EXPORTABLE, &hkey))
goto done;
// --- Bind key to cert, export PFX in memory ---
{
wchar_t containerW[64];
MultiByteToWideChar(CP_ACP, 0, container, -1, containerW, 64);
CRYPT_KEY_PROV_INFO prov_info = {0};
prov_info.pwszContainerName = containerW;
prov_info.pwszProvName = (LPWSTR) L"Microsoft Enhanced RSA and AES Cryptographic Provider";
prov_info.dwProvType = PROV_RSA_AES;
prov_info.dwKeySpec = AT_KEYEXCHANGE;
if (!CertSetCertificateContextProperty(cert_ctx,
CERT_KEY_PROV_INFO_PROP_ID, 0, &prov_info))
goto done;
}
mem_store = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, 0, 0, NULL);
if (!mem_store) goto done;
if (!CertAddCertificateContextToStore(mem_store, cert_ctx,
CERT_STORE_ADD_ALWAYS, NULL))
goto done;
// Two-pass PFX export: get size, then export
pfx.cbData = 0;
pfx.pbData = NULL;
if (!PFXExportCertStoreEx(mem_store, &pfx, L"", NULL, EXPORT_PRIVATE_KEYS))
goto done;
pfx.pbData = malloc(pfx.cbData);
if (!pfx.pbData) goto done;
if (!PFXExportCertStoreEx(mem_store, &pfx, L"", NULL, EXPORT_PRIVATE_KEYS))
goto done;
// --- Import PFX blob and acquire SChannel credential ---
result = load_pfx_blob(pfx.pbData, pfx.cbData, cred_out, cert_ctx_out);
done:
free(pfx.pbData);
if (mem_store) CertCloseStore(mem_store, 0);
if (hkey) CryptDestroyKey(hkey);
if (hprov) CryptReleaseContext(hprov, 0);
{ HCRYPTPROV tmp = 0;
CryptAcquireContextA(&tmp, container, MS_ENH_RSA_AES_PROV_A,
PROV_RSA_AES, CRYPT_DELETEKEYSET); }
if (pkcs8_buf) LocalFree(pkcs8_buf);
if (rsa_blob) LocalFree(rsa_blob);
free(key_der);
free(key_pem);
free(cert_der);
free(cert_pem);
if (cert_ctx) CertFreeCertificateContext(cert_ctx);
return result;
}
// ============================================================
// Global init/free (no-ops for SChannel)
// ============================================================
void tls_global_init(void)
{
}
void tls_global_free(void)
{
}
// ============================================================
// Server init/free
// ============================================================
int tls_server_init(TLS_Server *server, string cert_file, string key_file)
{
memset(server, 0, sizeof(*server));
server->num_certs = 0;
int ret = load_pem_credential(cert_file, key_file, &server->cred, &server->cert_ctx);
if (ret < 0)
return -1;
return 0;
}
void tls_server_free(TLS_Server *server)
{
FreeCredentialsHandle(&server->cred);
if (server->cert_ctx) {
HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
DWORD key_spec = 0;
BOOL caller_free = FALSE;
if (CryptAcquireCertificatePrivateKey(server->cert_ctx,
CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG, NULL, &hkey, &key_spec, &caller_free)) {
if (caller_free && hkey)
NCryptFreeObject(hkey);
}
CertFreeCertificateContext(server->cert_ctx);
}
for (int i = 0; i < server->num_certs; i++)
FreeCredentialsHandle(&server->certs[i].cred);
}
int tls_server_add_cert(TLS_Server *server, string domain, string cert_file, string key_file)
{
if (server->num_certs >= TLS_CERT_LIMIT)
return -1;
TLS_Cert *cert = &server->certs[server->num_certs];
if (domain.len >= (int) sizeof(cert->domain))
return -1;
PCCERT_CONTEXT cert_ctx = NULL;
int ret = load_pem_credential(cert_file, key_file, &cert->cred, &cert_ctx);
if (ret < 0)
return -1;
if (cert_ctx) {
HCRYPTPROV_OR_NCRYPT_KEY_HANDLE hkey = 0;
DWORD key_spec = 0;
BOOL caller_free = FALSE;
if (CryptAcquireCertificatePrivateKey(cert_ctx,
CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG, NULL, &hkey, &key_spec, &caller_free)) {
if (caller_free && hkey)
NCryptFreeObject(hkey);
}
CertFreeCertificateContext(cert_ctx);
}
memcpy(cert->domain, domain.ptr, domain.len);
cert->domain[domain.len] = '\0';
server->num_certs++;
return 0;
}
// ============================================================
// Connection init/free
// ============================================================
int tls_conn_init(TLS_Conn *conn, TLS_Server *server)
{
memset(conn, 0, sizeof(*conn));
conn->ctx_valid = false;
conn->handshake = true;
conn->cred = &server->cred;
byte_queue_init(&conn->in_buf, TLS_BUF_LIMIT);
byte_queue_init(&conn->out_buf, TLS_BUF_LIMIT);
conn->pending = NULL;
conn->pending_off = 0;
conn->pending_len = 0;
SecInvalidateHandle(&conn->ctx);
return 0;
}
void tls_conn_free(TLS_Conn *conn)
{
if (conn->ctx_valid)
DeleteSecurityContext(&conn->ctx);
byte_queue_free(&conn->in_buf);
byte_queue_free(&conn->out_buf);
free(conn->pending);
}
// ============================================================
// Handshake
// ============================================================
int tls_conn_handshake(TLS_Conn *conn)
{
assert(conn->handshake);
// Read available ciphertext from in_buf
ByteView in = byte_queue_read_buf(&conn->in_buf);
if (!in.ptr || in.len == 0) {
byte_queue_read_ack(&conn->in_buf, 0);
return 0;
}
int in_avail = (int) in.len;
// Input buffers
SecBuffer in_bufs[2];
in_bufs[0].BufferType = SECBUFFER_TOKEN;
in_bufs[0].pvBuffer = in.ptr;
in_bufs[0].cbBuffer = (unsigned long) in_avail;
in_bufs[1].BufferType = SECBUFFER_EMPTY;
in_bufs[1].pvBuffer = NULL;
in_bufs[1].cbBuffer = 0;
SecBufferDesc in_desc;
in_desc.ulVersion = SECBUFFER_VERSION;
in_desc.cBuffers = 2;
in_desc.pBuffers = in_bufs;
// Output buffers
SecBuffer out_bufs[1];
out_bufs[0].BufferType = SECBUFFER_TOKEN;
out_bufs[0].pvBuffer = NULL;
out_bufs[0].cbBuffer = 0;
SecBufferDesc out_desc;
out_desc.ulVersion = SECBUFFER_VERSION;
out_desc.cBuffers = 1;
out_desc.pBuffers = out_bufs;
DWORD flags = ASC_REQ_STREAM
| ASC_REQ_SEQUENCE_DETECT
| ASC_REQ_REPLAY_DETECT
| ASC_REQ_CONFIDENTIALITY
| ASC_REQ_ALLOCATE_MEMORY;
DWORD out_flags = 0;
TimeStamp expiry;
SECURITY_STATUS ss = AcceptSecurityContext(
conn->cred,
conn->ctx_valid ? &conn->ctx : NULL,
&in_desc,
flags,
0,
conn->ctx_valid ? NULL : &conn->ctx,
&out_desc,
&out_flags,
&expiry);
if (ss == SEC_E_OK || ss == SEC_I_CONTINUE_NEEDED)
conn->ctx_valid = true;
// Copy output token to out_buf
if (out_bufs[0].pvBuffer && out_bufs[0].cbBuffer > 0) {
byte_queue_write_setmincap(&conn->out_buf, out_bufs[0].cbBuffer);
byte_queue_write(&conn->out_buf, out_bufs[0].pvBuffer, out_bufs[0].cbBuffer);
FreeContextBuffer(out_bufs[0].pvBuffer);
}
// Calculate how much input was consumed
int consumed = in_avail;
if (in_bufs[1].BufferType == SECBUFFER_EXTRA && in_bufs[1].cbBuffer > 0)
consumed = in_avail - (int) in_bufs[1].cbBuffer;
if (ss == SEC_E_INCOMPLETE_MESSAGE) {
// SChannel didn't consume anything
byte_queue_read_ack(&conn->in_buf, 0);
return 0;
}
byte_queue_read_ack(&conn->in_buf, consumed);
if (ss == SEC_I_CONTINUE_NEEDED)
return 0;
if (ss == SEC_E_OK) {
conn->handshake = false;
ss = QueryContextAttributes(&conn->ctx, SECPKG_ATTR_STREAM_SIZES, &conn->sizes);
if (ss != SEC_E_OK)
return -1;
return 1;
}
return -1;
}
// ============================================================
// Network I/O (ciphertext ↔ socket)
// ============================================================
char *tls_conn_net_write_buf(TLS_Conn *conn, int *cap)
{
byte_queue_write_setmincap(&conn->in_buf, 4096);
ByteView bv = byte_queue_write_buf(&conn->in_buf);
if (!bv.ptr || bv.len == 0) {
byte_queue_write_ack(&conn->in_buf, 0);
return NULL;
}
*cap = (int) bv.len;
return (char *) bv.ptr;
}
void tls_conn_net_write_ack(TLS_Conn *conn, int num)
{
byte_queue_write_ack(&conn->in_buf, num);
}
char *tls_conn_net_read_buf(TLS_Conn *conn, int *num)
{
ByteView bv = byte_queue_read_buf(&conn->out_buf);
if (!bv.ptr || bv.len == 0) {
byte_queue_read_ack(&conn->out_buf, 0);
return NULL;
}
*num = (int) bv.len;
return (char *) bv.ptr;
}
void tls_conn_net_read_ack(TLS_Conn *conn, int num)
{
byte_queue_read_ack(&conn->out_buf, num);
}
// ============================================================
// Application I/O (encrypt/decrypt)
// ============================================================
int tls_conn_app_write(TLS_Conn *conn, char *src, int num)
{
assert(!conn->handshake);
if (num <= 0) return 0;
int max_msg = (int) conn->sizes.cbMaximumMessage;
if (num > max_msg)
num = max_msg;
int header_size = (int) conn->sizes.cbHeader;
int trailer_size = (int) conn->sizes.cbTrailer;
int total = header_size + num + trailer_size;
// Ensure output buffer has enough space
byte_queue_write_setmincap(&conn->out_buf, total);
ByteView bv = byte_queue_write_buf(&conn->out_buf);
if (!bv.ptr || (int) bv.len < total) {
// Try with less data
if (!bv.ptr || (int) bv.len < header_size + trailer_size + 1) {
byte_queue_write_ack(&conn->out_buf, 0);
return 0;
}
num = (int) bv.len - header_size - trailer_size;
total = header_size + num + trailer_size;
}
char *out_ptr = (char *) bv.ptr;
// Copy plaintext into the data portion
memcpy(out_ptr + header_size, src, num);
// Set up SecBuffers for in-place encryption
SecBuffer bufs[4];
bufs[0].BufferType = SECBUFFER_STREAM_HEADER;
bufs[0].pvBuffer = out_ptr;
bufs[0].cbBuffer = (unsigned long) header_size;
bufs[1].BufferType = SECBUFFER_DATA;
bufs[1].pvBuffer = out_ptr + header_size;
bufs[1].cbBuffer = (unsigned long) num;
bufs[2].BufferType = SECBUFFER_STREAM_TRAILER;
bufs[2].pvBuffer = out_ptr + header_size + num;
bufs[2].cbBuffer = (unsigned long) trailer_size;
bufs[3].BufferType = SECBUFFER_EMPTY;
bufs[3].pvBuffer = NULL;
bufs[3].cbBuffer = 0;
SecBufferDesc desc;
desc.ulVersion = SECBUFFER_VERSION;
desc.cBuffers = 4;
desc.pBuffers = bufs;
SECURITY_STATUS ss = EncryptMessage(&conn->ctx, 0, &desc, 0);
if (ss != SEC_E_OK) {
byte_queue_write_ack(&conn->out_buf, 0);
return -1;
}
int written = (int)(bufs[0].cbBuffer + bufs[1].cbBuffer + bufs[2].cbBuffer);
byte_queue_write_ack(&conn->out_buf, written);
return num;
}
int tls_conn_app_read(TLS_Conn *conn, char *dst, int cap)
{
assert(!conn->handshake);
// Drain any pending plaintext from a previous partial read
if (conn->pending_len > 0) {
int n = conn->pending_len;
if (n > cap) n = cap;
memcpy(dst, conn->pending + conn->pending_off, n);
conn->pending_off += n;
conn->pending_len -= n;
if (conn->pending_len == 0) {
free(conn->pending);
conn->pending = NULL;
conn->pending_off = 0;
}
return n;
}
ByteView in = byte_queue_read_buf(&conn->in_buf);
if (!in.ptr || in.len == 0) {
byte_queue_read_ack(&conn->in_buf, 0);
return 0;
}
int in_avail = (int) in.len;
// DecryptMessage operates in-place
SecBuffer bufs[4];
bufs[0].BufferType = SECBUFFER_DATA;
bufs[0].pvBuffer = in.ptr;
bufs[0].cbBuffer = (unsigned long) in_avail;
bufs[1].BufferType = SECBUFFER_EMPTY;
bufs[1].pvBuffer = NULL;
bufs[1].cbBuffer = 0;
bufs[2].BufferType = SECBUFFER_EMPTY;
bufs[2].pvBuffer = NULL;
bufs[2].cbBuffer = 0;
bufs[3].BufferType = SECBUFFER_EMPTY;
bufs[3].pvBuffer = NULL;
bufs[3].cbBuffer = 0;
SecBufferDesc desc;
desc.ulVersion = SECBUFFER_VERSION;
desc.cBuffers = 4;
desc.pBuffers = bufs;
SECURITY_STATUS ss = DecryptMessage(&conn->ctx, &desc, 0, NULL);
if (ss == SEC_E_INCOMPLETE_MESSAGE) {
byte_queue_read_ack(&conn->in_buf, 0);
return 0;
}
if (ss != SEC_E_OK && ss != SEC_I_RENEGOTIATE) {
byte_queue_read_ack(&conn->in_buf, 0);
return -1;
}
// Find decrypted data and extra ciphertext buffers
SecBuffer *data_buf = NULL;
SecBuffer *extra_buf = NULL;
for (int i = 0; i < 4; i++) {
if (bufs[i].BufferType == SECBUFFER_DATA)
data_buf = &bufs[i];
else if (bufs[i].BufferType == SECBUFFER_EXTRA)
extra_buf = &bufs[i];
}
int result = 0;
if (data_buf && data_buf->cbBuffer > 0) {
int total = (int) data_buf->cbBuffer;
int n = total;
if (n > cap) n = cap;
memcpy(dst, data_buf->pvBuffer, n);
result = n;
// Save excess plaintext for next call
int leftover = total - n;
if (leftover > 0) {
conn->pending = malloc(leftover);
if (conn->pending) {
memcpy(conn->pending, (char *) data_buf->pvBuffer + n, leftover);
conn->pending_off = 0;
conn->pending_len = leftover;
}
}
}
// Consume processed input, keeping any extra ciphertext
int consumed = in_avail;
if (extra_buf && extra_buf->cbBuffer > 0)
consumed = in_avail - (int) extra_buf->cbBuffer;
byte_queue_read_ack(&conn->in_buf, consumed);
if (ss == SEC_I_RENEGOTIATE)
return result > 0 ? result : 0;
return result;
}
int tls_conn_needs_flushing(TLS_Conn *conn)
{
return !byte_queue_empty(&conn->out_buf);
}
#endif // TLS_SCHANNEL
#endif // TLS_ENABLED
quakey/include/quakey.h
+30 -38
View File
@@ -4,6 +4,7 @@
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#ifdef _WIN32
#include <winsock2.h>
#include <ws2tcpip.h>
@@ -26,8 +27,6 @@
#include <pthread.h>
#endif
#include <stdbool.h>
typedef struct {} Quakey;
// Function pointers to a simulated program's code
@@ -35,11 +34,6 @@ typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, s
typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
typedef int (*QuakeyFreeFunc)(void *state);
typedef enum {
QUAKEY_LINUX,
QUAKEY_WINDOWS,
} QuakeyPlatform;
typedef struct {
// Label associated to the process for debugging
@@ -61,9 +55,6 @@ typedef struct {
// Disk size for the process
int disk_size;
// Platform used by this program
QuakeyPlatform platform;
} QuakeySpawn;
typedef unsigned long long QuakeyUInt64;
@@ -84,9 +75,6 @@ void *quakey_node_state(QuakeyNode node);
// Schedule and executes one program until it would block, then returns
int quakey_schedule_one(Quakey *quakey);
// Get the current simulated time in nanoseconds
QuakeyUInt64 quakey_current_time(Quakey *quakey);
// Generate a random u64
QuakeyUInt64 quakey_random(void);
@@ -98,30 +86,40 @@ typedef struct {
void quakey_signal(char *name);
int quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
// Limit the number of hosts that can be crashed at the same time.
// Set to 0 for no limit (default).
void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
void quakey_network_partitioning(Quakey *quakey, bool enable);
// Access spawned host information
//
// TODO: The following are to be refactored as the host index is not stable
int quakey_num_hosts(Quakey *quakey);
void *quakey_host_state(Quakey *quakey, int idx); // Returns NULL if host is dead
int quakey_host_is_dead(Quakey *quakey, int idx);
const char *quakey_host_name(Quakey *quakey, int idx);
// Enter/leave a host's context so that mock_xxx functions (file I/O,
// etc.) operate on that host's resources. Use from code that runs
// outside of the normal init/tick/free callbacks (e.g. invariant
// checkers).
// Fault injection control
void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
void quakey_network_partitioning(Quakey *quakey, bool enabled);
// Simulation time
QuakeyUInt64 quakey_current_time(Quakey *quakey);
// Host context (for accessing mock filesystem from outside scheduled context)
void quakey_enter_host(QuakeyNode node);
void quakey_leave_host(void);
int *mock_errno_ptr(void);
// Network mocks (POSIX-style, available on all platforms)
int mock_socket(int domain, int type, int protocol);
int mock_bind(int fd, void *addr, unsigned long addr_len);
int mock_connect(int fd, void *addr, unsigned long addr_len);
int mock_getsockopt(int fd, int level, int optname, void *optval, unsigned int *optlen);
int mock_listen(int fd, int backlog);
int mock_accept(int fd, void *addr, unsigned int *addr_len);
int mock_pipe(int *fds);
int mock_recv(int fd, char *dst, int len, int flags);
int mock_send(int fd, char *src, int len, int flags);
#ifdef _WIN32
// Windows-specific socket functions (use SOCKET type)
int mock_closesocket(SOCKET fd);
int mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp);
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
@@ -139,23 +137,18 @@ HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindClose(HANDLE hFindFile);
int mock__mkdir(char *path);
int mock_open(char *path, int flags, int mode);
int mock_close(int fd);
int mock_read(int fd, char *dst, int len);
int mock_write(int fd, char *src, int len);
int mock_remove(char *path);
int mock_rename(char *oldpath, char *newpath);
int mock_mkdir(char *path, int mode);
#else
int mock_socket(int domain, int type, int protocol);
int mock_closesocket(int fd);
int mock_ioctlsocket(int fd, long cmd, unsigned long *argp);
int mock_bind(int fd, void *addr, unsigned long addr_len);
int mock_connect(int fd, void *addr, unsigned long addr_len);
int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
int mock_listen(int fd, int backlog);
int mock_accept(int fd, void *addr, socklen_t *addr_len);
int mock_pipe(int *fds);
int mock_recv(int fd, char *dst, int len, int flags);
int mock_send(int fd, char *src, int len, int flags);
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_fcntl(int fd, int cmd, int flags);
int mock_close(int fd);
int mock_access(const char *path, int mode);
int mock_ftruncate(int fd, size_t new_size);
int mock_fstat(int fd, struct stat *buf);
int mock_read(int fd, char *dst, int len);
@@ -204,7 +197,6 @@ void mock_free(void *ptr);
#define open mock_open
#define fcntl mock_fcntl
#define close mock_close
#define access mock_access
#define ftruncate mock_ftruncate
#define CreateFileW mock_CreateFileW
#define CloseHandle mock_CloseHandle
quakey/src/quakey.c
+111 -199
View File
@@ -5,6 +5,7 @@
#include <quakey.h>
#include <stdint.h>
#include <assert.h>
#include <fcntl.h>
/////////////////////////////////////////////////////////////////
// Utilities
@@ -244,9 +245,6 @@ struct Host {
char *name;
// Platform used by this host
QuakeyPlatform platform;
// State of the ephimeral port allocation
u16 next_ephemeral_port;
@@ -402,11 +400,6 @@ struct Sim {
// reached, a new random target is generated.
HostPairList partition;
HostPairList target_partition;
// Maximum number of hosts that can be dead at the same time.
int max_crashes;
bool network_partitioning;
};
static void time_event_wakeup(Sim *sim, Nanos time, Host *host);
@@ -745,6 +738,11 @@ static b32 socket_queue_empty(SocketQueue *queue)
return queue->used == 0;
}
static b32 socket_queue_used(SocketQueue *queue)
{
return queue->used;
}
/////////////////////////////////////////////////////////////////
// Accept Queue Code
@@ -940,7 +938,6 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg)
{
host->sim = sim;
host->name = config.name;
host->platform = config.platform;
host->next_ephemeral_port = FIRST_EPHEMERAL_PORT;
// Save spawn config for restart after crash
@@ -1062,11 +1059,8 @@ static void host_free(Host *host)
}
// Remove all events associated with a host (WAKEUP events and events
// whose source or destination descriptors belong to this host).
// If skip_restart is true, RESTART events are kept (used during
// restart failure to avoid removing the event currently being
// processed by process_events_at_current_time).
static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
// whose source or destination descriptors belong to this host)
static void remove_events_for_host(Sim *sim, Host *host)
{
int i = 0;
while (i < sim->num_events) {
@@ -1075,7 +1069,7 @@ static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
if (event->type == EVENT_TYPE_WAKEUP && event->host == host)
remove = true;
if (!skip_restart && event->type == EVENT_TYPE_RESTART && event->host == host)
if (event->type == EVENT_TYPE_RESTART && event->host == host)
remove = true;
if (event->src_desc && event->src_desc->host == host)
remove = true;
@@ -1091,11 +1085,6 @@ static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart)
}
}
static void remove_events_for_host(Sim *sim, Host *host)
{
remove_events_for_host_ex(sim, host, false);
}
static int count_dead(Sim *sim)
{
int n = 0;
@@ -1123,11 +1112,6 @@ static void host_crash(Host *host)
// Remove all pending events for this host
remove_events_for_host(sim, host);
// Let the application clean up (e.g. log the crash)
host___ = host;
host->free_func(host->state);
host___ = NULL;
// Free application state
free(host->state);
host->state = NULL;
@@ -1138,6 +1122,7 @@ static void host_crash(Host *host)
host->poll_count = 0;
host->poll_timeout = -1;
printf("Quakey: Host crash (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
}
// Restart a crashed host: re-initialize from saved config,
@@ -1208,22 +1193,10 @@ static void host_restart(Host *host)
desc_free(sim, &host->desc[i], true);
}
host->num_desc = 0;
// Skip removing RESTART events: this function is called from
// time_event_process (EVENT_TYPE_RESTART), so the current
// restart event is still in the array being iterated by
// process_events_at_current_time. Removing it here would cause
// the outer loop's swap-removal to discard an unrelated event.
remove_events_for_host_ex(sim, host, true);
remove_events_for_host(sim, host);
free(host->state);
host->state = NULL;
host->dead = true;
// Schedule another restart attempt. Init can fail due to
// transient I/O errors from fault injection; permanently
// killing the host would be unrealistic since a real
// process supervisor would keep retrying.
Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL);
time_event_restart(sim, sim->current_time + restart_delay, host);
return;
}
@@ -1233,17 +1206,9 @@ static void host_restart(Host *host)
} else if (host->poll_timeout == 0)
host->timedout = true;
printf("Quakey: Host restart (%d/%d dead)\n", count_dead(sim), sim->num_hosts);
}
static b32 host_is_linux(Host *host)
{
return host->platform == QUAKEY_LINUX;
}
static b32 host_is_windows(Host *host)
{
return host->platform == QUAKEY_WINDOWS;
}
static Nanos host_time(Host *host)
{
@@ -1402,10 +1367,7 @@ static void host_update(Host *host)
);
host___ = NULL;
if (ret < 0) {
host_crash(host);
Nanos restart_delay = 1000000000ULL + (sim_random(host->sim) % 9000000000ULL);
time_event_restart(host->sim, host->sim->current_time + restart_delay, host);
return;
TODO;
}
if (host->poll_timeout > 0) {
@@ -1985,7 +1947,7 @@ static int host_read(Host *host, int desc_idx, char *dst, int len)
desc->type == DESC_PIPE) {
num = recv_inner(desc, dst, len);
} else if (desc->type == DESC_FILE) {
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
#ifdef FAULT_INJECTION
uint64_t roll = sim_random(host->sim) % 1000;
if (roll == 0) return HOST_ERROR_IO;
#endif
@@ -2029,7 +1991,7 @@ static int host_write(Host *host, int desc_idx, char *src, int len)
desc->type == DESC_PIPE) {
num = send_inner(desc, src, len);
} else if (desc->type == DESC_FILE) {
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
#ifdef FAULT_INJECTION
uint64_t roll = sim_random(host->sim) % 1000;
if (roll == 0) return HOST_ERROR_IO;
if (roll == 1) return HOST_ERROR_NOSPC;
@@ -2205,7 +2167,7 @@ static int host_fsync(Host *host, int desc_idx)
if (desc->type != DESC_FILE)
return HOST_ERROR_BADIDX;
#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS)
#ifdef FAULT_INJECTION
uint64_t roll = sim_random(host->sim) % 100;
if (roll == 0)
return HOST_ERROR_IO;
@@ -2430,7 +2392,7 @@ static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b);
static Nanos pick_partition_delay(Sim *sim, bool longer)
{
Nanos min_delay = 1000000000; // 1s
Nanos min_delay = 10000000; // 10ms
Nanos max_delay = 10000000000; // 10s
if (longer) {
min_delay = 10000000000;
@@ -2447,13 +2409,10 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
{
// Try to move one step toward the target partition.
// If current already matches target, pick a new target.
if (!break_or_repair_link(sim)) {
if (sim->network_partitioning) {
if (change_target_partition(sim) < 0) {
TODO;
}
if (!break_or_repair_link(sim))
if (change_target_partition(sim) < 0) {
TODO;
}
}
// Reschedule for the next partition step
event->time = sim->current_time + pick_partition_delay(sim, true);
@@ -2473,8 +2432,8 @@ static b32 time_event_process(TimeEvent *event, Sim *sim)
Host *peer_host = sim->hosts[idx];
if (hosts_are_partitioned(sim, src_desc->host, peer_host)) {
//SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
// src_desc->host->name, peer_host->name);
SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s",
src_desc->host->name, peer_host->name);
src_desc->connect_status = CONNECT_STATUS_NOHOST;
break;
}
@@ -2791,22 +2750,11 @@ static void sim_init(Sim *sim, uint64_t seed)
sim->events = NULL;
sim->num_signals = 0;
sim->head_signal = 0;
sim->max_crashes = 0;
sim->network_partitioning = true;
host_pair_list_init(&sim->partition);
host_pair_list_init(&sim->target_partition);
time_event_partition(sim, pick_partition_delay(sim, false));
}
static void sim_network_partitioning(Sim *sim, bool enable)
{
sim->network_partitioning = enable;
if (!enable) {
host_pair_list_free(&sim->target_partition);
host_pair_list_init(&sim->target_partition);
}
}
static void sim_free(Sim *sim)
{
host_pair_list_free(&sim->target_partition);
@@ -2890,8 +2838,8 @@ static void process_events_at_current_time(Sim *sim)
}
}
//if (sim->num_events > 10000)
// SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
if (sim->num_events > 10000)
SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events);
}
static int find_first_ready_host(Sim *sim)
@@ -2954,14 +2902,14 @@ static b32 sim_update(Sim *sim)
if (sim->num_hosts > 1) {
uint64_t rng = sim_random(sim);
if ((rng % 10000) == 0) {
// Pick a random live host to crash
int live_count = 0;
for (int i = 0; i < sim->num_hosts; i++)
if (!sim->hosts[i]->dead)
live_count++;
int dead_count = sim->num_hosts - live_count;
if (dead_count < sim->max_crashes) {
// Only crash if at least 2 hosts are alive (keep a majority)
if (live_count >= 2) {
// Pick one of the live hosts at random
int target = sim_random(sim) % live_count;
int j = 0;
@@ -3053,18 +3001,6 @@ void quakey_free(Quakey *quakey)
}
}
void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
{
Sim *sim = (Sim*) quakey;
sim->max_crashes = max_crashes;
}
void quakey_network_partitioning(Quakey *quakey, bool enable)
{
Sim *sim = (Sim*) quakey;
sim_network_partitioning(sim, enable);
}
QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg)
{
return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg);
@@ -3076,28 +3012,11 @@ void *quakey_node_state(QuakeyNode node)
return host->state;
}
void quakey_enter_host(QuakeyNode node)
{
Host *host = (void*) node;
host___ = host;
}
void quakey_leave_host(void)
{
host___ = NULL;
}
int quakey_schedule_one(Quakey *quakey)
{
return sim_update((Sim*) quakey);
}
QuakeyUInt64 quakey_current_time(Quakey *quakey)
{
Sim *sim = (Sim*) quakey;
return sim->current_time;
}
QuakeyUInt64 quakey_random(void)
{
Host *host = host___;
@@ -3157,6 +3076,39 @@ const char *quakey_host_name(Quakey *quakey, int idx)
return sim->hosts[idx]->name;
}
void quakey_set_max_crashes(Quakey *quakey, int max_crashes)
{
// TODO: Implement crash limiting. For now this is a no-op
// that allows the simulation to run without the feature.
(void) quakey;
(void) max_crashes;
}
void quakey_network_partitioning(Quakey *quakey, bool enabled)
{
// TODO: Implement network partitioning toggle. For now this is
// a no-op that allows the simulation to run without the feature.
(void) quakey;
(void) enabled;
}
QuakeyUInt64 quakey_current_time(Quakey *quakey)
{
Sim *sim = (Sim*) quakey;
return (QuakeyUInt64) sim->current_time;
}
void quakey_enter_host(QuakeyNode node)
{
Host *host = (void*) node;
host___ = host;
}
void quakey_leave_host(void)
{
host___ = NULL;
}
/////////////////////////////////////////////////////////////////
// Mock System Calls
@@ -3169,7 +3121,7 @@ int *mock_errno_ptr(void)
return host_errno_ptr(host);
}
#ifndef _WIN32
// Platform-independent mock functions (used by lib/ on all platforms)
int mock_socket(int domain, int type, int protocol)
{
@@ -3221,8 +3173,6 @@ int mock_close(int fd)
if (host == NULL)
abort_("Call to mock_close() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_close() not from Linux\n");
int desc_idx = fd;
int ret = host_close(host, desc_idx, false);
@@ -3241,17 +3191,6 @@ int mock_close(int fd)
return 0;
}
// NOTE: mock_access must be implemented for simulation to work correctly.
// chunk_store_exists() uses file_exists() which calls access(). Without a
// working mock_access, the simulation cannot check chunk existence and
// must fall back to file_open (O_CREAT), which creates empty files as a
// side effect and leads to data corruption (servers return uninitialized
// data for chunks they don't actually have).
int mock_access(const char *path, int mode)
{
assert(0); // TODO
}
static int convert_addr(void *addr, size_t addr_len,
Addr *converted_addr, uint16_t *converted_port)
{
@@ -3443,8 +3382,6 @@ int mock_open(char *path, int flags, int mode)
if (host == NULL)
abort_("Call to mock_open() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_open() not from Linux\n");
int converted_flags = convert_linux_open_flags_to_mockfs(flags);
@@ -3474,8 +3411,6 @@ int mock_read(int fd, char *dst, int len)
if (host == NULL)
abort_("Call to mock_read() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_read() not from Linux\n");
int ret = host_read(host, fd, dst, len);
if (ret < 0) {
@@ -3506,8 +3441,6 @@ int mock_write(int fd, char *src, int len)
if (host == NULL)
abort_("Call to mock_write() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_write() not from Linux\n");
int ret = host_write(host, fd, src, len);
if (ret < 0) {
@@ -3603,7 +3536,7 @@ int mock_send(int fd, char *src, int len, int flags)
return ret;
}
int mock_accept(int fd, void *addr, socklen_t *addr_len)
int mock_accept(int fd, void *addr, unsigned int *addr_len)
{
Host *host = host___;
if (host == NULL)
@@ -3661,7 +3594,7 @@ int mock_accept(int fd, void *addr, socklen_t *addr_len)
return new_fd;
}
int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
int mock_getsockopt(int fd, int level, int optname, void *optval, unsigned int *optlen)
{
if (level != SOL_SOCKET)
abort_("Call to mock_getsockopt() with level other than SOL_SOCKET\n");
@@ -3716,8 +3649,6 @@ int mock_remove(char *path)
if (host == NULL)
abort_("Call to mock_remove() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_remove() not from Linux\n");
int ret = host_remove(host, path);
if (ret < 0) {
@@ -3744,8 +3675,6 @@ int mock_rename(char *oldpath, char *newpath)
if (host == NULL)
abort_("Call to mock_rename() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_rename() not from Linux\n");
int ret = host_rename(host, oldpath, newpath);
if (ret < 0) {
@@ -3772,14 +3701,14 @@ int mock_rename(char *oldpath, char *newpath)
return 0;
}
#ifndef _WIN32
int mock_clock_gettime(clockid_t clockid, struct timespec *tp)
{
Host *host = host___;
if (host == NULL)
abort_("Call to mock_clock_gettime() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_clock_gettime() not from Linux\n");
if (tp == NULL) {
*host_errno_ptr(host) = EINVAL;
@@ -3817,8 +3746,6 @@ int mock_fsync(int fd)
if (host == NULL)
abort_("Call to mock_fsync() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_fsync() not from Linux\n");
int ret = host_fsync(host, fd);
if (ret < 0) {
@@ -3838,8 +3765,6 @@ int mock_ftruncate(int fd, size_t new_size)
if (host == NULL)
abort_("Call to mock_ftruncate() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_ftruncate() not from Linux\n");
int ret = host_ftruncate(host, fd, (int) new_size);
if (ret < 0) {
@@ -3861,8 +3786,6 @@ off_t mock_lseek(int fd, off_t offset, int whence)
if (host == NULL)
abort_("Call to mock_lseek() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_lseek() not from Linux\n");
// Convert POSIX whence to HOST whence
int host_whence;
@@ -3899,8 +3822,6 @@ int mock_fstat(int fd, struct stat *buf)
if (host == NULL)
abort_("Call to mock_fstat() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_fstat() not from Linux\n");
if (buf == NULL) {
*host_errno_ptr(host) = EINVAL;
@@ -3942,8 +3863,6 @@ char *mock_realpath(char *path, char *dst)
if (host == NULL)
abort_("Call to mock_realpath() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_realpath() not from Linux\n");
if (path == NULL) {
*host_errno_ptr(host) = EINVAL;
@@ -4057,8 +3976,6 @@ int mock_mkdir(char *path, mode_t mode)
if (host == NULL)
abort_("Call to mock_mkdir() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_mkdir() not from Linux\n");
// LittleFS doesn't use mode, but we accept it for API compatibility
(void) mode;
@@ -4088,8 +4005,6 @@ int mock_fcntl(int fd, int cmd, int flags)
if (host == NULL)
abort_("Call to mock_fcntl() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_fcntl() not from Linux\n");
switch (cmd) {
@@ -4142,8 +4057,6 @@ DIR *mock_opendir(char *name)
if (host == NULL)
abort_("Call to mock_opendir() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_opendir() not from Linux\n");
int ret = host_open_dir(host, name);
if (ret < 0) {
@@ -4180,8 +4093,6 @@ struct dirent* mock_readdir(DIR *dirp)
if (host == NULL)
abort_("Call to mock_readdir() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_readdir() not from Linux\n");
DIR_ *dirp_ = (DIR_*) dirp;
@@ -4230,8 +4141,6 @@ int mock_closedir(DIR *dirp)
if (host == NULL)
abort_("Call to mock_closedir() with no node scheduled\n");
if (!host_is_linux(host))
abort_("Call to mock_closedir() not from Linux\n");
DIR_ *dirp_ = (DIR_*) dirp;
@@ -4266,8 +4175,6 @@ int mock_GetLastError(void)
if (host == NULL)
abort_("Call to mock_GetLastError() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_GetLastError() not from Windows\n");
// Note that technically on windows errno and GetLastError
// are different things. Here we use errno_ to store the
@@ -4287,8 +4194,6 @@ void mock_SetLastError(int err)
if (host == NULL)
abort_("Call to mock_SetLastError() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_SetLastError() not from Windows\n");
*host_errno_ptr(host) = err;
}
@@ -4304,8 +4209,6 @@ int mock_closesocket(SOCKET fd)
if (host == NULL)
abort_("Call to mock_closesocket() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_closesocket() not from Windows\n");
int desc_idx = fd;
int ret = host_close(host, desc_idx, true); // expect_socket = true
@@ -4326,7 +4229,19 @@ int mock_closesocket(SOCKET fd)
int mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp)
{
TODO;
Host *host = host___;
if (host == NULL)
abort_("Call to mock_ioctlsocket() with no node scheduled\n");
if (cmd == FIONBIO) {
int host_flags = (*argp != 0) ? HOST_FLAG_NONBLOCK : 0;
int ret = host_setdescflags(host, (int) fd, host_flags);
if (ret < 0)
return SOCKET_ERROR;
return 0;
}
return SOCKET_ERROR;
}
// Helper function to convert wide string to narrow string (ASCII subset)
@@ -4353,11 +4268,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
// Convert access mode
if ((dwDesiredAccess & GENERIC_READ) && (dwDesiredAccess & GENERIC_WRITE))
lfs_flags = LFS_O_RDWR;
lfs_flags = MOCKFS_O_RDWR;
else if (dwDesiredAccess & GENERIC_WRITE)
lfs_flags = LFS_O_WRONLY;
lfs_flags = MOCKFS_O_WRONLY;
else
lfs_flags = LFS_O_RDONLY;
lfs_flags = MOCKFS_O_RDONLY;
*truncate = false;
@@ -4365,11 +4280,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
switch (dwCreationDisposition) {
case CREATE_NEW:
// Creates a new file, fails if file exists
lfs_flags |= LFS_O_CREAT | LFS_O_EXCL;
lfs_flags |= MOCKFS_O_CREAT | MOCKFS_O_EXCL;
break;
case CREATE_ALWAYS:
// Creates a new file, always (truncates if exists)
lfs_flags |= LFS_O_CREAT | LFS_O_TRUNC;
lfs_flags |= MOCKFS_O_CREAT | MOCKFS_O_TRUNC;
*truncate = true;
break;
case OPEN_EXISTING:
@@ -4378,11 +4293,11 @@ static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess,
break;
case OPEN_ALWAYS:
// Opens file if it exists, creates if it doesn't
lfs_flags |= LFS_O_CREAT;
lfs_flags |= MOCKFS_O_CREAT;
break;
case TRUNCATE_EXISTING:
// Opens and truncates, fails if file doesn't exist
lfs_flags |= LFS_O_TRUNC;
lfs_flags |= MOCKFS_O_TRUNC;
*truncate = true;
break;
default:
@@ -4402,8 +4317,6 @@ HANDLE mock_CreateFileW(WCHAR *lpFileName,
if (host == NULL)
abort_("Call to mock_CreateFileW() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_CreateFileW() not from Windows\n");
// lpSecurityAttributes and hTemplateFile are typically NULL
(void) lpSecurityAttributes;
@@ -4464,8 +4377,6 @@ BOOL mock_CloseHandle(HANDLE handle)
if (host == NULL)
abort_("Call to mock_CloseHandle() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_CloseHandle() not from Windows\n");
if (handle == INVALID_HANDLE_VALUE || handle == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4513,8 +4424,6 @@ BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *
if (host == NULL)
abort_("Call to mock_ReadFile() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_ReadFile() not from Windows\n");
// We don't support overlapped (async) I/O
if (ov != NULL)
@@ -4562,8 +4471,6 @@ BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED
if (host == NULL)
abort_("Call to mock_WriteFile() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_WriteFile() not from Windows\n");
// We don't support overlapped (async) I/O
if (ov != NULL)
@@ -4607,8 +4514,6 @@ DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceTo
if (host == NULL)
abort_("Call to mock_SetFilePointer() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_SetFilePointer() not from Windows\n");
if (hFile == INVALID_HANDLE_VALUE || hFile == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4675,8 +4580,6 @@ BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf)
if (host == NULL)
abort_("Call to mock_GetFileSizeEx() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_GetFileSizeEx() not from Windows\n");
if (handle == INVALID_HANDLE_VALUE || handle == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -4716,8 +4619,6 @@ BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount)
if (host == NULL)
abort_("Call to mock_QueryPerformanceCounter() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_QueryPerformanceCounter() not from Windows\n");
if (lpPerformanceCount == NULL)
return 0; // FALSE
@@ -4736,8 +4637,6 @@ BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency)
if (host == NULL)
abort_("Call to mock_QueryPerformanceFrequency() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_QueryPerformanceFrequency() not from Windows\n");
if (lpFrequency == NULL)
return 0; // FALSE
@@ -4755,8 +4654,6 @@ char *mock__fullpath(char *path, char *dst, int cap)
if (host == NULL)
abort_("Call to mock__fullpath() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock__fullpath() not from Windows\n");
if (path == NULL) {
*host_errno_ptr(host) = EINVAL;
@@ -4864,8 +4761,6 @@ int mock__mkdir(char *path)
if (host == NULL)
abort_("Call to mock__mkdir() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock__mkdir() not from Windows\n");
int ret = host_mkdir(host, path);
if (ret < 0) {
@@ -4916,8 +4811,6 @@ HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData)
if (host == NULL)
abort_("Call to mock_FindFirstFileA() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_FindFirstFileA() not from Windows\n");
if (lpFileName == NULL || lpFindFileData == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_PARAMETER;
@@ -5016,8 +4909,6 @@ BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData)
if (host == NULL)
abort_("Call to mock_FindNextFileA() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_FindNextFileA() not from Windows\n");
if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL || lpFindFileData == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -5060,8 +4951,6 @@ BOOL mock_FindClose(HANDLE hFindFile)
if (host == NULL)
abort_("Call to mock_FindClose() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_FindClose() not from Windows\n");
if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL) {
*host_errno_ptr(host) = ERROR_INVALID_HANDLE;
@@ -5094,8 +4983,6 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
if (host == NULL)
abort_("Call to mock_MoveFileExW() with no node scheduled\n");
if (!host_is_windows(host))
abort_("Call to mock_MoveFileExW() not from Windows\n");
// Validate parameters
if (lpExistingFileName == NULL) {
@@ -5133,7 +5020,7 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
// We need to check this before calling host_rename
if (!(dwFlags & MOVEFILE_REPLACE_EXISTING)) {
// Try to check if destination exists by attempting to open it
int check = host_open_file(host, newpath, LFS_O_RDONLY);
int check = host_open_file(host, newpath, MOCKFS_O_RDONLY);
if (check >= 0) {
// File exists, close it and return error
host_close(host, check, false);
@@ -5168,6 +5055,31 @@ BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwF
return 1; // TRUE
}
int mock_mkdir(char *path, int mode)
{
(void) mode;
Host *host = host___;
if (host == NULL)
abort_("Call to mock_mkdir() with no node scheduled\n");
int ret = host_mkdir(host, path);
if (ret < 0) {
switch (ret) {
case HOST_ERROR_EXIST:
*host_errno_ptr(host) = EEXIST;
return -1;
case HOST_ERROR_NOENT:
*host_errno_ptr(host) = ENOENT;
return -1;
default:
*host_errno_ptr(host) = EIO;
return -1;
}
}
return 0;
}
#endif
void *mock_malloc(size_t size)
src/chunk_store.c
+1 -1
View File
@@ -6,7 +6,7 @@
#include <stdio.h>
#include "chunk_store.h"
#include "file_system.h"
#include <lib/file_system.h>
// Build the full path for a chunk: "base_path/HEX_HASH"
// SHA256 hex = 64 chars. Returns string pointing into buf.
src/chunk_store.h
+1 -1
View File
@@ -4,7 +4,7 @@
#include <stdint.h>
#include <stdbool.h>
#include "basic.h"
#include <lib/basic.h>
typedef struct {
char base_path[256];
src/client.c
+37 -98
View File
@@ -6,16 +6,17 @@
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <poll.h>
#include "tcp.h"
#include "basic.h"
#include <lib/basic.h>
#include "config.h"
#include "metadata.h"
#include "server.h"
#include <toastyfs.h>
#include <stdio.h>
#define POLL_CAPACITY (NODE_LIMIT * 2 + 4)
typedef enum {
STEP_IDLE,
STEP_STORE_CHUNK,
@@ -45,7 +46,7 @@ typedef struct {
struct ToastyFS {
TCP tcp;
MessageSystem msys;
Address server_addrs[NODE_LIMIT];
int num_servers;
@@ -74,8 +75,6 @@ struct ToastyFS {
int put_data_len;
};
// ---- Client logging infrastructure (mirrors server node_log) ----
#define TIME_FMT "%7.3fs"
#define TIME_VAL(t) ((double)(t) / 1000000000.0)
@@ -147,7 +146,7 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs)
tfs->num_servers = num_addrs;
addr_sort(tfs->server_addrs, tfs->num_servers);
if (tcp_context_init(&tfs->tcp) < 0) {
if (message_system_init(&tfs->msys, tfs->server_addrs, num_addrs) < 0) {
free(tfs);
return NULL;
}
@@ -161,7 +160,7 @@ ToastyFS *toastyfs_init(uint64_t client_id, char **addrs, int num_addrs)
void toastyfs_free(ToastyFS *tfs)
{
tcp_context_free(&tfs->tcp);
message_system_free(&tfs->msys);
free(tfs->put_data);
}
@@ -203,40 +202,6 @@ static int leader_idx(ToastyFS *tfs)
return tfs->view_number % tfs->num_servers;
}
static void send_message_to_server(ToastyFS *tfs, int server_idx, MessageHeader *msg)
{
ByteQueue *output;
int conn_idx = tcp_index_from_tag(&tfs->tcp, server_idx);
if (conn_idx < 0) {
if (tcp_connect(&tfs->tcp, tfs->server_addrs[server_idx], server_idx, &output) < 0)
return;
} else {
output = tcp_output_buffer(&tfs->tcp, conn_idx);
if (output == NULL)
return;
}
byte_queue_write(output, msg, msg->length);
}
static void send_message_to_server_ex(ToastyFS *tfs, int server_idx,
MessageHeader *msg, void *extra, int extra_len)
{
ByteQueue *output;
int conn_idx = tcp_index_from_tag(&tfs->tcp, server_idx);
if (conn_idx < 0) {
if (tcp_connect(&tfs->tcp, tfs->server_addrs[server_idx], server_idx, &output) < 0)
return;
} else {
output = tcp_output_buffer(&tfs->tcp, conn_idx);
if (output == NULL)
return;
}
byte_queue_write(output, msg, msg->length - extra_len);
byte_queue_write(output, extra, extra_len);
}
static int begin_transfers(ToastyFS *tfs)
{
// Count started transfers
@@ -264,7 +229,7 @@ static int begin_transfers(ToastyFS *tfs)
.hash = tfs->transfers[i].hash,
.size = tfs->transfers[i].size,
};
send_message_to_server_ex(tfs, tfs->transfers[i].location,
send_message_ex(&tfs->msys, tfs->transfers[i].location,
&msg.base, tfs->transfers[i].data, tfs->transfers[i].size);
} else {
FetchChunkMessage msg = {
@@ -276,7 +241,7 @@ static int begin_transfers(ToastyFS *tfs)
.hash = tfs->transfers[i].hash,
.sender_idx = -1, // Client (not a peer server)
};
send_message_to_server(tfs, tfs->transfers[i].location, &msg.base);
send_message(&tfs->msys, tfs->transfers[i].location, &msg.base);
}
tfs->transfers[i].state = TRANSFER_STARTED;
@@ -340,7 +305,7 @@ static void replay_request(ToastyFS *tfs)
msg.oper.chunks[i].hash = tfs->chunks[i];
msg.oper.chunks[i].size = tfs->chunk_sizes[i];
}
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
}
break;
case STEP_DELETE:
@@ -359,7 +324,7 @@ static void replay_request(ToastyFS *tfs)
};
memcpy(msg.oper.bucket, tfs->bucket, META_BUCKET_MAX);
memcpy(msg.oper.key, tfs->key, META_KEY_MAX);
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
}
break;
case STEP_GET:
@@ -373,7 +338,7 @@ static void replay_request(ToastyFS *tfs)
};
memcpy(msg.bucket, tfs->bucket, META_BUCKET_MAX);
memcpy(msg.key, tfs->key, META_KEY_MAX);
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
}
break;
default:
@@ -381,11 +346,8 @@ static void replay_request(ToastyFS *tfs)
}
}
static int process_message(ToastyFS *tfs,
int conn_idx, uint8_t type, ByteView msg)
static int process_message(ToastyFS *tfs, uint16_t type, ByteView msg)
{
(void) conn_idx;
switch (tfs->step) {
case STEP_FETCH_CHUNK:
{
@@ -477,7 +439,7 @@ static int process_message(ToastyFS *tfs,
msg.oper.chunks[i].hash = tfs->chunks[i];
msg.oper.chunks[i].size = tfs->chunk_sizes[i];
}
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
tfs->step = STEP_COMMIT;
client_log(tfs, "SEND COMMIT_PUT", "key=%s chunks=%d req=%lu",
tfs->key, tfs->num_chunks, tfs->request_id);
@@ -513,8 +475,8 @@ static int process_message(ToastyFS *tfs,
} else if (type == MESSAGE_TYPE_REPLY) {
ReplyMessage reply;
if (msg.len != sizeof(ReplyMessage))
VsrReplyMessage reply;
if (msg.len != sizeof(VsrReplyMessage))
return -1;
memcpy(&reply, msg.ptr, sizeof(reply));
@@ -564,8 +526,8 @@ static int process_message(ToastyFS *tfs,
} else if (type == MESSAGE_TYPE_REPLY) {
ReplyMessage reply;
if (msg.len != sizeof(ReplyMessage))
VsrReplyMessage reply;
if (msg.len != sizeof(VsrReplyMessage))
return -1;
memcpy(&reply, msg.ptr, sizeof(reply));
@@ -675,44 +637,17 @@ static int process_message(ToastyFS *tfs,
void toastyfs_process_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, int pnum)
{
Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&tfs->tcp, events, ctxs, pdata, pnum);
message_system_process_events(&tfs->msys, ctxs, pdata, pnum);
for (int i = 0; i < num_events; i++) {
if (events[i].type == EVENT_DISCONNECT) {
int conn_idx = events[i].conn_idx;
client_log(tfs, "DISCONNECT", "conn=%d", conn_idx);
tcp_close(&tfs->tcp, conn_idx);
continue;
}
if (events[i].type != EVENT_MESSAGE)
continue;
int conn_idx = events[i].conn_idx;
for (;;) {
ByteView msg;
uint16_t msg_type;
int ret = tcp_next_message(&tfs->tcp, conn_idx, &msg, &msg_type);
if (ret == 0)
break;
if (ret < 0) {
tcp_close(&tfs->tcp, conn_idx);
break;
}
ret = process_message(tfs, conn_idx, msg_type, msg);
if (ret < 0) {
tcp_close(&tfs->tcp, conn_idx);
break;
}
tcp_consume_message(&tfs->tcp, conn_idx);
}
void *raw;
while ((raw = get_next_message(&tfs->msys)) != NULL) {
Message *header = (Message *)raw;
ByteView msg_view = { .ptr = raw, .len = header->length };
process_message(tfs, header->type, msg_view);
consume_message(&tfs->msys, raw);
}
// Check for operation timeout — retry the current operation if the
// Check for operation timeout -- retry the current operation if the
// deadline has passed (handles initial sends that were dropped because
// the TCP connection wasn't established yet, and unresponsive servers).
if (tfs->step != STEP_IDLE
@@ -758,9 +693,9 @@ int toastyfs_register_events(ToastyFS *tfs, void **ctxs, struct pollfd *pdata, i
}
*timeout = deadline_to_timeout(deadline, now);
if (pcap < TCP_POLL_CAPACITY)
if (pcap < POLL_CAPACITY)
return -1;
return tcp_register_events(&tfs->tcp, ctxs, pdata);
return message_system_register_events(&tfs->msys, ctxs, pdata, pcap);
}
static void
@@ -866,7 +801,7 @@ int toastyfs_async_get(ToastyFS *tfs, char *key, int key_len)
tfs->step_time = get_current_time(); // TODO: Handle INVALID_TIME error
tfs->step = STEP_GET;
client_log(tfs, "ASYNC GET", "key=%s leader=%d", tfs->key, leader_idx(tfs));
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
return 0;
}
@@ -900,7 +835,7 @@ int toastyfs_async_delete(ToastyFS *tfs, char *key, int key_len)
tfs->step_time = get_current_time(); // TODO: Handle INVALID_TIME error
tfs->step = STEP_DELETE;
client_log(tfs, "ASYNC DELETE", "key=%s req=%lu leader=%d", tfs->key, tfs->request_id, leader_idx(tfs));
send_message_to_server(tfs, leader_idx(tfs), &msg.base);
send_message(&tfs->msys, leader_idx(tfs), &msg.base);
return 0;
}
@@ -1073,14 +1008,18 @@ ToastyFS_Result toastyfs_get_result(ToastyFS *tfs)
static int wait_until_result(ToastyFS *tfs, ToastyFS_Result *res)
{
for (;;) {
void *ctxs[TCP_POLL_CAPACITY];
struct pollfd arr[TCP_POLL_CAPACITY];
void *ctxs[POLL_CAPACITY];
struct pollfd arr[POLL_CAPACITY];
int poll_timeout;
int num = toastyfs_register_events(tfs, ctxs, arr, TCP_POLL_CAPACITY, &poll_timeout);
int num = toastyfs_register_events(tfs, ctxs, arr, POLL_CAPACITY, &poll_timeout);
if (num < 0)
return num;
#ifdef _WIN32
WSAPoll(arr, num, poll_timeout);
#else
poll(arr, num, poll_timeout);
#endif
toastyfs_process_events(tfs, ctxs, arr, num);
*res = toastyfs_get_result(tfs);
src/client_table.c
+2 -2
View File
@@ -28,7 +28,7 @@ ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_i
return NULL;
}
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, void *pending_message)
{
if (client_table->count == client_table->capacity) {
int n = 2 * client_table->capacity;
@@ -44,7 +44,7 @@ int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t req
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_tag = conn_tag,
.pending_message = pending_message,
};
return 0;
}
src/client_table.h
+2 -2
View File
@@ -13,7 +13,7 @@ typedef struct {
// the REQUEST. After a view change, a new leader
// may find stale entries with pending=false from
// a previous view when it was leader before.
int conn_tag;
void *pending_message; // Raw message pointer for deferred reply
} ClientTableEntry;
typedef struct {
@@ -25,7 +25,7 @@ typedef struct {
void client_table_init(ClientTable *client_table);
void client_table_free(ClientTable *client_table);
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, void *pending_message);
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
#endif // CLIENT_TABLE_INCLUDED
src/config.h
+2
View File
@@ -1,6 +1,8 @@
#ifndef CONFIG_INCLUDED
#define CONFIG_INCLUDED
#define CEIL(a, b) (((a) + (b) - 1) / (b))
#define NODE_LIMIT 32
#define FUTURE_LIMIT 32
#define HEARTBEAT_INTERVAL_SEC 1
src/http_proxy.c
+41
View File
@@ -0,0 +1,41 @@
#include "http_proxy.h"
int http_proxy_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
HTTPProxy *proxy = state;
if (http_server_init(&proxy->http_server) < 0)
return -1;
if (http_server_listen_tcp(&proxy->http_server, xxx, yyy) < 0)
return -1;
return 0;
}
int http_proxy_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{
http_server_process_events(&proxy->http_server, ctxs, pdata, *pnum);
HTTP_Request *request;
HTTP_ResponseBuilder builder;
while (http_server_next_request(&proxy->http_server, &request, &builder)) {
// TODO
http_response_builder_status(builder, 200);
http_response_builder_submit(builder);
}
*timeout = -1;
*pnum = http_server_register_events(&proxy->http_server, ctxs, pdata, pcap);
return 0;
}
int http_proxy_free(void *state)
{
HTTPProxy *proxy = state;
http_server_free(&proxy->http_server);
}
src/http_proxy.h
+23
View File
@@ -0,0 +1,23 @@
#ifndef HTTP_PROXY_INCLUDED
#define HTTP_PROXY_INCLUDED
#include <toastyfs.h>
#include <lib/http_server.h>
typedef struct {
HTTP_Server http_server;
ToastyFS *toastyfs;
} HTTPProxy;
struct pollfd;
int http_proxy_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int http_proxy_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int http_proxy_free(void *state);
#endif // HTTP_PROXY_INCLUDED
src/invariant_checker.c
+14 -7
View File
@@ -9,16 +9,23 @@
#include "chunk_store.h"
#include <assert.h>
#include <stdio.h>
// Restore real allocators (see checker/linearizability.c for precedent).
#undef malloc
#undef realloc
#undef free
#include <stdlib.h>
#include <string.h>
// Forward declarations for quakey host context functions
// (we can't include quakey.h because it would re-mock malloc/realloc/free)
void quakey_enter_host(unsigned long long node);
void quakey_leave_host(void);
// These helpers are static in node.c; duplicated here for the checker.
static int self_idx(ServerState *state)
static int self_idx(Server *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
@@ -26,12 +33,12 @@ static int self_idx(ServerState *state)
UNREACHABLE;
}
static int leader_idx(ServerState *state)
static int leader_idx(Server *state)
{
return state->view_number % state->num_nodes;
}
static bool is_leader(ServerState *state)
static bool is_leader(Server *state)
{
if (state->status == STATUS_RECOVERY)
return false;
@@ -72,7 +79,7 @@ void invariant_checker_free(InvariantChecker *ic)
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
unsigned long long *node_handles)
{
int min_commit = -1;
@@ -90,7 +97,7 @@ void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_no
}
for (int i = 0; i < num_nodes; i++) {
ServerState *n = nodes[i];
Server *n = nodes[i];
if (n == NULL || n->status == STATUS_RECOVERY)
continue;
@@ -158,7 +165,7 @@ void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_no
}
for (int i = 0; i < num_nodes; i++) {
ServerState *s = nodes[i];
Server *s = nodes[i];
if (s == NULL)
continue;
@@ -297,7 +304,7 @@ void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_no
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
ServerState *source = nodes[source_node_idx];
Server *source = nodes[source_node_idx];
assert(source->log.count >= observed_max_commit);
for (int k = ic->shadow_count; k < observed_max_commit; k++) {
src/main.c
+17 -13
View File
@@ -52,7 +52,6 @@ int main(int argc, char **argv)
.addrs = (char*[]) { "127.0.0.2" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
(void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
@@ -68,7 +67,6 @@ int main(int argc, char **argv)
.addrs = (char*[]) { "127.0.0.3" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
(void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
@@ -84,7 +82,6 @@ int main(int argc, char **argv)
.addrs = (char*[]) { "127.0.0.7" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
(void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
@@ -93,14 +90,13 @@ int main(int argc, char **argv)
{
QuakeySpawn config = {
.name = "server1",
.state_size = sizeof(ServerState),
.state_size = sizeof(Server),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.4" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
@@ -109,14 +105,13 @@ int main(int argc, char **argv)
{
QuakeySpawn config = {
.name = "server2",
.state_size = sizeof(ServerState),
.state_size = sizeof(Server),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.5" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
@@ -125,14 +120,13 @@ int main(int argc, char **argv)
{
QuakeySpawn config = {
.name = "server3",
.state_size = sizeof(ServerState),
.state_size = sizeof(Server),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.6" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
@@ -151,7 +145,7 @@ int main(int argc, char **argv)
quakey_schedule_one(quakey);
ServerState *arr[] = {
Server *arr[] = {
quakey_node_state(node_1),
quakey_node_state(node_2),
quakey_node_state(node_3),
@@ -182,7 +176,11 @@ int main(int argc, char **argv)
////////////////////////////////////////////////////////////////////
#ifdef MAIN_CLIENT
#ifdef _WIN32
#include <winsock2.h>
#else
#include <poll.h>
#endif
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
@@ -246,8 +244,14 @@ int main(int argc, char **argv)
////////////////////////////////////////////////////////////////////
#ifdef MAIN_SERVER
#ifdef _WIN32
#include <winsock2.h>
#else
#include <poll.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include "server.h"
#define POLL_CAPACITY 1024
@@ -256,11 +260,11 @@ int main(int argc, char **argv)
{
int ret;
// ServerState is ~40 MB (MetaStore holds 4096 ObjectMeta entries),
// Server is ~40 MB (MetaStore holds 4096 ObjectMeta entries),
// which exceeds the default stack size. Heap-allocate it.
ServerState *state = malloc(sizeof(ServerState));
Server *state = malloc(sizeof(Server));
if (state == NULL) {
fprintf(stderr, "Failed to allocate ServerState\n");
fprintf(stderr, "Failed to allocate Server\n");
return -1;
}
src/message.c
-81
View File
@@ -1,81 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "message.h"
bool binary_read(BinaryReader *reader, void *dst, int len)
{
if (reader->len - reader->cur < len)
return false;
if (dst)
memcpy(dst, reader->src + reader->cur, len);
reader->cur += len;
return true;
}
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type)
{
uint16_t version = MESSAGE_VERSION;
uint32_t dummy = 0; // Dummy value
writer->output = output;
writer->start = byte_queue_offset(output);
byte_queue_write(output, &version, sizeof(version));
byte_queue_write(output, &type, sizeof(type));
writer->patch = byte_queue_offset(output);
byte_queue_write(output, &dummy, sizeof(dummy));
}
bool message_writer_free(MessageWriter *writer)
{
uint32_t length = byte_queue_size_from_offset(writer->output, writer->start);
byte_queue_patch(writer->output, writer->patch, &length, sizeof(length));
if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure?
return false;
return true;
}
void message_write(MessageWriter *writer, void *mem, int len)
{
byte_queue_write(writer->output, mem, len);
}
void message_write_u8(MessageWriter *writer, uint8_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_u32(MessageWriter *writer, uint32_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_hash(MessageWriter *writer, SHA256 value)
{
message_write(writer, &value, (int) sizeof(value));
}
int message_peek(ByteView msg, uint16_t *type, uint32_t *len)
{
if (msg.len < (int) sizeof(MessageHeader))
return 0;
MessageHeader header;
memcpy(&header, msg.ptr, sizeof(header));
// (We ignore endianess for now)
if (header.version != MESSAGE_VERSION)
return -1;
if (header.length > msg.len)
return 0;
if (type) *type = header.type;
if (len) *len = header.length;
return 1;
}
src/message.h
-46
View File
@@ -1,46 +0,0 @@
#ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
#include "byte_queue.h"
#define MESSAGE_VERSION 1
typedef struct {
uint8_t *src;
int len;
int cur;
} BinaryReader;
typedef struct {
uint16_t version;
uint16_t type;
uint32_t length;
} MessageHeader;
typedef struct {
ByteQueue *output;
ByteQueueOffset start;
ByteQueueOffset patch;
} MessageWriter;
bool binary_read(BinaryReader *reader, void *dst, int len);
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type);
bool message_writer_free(MessageWriter *writer);
void message_write(MessageWriter *writer, void *mem, int len);
void message_write_u8(MessageWriter *writer, uint8_t value);
void message_write_u32(MessageWriter *writer, uint32_t value);
void message_write_hash(MessageWriter *writer, SHA256 value);
int message_peek(ByteView msg, uint16_t *type, uint32_t *len);
void message_dump(FILE *stream, ByteView msg);
#endif // MESSAGE_INCLUDED
src/metadata.h
+1 -1
View File
@@ -4,7 +4,7 @@
#include <stdint.h>
#include <stdbool.h>
#include "basic.h"
#include <lib/basic.h>
typedef struct {
SHA256 hash;
src/random_client.c
+1 -4
View File
@@ -82,10 +82,7 @@ int random_client_init(void *state_, int argc, char **argv,
return -1;
state->started = false;
if (pcap < TCP_POLL_CAPACITY) {
fprintf(stderr, "Random client :: Not enough poll capacity\n");
return -1;
}
(void) pcap;
*pnum = toastyfs_register_events(state->tfs, ctxs, pdata, pcap, timeout);
*timeout = 0; // Ensure first tick fires immediately to start an operation
return 0;
src/random_client.h
+2 -2
View File
@@ -3,8 +3,8 @@
#include <toastyfs.h>
#include "tcp.h"
#include "basic.h"
#include <lib/tcp.h>
#include <lib/basic.h>
#include "config.h"
#include "metadata.h"
src/server.c
+252 -595
View File
File diff suppressed because it is too large Load Diff
src/server.h
+62 -62
View File
@@ -1,15 +1,16 @@
#ifndef NODE_INCLUDED
#define NODE_INCLUDED
#ifndef SERVER_INCLUDED
#define SERVER_INCLUDED
#include <lib/message.h>
#include "tcp.h"
#include "basic.h"
#include "message.h"
#include "log.h"
#include "config.h"
#include "metadata.h"
#include "chunk_store.h"
#include "client_table.h"
#define MESSAGE_VERSION 1
enum {
// Normal Protocol
@@ -48,107 +49,107 @@ enum {
};
typedef struct {
MessageHeader base;
Message base;
MetaOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
Message base;
MetaOper oper;
int sender_idx;
int log_index;
int commit_index;
int sender_idx;
int log_index;
int commit_index;
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
} PrepareMessage;
typedef struct {
MessageHeader base;
int sender_idx;
int log_index;
Message base;
int sender_idx;
int log_index;
uint64_t view_number;
} PrepareOKMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int sender_idx;
int commit_index;
int sender_idx;
int commit_index;
} CommitMessage;
typedef struct {
MessageHeader base;
bool rejected;
Message base;
bool rejected;
MetaResult result;
uint64_t request_id;
} ReplyMessage;
uint64_t request_id;
} VsrReplyMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int sender_idx;
int sender_idx;
} BeginViewChangeMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number; // The new view number
uint64_t old_view_number; // Last view number when replica was in normal status
int op_number; // Number of entries in the log
int commit_index;
int sender_idx;
int op_number; // Number of entries in the log
int commit_index;
int sender_idx;
// Followed by: LogEntry log[op_number]
} DoViewChangeMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int commit_index;
int op_number; // Number of log entries that follow
int commit_index;
int op_number; // Number of log entries that follow
// Followed by: LogEntry log[op_number]
} BeginViewMessage;
typedef struct {
MessageHeader base;
int sender_idx;
Message base;
int sender_idx;
uint64_t nonce;
} RecoveryMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int op_number;
int op_number;
uint64_t nonce;
int commit_index;
int sender_idx;
int commit_index;
int sender_idx;
} RecoveryResponseMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int op_number; // Requester's current log count
int sender_idx;
int op_number; // Requester's current log count
int sender_idx;
} GetStateMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
int op_number; // Number of log entries that follow
int commit_index;
int start_index; // Global log index of the first entry in the suffix
int op_number; // Number of log entries that follow
int commit_index;
int start_index; // Global log index of the first entry in the suffix
// Followed by: LogEntry log[op_number]
} NewStateMessage;
typedef struct {
MessageHeader base;
Message base;
uint64_t view_number;
} RedirectMessage;
// StoreChunk: client -> any server. Carries chunk data as trailing bytes.
typedef struct {
MessageHeader base;
Message base;
SHA256 hash;
uint32_t size;
// Followed by: uint8_t data[size]
@@ -156,21 +157,21 @@ typedef struct {
// StoreChunkAck: server -> client.
typedef struct {
MessageHeader base;
SHA256 hash;
bool success;
Message base;
SHA256 hash;
bool success;
} StoreChunkAckMessage;
// FetchChunk: client/server -> server. Request a chunk by hash.
typedef struct {
MessageHeader base;
SHA256 hash;
int sender_idx; // -1 if from a client
Message base;
SHA256 hash;
int sender_idx; // -1 if from a client
} FetchChunkMessage;
// FetchChunkResponse: server -> client/server. Chunk data as trailing bytes.
typedef struct {
MessageHeader base;
Message base;
SHA256 hash;
uint32_t size; // 0 if chunk not found
// Followed by: uint8_t data[size]
@@ -179,7 +180,7 @@ typedef struct {
// CommitPut: blob client -> leader. Commits metadata after chunks uploaded.
// Processed like a REQUEST (goes through VSR log).
typedef struct {
MessageHeader base;
Message base;
MetaOper oper;
uint64_t client_id;
uint64_t request_id;
@@ -187,14 +188,14 @@ typedef struct {
// GetBlob: client -> any server. Request metadata for a blob.
typedef struct {
MessageHeader base;
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
Message base;
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
} GetBlobMessage;
// GetBlobResponse: server -> client. Returns blob metadata.
typedef struct {
MessageHeader base;
Message base;
bool found;
uint64_t size;
SHA256 content_hash;
@@ -210,7 +211,7 @@ typedef enum {
typedef struct {
TCP tcp;
MessageSystem msys;
Address self_addr;
Address node_addrs[NODE_LIMIT];
@@ -219,10 +220,9 @@ typedef struct {
Status status;
ClientTable client_table;
int next_client_tag;
uint64_t view_number;
uint64_t last_normal_view; // Latest view where status was NORMAL
uint64_t last_normal_view; // Latest view where status was NORMAL
// These fields are used in recovery mode
uint32_t recovery_votes;
@@ -265,7 +265,7 @@ typedef struct {
// Set at each wakeup
Time now;
} ServerState;
} Server;
struct pollfd;
@@ -294,7 +294,7 @@ void invariant_checker_free(InvariantChecker *ic);
// node_handles: opaque QuakeyNode handles for entering host context.
// Pass NULL when running outside the simulation (real mode).
void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
void invariant_checker_run(InvariantChecker *ic, Server **nodes, int num_nodes,
unsigned long long *node_handles);
#endif // NODE_INCLUDED
#endif // SERVER_INCLUDED
src/tcp.c
-504
View File
@@ -1,504 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include "tcp.h"
#include "message.h"
static int set_socket_blocking(SOCKET sock, bool value)
{
#ifdef _WIN32
u_long mode = !value;
if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
return -1;
#else
int flags = fcntl(sock, F_GETFL, 0);
if (flags < 0)
return -1;
if (value) flags &= ~O_NONBLOCK;
else flags |= O_NONBLOCK;
if (fcntl(sock, F_SETFL, flags) < 0)
return -1;
#endif
return 0;
}
static SOCKET create_listen_socket(Address addr)
{
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET)
return INVALID_SOCKET;
if (set_socket_blocking(fd, false) < 0) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
#ifndef MAIN_SIMULATION
int reuse = 1;
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
#endif
if (!addr.is_ipv4) {
assert(0); // TODO
}
struct sockaddr_in bind_buf;
bind_buf.sin_family = AF_INET;
bind_buf.sin_port = htons(addr.port);
memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4));
if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
int backlog = 32;
if (listen(fd, backlog) < 0) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
return fd;
}
static int create_socket_pair(SOCKET *a, SOCKET *b)
{
#ifdef _WIN32
SOCKET sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (sock == INVALID_SOCKET)
return -1;
// Bind to loopback address with port 0 (dynamic port assignment)
struct sockaddr_in addr;
int addr_len = sizeof(addr);
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1
addr.sin_port = 0; // Let system choose port
if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
closesocket(sock);
return -1;
}
if (getsockname(sock, (struct sockaddr*)&addr, &addr_len) == SOCKET_ERROR) {
closesocket(sock);
return -1;
}
if (connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) {
closesocket(sock);
return -1;
}
*a = sock;
*b = sock;
// Optional: Set socket to non-blocking mode
// This prevents send() from blocking if the receive buffer is full
u_long mode = 1;
ioctlsocket(sock, FIONBIO, &mode); // TODO: does this fail?
return 0;
#else
int fds[2];
if (pipe(fds) < 0)
return -1;
*a = fds[0];
*b = fds[1];
return 0;
#endif
}
static void close_socket_pair(SOCKET a, SOCKET b)
{
#ifdef _WIN32
closesocket(a);
(void) b;
#else
close(a);
close(b);
#endif
}
static void conn_init(Connection *conn, SOCKET fd, bool connecting)
{
conn->fd = fd;
conn->tag = -1;
conn->connecting = connecting;
conn->closing = false;
conn->msglen = 0;
byte_queue_init(&conn->input, 1<<30);
byte_queue_init(&conn->output, 1<<30);
}
static void conn_free(Connection *conn)
{
CLOSE_SOCKET(conn->fd);
byte_queue_free(&conn->input);
byte_queue_free(&conn->output);
}
static int conn_events(Connection *conn)
{
int events = 0;
if (conn->connecting)
events |= POLLOUT;
else {
assert(!byte_queue_full(&conn->input));
if (!conn->closing)
events |= POLLIN;
if (!byte_queue_empty(&conn->output))
events |= POLLOUT;
}
return events;
}
int tcp_context_init(TCP *tcp)
{
tcp->listen_fd = INVALID_SOCKET;
tcp->num_conns = 0;
if (create_socket_pair(&tcp->wait_fd, &tcp->signal_fd) < 0)
return -1;
return 0;
}
void tcp_context_free(TCP *tcp)
{
// Free all connection byte queues without closing sockets
// (sockets are managed by the simulation and will be cleaned up separately)
for (int i = 0; i < tcp->num_conns; i++) {
byte_queue_free(&tcp->conns[i].input);
byte_queue_free(&tcp->conns[i].output);
}
tcp->num_conns = 0;
if (tcp->listen_fd != INVALID_SOCKET)
CLOSE_SOCKET(tcp->listen_fd);
close_socket_pair(tcp->wait_fd, tcp->signal_fd);
}
int tcp_wakeup(TCP *tcp)
{
send(tcp->signal_fd, "0", 1, 0); // TODO: Handle error
return 0;
}
int tcp_index_from_tag(TCP *tcp, int tag)
{
for (int i = 0; i < tcp->num_conns; i++)
if (tcp->conns[i].tag == tag)
return i;
return -1;
}
int tcp_listen(TCP *tcp, Address addr)
{
SOCKET listen_fd = create_listen_socket(addr);
if (listen_fd == INVALID_SOCKET)
return -1;
tcp->listen_fd = listen_fd;
return 0;
}
int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type)
{
*msg = byte_queue_read_buf(&tcp->conns[conn_idx].input);
uint32_t len;
int ret = message_peek(*msg, type, &len);
// Invalid message?
if (ret < 0) {
byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
return -1;
}
// Still buffering header?
if (ret == 0) {
byte_queue_read_ack(&tcp->conns[conn_idx].input, 0);
if (byte_queue_full(&tcp->conns[conn_idx].input))
return -1;
return 0;
}
// Message received
assert(ret > 0);
msg->len = len;
tcp->conns[conn_idx].msglen = len;
return 1;
}
void tcp_consume_message(TCP *tcp, int conn_idx)
{
byte_queue_read_ack(&tcp->conns[conn_idx].input, tcp->conns[conn_idx].msglen);
tcp->conns[conn_idx].msglen = 0;
}
int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled)
{
int num_polled = 0;
polled[num_polled].fd = tcp->wait_fd;
polled[num_polled].events = POLLIN;
polled[num_polled].revents = 0;
contexts[num_polled] = NULL;
num_polled++;
if (tcp->listen_fd != INVALID_SOCKET && tcp->num_conns < TCP_CONNECTION_LIMIT) {
polled[num_polled].fd = tcp->listen_fd;
polled[num_polled].events = POLLIN;
polled[num_polled].revents = 0;
contexts[num_polled] = NULL;
num_polled++;
}
for (int i = 0; i < tcp->num_conns; i++) {
int events = conn_events(&tcp->conns[i]);
if (events) {
polled[num_polled].fd = tcp->conns[i].fd;
polled[num_polled].events = events;
polled[num_polled].revents = 0;
contexts[num_polled] = &tcp->conns[i];
num_polled++;
}
}
return num_polled;
}
// The "events" array must be an array of capacity TCP_EVENT_CAPACITY,
// while "contexts" and "polled" must have capacity TCP_POLL_CAPACITY.
int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled)
{
bool removed[TCP_POLL_CAPACITY];
for (int i = 0; i < TCP_POLL_CAPACITY; i++)
removed[i] = false;
int num_events = 0;
for (int i = 1; i < num_polled; i++) {
if (polled[i].fd == tcp->wait_fd) {
if (polled[i].revents & POLLIN) {
char buf[100];
recv(tcp->wait_fd, buf, sizeof(buf), 0); // TODO: Make sure all bytes are consumed
events[num_events++] = (Event) { EVENT_WAKEUP, -1, -1 };
}
} else if (polled[i].fd == tcp->listen_fd) {
assert(contexts[i] == NULL);
if (polled[i].revents & POLLIN) {
SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
if (new_fd != INVALID_SOCKET) {
if (set_socket_blocking(new_fd, false) < 0)
CLOSE_SOCKET(new_fd);
else {
conn_init(&tcp->conns[tcp->num_conns++], new_fd, false);
events[num_events++] = (Event) { EVENT_CONNECT, tcp->num_conns-1, tcp->conns[tcp->num_conns-1].tag };
}
}
}
removed[i] = false;
} else {
Connection *conn = contexts[i];
bool defer_close = false;
bool defer_ready = false;
if (conn->connecting) {
// Check for error conditions on the socket
if (polled[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
defer_close = true;
} else if (polled[i].revents & POLLOUT) {
int err = 0;
socklen_t len = sizeof(err);
if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
defer_close = true;
else {
conn->connecting = false;
events[num_events++] = (Event) { EVENT_CONNECT, conn - tcp->conns, conn->tag };
}
}
} else {
if (polled[i].revents & POLLIN) {
byte_queue_write_setmincap(&conn->input, 1<<9);
ByteView buf = byte_queue_write_buf(&conn->input);
int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num == 0)
defer_close = true;
else if (num < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) // TODO: does Windows return these error codes or not?
defer_close = true;
num = 0;
}
byte_queue_write_ack(&conn->input, num);
ByteView msg = byte_queue_read_buf(&conn->input);
int ret = message_peek(msg, NULL, NULL);
byte_queue_read_ack(&conn->input, 0);
if (ret < 0) {
// Invalid message
defer_close = true;
} else if (ret == 0) {
// Still buffering
if (byte_queue_full(&conn->input))
defer_close = true;
} else {
// Message received
assert(ret > 0);
defer_ready = true;
}
}
if (polled[i].revents & POLLOUT) {
ByteView buf = byte_queue_read_buf(&conn->output);
int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
defer_close = true;
num = 0;
}
byte_queue_read_ack(&conn->output, num);
if (conn->closing && byte_queue_empty(&conn->output))
defer_close = true;
}
}
// TODO: byte_queue_error here?
removed[i] = defer_close;
if (0) {}
else if (defer_close) events[num_events++] = (Event) { EVENT_DISCONNECT, conn - tcp->conns, conn->tag };
else if (defer_ready) events[num_events++] = (Event) { EVENT_MESSAGE, conn - tcp->conns, conn->tag };
}
}
for (int i = 1; i < num_polled; i++) {
if (removed[i]) {
Connection *conn = contexts[i];
assert(conn);
int removed_idx = conn - tcp->conns;
conn_free(conn);
int last_idx = --tcp->num_conns;
if (removed_idx != last_idx) {
*conn = tcp->conns[last_idx];
// Update event conn_idx values to reflect the swap
for (int j = 0; j < num_events; j++) {
if (events[j].conn_idx == last_idx)
events[j].conn_idx = removed_idx;
}
// Update contexts pointers for remaining iterations
for (int j = i + 1; j < num_polled; j++) {
if (contexts[j] == &tcp->conns[last_idx])
contexts[j] = conn;
}
}
}
}
return num_events;
}
ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx)
{
return &tcp->conns[conn_idx].output;
}
int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
{
if (tcp->num_conns == TCP_CONNECTION_LIMIT)
return -1;
int conn_idx = tcp->num_conns;
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET)
return -1;
if (set_socket_blocking(fd, false) < 0) {
CLOSE_SOCKET(fd);
return -1;
}
int ret;
if (addr.is_ipv4) {
struct sockaddr_in buf;
buf.sin_family = AF_INET;
buf.sin_port = htons(addr.port);
memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
} else {
struct sockaddr_in6 buf;
buf.sin6_family = AF_INET6;
buf.sin6_port = htons(addr.port);
memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
}
bool connecting;
if (ret == 0) {
connecting = false;
} else {
if (errno != EINPROGRESS) {
CLOSE_SOCKET(fd);
return -1;
}
connecting = true;
}
// Check that this tag wasn't already used
for (int i = 0; i < tcp->num_conns; i++)
assert(tcp->conns[i].tag != tag);
conn_init(&tcp->conns[conn_idx], fd, connecting);
tcp->conns[conn_idx].tag = tag;
if (output)
*output = &tcp->conns[conn_idx].output;
tcp->num_conns++;
return 0;
}
void tcp_close(TCP *tcp, int conn_idx)
{
tcp->conns[conn_idx].closing = true;
tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection
// TODO: if no event will be triggered, the connection will not be closed
// if the output buffer is empty, the connection should be closed here.
}
void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique)
{
assert(tag != -1);
if (unique) {
for (int i = 0; i < tcp->num_conns; i++)
assert(tcp->conns[i].tag != tag);
}
tcp->conns[conn_idx].tag = tag;
}
int tcp_get_tag(TCP *tcp, int conn_idx)
{
return tcp->conns[conn_idx].tag;
}
src/tcp.h
-87
View File
@@ -1,87 +0,0 @@
#ifndef TCP_INCLUDED
#define TCP_INCLUDED
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
# define QUAKEY_ENABLE_MOCKS
# include <quakey.h>
#else
# ifdef _WIN32
# include <winsock2.h>
# endif
#endif
#include "byte_queue.h"
#ifdef _WIN32
#define CLOSE_SOCKET closesocket
#else
#define SOCKET int
#define INVALID_SOCKET -1
#define CLOSE_SOCKET close
#endif
#ifndef TCP_CONNECTION_LIMIT
// Maximum number of connections that can be managed
// simultaneously.
#define TCP_CONNECTION_LIMIT 512
#endif
// This is the maximum number of descriptors that the
// TCP system will want to wait at any given time.
// One descriptor per connection plus a listener socket
// and a self-pipe handle for wakeup.
#define TCP_POLL_CAPACITY (TCP_CONNECTION_LIMIT+2)
// Number of TCP events that can be returned at a given
// time by "tcp_translate_events". There may be a single
// event per connection (MESSAGE, CONNECT, DISCONNECT)
// plus a general WAKEUP event.
#define TCP_EVENT_CAPACITY (TCP_CONNECTION_LIMIT+1)
typedef enum {
EVENT_WAKEUP,
EVENT_MESSAGE,
EVENT_CONNECT,
EVENT_DISCONNECT,
} EventType;
typedef struct {
EventType type;
int conn_idx;
int tag;
} Event;
typedef struct {
SOCKET fd;
int tag;
bool connecting;
bool closing;
uint32_t msglen;
ByteQueue input;
ByteQueue output;
} Connection;
typedef struct {
SOCKET listen_fd;
SOCKET wait_fd;
SOCKET signal_fd;
int num_conns;
Connection conns[TCP_CONNECTION_LIMIT];
} TCP;
int tcp_context_init(TCP *tcp);
void tcp_context_free(TCP *tcp);
int tcp_wakeup(TCP *tcp);
int tcp_index_from_tag(TCP *tcp, int tag);
int tcp_listen(TCP *tcp, Address addr);
int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type);
void tcp_consume_message(TCP *tcp, int conn_idx);
int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled);
int tcp_register_events(TCP *tcp, void **contexts, struct pollfd *polled);
ByteQueue *tcp_output_buffer(TCP *tcp, int conn_idx);
int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output);
void tcp_close(TCP *tcp, int conn_idx);
void tcp_set_tag(TCP *tcp, int conn_idx, int tag, bool unique);
int tcp_get_tag(TCP *tcp, int conn_idx);
#endif // TCP_INCLUDED