cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full rewrite with VSR support

Browse Source
This commit is contained in:
cozis
2026-02-20 16:55:29 +01:00
parent 2460817fa4
commit b64517c20e
59 changed files with 5790 additions and 10339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+2
View File
@@ -0,0 +1,2 @@
# Enforce LF line endings for all text files
* text eol=lf
.gitignore
+4 -6
View File
@@ -1,11 +1,9 @@
toasty_metadata_server
toasty_chunk_server
toasty_test
toasty_test_coverage
toasty_simulation
toasty_simulation_coverage
toastyfs
toastyfs_client
toastyfs_simulation
*.gcno
*.gcda
*.gcov
*.info
coverage_html/
*.log
TODO.txt
-28
View File
@@ -1,28 +0,0 @@
[X] Implement proper operation handles that allow users to wait for specific events
[X] Implement metadata server WAL
[X] Check that the corner case where read and writes go over the length of the file work correctly
[X] Return the number of bytes read or written in the ToastyFS_Result struct
[X] Add a change counter to FileTree. The counter is initialized to 0 and updated any time the tree is changed. When a new file is created, the current change counter is assigned to it. This makes it possible to verify that the file didn't change in between read and write operations without that clumsy previous hash list and previous chunk size.
[X] Add a flag for write operations to create the file if it doesn't exist alraedy
[X] Add failt injections to the simulation
[ ] Test node crash
[ ] Reference count chunks
[ ] Clean up the read operation for clients
[ ] Make parallel uploads/downloads configurable
[ ] Recalculate next write locations whenever a write occurs, not at each read
[ ] add logging (when chunk servers connect and disconnect to the metadata server)
[ ] Should list scenarios that need testing, like those where chunks would be dropped
[ ] Update DESIGN.txt and the code to remove the chunk list message. The information of chunks held by chunk servers is now transmitted to the metadata server during state updates
[ ] clean up the replication factor business
[ ] Add authentication
[ ] Add release build and allow switching between debug, release, coverage, sanitizer builds while calling make
[ ] When an operation is committed to the WAL, it may then fail due to not deterministic reasons (out of memory). When the metadata server restarts and replays the log, the operation may succede and cause the system to diverge from the last instance of the process. Is this not a problem? How do we solve it?
[ ] Add notes on who inspired the testing approach
[ ] Add notes on benchmarks and that we don't do batching
[ ] Fix endianess when writing to network and the WAL
[ ] WAL entry checksum
[ ] Check write() errors when appending WAL entries
[ ] When WAL entries are partially written, remove partial data and fail
[ ] Find a way to ensure listing operations of large directories are handled
[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication?
[ ] Rename generation counters to version numbers
build.sh
+3 -4
View File
@@ -1,4 +1,3 @@
#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c quakey/src/mockfs.c quakey/src/quakey.c -o toasty_simulation -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c quakey/src/mockfs.c quakey/src/quakey.c -o toasty_test -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_TEST
#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c -o toasty_metadata_server -DMAIN_METADATA_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c -o toasty_chunk_server -DMAIN_CHUNK_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/invariant_checker.c src/chunk_store.c src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c -o toastyfs_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER
gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT
build_coverage.sh
-37
View File
@@ -1,37 +0,0 @@
#!/bin/bash
set -e
SIMULATION_BINARY="toasty_simulation_coverage"
RUN_DURATION=60
COVERAGE_DIR="coverage_html"
echo "=== Step 1: Building simulation with coverage instrumentation ==="
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c \
src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c \
src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c \
quakey/src/mockfs.c quakey/src/quakey.c \
-o "$SIMULATION_BINARY" \
-Iquakey/include -Iinclude \
-Wall -Wextra -ggdb -O0 \
-DMAIN_SIMULATION \
--coverage -fprofile-arcs -ftest-coverage
echo "Build complete: $SIMULATION_BINARY"
echo ""
echo "=== Step 2: Running simulation for ${RUN_DURATION}s ==="
lcov --zerocounters --directory . --quiet
lcov --capture --initial --directory . --output-file coverage_base.info --rc lcov_branch_coverage=1 --quiet
timeout --signal=INT --kill-after=5 "$RUN_DURATION" "./$SIMULATION_BINARY" || true
echo "Simulation finished"
echo ""
echo "=== Step 3: Generating coverage report ==="
lcov --capture --directory . --output-file coverage_test.info --rc lcov_branch_coverage=1 --quiet
lcov --add-tracefile coverage_base.info --add-tracefile coverage_test.info --output-file coverage.info --rc lcov_branch_coverage=1 --quiet
lcov --remove coverage.info '/usr/*' 'quakey/*' --output-file coverage.info --rc lcov_branch_coverage=1 --quiet
genhtml coverage.info --output-directory "$COVERAGE_DIR" --branch-coverage --quiet
rm -f coverage_base.info coverage_test.info
echo ""
echo "=== Done ==="
echo "Coverage report: $COVERAGE_DIR/index.html"
build_feature.sh
-36
View File
@@ -1,36 +0,0 @@
#!/bin/bash
set -e
TEST_BINARY="toasty_test_coverage"
COVERAGE_DIR="coverage_html"
echo "=== Step 1: Building test with coverage instrumentation ==="
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c \
src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c \
src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c \
quakey/src/mockfs.c quakey/src/quakey.c \
-o "$TEST_BINARY" \
-Iquakey/include -Iinclude \
-Wall -Wextra -ggdb -O0 \
-DMAIN_TEST \
--coverage -fprofile-arcs -ftest-coverage
echo "Build complete: $TEST_BINARY"
echo ""
echo "=== Step 2: Running tests ==="
lcov --zerocounters --directory . --quiet
lcov --capture --initial --directory . --output-file coverage_base.info --rc lcov_branch_coverage=1 --quiet
"./$TEST_BINARY"
echo "Tests finished"
echo ""
echo "=== Step 3: Generating coverage report ==="
lcov --capture --directory . --output-file coverage_test.info --rc lcov_branch_coverage=1 --quiet
lcov --add-tracefile coverage_base.info --add-tracefile coverage_test.info --output-file coverage.info --rc lcov_branch_coverage=1 --quiet
lcov --remove coverage.info '/usr/*' 'quakey/*' --output-file coverage.info --rc lcov_branch_coverage=1 --quiet
genhtml coverage.info --output-directory "$COVERAGE_DIR" --branch-coverage --quiet
rm -f coverage_base.info coverage_test.info
echo ""
echo "=== Done ==="
echo "Coverage report: $COVERAGE_DIR/index.html"
include/ToastyFS.h
-263
View File
@@ -1,263 +0,0 @@
#ifndef TOASTY_INCLUDED
#define TOASTY_INCLUDED
//////////////////////////////////////////////////////////////////////////////////
// INCLUDES
//////////////////////////////////////////////////////////////////////////////////
#include <stdint.h>
#include <stdbool.h>
//////////////////////////////////////////////////////////////////////////////////
// TYPES & UTILITIES
//////////////////////////////////////////////////////////////////////////////////
typedef struct ToastyFS ToastyFS;
// Helper type to avoid zero-terminated strings
typedef struct {
char *ptr;
int len;
} ToastyString;
// Macro to convert string literals to ToastyStrings
#define TOASTY_STR(X) ((ToastyString) { (X), (int) sizeof(X)-1 })
//////////////////////////////////////////////////////////////////////////////////
// PRIMARY
//////////////////////////////////////////////////////////////////////////////////
// Instanciate a ToastyFS client object. The "addr" and "port"
// arguments refer to the address and port of the cluster's
// metadata server.
ToastyFS *toasty_connect(ToastyString addr, uint16_t port);
// Release all resources associated to this client
void toasty_disconnect(ToastyFS *toasty);
// Threads can call this function to wake up a thread blocked
// inside "toasty_wait_result".
// Return 0 on success, -1 on error.
int toasty_wakeup(ToastyFS *toasty);
//////////////////////////////////////////////////////////////////////////////////
// BLOCKING API
//////////////////////////////////////////////////////////////////////////////////
// TODO: comment
typedef uint64_t ToastyVersionTag;
// TODO: comment
#define TOASTY_VERSION_TAG_EMPTY ((ToastyVersionTag) 0)
// Write operation flags
#define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) // Create file if it doesn't exist
#define TOASTY_WRITE_TRUNCATE_AFTER (1 << 1) // Truncate file after write offset+length
// Creates a directory at the specified path.
// Returns 0 on success, -1 on error.
//
// If the version tag is not NULL, it's used to
// return the version tag associated to the newly
// created file.
int toasty_create_dir(ToastyFS *toasty, ToastyString path,
ToastyVersionTag *vtag);
// Creates a file with the given chunk size at
// the specified path. Returns 0 on success, -1
// on error. The chunk size can't be 0.
//
// If the version tag is not NULL, it's used to
// return the version tag associated to the newly
// created file.
int toasty_create_file(ToastyFS *toasty, ToastyString path,
unsigned int chunk_size, ToastyVersionTag *vtag);
// Deletes a file or directory at the specified path.
// Returns 0 on success, -1 on error.
//
// If the version tag is not 0, the file/directory
// is only deleted if the tags match.
int toasty_delete(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag);
typedef struct {
char name[128]; // TODO: Implement a proper name length
bool is_dir;
ToastyVersionTag vtag;
} ToastyListingEntry;
typedef struct {
int count;
ToastyListingEntry *items;
} ToastyListing;
// Lists all files and directories within the given
// path. Returns 0 and fills up the listing argument
// on success, returns -1 on error. The listing is
// a dynamic array that needs to be freed using
// "toasy_free_listing".
//
// If the version tag is not 0, the listing only
// succedes if the tag matches the remote one.
// If the operation succedes, the vtag is set to
// the remote tag.
int toasty_list(ToastyFS *toasty, ToastyString path,
ToastyListing *listing, ToastyVersionTag *vtag);
// Frees a listing created by "toasty_list".
void toasty_free_listing(ToastyListing *listing);
// Reads "len" bytes at offset "off" from the file at
// the given path. Returns the number of bytes read on
// success, or -1 on error.
//
// If vtag is not NULL, the read only succedes if the
// target version tag matches vtag or vtag was 0. If
// the operation succedes, the version tag is set to
// the remote entity's.
int toasty_read(ToastyFS *toasty, ToastyString path, int off,
void *dst, int len, ToastyVersionTag *vtag);
// Writes "len" bytes at offset "off" to the file at
// the given path. Returns the number of bytes written
// on success, or -1 on error.
//
// For how vtag works, see toasty_read.
//
// The flags parameter accepts the following values:
// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it
// doesn't exist. A default chunk size of 4096 bytes will be used.
// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len,
// discarding any data beyond the write range. Useful for HTTP PUT semantics.
int toasty_write(ToastyFS *toasty, ToastyString path, int off,
void *src, int len, ToastyVersionTag *vtag, uint32_t flags);
//////////////////////////////////////////////////////////////////////////////////
// ASYNCHRONOUS API
//////////////////////////////////////////////////////////////////////////////////
// Handle representing a pending operation
typedef uint32_t ToastyHandle;
// Invalid value for a ToastyHandle
#define TOASTY_INVALID ((ToastyHandle) 0)
// Begins a directory creation operation and returns
// a handle to it. On error, TOASTY_INVALID is returned.
ToastyHandle toasty_begin_create_dir(ToastyFS *toasty, ToastyString path);
// Begins a file creation operation and returns a
// handle to it. On error, TOASTY_INVALID is returned.
ToastyHandle toasty_begin_create_file(ToastyFS *toasty, ToastyString path,
unsigned int chunk_size);
// Begins a file or directory deletion operation and
// returns a handle to it. On error, TOASTY_INVALID is
// returned.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path,
ToastyVersionTag vtag);
// Begins a directory listing operation and returns
// a handle to it. On error, TOASTY_INVALID is returned.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag);
// Begins a read operation and returns a handle to it.
// On error, TOASTY_INVALID is returned.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path,
int off, void *dst, int len, ToastyVersionTag vtag);
// Begins a write operation and returns a handle to it.
// On error, TOASTY_INVALID is returned. Note that the source
// buffer must be valid until the operation completes.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
//
// The flags parameter accepts the following values:
// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it
// doesn't exist. A default chunk size of 4096 bytes will be used.
// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len,
// discarding any data beyond the write range. Useful for HTTP PUT semantics.
ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path,
int off, void *src, int len, ToastyVersionTag vtag, uint32_t flags);
// Associate the pointer "user" to the handle. The user
// pointer will be returned in the ToastyResult when the
// operation compltes.
void toasty_set_user(ToastyFS *toasty, ToastyHandle handle, void *user);
typedef enum {
TOASTY_RESULT_EMPTY,
TOASTY_RESULT_CREATE_ERROR,
TOASTY_RESULT_CREATE_SUCCESS,
TOASTY_RESULT_DELETE_ERROR,
TOASTY_RESULT_DELETE_SUCCESS,
TOASTY_RESULT_LIST_ERROR,
TOASTY_RESULT_LIST_SUCCESS,
TOASTY_RESULT_READ_ERROR,
TOASTY_RESULT_READ_SUCCESS,
TOASTY_RESULT_WRITE_ERROR,
TOASTY_RESULT_WRITE_SUCCESS,
} ToastyResultType;
typedef struct {
ToastyResultType type;
ToastyListing listing;
ToastyVersionTag vtag;
void* user;
int bytes_read; // For read operations: actual number of bytes read
} ToastyResult;
// If the operation specified by "handle" is complete,
// its result is stored in "result" and 0 is returned.
// If the operation is still in progress, 1 is returned.
// On error, -1 is returned. If the handle is TOASTY_INVALID,
// then the result of the first complete operation is
// returned.
// Note that if a result is returned, handles to that
// operation are invalidated.
// The "result" must be freed using "toasty_free_result".
//
// Note:
// If you call this function in a loop, the state of the
// operation will not progress. You can't wait for completion
// by calling this in a loop. You need to use "toasty_wait_result"
int toasty_get_result(ToastyFS *toasty, ToastyHandle handle,
ToastyResult *result);
// Blocks execution until an operation is complete. This works
// like "toasty_get_result", except it waits for "timeout" milliseconds
// if the result isn't available. If "timeout" is -1, it waits
// indefinitely.
// The "result" must be freed using "toasty_free_result".
int toasty_wait_result(ToastyFS *toasty, ToastyHandle handle,
ToastyResult *result, int timeout);
// Frees resources of a "ToastyResult" previously initialized
// by "toasty_get_result" or "toasty_wait_result".
void toasty_free_result(ToastyResult *result);
//////////////////////////////////////////////////////////////////////////////////
// OTHER
//////////////////////////////////////////////////////////////////////////////////
#define TOASTY_POLL_CAPACITY 514
struct pollfd;
// TODO: comment
int toasty_process_events(ToastyFS *toasty, void **contexts,
struct pollfd *polled, int num_polled);
//////////////////////////////////////////////////////////////////////////////////
// THE END
//////////////////////////////////////////////////////////////////////////////////
#endif // TOASTY_INCLUDED
src/basic.c → lib/basic.c
+104
View File
@@ -126,3 +126,107 @@ void append_hex_as_str(char *out, SHA256 hash)
out[(i << 1) + 1] = table[(uint8_t) hash.data[i] & 0xF];
}
}
// TODO: check this function
bool addr_lower(Address a, Address b)
{
if (a.is_ipv4) {
if (!b.is_ipv4)
return true;
if (a.ipv4.data < b.ipv4.data)
return true;
if (a.ipv4.data == b.ipv4.data &&
a.port < b.port)
return true;
return false;
} else {
if (b.is_ipv4)
return false;
for (int i = 0; i < 8; i++) {
if (a.ipv6.data[i] < b.ipv6.data[i])
return true;
if (a.ipv6.data[i] > b.ipv6.data[i])
return false;
}
if (a.port < b.port)
return true;
return false;
}
}
bool addr_eql(Address a, Address b)
{
if (a.is_ipv4 != b.is_ipv4)
return false;
if (a.port != b.port)
return false;
if (a.is_ipv4) {
if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4)))
return false;
} else {
if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6)))
return false;
}
return true;
}
int parse_addr_arg(char *arg, Address *out)
{
int len = strlen(arg);
int i = 0;
while (i < len && arg[i] != ':')
i++;
if (i == len)
return -1; // No ':' character.
arg[i] = '\0';
IPv4 ipv4;
int ret = inet_pton(AF_INET, arg, &ipv4);
arg[i] = ':';
if (ret != 1)
return -1;
errno = 0;
ret = atoi(arg + i + 1);
if (ret == 0 && errno != 0)
return -1;
out->ipv4 = ipv4;
out->is_ipv4 = true;
out->port = ret;
return 0;
}
void addr_sort(Address *addrs, int count)
{
for (int i = 0; i < count; i++) {
int k = i; // Index of the lowest address in [i, num_nodes-1]
for (int j = i+1; j < count; j++) {
if (addr_lower(addrs[j], addrs[k]))
k = j;
}
Address tmp = addrs[i];
addrs[i] = addrs[k];
addrs[k] = tmp;
}
}
src/basic.h → lib/basic.h
+7
View File
@@ -36,6 +36,7 @@ typedef uint64_t Time;
#define S(X) ((string) { (X), (int) sizeof(X)-1 })
#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
#define UNREACHABLE __builtin_trap();
@@ -51,4 +52,10 @@ int getargi(int argc, char **argv, char *name, int fallback);
void append_hex_as_str(char *out, SHA256 hash);
bool addr_eql(Address a, Address b);
bool addr_lower(Address a, Address b);
int parse_addr_arg(char *arg, Address *out);
void addr_sort(Address *addrs, int count);
#endif // BASIC_INCLUDED
src/byte_queue.c → lib/byte_queue.c
View File
src/byte_queue.h → lib/byte_queue.h
View File
src/file_system.c → lib/file_system.c
+58 -1
View File
@@ -9,6 +9,24 @@
int rename_file_or_dir(string oldpath, string newpath);
bool file_exists(string path)
{
char zt[1<<10];
if (path.len >= (int) sizeof(zt))
return false;
memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0';
#ifdef __linux__
return access(zt, F_OK) == 0;
#endif
#ifdef _WIN32
DWORD attrs = GetFileAttributesA(zt);
return attrs != INVALID_FILE_ATTRIBUTES;
#endif
}
int file_open(string path, Handle *fd)
{
#ifdef __linux__
@@ -18,7 +36,7 @@ int file_open(string path, Handle *fd)
memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0';
int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
int ret = open(zt, O_RDWR | O_CREAT, 0644);
if (ret < 0)
return -1;
@@ -59,6 +77,19 @@ void file_close(Handle fd)
#endif
}
int file_truncate(Handle fd, size_t new_size)
{
#ifdef __linux__
if (ftruncate((int) fd.data, new_size) < 0)
return -1;
return 0;
#endif
#ifdef _WIN32
return -1; // TODO: Not implemented
#endif
}
int file_set_offset(Handle fd, int off)
{
#ifdef __linux__
@@ -400,3 +431,29 @@ void directory_scanner_free(DirectoryScanner *scanner)
}
#endif
int file_read_exact(Handle handle, char *dst, int len)
{
int copied = 0;
while (copied < len) {
int ret = file_read(handle, dst + copied, len - copied);
if (ret < 0)
return -1;
if (ret == 0)
return 0; // EOF
copied += ret;
}
return copied;
}
int file_write_exact(Handle handle, char *src, int len)
{
int copied = 0;
while (copied < len) {
int ret = file_write(handle, src + copied, len - copied);
if (ret < 0)
return -1;
copied += ret;
}
return 0;
}
src/file_system.h → lib/file_system.h
+5
View File
@@ -29,8 +29,10 @@ typedef struct {
} DirectoryScanner;
#endif
bool file_exists(string path);
int file_open(string path, Handle *fd);
void file_close(Handle fd);
int file_truncate(Handle fd, size_t new_size);
int file_set_offset(Handle fd, int off);
int file_get_offset(Handle fd, int *off);
int file_lock(Handle fd);
@@ -50,4 +52,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path);
int directory_scanner_next(DirectoryScanner *scanner, string *name);
void directory_scanner_free(DirectoryScanner *scanner);
int file_read_exact(Handle handle, char *dst, int len);
int file_write_exact(Handle handle, char *src, int len);
#endif // FILE_SYSTEM_INCLUDED
lib/message.c
+81
View File
@@ -0,0 +1,81 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "message.h"
bool binary_read(BinaryReader *reader, void *dst, int len)
{
if (reader->len - reader->cur < len)
return false;
if (dst)
memcpy(dst, reader->src + reader->cur, len);
reader->cur += len;
return true;
}
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type)
{
uint16_t version = MESSAGE_VERSION;
uint32_t dummy = 0; // Dummy value
writer->output = output;
writer->start = byte_queue_offset(output);
byte_queue_write(output, &version, sizeof(version));
byte_queue_write(output, &type, sizeof(type));
writer->patch = byte_queue_offset(output);
byte_queue_write(output, &dummy, sizeof(dummy));
}
bool message_writer_free(MessageWriter *writer)
{
uint32_t length = byte_queue_size_from_offset(writer->output, writer->start);
byte_queue_patch(writer->output, writer->patch, &length, sizeof(length));
if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure?
return false;
return true;
}
void message_write(MessageWriter *writer, void *mem, int len)
{
byte_queue_write(writer->output, mem, len);
}
void message_write_u8(MessageWriter *writer, uint8_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_u32(MessageWriter *writer, uint32_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_hash(MessageWriter *writer, SHA256 value)
{
message_write(writer, &value, (int) sizeof(value));
}
int message_peek(ByteView msg, uint16_t *type, uint32_t *len)
{
if (msg.len < (int) sizeof(MessageHeader))
return 0;
MessageHeader header;
memcpy(&header, msg.ptr, sizeof(header));
// (We ignore endianess for now)
if (header.version != MESSAGE_VERSION)
return -1;
if (header.length > msg.len)
return 0;
if (type) *type = header.type;
if (len) *len = header.length;
return 1;
}
lib/message.h
+46
View File
@@ -0,0 +1,46 @@
#ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
#include "byte_queue.h"
#define MESSAGE_VERSION 1
typedef struct {
uint8_t *src;
int len;
int cur;
} BinaryReader;
typedef struct {
uint16_t version;
uint16_t type;
uint32_t length;
} MessageHeader;
typedef struct {
ByteQueue *output;
ByteQueueOffset start;
ByteQueueOffset patch;
} MessageWriter;
bool binary_read(BinaryReader *reader, void *dst, int len);
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type);
bool message_writer_free(MessageWriter *writer);
void message_write(MessageWriter *writer, void *mem, int len);
void message_write_u8(MessageWriter *writer, uint8_t value);
void message_write_u32(MessageWriter *writer, uint32_t value);
void message_write_hash(MessageWriter *writer, SHA256 value);
int message_peek(ByteView msg, uint16_t *type, uint32_t *len);
void message_dump(FILE *stream, ByteView msg);
#endif // MESSAGE_INCLUDED
src/tcp.c → lib/tcp.c
+25 -37
View File
@@ -8,25 +8,6 @@
#include "tcp.h"
#include "message.h"
bool addr_eql(Address a, Address b)
{
if (a.is_ipv4 != b.is_ipv4)
return false;
if (a.port != b.port)
return false;
if (a.is_ipv4) {
if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4)))
return false;
} else {
if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6)))
return false;
}
return true;
}
static int set_socket_blocking(SOCKET sock, bool value)
{
#ifdef _WIN32
@@ -46,7 +27,7 @@ static int set_socket_blocking(SOCKET sock, bool value)
return 0;
}
static SOCKET create_listen_socket(string addr, uint16_t port)
static SOCKET create_listen_socket(Address addr)
{
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET)
@@ -59,22 +40,14 @@ static SOCKET create_listen_socket(string addr, uint16_t port)
// TODO: mark address as reusable in debug builds
char tmp[1<<10];
if (addr.len >= (int) sizeof(tmp)) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
if (!addr.is_ipv4) {
assert(0); // TODO
}
memcpy(tmp, addr.ptr, addr.len);
tmp[addr.len] = '\0';
struct sockaddr_in bind_buf;
bind_buf.sin_family = AF_INET;
bind_buf.sin_port = htons(port);
if (inet_pton(AF_INET, tmp, &bind_buf.sin_addr) != 1) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
bind_buf.sin_port = htons(addr.port);
memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4));
if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
@@ -155,8 +128,8 @@ static void conn_init(Connection *conn, SOCKET fd, bool connecting)
conn->connecting = connecting;
conn->closing = false;
conn->msglen = 0;
byte_queue_init(&conn->input, 1<<20);
byte_queue_init(&conn->output, 1<<20);
byte_queue_init(&conn->input, 1<<30);
byte_queue_init(&conn->output, 1<<30);
}
static void conn_free(Connection *conn)
@@ -225,9 +198,9 @@ int tcp_index_from_tag(TCP *tcp, int tag)
return -1;
}
int tcp_listen(TCP *tcp, string addr, uint16_t port)
int tcp_listen(TCP *tcp, Address addr)
{
SOCKET listen_fd = create_listen_socket(addr, port);
SOCKET listen_fd = create_listen_socket(addr);
if (listen_fd == INVALID_SOCKET)
return -1;
@@ -420,8 +393,22 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
if (removed[i]) {
Connection *conn = contexts[i];
assert(conn);
int removed_idx = conn - tcp->conns;
conn_free(conn);
*conn = tcp->conns[--tcp->num_conns];
int last_idx = --tcp->num_conns;
if (removed_idx != last_idx) {
*conn = tcp->conns[last_idx];
// Update event conn_idx values to reflect the swap
for (int j = 0; j < num_events; j++) {
if (events[j].conn_idx == last_idx)
events[j].conn_idx = removed_idx;
}
// Update contexts pointers for remaining iterations
for (int j = i + 1; j < num_polled; j++) {
if (contexts[j] == &tcp->conns[last_idx])
contexts[j] = conn;
}
}
}
}
@@ -491,6 +478,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
void tcp_close(TCP *tcp, int conn_idx)
{
tcp->conns[conn_idx].closing = true;
tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection
// TODO: if no event will be triggered, the connection will not be closed
// if the output buffer is empty, the connection should be closed here.
}
src/tcp.h → lib/tcp.h
+1 -2
View File
@@ -69,12 +69,11 @@ typedef struct {
Connection conns[TCP_CONNECTION_LIMIT];
} TCP;
bool addr_eql(Address a, Address b);
int tcp_context_init(TCP *tcp);
void tcp_context_free(TCP *tcp);
int tcp_wakeup(TCP *tcp);
int tcp_index_from_tag(TCP *tcp, int tag);
int tcp_listen(TCP *tcp, string addr, uint16_t port);
int tcp_listen(TCP *tcp, Address addr);
int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type);
void tcp_consume_message(TCP *tcp, int conn_idx);
int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled);
quakey/include/quakey.h
+33 -1
View File
@@ -26,6 +26,8 @@
#include <pthread.h>
#endif
#include <stdbool.h>
typedef struct {} Quakey;
// Function pointers to a simulated program's code
@@ -72,12 +74,19 @@ int quakey_init(Quakey **quakey, QuakeyUInt64 seed);
// Stop a simulation
void quakey_free(Quakey *quakey);
typedef unsigned long long QuakeyNode;
// Add a program to the simulation
void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg);
QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg);
void *quakey_node_state(QuakeyNode node);
// Schedule and executes one program until it would block, then returns
int quakey_schedule_one(Quakey *quakey);
// Get the current simulated time in nanoseconds
QuakeyUInt64 quakey_current_time(Quakey *quakey);
// Generate a random u64
QuakeyUInt64 quakey_random(void);
@@ -89,6 +98,25 @@ typedef struct {
void quakey_signal(char *name);
int quakey_get_signal(Quakey *quakey, QuakeySignal *signal);
// Limit the number of hosts that can be crashed at the same time.
// Set to 0 for no limit (default).
void quakey_set_max_crashes(Quakey *quakey, int max_crashes);
void quakey_network_partitioning(Quakey *quakey, bool enable);
// Access spawned host information
int quakey_num_hosts(Quakey *quakey);
void *quakey_host_state(Quakey *quakey, int idx); // Returns NULL if host is dead
int quakey_host_is_dead(Quakey *quakey, int idx);
const char *quakey_host_name(Quakey *quakey, int idx);
// Enter/leave a host's context so that mock_xxx functions (file I/O,
// etc.) operate on that host's resources. Use from code that runs
// outside of the normal init/tick/free callbacks (e.g. invariant
// checkers).
void quakey_enter_host(QuakeyNode node);
void quakey_leave_host(void);
int *mock_errno_ptr(void);
#ifdef _WIN32
@@ -125,6 +153,8 @@ int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_fcntl(int fd, int cmd, int flags);
int mock_close(int fd);
int mock_access(const char *path, int mode);
int mock_ftruncate(int fd, size_t new_size);
int mock_fstat(int fd, struct stat *buf);
int mock_read(int fd, char *dst, int len);
int mock_write(int fd, char *src, int len);
@@ -172,6 +202,8 @@ void mock_free(void *ptr);
#define open mock_open
#define fcntl mock_fcntl
#define close mock_close
#define access mock_access
#define ftruncate mock_ftruncate
#define CreateFileW mock_CreateFileW
#define CloseHandle mock_CloseHandle
#define ReadFile mock_ReadFile
quakey/src/mockfs.c
+57
View File
@@ -657,6 +657,63 @@ int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence)
return new_offset;
}
static void byte_buffer_truncate(ByteBuffer *byte_buffer, int new_size, ByteChunk **free_list)
{
if (new_size == 0) {
byte_buffer_free(byte_buffer, free_list);
byte_buffer_init(byte_buffer);
return;
}
// Walk through chunks to find the one containing the new end
int remaining = new_size;
ByteChunk *chunk = byte_buffer->head;
while (chunk) {
int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used);
if (remaining <= chunk_used) {
// This chunk becomes the new tail
// Free all subsequent chunks
ByteChunk *to_free = chunk->next;
if (to_free) {
// Find end of chain to free
ByteChunk *last = to_free;
while (last->next)
last = last->next;
last->next = *free_list;
*free_list = to_free;
}
chunk->next = NULL;
byte_buffer->tail = chunk;
byte_buffer->tail_used = remaining;
return;
}
remaining -= BYTE_CHUNK_SIZE;
chunk = chunk->next;
}
}
int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size)
{
if (new_size < 0)
return MOCKFS_ERRNO_INVAL;
if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR)))
return MOCKFS_ERRNO_BADF;
ByteBuffer *bb = &open_file->entity->byte_buffer;
int current_size = byte_buffer_size(bb);
if (new_size < current_size) {
byte_buffer_truncate(bb, new_size, &open_file->mfs->chunk_free_list);
} else if (new_size > current_size) {
int ret = byte_buffer_extend(bb, new_size, open_file->mfs);
if (ret < 0)
return ret;
}
return 0;
}
static int remove_inner(MockFS *mfs, MockFS_Entity *entity, bool recursive)
{
if (entity->parent == NULL)
quakey/src/mockfs.h
+1
View File
@@ -99,6 +99,7 @@ int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent);
int mockfs_sync(MockFS_OpenFile *open_file);
int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence);
int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size);
int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive);
quakey/src/quakey.c
+749 -26
View File
File diff suppressed because it is too large Load Diff
spec/design.txt
-233
View File
@@ -1,233 +0,0 @@
Architecture
A ToastyFS instance is composed by a metadata server, a number
of chunk servers, and a number of clients.
The metadata server stores the full file system hieararchy,
except instead of storing the file contents, it stores an
array of hashes of the chunks of each file. A "chunk" is a
file range that is fixed for a single file but may vary
between files. Chunk servers hold an array of chunks that
are identified by their hash. The metadata server keeps
track of which chunks each chunk server is holding.
Clients are users of the file system that can read and
write metadata and files. They are assumed to behave
correctly.
Any read and write operation that doesn't involve file
contents can be performed by clients by talking to the
metadata server directly. Such operations include creating
an empty file or a directory, deleting a file or directory,
listing files.
If a client wants to read a range of bytes from a file,
it sends the metadata server the file name and range.
The metadata server responds with the chunk size of that
file, the list of hashes for the chunks involved in the
read, and the IP addresses of the chunk servers that hold
each chunk. The metadata server also adds the IP addresses
of three chunk servers any new chunks should be written
to. The client can then download the chunks from the chunk
servers and reassemble the result.
If a client wants to write at a range of bytes of a file,
it starts by reading that range from the metadata server,
getting the list of hashes it will modify, their locations,
and locations for any new chunks. The client then modifies
the chunk by sending to each chunk server the hash to modify
and the patch (a range of bytes within a chunk plus the new
data). The chunk server creates a new modified chunk and
keeps the old version, then returns the new hash. If all
modifications are successful, the client holds the set of
old hashes and new hashes for that file range. It completes
the write by telling the metadata server to swap the old
hashes with the new ones (optionally including write flags
such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes
don't match, another write succeded in the mean time and
touched that range, therefore the write fails. If the old
hashes match, the write succeded. If the client fails to
modify any chunks, it doesn't commit the write with the
metadata server.
If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING
flag is set, the metadata server atomically creates the file
with a default chunk size (4096 bytes) and retries the write.
This operation is logged to the WAL to ensure crash consistency.
If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is
truncated after the write, setting its size to exactly offset+length
and discarding any data beyond that point. This is useful for HTTP
PUT semantics where the entire file content should be replaced.
Note that write failures may cause chunks to be orphaned
on chunk servers. This is solved by a garbage collection
algorithm implemented by the synchronization messages
between metadata and chunk server.
Note that clients may cache chunks and index them by their
hash. When they read a file and receive its hashes, they may
avoid reaching for the chunk servers if they already cached
the chunks with those hashes. This allows reading files with
only one round trip at no cost of correctness. If getting
the up-to-date contents is not a concern, clients may also
cache file metadata.
Metadata and chunk server exchange:
The metadata server is only aware of each chunk server
as long as they have a TCP connection. When a chunk server
first connects to the metadata server, it authenticates
itself and sends its own IP addresses. If the server is
authentic, the metadata server requests the full list
of chunks the chunk server is holding. Upon receiving the
state of chunk server, the metadata server adds all useful
chunks to the "old_list" and all useless chunks to the
"rem_list", then sends the rem_list to the chunk server
which removes those chunks.
When writes are committed to the metadata server involving
new chunks to a chunk server, the metadata server adds those
hashes to an "add_list" and any hashes that are not useful
anymore to the rem_list.
Periodically, the metadata server sends the add_list and
rem_list to the chunk server. These list tell the chunk
server the ideal state it should have from the point of
view of the metadata server. Elements in the add_list should
already be in the chunk servers, and elements from the
rem_list are to be removed. A chunk server marks any chunk
in the rem_list as to be removed and checks that hashes
in the add list are present. If a chunk in the add list
is marked as to be removed, it is unmarked. When a chunk
is marked as to be removed for a certain amount of time,
it is permanently deleted. When the synchronization is
complete, the metadata server merges the add_list into
the old_list and clears the rem_list. If chunks in the
add_list are not present in the chunk server, it responds
with an error message containing the list of missing chunks.
The metadata server then responds with a list of chunk
server addresses where the chunk server with the missing
chunk can download it from. Each chunk server goes
through its download list one at the time downloading
the missing chunks.
Note that if the chunk server finds that its holding some
chunks that are not in the hash list of the metadata server,
that does not mean they are orphaned. It's possible that
some writes are being performed by clients that have uploaded
chunks to that chunk server but didn't yet acknowledge it
to the metadata server. If all goes well and the write
succeded, the metadata server will add those hashes to the
hash list. Chunk servers should only drop chunks if they
are not referenced by the metadata server for a period of
time (say, 30 minutes).
Security
All nodes of the system share a secret key and use it to
authenticate each other and encrypt messages. This allows
the server to accept new chunk servers and clients with
no prior setup
Reliability
The metadata server is a single point of failure. To reduce
the impact of crashes, the metadata server stores all write
operations into a write-ahead log that is replayed any time
the process goes online.
Chunk Management:
Chunks are added to the system when:
1. A chunk server connects
2. A write operation on metadata occurs (adding chunks)
They are removed when:
1. A chunk server disconnects
2. A write operation on metadata occurs (overwriting old chunks)
3. A delete operation on metadata occurs
4. A chunk is corrupted or removed forcefully from the chunk server
The system must make sure that chunks are not over-replicated
or under-replicated. If they are over-replicated, some chunk
servers need to forget some copies. If they are under-replicated,
some chunk servers need to copy chunks from elsewhere.
Metadata server variables for a chunk server:
ms_old_list: List of chunks that are known to be held by CS
ms_add_list: List of chunks that should be held by CS
ms_rem_list: List of chunks that may be held by CS but should removed from it
Chunk server variables:
cs_add_list: List of chunks added since the last update
cs_rem_list: List of chunks marked for removal after a timeout
cs_lst_list: List of chunks that were lost due to errors or forceful removals of chunk files
Metadata change for write:
When clients commit a write by adding new hashes to the metadata,
MS adds those hashes to the ms_add_list for the CS where the client
uploaded the chunks. If a hash was overwritten and became useless,
that hash is added to the ms_rem_list for all CS holding it.
Metadata change for delete:
All hashes that are no longer reachable by the file tree are added
to the ms_rem_list of their holders
Chunk upload:
When a chunk is uploaded to a chunk server, its hash is added to
the cs_add_list.
Periodically on the chunk server:
Elements in the cs_rem_list have a 30 minute timeout after which they are deleted
permanently.
Periodically:
CS sends cs_add_list to MS
MS may add a subset of cs_add_list to ms_add_list based on the chunk replication and distribution policy
MS sends ms_add_list and ms_rem_list to CS
CS (1) Adds all elements from ms_rem_list to cs_rem_list
(2) Elements in ms_add_list that are not held by the chunk server are added to
a temporary list tmp_list
(3) Removes elements in ms_add_list from cs_add_list and cs_rem_list, then merges cs_add_list into cs_rem_list and clears cs_add_list.
(4) Elements in cs_lst_list are added to tmp_list, then cs_lst_list is cleared.
(6) tmp_list is sent to MS
(7) cs_add_list is cleared
MS (1) Receives tmp_list and sends download locations to CS for those chunks
(2) Merges ms_add_list into ms_old_list, then removes all items in tmp_list from ms_old_list
(3) Sets ms_add_list equal to tmp_list
Chunk replication and distribution policy:
During an update, when CS reports a new chunk to MS, MS has to decide whether
to allow the CS to keep it or not.
There are 4 cases:
- The chunk is useless (not referenced by any file)
- The chunk is under-replicated even counting the new copy
- The chunk is properly replicated with the new copy
- The chunk is over-replicated with the new copy
If the chunk is not referenced by the file tree, do nothing.
If the chunk is properly replicated or under-replicated, add it to the ms_add_list.
If the chunk is over-replicated, either don't add it to the ms_add_list or add it to the ms_rem_list of some other holder.
TODO: The way chunk servers tell about the chunks they are holding
recently changed. Now instead of sending a full chunk list when
connecting, they send batches of chunks to the metadata server
during state updates. It also changed that now chunk servers
initiate updates.
When a chunk server connects (and authenticates)
it sends alongside the auth message the hash of
the hash list of all chunks. The metadata server
then replies with a message saying whether it
already cached that chunk list or not. If it didn't, the chunk server sends the entire list
in chunks during state updates, with an increased
update frequency.
Metadata Persistence & Crash Recovery:
The metadata server uses a write-ahead log (WAL) file to store its state on disk.
Log files start with a full snapshot of the metadata and continues with operation
log entries.
When a file gets too big, the metadata server creates a new WAL file by writing
a snapshot to it and continuing logging there.
spec/protocol.txt
-281
View File
@@ -1,281 +0,0 @@
1. Introduction & Notation
The DESIGN.txt file gives an overview of the system
and how nodes of a cluster interact with each other.
This file documents the specific binary format used
to exchange information between nodes.
All messages start with a shared header, defined as:
struct Header {
uint16_t version;
uint16_t type;
uint32_t length;
};
1.1 Generation Counter Special Values
Generation counters (uint64_t) have two special values:
NO_GENERATION (0):
When used in expect_gen, means "skip generation check entirely".
Files/directories are never assigned this generation value.
MISSING_FILE_GENERATION (UINT64_MAX):
When used in expect_gen, means "expect file/directory to NOT exist".
- For DELETE: succeeds if file doesn't exist (no-op)
- For WRITE: fails with BADGEN if file exists, fails with NOENT if missing
- For existing operations: causes generation mismatch if file exists
Files/directories are never assigned this generation value.
2. Client Messages
2.1 Client to Metadata Server messages
Let's start from the interactions between a client and
the metadata server:
[ CREATE | C -> MS ]
Upon file creation, the client sends a "CREATE" message with the following layout:
struct CreateMessage {
Header header; // type=CREATE
uint16_t path_len;
char path[path_len];
uint8_t is_dir;
if (is_dir != 0) {
uint32_t chunk_size;
}
};
Note that in general only paths up to 65K bytes are
supported and that the layout of fields may depend
on the value of others.
The server then responds with a CREATE_SUCCESS or CREATE_ERROR message.
[ DELETE | C -> MS ]
When a client deletes a file, it sends a DELETE
message with the following layout:
struct DeleteMessage {
Header header; // type=DELETE
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
};
The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless.
The server then responds with a DELETE_SUCCESS or DELETE_ERROR message.
[ LIST | C -> MS ]
When a client requests a directory listing, it sends a LIST message whith the following format:
struct ListMessage {
Header header; // type=LIST
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
};
If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped.
The server then responds with a LIST_SUCCESS or LIST_ERROR message.
[ READ | C -> MS ]
When a client requests to read a file, it sends a READ message with the following format:
struct ReadMessage {
Header header; // type=READ
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
uint32_t offset;
uint32_t length;
};
The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped.
The offset and length fields determine the region to be read from the file.
[ WRITE | C -> MS ]
When a client wants to write to a file, it sends a WRITE message with the following layout:
struct Location {
uint8_t is_ipv4;
if (is_ipv4) {
uint32_t ipv4;
} else {
uint128_t ipv6;
}
uint16_t port;
};
struct WriteChunk {
SHA256 hash;
uint32_t num_locations;
Location locations[num_locations];
};
struct WriteMessage {
Header header; // type=WRITE
uint64_t expect_gen;
uint32_t flags;
uint16_t path_len;
char path[path_len];
uint32_t offset;
uint32_t length;
uint32_t num_chunks;
WriteChunk chunks[num_chunks];
};
If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata.
The flags field contains write operation flags. Currently defined flags:
- TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist,
the metadata server will atomically create it with a default chunk
size of 4096 bytes and retry the write operation. The creation is
logged to the WAL for crash consistency.
- TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write
operation, setting the file size to exactly offset+length. Any data
beyond this point is discarded. Useful for HTTP PUT semantics where
the entire file content should be replaced.
The offset and length mark the region that is being written to.
Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to
num_chunks == length / chunk_size
Where chunk_size is the one for the target file at the specified generation.
Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it.
2.2 Client to Chunk Server messages
TODO
3. Metadata Server messages
The following is the list of messages the Metadata Server may send to a Client:
[ CREATE_ERROR | MS -> C ]
When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout:
struct CreateErrorMessage {
Header header; // type=CREATE_ERROR
uint16_t message_len;
char message[message_len];
};
[ CREATE_SUCCESS | MS -> C ]
When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout:
struct CreateSuccessMessage {
Header header; // type=CREATE_SUCCESS
uint64_t gen;
};
The gen field is the generation counter given to the file.
[ DELETE_ERROR | MS -> C ]
See CREATE_ERROR
[ DELETE_SUCCESS | MS -> C ]
When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout:
struct DeleteSuccessMessage {
Header header; // type=DELETE_SUCCESS
};
It does not store any fields other than the header.
[ LIST_ERROR | MS -> C ]
When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout:
struct ListErrorMessage {
Header header; // type=LIST_ERROR
uint16_t message_len;
char message[message_len];
};
[ LIST_SUCCESS | MS -> C ]
When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout:
struct ListItem {
uint64_t gen;
uint8_t is_dir;
uint16_t name_len;
char name[name_len];
};
struct ListSuccessMessage {
Header header; // type=LIST_SUCCESS
uint64_t gen;
uint8_t truncated;
uint32_t item_count;
ListItem items[item_count];
};
The ListSuccessMessage gen field contains the generation counter for the directory, while the gen field in ListItem is the counter for that specific child.
If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message.
[ READ_ERROR | MS -> C ]
TODO
[ READ_SUCCESS | MS -> C ]
struct IPv4AndPort {
uint32_t ipv4;
uint16_t port;
};
struct IPv6AndPort {
uint128_t ipv6;
uint16_t port;
};
struct ChunkServerAddrList {
uint32_t num_ipv4;
IPv4AndPort ipv4s[num_ipv4];
uint32_t num_ipv6;
IPv6AndPort ipv6s[num_ipv6];
};
struct ReadChunk {
SHA256 hash;
uint32_t num_holders;
ChunkServerAddrList holders[num_holders];
};
struct ReadSuccessMessage {
Header header; // type=READ_SUCCESS
uint64_t gen;
uint32_t chunk_size;
uint32_t file_length;
uint32_t num_hashes;
ReadChunk chunks[num_hashes];
uint32_t num_write_locations;
ChunkServerAddrList write_locations[num_write_locations];
};
The message returns general information about the file such as its generation counter, length in bytes, and chunk size.
The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them.
The write locations are a list of chunk servers that clients may write new chunks to.
[ WRITE_ERROR | MS -> C ]
See CREATE_ERROR
[ WRITE_SUCCESS | MS -> C ]
When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout:
struct WriteSuccessMessage {
Header header; // type=WRITE_SUCCESS
uint64_t gen;
};
The gen field is the new generation counter for that file.
spec/spec.txt
-482
View File
@@ -1,482 +0,0 @@
note that:
MS = metadata server
CS = chunk server
CLI = client
=========================================================
=== METADATA SERVER PROCESS =============================
=========================================================
metadata_server_init:
Start a TCP server at the interface and port specified using options --addr and --port (default 127.0.0.1 and 8080)
Initialize the file tree as empty
Initialize the WAL and, if it exists already, replay its contents into the file tree
metadata_server_free:
Free resources associated to the MS
metadata_server_tick:
for each network event:
if connection established:
mark the connection as UNKNOWN
if disconnect:
if the connection was marked as CS:
Free the information associated to it
if message received:
if the connection is marked as UNKNOWN:
if the message is from a CS:
mark the connection as CS and associated to it a struct for the information associated to it.
else
mark the connection as CLI
if the peer is a client:
if the message has type CREATE:
Read path_len, path, and is_dir from the message
If is_dir is false:
Read chunk_size from message
Else:
Set chunk_size to 0
Append the create operation to the WAL
Attempt to create entity in the file_tree
If file_tree creation fails:
Send CREATE_ERROR with the error description
Else:
Send CREATE_SUCCESS containing the new generation number
if the message has type DELETE:
Read expect_gen, path_len, and path from message
Append delete operation to the WAL
Attempt to delete entity in the file_tree
If file_tree deletion fails:
Send DELETE_ERROR with the error description
Else:
Send DELETE_SUCCESS (no payload)
if the message has type LIST:
Read expect_gen, path_len, and path from message
Attempt to list directory contents from file_tree
If listing fails:
Send LIST_ERROR with error description
Else:
Send LIST_SUCCESS containing:
- Directory generation
- Truncated flag (if results exceed MAX_LIST_SIZE)
- Item count
- List of items (generation, is_dir, name_len, name)
if the message has type READ:
Read expect_gen, path_len, path, offset, and length
Attempt to read file info from file_tree
If read fails:
Send READ_ERROR containing:
- Error description
- A list of suggested chunk server addresses for writing (load balanced)
NOTE: Providing write locations on read error allows clients to handle
"missing file" scenarios by immediately writing if desired.
Else:
Send READ_SUCCESS containing:
- File generation
- Chunk size
- Actual bytes available (may be less than requested length)
- Number of chunks
- List of chunks, where each contains:
- SHA256 Hash
- List of chunk servers holding this chunk (holders)
- A list of suggested chunk server addresses for writing new chunks
(load balanced via choose_servers_for_write)
if the message has type WRITE:
Read expect_gen, flags, path_len, path
Read offset, length, num_chunks
For each chunk in num_chunks:
Read the hash and the list of Chunk Server addresses (locations) that hold it
(This implies the client has already uploaded the data to these servers)
Append write operation to WAL
Attempt to apply write to file_tree (returns new_gen and a list of removed_hashes)
If result is NOENT (File Not Found) AND flags has TOASTY_WRITE_CREATE_IF_MISSING:
- Append create operation (default chunk_size 4096) to WAL
- Create entity in file_tree
- Retry the write operation to file_tree
If write fails:
Send WRITE_ERROR with error description
Else:
For each new chunk hash provided in the message:
Find the chunk servers associated with that hash in the message
Add the hash to those servers' ms_add_list
For each removed_hash returned by the file_tree:
Find all chunk servers currently holding this hash
Add the hash to their ms_rem_list (marking it for deletion)
Send WRITE_SUCCESS containing the new generation number
if the peer is a CS:
if the message has type AUTH:
Add all IPv4/port and IPv6/port pairs to the data associated to this connection.
Then, respond with a AUTH_RESPONSE message.
if the message has type SYNC:
For each hash received:
If it's not in use by the file tree:
ignore it
If less than REPLICATION_FACTOR CS are holding it:
Add this hash to this CS's add_list
Respond with a SYNC_2 message containing the add_list and the rem_list.
if the message has type SYNC_3:
Create an empty tmp_list
For each hash received:
We expect the hash not to be in the add_list or the old_list. Only hashes that were actually expected to be in on the server should be recovered.
Add the hash to tmp_list.
For each CS holding this hash in their old_list or add_list:
Write its address to the output message
Merge add_list into old_list
Remove elements in tmp_list from old_list
Set old_list equal to tmp_list
Send the message as SYNC_4
For each chunk server:
If its last response was more than RESPONSE_TIME_LIMIT seconds ago:
Drop the connection
Set the current timeout to the next most imminent event
=========================================================
=== CHUNK SERVER PROCESS ================================
=========================================================
chunk_server_init:
Start a TCP server at the interface and port specified using options --addr and --port (default 127.0.0.1 and 8081)
Create a directory for chunk files at path specified using option --path (default "chunk_server_data")
Connect to MS using the remote address and port specified using options --remote-addr and --remote-port (default 127.0.0.1, 8080) and send an AUTH message containing the list of local IPv4/port and IPv6/port pairs.
chunk_server_free:
free all resources
start_download:
if not downloading already:
get a descriptor from the download_targets array and send DOWNLOAD_CHUNK message with the specified hash.
chunk_server_tick:
for each network event:
if connection established:
do nothing
if disconnect:
if the connection was to MS:
disconnect_time = current_time
if the connection was to a CS:
downloading = false
if message received:
if from MS:
if tagged as AUTH_RESPONSE:
Read all chunk file names from the data directory and convert them into hashes by decoding them as hexadecimal. Add all of these hashes to the add_list.
No message is sent back to MS in response.
if tagged as SYNC_2:
For each hash in the first list received by MS:
If no chunk file is associated to it:
add it to a tmp_list
else:
remove it from the rem_list and from the add_list.
For each hash still in the add_list:
Add it to the rem_list
Mark the add_list as empty
For each hash in the second list received by MS:
Add it to rem_list
Send SYNC_3 message with a list of hashes made by those in tmp_list and lst_list.
if tagged as SYNC_4:
At this point MS knows which chunks this CS is missing, so it sent back information for where to download them.
For each element in the list:
Add a download descrptor to download_targets list.
start_download()
(other cases are not allowed)
if from CS:
We may get messages from other CS when trying to download a chunk from it that was lost locally.
if tagged DOWNLOAD_CHUNK_ERROR:
Start the next download
if tagged DOWNLOAD_CHUNK_SUCCESS:
Create a chunk file with the received data, then add its hash to the add_list.
start_download()
if from CLI:
if tagged CREATE_CHUNK:
Take the received data and make a chunk file for it, then add its hash to the add_list.
Respond with a CREATE_CHUNK_SUCCESS message.
if tagged UPLOAD_CHUNK:
Load the chunk referred by the client and patch it with the data it sent at the specified offset, then calculate its hash and store it as a new chunk.
Add its hash to the add_list.
Respond with a UPLOAD_CHUNK_SUCCESS message.
if tagged DOWNLOAD_CHUNK:
Load the chunk referred by the client and respond with its data in a DOWNLOAD_CHUNK_SUCCESS message.
For each expired item in rem_list:
Delete its associated chunk file
If the connection to MS is intact and more than SYNC_INTERVAL seconds elapsed since the last sync:
Send a SYNC message containing all add_list items.
start_download()
If there is no connection to MS and more than RECONNECT_DELAY seconds passed:
Connect to the metadata server and send a message of type AUTH containing the list of our IPv4s/port pairs and IPv6/port pairs.
Set the current timeout to the earliest time, considering the next reconnect attempt timeto MS if disconnected and the most imminent rem_list deadline.
=========================================================
=== CLIENT LIBRARY ======================================
=========================================================
# ------------------------------------------------------------------------------
# 1. DATA STRUCTURES
# ------------------------------------------------------------------------------
class ToastyClient:
metadata_server_conn: Connection
chunk_server_pool: Map<Address, Connection>
pending_operations: Map<OpID, Operation>
class Operation:
id: Integer
type: Enum(READ, WRITE, CREATE, DELETE, LIST)
status: Enum(PENDING, COMPLETED, FAILED)
# Context
path: String
user_buffer: Byte[]
offset: Integer
length: Integer
flags: Bitmask
expected_generation: Integer
# Internal State
chunks_to_process: List<ChunkTask>
bytes_transferred: Integer
result_data: Any
class ChunkTask:
# Used for tracking individual chunk uploads/downloads
chunk_index: Integer
hash: String
target_servers: List<Address>
status: Enum(WAITING, IN_PROGRESS, DONE)
final_hash: String # Returned by CS after upload/patch
# ------------------------------------------------------------------------------
# 2. PUBLIC BLOCKING API (User Facing)
# ------------------------------------------------------------------------------
function toasty_connect(address, port):
client = new ToastyClient()
client.metadata_server_conn = tcp_connect(address, port)
return client
function toasty_read(client, path, offset, length):
op_handle = begin_read_op(client, path, offset, length)
return wait_for_result(client, op_handle)
function toasty_write(client, path, data, offset, flags):
op_handle = begin_write_op(client, path, data, offset, flags)
return wait_for_result(client, op_handle)
# (Create, Delete, and List follow the same pattern: begin_op -> wait_for_result)
# ------------------------------------------------------------------------------
# 3. INTERNAL ASYNCHRONOUS LOGIC
# ------------------------------------------------------------------------------
# --- READ LOGIC ---
function begin_read_op(client, path, offset, length):
op = new Operation(type=READ, path, offset, length)
# Step 1: Ask Metadata Server (MS) for the file layout
send_message(client.metadata_server_conn,
type=READ_REQUEST,
path=path, range=(offset, length))
return register_operation(client, op)
function handle_read_metadata_response(op, msg):
if msg.type == ERROR:
op.status = FAILED
return
# Step 2: Calculate chunk plan
chunk_size = msg.chunk_size
op.chunks_to_process = calculate_required_chunks(op.offset, op.length, chunk_size)
# Step 3: Queue downloads from Chunk Servers (CS)
for chunk in op.chunks_to_process:
# MS returns a list of holders for each chunk hash
best_cs = load_balance_select(chunk.holders)
conn = get_cs_connection(best_cs)
send_message(conn,
type=DOWNLOAD_CHUNK,
hash=chunk.hash)
function handle_chunk_download_success(op, msg):
# Step 4: Assemble data
copy_bytes(src=msg.data, dest=op.user_buffer, at_offset=msg.chunk_index)
mark_task_complete(op, msg.chunk_index)
if all_tasks_complete(op):
op.status = COMPLETED
# --- WRITE LOGIC (The Complex Part) ---
function begin_write_op(client, path, data, offset, flags):
op = new Operation(type=WRITE, path, data, offset, flags)
# Step 1: Send READ to MS first.
# We need the current generation number and existing chunk hashes to perform
# patches or ensure consistency, even if we are overwriting.
send_message(client.metadata_server_conn,
type=READ_REQUEST,
path=path, range=(offset, len(data)))
return register_operation(client, op)
function handle_write_metadata_ready(op, msg):
# Step 2: Prepare Upload Tasks
# If file missing & CREATE_IF_MISSING flag is set, we use defaults (size 0, gen 0).
current_generation = msg.generation
chunk_size = msg.chunk_size
write_targets = msg.write_locations # Recommended CS IPs from MS
# Slice user data into chunks
sliced_chunks = split_into_chunks(op.user_buffer, chunk_size)
for slice in sliced_chunks:
task = new ChunkTask()
task.target_servers = write_targets # We must replicate to 3 servers
# Determine if this is a Create (New) or Update (Patch)
if slice.index >= len(msg.existing_hashes):
task.type = NEW_CHUNK
else:
task.type = PATCH_CHUNK
task.base_hash = msg.existing_hashes[slice.index]
op.chunks_to_process.add(task)
# Step 3: Start Uploads (Parallel up to limit)
pump_upload_queue(op)
function pump_upload_queue(op):
while active_uploads(op) < MAX_PARALLEL_UPLOADS:
task = get_next_waiting_task(op)
if not task: break
# We must upload to N replicas (usually 3)
for replica_addr in task.target_servers:
conn = get_cs_connection(replica_addr)
if task.type == NEW_CHUNK:
# Full upload
send_message(conn, type=CREATE_CHUNK, data=task.data)
else:
# Differential patch (Rsync-style or offset-based)
send_message(conn, type=UPLOAD_CHUNK,
base_hash=task.base_hash,
patch_data=task.data)
function handle_upload_ack(op, msg):
# Step 4: Collect new hashes
task = get_task(op, msg.task_id)
# The CS computes the new SHA256 after applying the patch/data
task.final_hash = msg.new_hash
task.replicas_confirmed++
if task.replicas_confirmed >= REQUIRED_REPLICATION:
task.status = DONE
if all_tasks_complete(op):
commit_write(op)
else:
pump_upload_queue(op) # Start next batch
function commit_write(op):
# Step 5: Atomic Commit at Metadata Server
commit_msg = new Message(type=WRITE_REQUEST)
commit_msg.path = op.path
commit_msg.expected_generation = op.expected_generation # Optimistic Locking
commit_msg.flags = op.flags
# Map the linear chunk index to the new Hash + Location list
commit_msg.chunks = map_chunks_to_hashes_and_locations(op.chunks_to_process)
send_message(client.metadata_server_conn, commit_msg)
function handle_commit_response(op, msg):
if msg.type == WRITE_SUCCESS:
op.status = COMPLETED
op.result_data = msg.new_generation
elif msg.type == ERROR and msg.code == GENERATION_MISMATCH:
# Optimistic concurrency failed (someone else wrote).
# Retry the whole operation from Step 1 (READ).
reset_and_retry(op)
else:
op.status = FAILED
# ------------------------------------------------------------------------------
# 4. EVENT LOOP (Driver)
# ------------------------------------------------------------------------------
function wait_for_result(client, op_handle):
while get_op_status(op_handle) == PENDING:
# Wait for network activity (Poll/Select)
events = wait_for_network_events(timeout=DEFAULT_TIMEOUT)
for event in events:
if event.type == DISCONNECT:
handle_disconnect(client, event.conn)
continue
# Route message to the specific Operation ID embedded in the packet tag
op = client.pending_operations[event.tag.op_id]
msg = read_message(event.conn)
# Dispatcher
switch (op.type, msg.type):
case (READ, READ_SUCCESS): handle_read_metadata_response(op, msg)
case (READ, CHUNK_DATA): handle_chunk_download_success(op, msg)
case (WRITE, READ_SUCCESS): handle_write_metadata_ready(op, msg)
case (WRITE, UPLOAD_ACK): handle_upload_ack(op, msg)
case (WRITE, WRITE_SUCCESS): handle_commit_response(op, msg)
case (_, ERROR): handle_error(op, msg)
return get_op_result(op_handle)
}
src/blob_client.c
+675
View File
@@ -0,0 +1,675 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdint.h>
#include <assert.h>
#include "blob_client.h"
#include "server.h"
#define TIME_FMT "%7.3fs"
#define TIME_VAL(t) ((double)(t) / 1000000000.0)
static uint64_t next_blob_client_id = 100;
static uint64_t blob_random(void)
{
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
return quakey_random();
#else
return (uint64_t)rand();
#endif
}
static void blob_log_impl(BlobClientState *state, Time now, const char *event, const char *detail)
{
printf("[" TIME_FMT "] BLOB %lu | %-20s %s\n",
TIME_VAL(now),
state->client_id,
event,
detail ? detail : "");
}
#define blob_log(state, now, event, fmt, ...) do { \
char _detail[256]; \
snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \
blob_log_impl(state, now, event, _detail); \
} while (0)
#define blob_log_simple(state, now, event) \
blob_log_impl(state, now, event, NULL)
static int leader_idx(BlobClientState *state)
{
return state->view_number % state->num_servers;
}
///////////////////////////////////////////////////////////////
// Chunk ack counting
///////////////////////////////////////////////////////////////
static int count_acks(uint32_t mask)
{
int n = 0;
for (int i = 0; i < 32; i++)
if (mask & (1u << i)) n++;
return n;
}
static bool all_chunks_acked(BlobClientState *state)
{
for (int i = 0; i < state->num_chunks; i++) {
if (count_acks(state->chunks[i].ack_mask) < state->f_plus_one)
return false;
}
return true;
}
static bool all_chunks_fetched(BlobClientState *state)
{
for (int i = 0; i < state->num_chunks; i++) {
if (!state->chunks[i].fetched)
return false;
}
return true;
}
///////////////////////////////////////////////////////////////
// Generate a test blob
///////////////////////////////////////////////////////////////
static void generate_test_blob(BlobClientState *state)
{
// Generate random bucket/key
snprintf(state->bucket, META_BUCKET_MAX, "blob-b%d", (int)(blob_random() % 4));
snprintf(state->key, META_KEY_MAX, "blob-k%d", (int)(blob_random() % 64));
// 1-3 chunks per blob
state->num_chunks = 1 + blob_random() % 3;
state->blob_size = 0;
for (int i = 0; i < state->num_chunks; i++) {
// Generate random hash for each chunk
for (int j = 0; j < 32; j++)
state->chunks[i].hash.data[j] = blob_random() & 0xFF;
state->chunks[i].size = BLOB_TEST_CHUNK_SIZE;
state->chunks[i].ack_mask = 0;
state->chunks[i].fetched = false;
state->blob_size += BLOB_TEST_CHUNK_SIZE;
}
// Content hash (random for now)
for (int j = 0; j < 32; j++)
state->content_hash.data[j] = blob_random() & 0xFF;
}
///////////////////////////////////////////////////////////////
// Send operations
///////////////////////////////////////////////////////////////
// Send StoreChunk for a given chunk to a specific server.
// The chunk data is the hash bytes (32 bytes).
static void send_store_chunk(BlobClientState *state, int chunk_idx, int server_idx)
{
int conn_idx = tcp_index_from_tag(&state->tcp, server_idx);
if (conn_idx < 0) {
tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL);
return; // Will retry next tick
}
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output == NULL) return;
StoreChunkMessage msg = {
.base = {
.version = MESSAGE_VERSION,
.type = MESSAGE_TYPE_STORE_CHUNK,
.length = sizeof(StoreChunkMessage) + state->chunks[chunk_idx].size,
},
.hash = state->chunks[chunk_idx].hash,
.size = state->chunks[chunk_idx].size,
};
byte_queue_write(output, &msg, sizeof(msg));
// Chunk data = hash bytes (BLOB_TEST_CHUNK_SIZE = 32 = sizeof(SHA256))
byte_queue_write(output, state->chunks[chunk_idx].hash.data, state->chunks[chunk_idx].size);
}
static void send_commit_put(BlobClientState *state)
{
int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state));
if (conn_idx < 0) {
tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL);
return;
}
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output == NULL) return;
CommitPutMessage msg = {
.base = {
.version = MESSAGE_VERSION,
.type = MESSAGE_TYPE_COMMIT_PUT,
.length = sizeof(CommitPutMessage),
},
.oper = {
.type = META_OPER_PUT,
.size = state->blob_size,
.content_hash = state->content_hash,
.num_chunks = state->num_chunks,
},
.client_id = state->client_id,
.request_id = state->request_id,
};
memcpy(msg.oper.bucket, state->bucket, META_BUCKET_MAX);
memcpy(msg.oper.key, state->key, META_KEY_MAX);
for (int i = 0; i < state->num_chunks; i++) {
msg.oper.chunks[i].hash = state->chunks[i].hash;
msg.oper.chunks[i].size = state->chunks[i].size;
}
byte_queue_write(output, &msg, sizeof(msg));
}
static void send_get_blob(BlobClientState *state, int server_idx)
{
int conn_idx = tcp_index_from_tag(&state->tcp, server_idx);
if (conn_idx < 0) {
tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL);
return;
}
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output == NULL) return;
GetBlobMessage msg = {
.base = {
.version = MESSAGE_VERSION,
.type = MESSAGE_TYPE_GET_BLOB,
.length = sizeof(GetBlobMessage),
},
};
memcpy(msg.bucket, state->bucket, META_BUCKET_MAX);
memcpy(msg.key, state->key, META_KEY_MAX);
byte_queue_write(output, &msg, sizeof(msg));
}
static void send_fetch_chunk(BlobClientState *state, int chunk_idx, int server_idx)
{
int conn_idx = tcp_index_from_tag(&state->tcp, server_idx);
if (conn_idx < 0) {
tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL);
return;
}
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
if (output == NULL) return;
FetchChunkMessage msg = {
.base = {
.version = MESSAGE_VERSION,
.type = MESSAGE_TYPE_FETCH_CHUNK,
.length = sizeof(FetchChunkMessage),
},
.hash = state->chunks[chunk_idx].hash,
.sender_idx = -1, // Client (not a peer server)
};
byte_queue_write(output, &msg, sizeof(msg));
}
///////////////////////////////////////////////////////////////
// Message processing
///////////////////////////////////////////////////////////////
static int
process_message(BlobClientState *state,
int conn_idx, uint8_t type, ByteView msg)
{
(void) conn_idx;
Time now = get_current_time();
switch (type) {
case MESSAGE_TYPE_REDIRECT: {
RedirectMessage redirect;
if (msg.len != sizeof(RedirectMessage))
return -1;
memcpy(&redirect, msg.ptr, sizeof(redirect));
if (redirect.view_number > state->view_number) {
blob_log(state, now, "RECV REDIRECT", "view=%lu -> %lu",
(unsigned long)state->view_number,
(unsigned long)redirect.view_number);
state->view_number = redirect.view_number;
// Re-send CommitPut to new leader
if (state->phase == BLOB_COMMITTING) {
send_commit_put(state);
state->phase_time = now;
}
}
return 0;
}
case MESSAGE_TYPE_STORE_CHUNK_ACK: {
if (state->phase != BLOB_UPLOADING)
return 0;
StoreChunkAckMessage ack;
if (msg.len != sizeof(StoreChunkAckMessage))
return -1;
memcpy(&ack, msg.ptr, sizeof(ack));
// Find which chunk this ack is for
for (int i = 0; i < state->num_chunks; i++) {
if (memcmp(&state->chunks[i].hash, &ack.hash, sizeof(SHA256)) == 0) {
int tag = tcp_get_tag(&state->tcp, conn_idx);
if (tag >= 0 && tag < 32 && ack.success)
state->chunks[i].ack_mask |= (1u << tag);
blob_log(state, now, "RECV STORE_ACK", "chunk=%d server=%d ok=%d acks=%d/%d",
i, tag, ack.success,
count_acks(state->chunks[i].ack_mask), state->f_plus_one);
break;
}
}
// Check if all chunks have f+1 acks
if (all_chunks_acked(state)) {
blob_log(state, now, "UPLOAD DONE", "%s/%s chunks=%d",
state->bucket, state->key, state->num_chunks);
// Move to commit phase
state->phase = BLOB_COMMITTING;
state->phase_time = now;
state->request_id++;
send_commit_put(state);
blob_log(state, now, "SEND COMMIT_PUT", "%s/%s req=%lu",
state->bucket, state->key, state->request_id);
}
return 0;
}
case MESSAGE_TYPE_REPLY: {
if (state->phase != BLOB_COMMITTING)
return 0;
ReplyMessage reply;
if (msg.len != sizeof(ReplyMessage))
return -1;
memcpy(&reply, msg.ptr, sizeof(reply));
if (reply.request_id != state->request_id)
return 0;
if (reply.rejected) {
blob_log(state, now, "RECV REPLY", "REJECTED, retrying");
state->phase_time = now;
send_commit_put(state);
return 0;
}
state->puts_completed++;
blob_log(state, now, "PUT DONE", "%s/%s puts=%d",
state->bucket, state->key, state->puts_completed);
// Next: GET this blob back to verify
state->do_get_next = true;
state->phase = BLOB_IDLE;
return 0;
}
case MESSAGE_TYPE_GET_BLOB_RESPONSE: {
if (state->phase != BLOB_FETCHING_META)
return 0;
GetBlobResponseMessage resp;
if (msg.len != sizeof(GetBlobResponseMessage))
return -1;
memcpy(&resp, msg.ptr, sizeof(resp));
if (!resp.found) {
// Blob not yet committed on this server, retry another
blob_log(state, now, "RECV GET_BLOB", "NOT_FOUND, retrying");
state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers;
state->phase_time = now;
send_get_blob(state, state->fetch_server_idx);
return 0;
}
blob_log(state, now, "RECV GET_BLOB", "%s/%s chunks=%u",
state->bucket, state->key, resp.num_chunks);
// Verify metadata matches what we uploaded
if (resp.num_chunks != (uint32_t)state->num_chunks) {
fprintf(stderr, "BLOB CLIENT: metadata mismatch! expected %d chunks, got %u\n",
state->num_chunks, resp.num_chunks);
return -1;
}
for (int i = 0; i < state->num_chunks; i++) {
if (memcmp(&resp.chunks[i].hash, &state->chunks[i].hash, sizeof(SHA256)) != 0) {
fprintf(stderr, "BLOB CLIENT: chunk %d hash mismatch!\n", i);
return -1;
}
}
// Start fetching chunk data
state->phase = BLOB_FETCHING_DATA;
state->phase_time = now;
state->fetch_chunk_idx = 0;
state->fetch_server_idx = 0;
for (int i = 0; i < state->num_chunks; i++)
state->chunks[i].fetched = false;
// Send FetchChunk for the first chunk
send_fetch_chunk(state, 0, state->fetch_server_idx);
blob_log(state, now, "SEND FETCH_CHUNK", "chunk=0 server=%d", state->fetch_server_idx);
return 0;
}
case MESSAGE_TYPE_FETCH_CHUNK_RESPONSE: {
if (state->phase != BLOB_FETCHING_DATA)
return 0;
FetchChunkResponseMessage resp;
if (msg.len < sizeof(FetchChunkResponseMessage))
return -1;
memcpy(&resp, msg.ptr, sizeof(resp));
// Find which chunk this is for
int chunk_idx = -1;
for (int i = 0; i < state->num_chunks; i++) {
if (memcmp(&state->chunks[i].hash, &resp.hash, sizeof(SHA256)) == 0) {
chunk_idx = i;
break;
}
}
if (chunk_idx < 0)
return 0; // Unknown chunk, ignore
if (resp.size == 0) {
// Server doesn't have the chunk. Try next server.
blob_log(state, now, "RECV FETCH_RESP", "chunk=%d NOT_FOUND, trying next server", chunk_idx);
state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers;
send_fetch_chunk(state, chunk_idx, state->fetch_server_idx);
return 0;
}
// Verify data size
uint32_t data_size = msg.len - sizeof(FetchChunkResponseMessage);
if (data_size != resp.size || resp.size != state->chunks[chunk_idx].size) {
fprintf(stderr, "BLOB CLIENT: chunk %d size mismatch! expected %u, got %u\n",
chunk_idx, state->chunks[chunk_idx].size, resp.size);
return -1;
}
// Verify data content (data should equal hash bytes)
uint8_t *data = (uint8_t *)(msg.ptr + sizeof(FetchChunkResponseMessage));
if (memcmp(data, state->chunks[chunk_idx].hash.data, BLOB_TEST_CHUNK_SIZE) != 0) {
fprintf(stderr, "BLOB CLIENT: chunk %d data verification FAILED!\n", chunk_idx);
return -1;
}
state->chunks[chunk_idx].fetched = true;
blob_log(state, now, "RECV FETCH_RESP", "chunk=%d VERIFIED", chunk_idx);
// Check if all chunks fetched
if (all_chunks_fetched(state)) {
state->gets_completed++;
state->gets_verified++;
blob_log(state, now, "GET DONE", "%s/%s gets=%d verified=%d",
state->bucket, state->key, state->gets_completed, state->gets_verified);
state->do_get_next = false;
state->phase = BLOB_IDLE;
return 0;
}
// Fetch next unfetched chunk
for (int i = 0; i < state->num_chunks; i++) {
if (!state->chunks[i].fetched) {
state->fetch_chunk_idx = i;
state->fetch_server_idx = 0;
send_fetch_chunk(state, i, 0);
blob_log(state, now, "SEND FETCH_CHUNK", "chunk=%d server=0", i);
break;
}
}
return 0;
}
default:
break;
}
return 0;
}
///////////////////////////////////////////////////////////////
// Init / Tick / Free
///////////////////////////////////////////////////////////////
int blob_client_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
BlobClientState *state = state_;
state->num_servers = 0;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--server")) {
i++;
if (i == argc) {
fprintf(stderr, "Option --server missing value\n");
return -1;
}
if (state->num_servers == NODE_LIMIT) {
fprintf(stderr, "Node limit reached\n");
return -1;
}
if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) {
fprintf(stderr, "Malformed address\n");
return -1;
}
} else {
// Ignore unknown options
}
}
addr_sort(state->server_addrs, state->num_servers);
if (tcp_context_init(&state->tcp) < 0) {
fprintf(stderr, "Blob client :: Couldn't setup TCP context\n");
return -1;
}
// f = (num_servers - 1) / 2, so f+1 = (num_servers + 1) / 2
state->f_plus_one = (state->num_servers + 1) / 2;
state->view_number = 0;
state->request_id = 0;
state->client_id = next_blob_client_id++;
state->phase = BLOB_IDLE;
state->phase_time = 0;
state->do_get_next = false;
state->puts_completed = 0;
state->gets_completed = 0;
state->gets_verified = 0;
state->reconnect_time = 0;
state->upload_server_cursor = 0;
// Connect to all servers
for (int i = 0; i < state->num_servers; i++) {
if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) {
fprintf(stderr, "Blob client :: Couldn't connect to server %d\n", i);
tcp_context_free(&state->tcp);
return -1;
}
}
{
Time now = get_current_time();
blob_log(state, now, "INIT", "servers=%d f+1=%d", state->num_servers, state->f_plus_one);
}
*timeout = 0;
if (pcap < TCP_POLL_CAPACITY) {
fprintf(stderr, "Blob client :: Not enough poll capacity\n");
return -1;
}
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int blob_client_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{
BlobClientState *state = state_;
Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
for (int i = 0; i < num_events; i++) {
if (events[i].type == EVENT_DISCONNECT) {
int conn_idx = events[i].conn_idx;
tcp_close(&state->tcp, conn_idx);
continue;
}
if (events[i].type != EVENT_MESSAGE)
continue;
int conn_idx = events[i].conn_idx;
for (;;) {
ByteView msg;
uint16_t msg_type;
int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type);
if (ret == 0)
break;
if (ret < 0) {
tcp_close(&state->tcp, conn_idx);
break;
}
ret = process_message(state, conn_idx, msg_type, msg);
if (ret < 0) {
tcp_close(&state->tcp, conn_idx);
break;
}
tcp_consume_message(&state->tcp, conn_idx);
}
}
Time now = get_current_time();
// Timeout handling: if we've been in any phase too long, retry
if (state->phase != BLOB_IDLE) {
Time phase_deadline = state->phase_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL;
if (phase_deadline <= now) {
blob_log(state, now, "TIMEOUT", "phase=%d, retrying", state->phase);
switch (state->phase) {
case BLOB_UPLOADING:
// Re-send StoreChunk for chunks that need more acks
for (int c = 0; c < state->num_chunks; c++) {
if (count_acks(state->chunks[c].ack_mask) < state->f_plus_one) {
for (int s = 0; s < state->f_plus_one; s++) {
if (!(state->chunks[c].ack_mask & (1u << s)))
send_store_chunk(state, c, s);
}
}
}
state->phase_time = now;
break;
case BLOB_COMMITTING:
state->view_number++;
send_commit_put(state);
state->phase_time = now;
break;
case BLOB_FETCHING_META:
state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers;
send_get_blob(state, state->fetch_server_idx);
state->phase_time = now;
break;
case BLOB_FETCHING_DATA:
state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers;
send_fetch_chunk(state, state->fetch_chunk_idx, state->fetch_server_idx);
state->phase_time = now;
break;
default:
break;
}
}
}
// Ensure connections to all servers (needed during all phases for
// retransmission after network partitions or server restarts)
for (int i = 0; i < state->num_servers; i++) {
int ci = tcp_index_from_tag(&state->tcp, i);
if (ci < 0)
tcp_connect(&state->tcp, state->server_addrs[i], i, NULL);
}
// Start new operation when idle
if (state->phase == BLOB_IDLE) {
if (state->do_get_next) {
// GET the blob we just PUT
state->phase = BLOB_FETCHING_META;
state->phase_time = now;
state->fetch_server_idx = 0;
send_get_blob(state, state->fetch_server_idx);
blob_log(state, now, "SEND GET_BLOB", "%s/%s server=%d",
state->bucket, state->key, state->fetch_server_idx);
} else {
// PUT a new blob
generate_test_blob(state);
state->phase = BLOB_UPLOADING;
state->phase_time = now;
blob_log(state, now, "START PUT", "%s/%s chunks=%d",
state->bucket, state->key, state->num_chunks);
// Send StoreChunk for each chunk to the first f+1 servers
for (int c = 0; c < state->num_chunks; c++) {
for (int s = 0; s < state->f_plus_one; s++) {
send_store_chunk(state, c, s);
}
}
}
}
// Set timeout
Time deadline = INVALID_TIME;
if (state->phase != BLOB_IDLE) {
nearest_deadline(&deadline, state->phase_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL);
}
*timeout = deadline_to_timeout(deadline, now);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int blob_client_free(void *state_)
{
BlobClientState *state = state_;
{
Time now = get_current_time();
blob_log(state, now, "SHUTDOWN", "puts=%d gets=%d verified=%d",
state->puts_completed, state->gets_completed, state->gets_verified);
}
tcp_context_free(&state->tcp);
return 0;
}
src/blob_client.h
+86
View File
@@ -0,0 +1,86 @@
#ifndef BLOB_CLIENT_INCLUDED
#define BLOB_CLIENT_INCLUDED
#include <lib/tcp.h>
#include <lib/basic.h>
#include "config.h"
#include "metadata.h"
// Maximum number of chunks per blob in the test client.
// Kept small for simulation; real blobs use META_CHUNKS_MAX.
#define BLOB_MAX_CHUNKS 8
// Chunk data size for the test client (32 bytes = SHA256 hash length).
// Each chunk's data is its hash bytes, making verification trivial.
#define BLOB_TEST_CHUNK_SIZE 32
typedef enum {
BLOB_IDLE, // Ready to start a new operation
BLOB_UPLOADING, // Sending StoreChunk, waiting for acks
BLOB_COMMITTING, // Sent CommitPut, waiting for REPLY
BLOB_FETCHING_META, // Sent GetBlob, waiting for response
BLOB_FETCHING_DATA, // Sending FetchChunk, waiting for responses
} BlobPhase;
typedef struct {
SHA256 hash;
uint32_t size;
uint32_t ack_mask; // Bitmask: which servers acked this chunk
bool fetched; // GET: whether chunk data was received and verified
} BlobChunkState;
typedef struct {
TCP tcp;
Address server_addrs[NODE_LIMIT];
int num_servers;
int f_plus_one; // Number of servers to upload each chunk to
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
BlobPhase phase;
Time phase_time; // When we entered this phase
// Current blob metadata
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
uint64_t blob_size;
SHA256 content_hash;
int num_chunks;
BlobChunkState chunks[BLOB_MAX_CHUNKS];
// Upload tracking
int upload_server_cursor; // Next server index to try for StoreChunk
// Download tracking
int fetch_chunk_idx; // Which chunk we're currently fetching
int fetch_server_idx; // Which server to try next
// Alternation between PUT and GET
bool do_get_next; // If true, next op is GET; else PUT
// Statistics
int puts_completed;
int gets_completed;
int gets_verified;
Time reconnect_time;
} BlobClientState;
struct pollfd;
int blob_client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int blob_client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int blob_client_free(void *state);
#endif // BLOB_CLIENT_INCLUDED
src/chunk_server.c
-1069
View File
File diff suppressed because it is too large Load Diff
src/chunk_server.h
-70
View File
@@ -1,70 +0,0 @@
#ifndef CHUNK_SERVER_INCLUDED
#define CHUNK_SERVER_INCLUDED
#include "tcp.h"
#include "basic.h"
#include "hash_set.h"
#define TAG_METADATA_SERVER 1
#define TAG_CHUNK_SERVER 2
typedef struct {
Address addr;
SHA256 hash;
} DownloadTarget;
typedef struct {
DownloadTarget *items;
int count;
int capacity;
} DownloadTargets;
typedef struct {
char path[PATH_MAX];
bool trace;
Address local_addr;
Address remote_addr;
Time disconnect_time;
Time last_sync_time;
int reconnect_delay; // In seconds
// --- Subsystems ---
TCP tcp;
// --- Download Management ---
bool downloading;
SHA256 current_download_target_hash;
DownloadTargets download_targets;
// --- Chunk Management ---
// List of chunks added since the last update
HashSet cs_add_list;
// List of chunks marked for removal after a timeout
TimedHashSet cs_rem_list;
// List of chunks that were lost due to errors or forceful removals of chunk files
HashSet cs_lst_list;
} ChunkServer;
struct pollfd;
int chunk_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_free(void *state);
#endif // CHUNK_SERVER_INCLUDED
src/chunk_store.c
+107
View File
@@ -0,0 +1,107 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdio.h>
#include "chunk_store.h"
#include <lib/file_system.h>
// Build the full path for a chunk: "base_path/HEX_HASH"
// SHA256 hex = 64 chars. Returns string pointing into buf.
static string chunk_path(ChunkStore *cs, SHA256 hash, char *buf, int buf_size)
{
int base_len = strlen(cs->base_path);
if (base_len + 1 + 64 + 1 > buf_size)
return (string){ buf, 0 };
memcpy(buf, cs->base_path, base_len);
buf[base_len] = '/';
append_hex_as_str(buf + base_len + 1, hash);
buf[base_len + 1 + 64] = '\0';
return (string){ buf, base_len + 1 + 64 };
}
int chunk_store_init(ChunkStore *cs, const char *base_path)
{
int len = strlen(base_path);
if (len >= (int) sizeof(cs->base_path))
return -1;
memcpy(cs->base_path, base_path, len + 1);
string path = { cs->base_path, len };
create_dir(path); // Ignore error if already exists
return 0;
}
void chunk_store_free(ChunkStore *cs)
{
(void) cs;
}
int chunk_store_write(ChunkStore *cs, SHA256 hash, char *data, uint32_t size)
{
char buf[512];
string path = chunk_path(cs, hash, buf, sizeof(buf));
if (path.len == 0)
return -1;
Handle fd;
if (file_open(path, &fd) < 0)
return -1;
if (file_truncate(fd, 0) < 0) {
file_close(fd);
return -1;
}
int ret = file_write_exact(fd, data, size);
file_close(fd);
return ret;
}
int chunk_store_read(ChunkStore *cs, SHA256 hash, char *dst, uint32_t size)
{
char buf[512];
string path = chunk_path(cs, hash, buf, sizeof(buf));
if (path.len == 0)
return -1;
Handle fd;
if (file_open(path, &fd) < 0)
return -1;
int ret = file_read_exact(fd, dst, size);
file_close(fd);
return ret;
}
bool chunk_store_exists(ChunkStore *cs, SHA256 hash)
{
char buf[512];
string path = chunk_path(cs, hash, buf, sizeof(buf));
if (path.len == 0)
return false;
// Use file_open instead of file_exists (access) because
// mock_access is not implemented in the Quakey simulation.
Handle fd;
if (file_open(path, &fd) < 0)
return false;
file_close(fd);
return true;
}
int chunk_store_delete(ChunkStore *cs, SHA256 hash)
{
char buf[512];
string path = chunk_path(cs, hash, buf, sizeof(buf));
if (path.len == 0)
return -1;
return remove_file_or_dir(path);
}
src/chunk_store.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef CHUNK_STORE_INCLUDED
#define CHUNK_STORE_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#include <lib/basic.h>
typedef struct {
char base_path[256];
} ChunkStore;
int chunk_store_init(ChunkStore *cs, const char *base_path);
void chunk_store_free(ChunkStore *cs);
int chunk_store_write(ChunkStore *cs, SHA256 hash, char *data, uint32_t size);
int chunk_store_read(ChunkStore *cs, SHA256 hash, char *dst, uint32_t size);
bool chunk_store_exists(ChunkStore *cs, SHA256 hash);
int chunk_store_delete(ChunkStore *cs, SHA256 hash);
#endif // CHUNK_STORE_INCLUDED
src/client.c
+252 -2434
View File
File diff suppressed because it is too large Load Diff
src/client.h
+49
View File
@@ -0,0 +1,49 @@
#ifndef CLIENT_INCLUDED
#define CLIENT_INCLUDED
#include <lib/tcp.h>
#include <lib/basic.h>
#include "config.h"
#include "metadata.h"
typedef struct {
TCP tcp;
// True if we are waiting for a response
bool pending;
Time request_time; // When the current request was sent
// The operation sent in the current pending request (for logging)
MetaOper last_oper;
// Checker support
MetaResult last_result;
bool last_was_timeout;
bool last_was_rejected;
Address server_addrs[NODE_LIMIT];
int num_servers;
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
Time reconnect_time; // Earliest time to retry connecting to leader
} ClientState;
struct pollfd;
int client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int client_free(void *state);
#endif // CLIENT_INCLUDED
src/client_table.c
+70
View File
@@ -0,0 +1,70 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdbool.h>
#include "client_table.h"
void client_table_init(ClientTable *client_table)
{
client_table->count = 0;
client_table->capacity = 0;
client_table->entries = NULL;
}
void client_table_free(ClientTable *client_table)
{
free(client_table->entries);
}
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id)
{
for (int i = 0; i < client_table->count; i++) {
if (client_table->entries[i].client_id == client_id)
return &client_table->entries[i];
}
return NULL;
}
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag)
{
if (client_table->count == client_table->capacity) {
int n = 2 * client_table->capacity;
if (n == 0) n = 8;
void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
if (p == NULL)
return -1;
client_table->capacity = n;
client_table->entries = p;
}
client_table->entries[client_table->count++] = (ClientTableEntry) {
.client_id = client_id,
.last_request_id = request_id,
.pending = true,
.conn_tag = conn_tag,
};
return 0;
}
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id)
{
if (client_table->count == client_table->capacity) {
int n = 2 * client_table->capacity;
if (n == 0) n = 8;
void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry));
if (p == NULL)
return -1;
client_table->capacity = n;
client_table->entries = p;
}
client_table->entries[client_table->count++] = (ClientTableEntry) {
.client_id=client_id,
.last_request_id=request_id,
.pending=true,
};
return 0;
}
src/client_table.h
+31
View File
@@ -0,0 +1,31 @@
#ifndef CLIENT_TABLE_INCLUDED
#define CLIENT_TABLE_INCLUDED
#include <stdint.h>
#include "metadata.h"
typedef struct {
uint64_t client_id;
uint64_t last_request_id;
MetaResult last_result;
bool pending; // Only meaningful on the leader that received
// the REQUEST. After a view change, a new leader
// may find stale entries with pending=false from
// a previous view when it was leader before.
int conn_tag;
} ClientTableEntry;
typedef struct {
int count;
int capacity;
ClientTableEntry *entries;
} ClientTable;
void client_table_init(ClientTable *client_table);
void client_table_free(ClientTable *client_table);
ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id);
int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag);
int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id);
#endif // CLIENT_TABLE_INCLUDED
src/config.h
+8 -8
View File
@@ -1,13 +1,13 @@
#ifndef CONFIG_INCLUDED
#define CONFIG_INCLUDED
#define MAX_SERVER_ADDRS 4
#define MAX_CHUNK_SERVERS 128
#define MAX_OPERATIONS 128
#define MAX_REQUESTS_PER_QUEUE 128
#define REPLICATION_FACTOR 3
#define SYNC_INTERVAL 10
#define RESPONSE_TIME_LIMIT 90
#define DELETION_TIMEOUT 30
#define NODE_LIMIT 32
#define FUTURE_LIMIT 32
#define HEARTBEAT_INTERVAL_SEC 1
#define PRIMARY_DEATH_TIMEOUT_SEC 5
#define RECOVERY_TIMEOUT_SEC 3
#define RECOVERY_ATTEMPT_LIMIT 10
#define STATE_TRANSFER_TIMEOUT_SEC 2
#define VIEW_CHANGE_TIMEOUT_SEC 20
#endif // CONFIG_INCLUDED
src/file_tree.c
-820
View File
@@ -1,820 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <assert.h>
#include <quakey.h>
#include "basic.h"
#include "file_tree.h"
static int parse_path(string path, string *comps, int max)
{
bool is_absolute = false;
if (path.len > 0 && path.ptr[0] == '/') {
is_absolute = true;
path.ptr++;
path.len--;
if (path.len == 0)
return 0; // Absolute paths with no components are allowed
}
int num = 0;
uint32_t i = 0;
for (;;) {
uint32_t off = i;
while (i < (uint32_t) path.len && path.ptr[i] != '/')
i++;
uint32_t len = i - off;
if (len == 0)
return -1; // Empty component
string comp = { path.ptr + off, len };
if (comp.len == 2 && comp.ptr[0] == '.' && comp.ptr[1] == '.') {
if (num == 0) {
// For absolute paths, ".." at root is ignored (stays at root)
// For relative paths, ".." with no components references parent, which is invalid
if (!is_absolute)
return -1;
// Otherwise, ignore the ".." (absolute path, already at root)
} else {
num--;
}
} else if (comp.len != 1 || comp.ptr[0] != '.') {
if (num == max)
return -1; // To many components
comps[num++] = comp;
}
if (i == (uint32_t) path.len)
break;
assert(path.ptr[i] == '/');
i++;
if (i == (uint32_t) path.len)
break;
}
return num;
}
static int dir_find(Dir *parent, string name)
{
for (uint64_t i = 0; i < parent->num_children; i++)
if (streq((string) { parent->children[i].name, parent->children[i].name_len }, name))
return i;
return -1;
}
static Entity *resolve_path(Entity *root, string *comps, int num_comps)
{
assert(root->is_dir);
Entity *current = root;
for (int i = 0; i < num_comps; i++) {
if (!current->is_dir)
return NULL;
int j = dir_find(&current->d, comps[i]);
if (j == -1)
return NULL;
current = &current->d.children[j];
}
return current;
}
static void entity_free(Entity *e);
static bool entity_uses_hash(Entity *e, SHA256 hash);
static void dir_init(Dir *d)
{
d->num_children = 0;
d->max_children = 0;
d->children = NULL;
}
static void dir_free(Dir *d)
{
for (uint64_t i = 0; i < d->num_children; i++)
entity_free(&d->children[i]);
free(d->children);
}
static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
{
assert(entity_gen != NO_GENERATION);
assert(entity_gen != MISSING_FILE_GENERATION);
// NO_GENERATION means "skip generation check"
if (expected_gen == NO_GENERATION)
return true;
// MISSING_FILE_GENERATION means "expect file to NOT exist"
// Since we're checking against an existing entity, this is a mismatch
if (expected_gen == MISSING_FILE_GENERATION)
return false;
return expected_gen == entity_gen;
}
static uint64_t create_generation(uint64_t *next_gen)
{
(*next_gen)++;
if (*next_gen == 0 || *next_gen == UINT64_MAX)
*next_gen = 1;
return *next_gen;
}
static int dir_remove(Dir *d, int idx, uint64_t expected_gen)
{
if (!gen_match(expected_gen, d->children[idx].gen))
return -1;
// TODO: pretty sure this leaks memory
d->children[idx] = d->children[--d->num_children];
return 0;
}
static bool dir_uses_hash(Dir *d, SHA256 hash)
{
for (uint64_t i = 0; i < d->num_children; i++)
if (entity_uses_hash(&d->children[i], hash))
return true;
return false;
}
static void file_init(File *f, uint64_t chunk_size)
{
f->chunk_size = chunk_size;
f->num_chunks = 0;
f->file_size = 0;
f->chunks = NULL;
}
static void file_free(File *f)
{
free(f->chunks);
f->chunks = NULL;
}
static bool file_uses_hash(File *f, SHA256 hash)
{
for (uint64_t i = 0; i < f->num_chunks; i++)
if (!memcmp(&f->chunks[i], &hash, sizeof(SHA256)))
return true;
return false;
}
// Fails when the name is too long
static int entity_init(Entity *e, char *name, int name_len,
bool is_dir, uint64_t chunk_size, uint64_t *next_gen)
{
if (name_len >= (int) sizeof(e->name))
return -1;
e->gen = create_generation(next_gen);
assert(e->gen != NO_GENERATION);
memcpy(e->name, name, name_len);
e->name[name_len] = '\0';
e->name_len = (uint16_t) name_len;
e->is_dir = is_dir;
if (is_dir)
dir_init(&e->d);
else
file_init(&e->f, chunk_size);
return 0;
}
static void entity_free(Entity *e)
{
if (e->is_dir)
dir_free(&e->d);
else
file_free(&e->f);
}
static bool entity_uses_hash(Entity *e, SHA256 hash)
{
if (e->is_dir)
return dir_uses_hash(&e->d, hash);
else
return file_uses_hash(&e->f, hash);
}
int file_tree_init(FileTree *ft)
{
ft->next_gen = 1;
int ret = entity_init(&ft->root, "", 0, true, 0, &ft->next_gen);
if (ret < 0) return -1;
return 0;
}
void file_tree_free(FileTree *ft)
{
entity_free(&ft->root);
}
bool file_tree_uses_hash(FileTree *ft, SHA256 hash)
{
return entity_uses_hash(&ft->root, hash);
}
int file_tree_list(FileTree *ft, string path,
ListItem *items, int max_items, uint64_t *gen)
{
int num_comps;
string comps[MAX_COMPS];
num_comps = parse_path(path, comps, MAX_COMPS);
if (num_comps < 0)
return FILETREE_BADPATH;
Entity *e = resolve_path(&ft->root, comps, num_comps);
if (e == NULL)
return FILETREE_NOENT;
if (!e->is_dir)
return FILETREE_NOTDIR;
Dir *d = &e->d;
int num_items = d->num_children;
if (num_items > max_items) num_items = max_items;
for (int i = 0; i < num_items; i++) {
Entity *c = &d->children[i];
int name_cpy = c->name_len;
if (name_cpy > (int) sizeof(items[i].name)-1)
name_cpy = (int) sizeof(items[i].name)-1;
memcpy(items[i].name, c->name, name_cpy);
items[i].name[name_cpy] = '\0';
items[i].name_len = name_cpy;
items[i].is_dir = c->is_dir;
items[i].gen = c->gen;
}
assert(e->gen != NO_GENERATION);
*gen = e->gen;
return d->num_children;
}
int file_tree_create_entity(FileTree *ft, string path,
bool is_dir, uint64_t chunk_size, uint64_t *gen)
{
int num_comps;
string comps[MAX_COMPS];
num_comps = parse_path(path, comps, MAX_COMPS);
if (num_comps < 0)
// Couldn't parse path
return FILETREE_BADPATH;
if (num_comps == 0)
// Path is empty, which means the caller is referencing the root,
// which exists already.
return FILETREE_EXISTS;
// Resolve the path up to the second last component
Entity *e = resolve_path(&ft->root, comps, num_comps-1);
if (e == NULL)
// Parent directory doesn't exist
return FILETREE_NOENT;
if (!e->is_dir)
// Parent entity is not a directory
return FILETREE_NOTDIR;
string name = comps[num_comps-1];
if (dir_find(&e->d, name) != -1)
return FILETREE_EXISTS;
Dir *d = &e->d;
if (d->num_children == d->max_children) {
int new_max = 2 * d->max_children;
if (new_max == 0)
new_max = 8;
Entity *p = malloc(sizeof(Entity) * new_max);
if (p == NULL)
return FILETREE_NOMEM;
for (uint64_t i = 0; i < d->num_children; i++)
p[i] = d->children[i];
free(d->children);
d->children = p;
d->max_children = new_max;
}
Entity *c = &d->children[d->num_children];
int ret = entity_init(c, (char*) name.ptr, name.len, is_dir, chunk_size, &ft->next_gen);
if (ret < 0)
// Invalid name for the new file
return FILETREE_BADPATH;
assert(e->gen != NO_GENERATION);
*gen = e->gen;
d->num_children++;
return 0;
}
// TODO: this should return the list of unreferenced hashes
int file_tree_delete_entity(FileTree *ft, string path,
uint64_t expected_gen)
{
int num_comps;
string comps[MAX_COMPS];
num_comps = parse_path(path, comps, MAX_COMPS);
if (num_comps < 0)
return FILETREE_BADPATH;
if (num_comps == 0)
return FILETREE_BADOP;
Entity *e = resolve_path(&ft->root, comps, num_comps-1);
if (e == NULL)
return FILETREE_NOENT;
if (!e->is_dir)
return FILETREE_NOTDIR;
int i = dir_find(&e->d, comps[num_comps-1]);
if (i == -1) {
// File doesn't exist
// If caller expected it not to exist (MISSING_FILE_GENERATION), succeed
if (expected_gen == MISSING_FILE_GENERATION)
return 0;
return FILETREE_NOENT;
}
// File exists - check generation
if (dir_remove(&e->d, i, expected_gen) < 0)
return FILETREE_BADGEN;
return 0;
}
int file_tree_write(
FileTree* ft,
string path,
uint64_t off,
uint64_t len,
uint32_t num_chunks,
uint64_t expect_gen,
uint64_t* new_gen,
SHA256* hashes,
SHA256* removed_hashes,
int* num_removed,
bool truncate_after)
{
// WRITE operations cannot use expect_gen=0
if (expect_gen == NO_GENERATION)
return FILETREE_BADGEN;
int num_comps;
string comps[MAX_COMPS];
num_comps = parse_path(path, comps, MAX_COMPS);
if (num_comps < 0)
return FILETREE_BADPATH;
Entity *e = resolve_path(&ft->root, comps, num_comps);
if (e == NULL) {
// File doesn't exist
// If caller expected it not to exist (MISSING_FILE_GENERATION), that's correct
// but we still can't write to a non-existent file (need CREATE_IF_MISSING flag in client layer)
if (expect_gen == MISSING_FILE_GENERATION)
return FILETREE_NOENT; // Expected behavior: file missing as expected, but can't write
return FILETREE_NOENT;
}
if (e->is_dir)
return FILETREE_ISDIR;
// Check generation - will fail if expect_gen is MISSING_FILE_GENERATION (expects missing but file exists)
if (!gen_match(expect_gen, e->gen))
return FILETREE_BADGEN;
File *f = &e->f;
uint64_t first_chunk_index = off / f->chunk_size;
uint64_t last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size;
assert(last_chunk_index - first_chunk_index + 1 == num_chunks);
if (last_chunk_index >= f->num_chunks) {
uint64_t old_num_chunks = f->num_chunks;
SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256));
if (new_chunks == NULL)
return FILETREE_NOMEM;
if (f->chunks) {
if (f->num_chunks > 0)
memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256));
free(f->chunks);
}
f->chunks = new_chunks;
f->num_chunks = last_chunk_index+1;
for (uint64_t i = old_num_chunks; i < last_chunk_index+1; i++)
memset(&f->chunks[i], 0, sizeof(SHA256));
}
int num_overwritten_hashes = 0;
SHA256 overwritten_hashes[100]; // TODO: fix this limit
if (num_chunks > 100) {
assert(0); // TODO
}
// Update chunks
for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++) {
overwritten_hashes[num_overwritten_hashes++] = f->chunks[i];
f->chunks[i] = hashes[i - first_chunk_index];
}
// Update file size (last byte written + 1)
uint64_t new_size = off + len;
if (truncate_after) {
// With truncation, set file size to exactly new_size and remove chunks beyond
uint64_t new_num_chunks = last_chunk_index + 1;
// Add any chunks beyond the write to the overwritten list (they'll be removed)
for (uint64_t i = new_num_chunks; i < f->num_chunks; i++) {
if (num_overwritten_hashes < 100) { // Respect the limit
overwritten_hashes[num_overwritten_hashes++] = f->chunks[i];
}
}
f->num_chunks = new_num_chunks;
f->file_size = new_size;
} else {
// Without truncation, only grow the file
if (new_size > f->file_size)
f->file_size = new_size;
}
// Now check which old hashes are no longer used
// anywhere in the tree
//
// NOTE: If removed_hashes is NULL, the caller isn't
// interested in which hashes are no longer reachable.
if (removed_hashes != NULL) {
*num_removed = 0;
for (int i = 0; i < num_overwritten_hashes; i++) {
SHA256 hash = overwritten_hashes[i];
// Skip zero hashes
bool is_zero = true;
for (int j = 0; j < (int) sizeof(SHA256); j++) {
if (hash.data[j] != 0) {
is_zero = false;
break;
}
}
if (is_zero)
continue;
// Check if this hash is still used anywhere in the tree
if (!entity_uses_hash(&ft->root, hash)) {
removed_hashes[*num_removed] = hash;
(*num_removed)++;
}
}
}
e->gen = create_generation(&ft->next_gen);
assert(e->gen != NO_GENERATION);
*new_gen = e->gen;
return 0;
}
int file_tree_read(FileTree *ft, string path,
uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size,
SHA256 *hashes, int max_hashes, uint64_t *actual_bytes)
{
// Initialize gen to NO_GENERATION so error paths have a well-defined value
*gen = NO_GENERATION;
int num_comps;
string comps[MAX_COMPS];
num_comps = parse_path(path, comps, MAX_COMPS);
if (num_comps < 0)
return FILETREE_BADPATH;
Entity *e = resolve_path(&ft->root, comps, num_comps);
if (e == NULL)
return FILETREE_NOENT;
if (e->is_dir)
return FILETREE_ISDIR;
File *f = &e->f;
*chunk_size = f->chunk_size;
// Calculate actual bytes that can be read based on actual file size
if (off >= f->file_size) {
*actual_bytes = 0;
} else if (off + len > f->file_size) {
*actual_bytes = f->file_size - off;
} else {
*actual_bytes = len;
}
if (len == 0) {
assert(e->gen != NO_GENERATION);
*gen = e->gen;
return 0;
}
uint64_t first_chunk_index = off / f->chunk_size;
uint64_t last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size;
if (first_chunk_index >= f->num_chunks) {
*gen = e->gen;
return 0;
}
if (last_chunk_index >= f->num_chunks) {
if (f->num_chunks == 0) {
assert(e->gen != NO_GENERATION);
*gen = e->gen;
return 0;
}
last_chunk_index = f->num_chunks-1;
}
int num_hashes = 0;
for (uint32_t i = first_chunk_index; i <= last_chunk_index; i++) {
SHA256 hash = f->chunks[i];
if (num_hashes < max_hashes)
hashes[num_hashes] = hash;
num_hashes++;
}
assert(e->gen != NO_GENERATION);
*gen = e->gen;
return num_hashes;
}
string file_tree_strerror(int code)
{
switch (code) {
case FILETREE_NOMEM : return S("Out of memory");
case FILETREE_NOENT : return S("No such file or directory");
case FILETREE_NOTDIR : return S("Entity is not a directory");
case FILETREE_ISDIR : return S("Entity is a directory");
case FILETREE_EXISTS : return S("File or directory already exists");
case FILETREE_BADPATH: return S("Invalid path");
case FILETREE_BADOP : return S("Invalid operation");
case FILETREE_BADGEN : return S("Generation counter mismatch or invalid value");
default:break;
}
return S("Unknown error");
}
typedef struct {
int (*write_fn)(char*,int,void*);
void *write_data;
char *buffer;
int buffer_size;
int buffer_used;
bool error;
} SerializeContext;
static void sc_flush(SerializeContext *sc)
{
if (sc->error)
return;
int ret = sc->write_fn(sc->buffer, sc->buffer_used, sc->write_data);
if (ret < 0) {
sc->error = true;
return;
}
sc->buffer_used = 0;
}
static void sc_write_mem(SerializeContext *sc, char *src, int len)
{
if (sc->error)
return;
if (sc->buffer_size - sc->buffer_used < len) {
if (len > sc->buffer_size) {
sc->error = true;
return;
}
sc_flush(sc);
if (sc->error)
return;
}
memcpy(sc->buffer + sc->buffer_used, src, len);
sc->buffer_used += len;
}
static void sc_write_u8 (SerializeContext *sc, uint8_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
static void sc_write_u16 (SerializeContext *sc, uint16_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
static void sc_write_u64 (SerializeContext *sc, uint64_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
static void sc_write_hash(SerializeContext *sc, SHA256 value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
static void file_serialize(SerializeContext *sc, File *f)
{
sc_write_u64(sc, f->chunk_size);
sc_write_u64(sc, f->num_chunks);
sc_write_u64(sc, f->file_size);
for (uint64_t i = 0; i < f->num_chunks; i++)
sc_write_hash(sc, f->chunks[i]);
}
static void entity_serialize(SerializeContext *sc, Entity *e);
static void dir_serialize(SerializeContext *sc, Dir *d)
{
sc_write_u64(sc, d->num_children);
for (uint64_t i = 0; i < d->num_children; i++)
entity_serialize(sc, &d->children[i]);
}
static void entity_serialize(SerializeContext *sc, Entity *e)
{
sc_write_u16(sc, e->name_len);
sc_write_mem(sc, e->name, e->name_len);
sc_write_u8(sc, e->is_dir);
if (e->is_dir)
dir_serialize(sc, &e->d);
else
file_serialize(sc, &e->f);
}
int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *write_data)
{
SerializeContext sc;
sc.write_fn = write_fn;
sc.write_data = write_data;
sc.buffer_used = 0;
sc.buffer_size = 1<<10;
sc.buffer = malloc(sc.buffer_size);
sc.error = false;
if (sc.buffer == NULL)
sc.error = true;
entity_serialize(&sc, &ft->root);
sc_flush(&sc);
free(sc.buffer);
if (sc.error)
return -1;
return 0;
}
typedef struct {
int (*read_fn)(char*,int,void*);
void *read_data;
char *buffer;
int buffer_size;
int buffer_used;
int buffer_head;
bool error;
uint64_t total_read;
} DeserializeContext;
static void dc_read_mem(DeserializeContext *dc, void *dst, int len)
{
if (dc->error)
return;
if (dc->buffer_used < len) {
if (dc->buffer_size < len) {
dc->error = true;
return;
}
memmove(dc->buffer, dc->buffer + dc->buffer_head, dc->buffer_used);
dc->buffer_head = 0;
int ret = dc->read_fn(
dc->buffer + dc->buffer_used,
dc->buffer_size - dc->buffer_used,
dc->read_data);
if (ret < 0) {
dc->error = true;
return;
}
dc->buffer_used += ret;
if (dc->buffer_used < len) {
dc->error = true;
return;
}
}
memcpy(dst, dc->buffer + dc->buffer_head, len);
dc->buffer_head += len;
dc->buffer_used -= len;
dc->total_read += len;
}
static void dc_read_u8 (DeserializeContext *dc, uint8_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
static void dc_read_u16(DeserializeContext *dc, uint16_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
static void dc_read_u64(DeserializeContext *dc, uint64_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
static void dc_read_hash(DeserializeContext *dc, SHA256 *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
static void file_deserialize(DeserializeContext *dc, File *f)
{
dc_read_u64(dc, &f->chunk_size);
dc_read_u64(dc, &f->num_chunks);
dc_read_u64(dc, &f->file_size);
f->chunks = malloc(f->num_chunks * sizeof(SHA256));
if (f->chunks == NULL) {
assert(0); // TODO
}
for (uint64_t i = 0; i < f->num_chunks; i++)
dc_read_hash(dc, &f->chunks[i]);
}
static void entity_deserialize(DeserializeContext *dc, Entity *e);
static void dir_deserialize(DeserializeContext *dc, Dir *d)
{
dc_read_u64(dc, &d->num_children);
d->max_children = d->num_children;
d->children = malloc(d->num_children * sizeof(Entity));
if (d->children == NULL) {
assert(0); // TODO
}
// TODO: not checking for errors is not okay as
// the code will branch based on garbage
// values.
for (uint64_t i = 0; i < d->num_children; i++)
entity_deserialize(dc, &d->children[i]);
}
static void entity_deserialize(DeserializeContext *dc, Entity *e)
{
dc_read_u16(dc, &e->name_len); // TODO: make sure this doesn't go over the static buffer
dc_read_mem(dc, e->name, e->name_len);
uint8_t is_dir;
dc_read_u8 (dc, &is_dir);
e->is_dir = (is_dir != 0);
if (e->is_dir)
dir_deserialize(dc, &e->d);
else
file_deserialize(dc, &e->f);
}
int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data)
{
DeserializeContext dc;
dc.read_fn = read_fn;
dc.read_data = read_data;
dc.buffer_head = 0;
dc.buffer_used = 0;
dc.buffer_size = 1<<10;
dc.buffer = malloc(dc.buffer_size);
dc.error = false;
if (dc.buffer == NULL)
dc.error = true;
dc.total_read = 0;
entity_deserialize(&dc, &ft->root);
free(dc.buffer);
if (dc.error)
return -1;
if (dc.total_read > INT_MAX) {
assert(0); // TODO
}
return dc.total_read;
}
src/file_tree.h
-97
View File
@@ -1,97 +0,0 @@
#ifndef FILE_TREE_INCLUDED
#define FILE_TREE_INCLUDED
#include "basic.h"
// Special generation counter values:
// NO_GENERATION (0): Skip generation check (accept any generation)
// MISSING_FILE_GENERATION (UINT64_MAX): Expect file/directory to NOT exist
#define NO_GENERATION ((uint64_t) 0)
#define MISSING_FILE_GENERATION ((uint64_t) UINT64_MAX)
enum {
FILETREE_NOMEM = -1,
FILETREE_NOENT = -2,
FILETREE_NOTDIR = -3,
FILETREE_ISDIR = -4,
FILETREE_EXISTS = -5,
FILETREE_BADPATH = -6,
FILETREE_BADOP = -7,
FILETREE_BADGEN = -8,
};
typedef struct Entity Entity;
typedef struct {
uint64_t chunk_size; // TODO: this should be an u32
uint64_t num_chunks; // TODO: and this too
uint64_t file_size; // Offset of last byte written + 1
SHA256 *chunks;
} File;
typedef struct {
uint64_t max_children;
uint64_t num_children;
Entity *children;
} Dir;
struct Entity {
uint64_t gen;
char name[1<<8];
uint16_t name_len;
bool is_dir;
union {
Dir d;
File f;
};
};
typedef struct {
Entity root;
uint64_t next_gen;
} FileTree;
typedef struct {
char name[1<<8];
int name_len;
bool is_dir;
uint64_t gen;
} ListItem;
#define MAX_COMPS 32
int file_tree_init(FileTree *ft);
void file_tree_free(FileTree *ft);
bool file_tree_uses_hash(FileTree *ft, SHA256 hash);
int file_tree_list(FileTree *ft, string path,
ListItem *items, int max_items, uint64_t *gen);
int file_tree_create_entity(FileTree *ft, string path,
bool is_dir, uint64_t chunk_size, uint64_t *gen);
int file_tree_delete_entity(FileTree *ft, string path,
uint64_t expected_gen);
int file_tree_write(FileTree *ft, string path,
uint64_t off, uint64_t len, uint32_t num_chunks,
uint64_t expect_gen,
uint64_t *new_gen,
SHA256 *hashes,
SHA256 *removed_hashes,
int *num_removed,
bool truncate_after);
int file_tree_read(FileTree *ft, string path,
uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size,
SHA256 *hashes, int max_hashes, uint64_t *actual_bytes);
string file_tree_strerror(int code);
int file_tree_serialize(FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data);
int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data);
#endif // FILE_TREE_INCLUDED
src/hash_set.c
-171
View File
@@ -1,171 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "hash_set.h"
void hash_set_init(HashSet *set)
{
set->items = NULL;
set->count = 0;
set->capacity = 0;
}
void hash_set_free(HashSet *set)
{
free(set->items);
set->items = NULL;
}
void hash_set_clear(HashSet *set)
{
free(set->items);
set->items = NULL;
set->count = 0;
set->capacity = 0;
}
int hash_set_insert(HashSet *set, SHA256 hash)
{
// Avoid duplicates
for (int i = 0; i < set->count; i++)
if (!memcmp(&set->items[i], &hash, sizeof(SHA256)))
return 0; // Already present
if (set->count == set->capacity) {
int new_capacity;
if (set->items == NULL)
new_capacity = 16;
else
new_capacity = 2 * set->capacity;
SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256));
if (new_items == NULL)
return -1;
set->items = new_items;
set->capacity = new_capacity;
}
set->items[set->count++] = hash;
return 0;
}
bool hash_set_remove(HashSet *set, SHA256 hash)
{
for (int i = 0; i < set->count; i++)
if (!memcmp(&hash, &set->items[i], sizeof(SHA256))) {
set->items[i] = set->items[--set->count];
return true;
}
return false;
}
bool hash_set_contains(HashSet *set, SHA256 hash)
{
for (int i = 0; i < set->count; i++)
if (!memcmp(&hash, &set->items[i], sizeof(SHA256)))
return true;
return false;
}
int hash_set_merge(HashSet *dst, HashSet src)
{
HashSet ret;
hash_set_init(&ret);
for (int i = 0; i < dst->count; i++) {
if (hash_set_insert(&ret, dst->items[i]) < 0)
goto error;
}
for (int i = 0; i < src.count; i++) {
if (hash_set_insert(&ret, src.items[i]) < 0)
goto error;
}
hash_set_free(dst);
*dst = ret;
return 0;
error:
hash_set_free(&ret);
return -1;
}
void hash_set_remove_set(HashSet *dst, HashSet src)
{
for (int i = 0; i < src.count; i++)
hash_set_remove(dst, src.items[i]);
}
void timed_hash_set_init(TimedHashSet *set)
{
set->items = NULL;
set->count = 0;
set->capacity = 0;
}
void timed_hash_set_free(TimedHashSet *set)
{
free(set->items);
set->items = NULL;
}
int timed_hash_set_find(TimedHashSet *set, SHA256 hash)
{
for (int i = 0; i < set->count; i++)
if (!memcmp(&set->items[i].hash, &hash, sizeof(SHA256)))
return i;
return -1;
}
int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time)
{
// Check if already in set
int idx = timed_hash_set_find(set, hash);
if (idx >= 0) {
// Already marked, keep the original time
return 0;
}
if (set->count == set->capacity) {
int new_capacity;
if (set->capacity == 0)
new_capacity = 8;
else
new_capacity = 2 * set->capacity;
TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash));
if (new_items == NULL)
return -1;
if (set->capacity > 0) {
memcpy(new_items, set->items, set->count * sizeof(set->items[0]));
free(set->items);
}
set->items = new_items;
set->capacity = new_capacity;
}
set->items[set->count++] = (TimedHash) { hash, time };
return 0;
}
void timed_hash_set_remove(TimedHashSet *set, SHA256 hash)
{
int idx = timed_hash_set_find(set, hash);
if (idx >= 0) {
// Remove by shifting remaining items
if (idx < set->count - 1) {
memmove(&set->items[idx], &set->items[idx + 1],
(set->count - idx - 1) * sizeof(set->items[0]));
}
set->count--;
}
}
src/hash_set.h
-38
View File
@@ -1,38 +0,0 @@
#ifndef HASH_SET_INCLUDED
#define HASH_SET_INCLUDED
#include "basic.h"
typedef struct {
SHA256 *items;
int count;
int capacity;
} HashSet;
typedef struct {
SHA256 hash;
Time time;
} TimedHash;
typedef struct {
TimedHash *items;
int count;
int capacity;
} TimedHashSet;
void hash_set_init (HashSet *set);
void hash_set_free (HashSet *set);
void hash_set_clear (HashSet *set);
int hash_set_insert (HashSet *set, SHA256 hash);
bool hash_set_remove (HashSet *set, SHA256 hash);
int hash_set_merge (HashSet *dst, HashSet src);
void hash_set_remove_set(HashSet *dst, HashSet src);
bool hash_set_contains (HashSet *set, SHA256 hash);
void timed_hash_set_init (TimedHashSet *set);
void timed_hash_set_free (TimedHashSet *set);
int timed_hash_set_find (TimedHashSet *set, SHA256 hash);
int timed_hash_set_insert (TimedHashSet *set, SHA256 hash, Time time);
void timed_hash_set_remove (TimedHashSet *set, SHA256 hash);
#endif // HASH_SET_INCLUDED
src/invariant_checker.c
+445
View File
@@ -0,0 +1,445 @@
// VSR invariant checker with external shadow log tracking.
//
// This file runs in the main simulation loop outside Quakey-scheduled
// processes. It includes node.h for struct definitions, then restores
// real allocators since mock_malloc/realloc/free abort outside process
// context.
#include "server.h"
#include "chunk_store.h"
#include <assert.h>
// Restore real allocators (see checker/linearizability.c for precedent).
#undef malloc
#undef realloc
#undef free
#include <stdlib.h>
// These helpers are static in node.c; duplicated here for the checker.
static int self_idx(ServerState *state)
{
for (int i = 0; i < state->num_nodes; i++)
if (addr_eql(state->node_addrs[i], state->self_addr))
return i;
UNREACHABLE;
}
static int leader_idx(ServerState *state)
{
return state->view_number % state->num_nodes;
}
static bool is_leader(ServerState *state)
{
if (state->status == STATUS_RECOVERY)
return false;
return self_idx(state) == leader_idx(state);
}
static int shadow_log_append(InvariantChecker *ic, MetaOper oper)
{
if (ic->shadow_count == ic->shadow_capacity) {
int n = 2 * ic->shadow_capacity;
if (n < 8)
n = 8;
MetaOper *p = realloc(ic->shadow_log, n * sizeof(MetaOper));
if (p == NULL)
return -1;
ic->shadow_log = p;
ic->shadow_capacity = n;
}
ic->shadow_log[ic->shadow_count++] = oper;
return 0;
}
void invariant_checker_init(InvariantChecker *ic)
{
ic->last_min_commit = -1;
ic->last_max_commit = -1;
for (int i = 0; i < NODE_LIMIT; i++)
ic->prev_status[i] = STATUS_NORMAL;
ic->shadow_log = NULL;
ic->shadow_count = 0;
ic->shadow_capacity = 0;
}
void invariant_checker_free(InvariantChecker *ic)
{
fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n",
ic->shadow_count);
free(ic->shadow_log);
}
void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
unsigned long long *node_handles)
{
int min_commit = -1;
int max_commit = -1;
bool primary = false;
uint64_t primary_view_number = 0;
bool min_commit_just_recovered = false;
uint64_t max_view_number = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
max_view_number = MAX(max_view_number, nodes[i]->view_number);
}
for (int i = 0; i < num_nodes; i++) {
ServerState *n = nodes[i];
if (n == NULL || n->status == STATUS_RECOVERY)
continue;
if (min_commit < 0 || min_commit > n->commit_index) {
min_commit = n->commit_index;
min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY);
}
if (max_commit < 0 || max_commit < n->commit_index) {
max_commit = n->commit_index;
}
if (is_leader(n) && n->view_number > primary_view_number) {
primary = true;
primary_view_number = n->view_number;
}
}
// If the primary isn't up to date, it's not the
// real primary.
if (primary_view_number < max_view_number)
primary = false;
if (min_commit < 0) {
assert(ic->last_min_commit == -1);
} else {
// The minimum number of committed entries should
// only increase, but there are some corner-cases
// when this is not true.
// When a node completes the recovery state, its
// log is technically outdated as it was sent some
// point in the past. If operations are committed
// while the recovery response is in transit over
// the network, the minimum number of committed
// entries will decrease.
if (!min_commit_just_recovered) {
assert(ic->last_min_commit <= min_commit);
}
}
if (max_commit < 0) {
assert(ic->last_max_commit == -1);
} else {
// The maximum number of committed entries
// should only increase. The primary generally
// has more committed entries than replicas. If
// the primary dies, the maximum commit number
// of the live nodes may decrease. This is still
// okay since the new primary will commit up to
// that point as the view change completes. This
// implies that the maximum commit number should
// always increase unless there is no primary.
if (!primary) {
max_commit = ic->last_max_commit;
} else {
//assert(ic->last_max_commit <= max_commit);
}
}
ic->last_min_commit = min_commit;
ic->last_max_commit = max_commit;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i])
ic->prev_status[i] = nodes[i]->status;
}
for (int i = 0; i < num_nodes; i++) {
ServerState *s = nodes[i];
if (s == NULL)
continue;
// 1. commit_index <= log.count
// A node cannot have committed more entries than it has in its log.
if (s->commit_index > s->log.count) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n",
i, s->commit_index, s->log.count);
__builtin_trap();
}
// 2. commit_index >= 0
if (s->commit_index < 0) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n",
i, s->commit_index);
__builtin_trap();
}
// 4. Future buffer count is in valid range.
if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n",
i, s->num_future, FUTURE_LIMIT);
__builtin_trap();
}
// 7. last_normal_view <= view_number
// The most recent view in which this node was in NORMAL status
// cannot exceed its current view number.
if (s->last_normal_view > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: last_normal_view (%lu) > view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 8. When status is NORMAL, last_normal_view must equal view_number.
// Every transition to NORMAL status sets last_normal_view = view_number.
// If they diverge while in NORMAL, a transition forgot to update it.
if (s->status == STATUS_NORMAL && s->last_normal_view != s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: status is NORMAL but "
"last_normal_view (%lu) != view_number (%lu)\n",
i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number);
__builtin_trap();
}
// 9. Log entry view numbers must not exceed the node's current view.
// Entries are created in the view they were proposed. No entry
// should carry a view number from the future.
for (int k = 0; k < s->log.count; k++) {
if ((uint64_t)s->log.entries[k].view_number > s->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) "
"> view_number (%lu)\n",
i, k, s->log.entries[k].view_number,
(unsigned long)s->view_number);
__builtin_trap();
}
}
// 10. For a leader in NORMAL status, every uncommitted log entry
// must have the leader's own vote bit set. The leader always
// votes for its own entries.
if (s->status == STATUS_NORMAL && is_leader(s)) {
int idx = self_idx(s);
for (int k = s->commit_index; k < s->log.count; k++) {
if (!(s->log.entries[k].votes & (1 << idx))) {
fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): "
"uncommitted log[%d] missing leader's own vote bit\n",
i, k);
__builtin_trap();
}
}
}
}
// Cross-node invariants
// 5. At most one leader in normal status per view.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i]))
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j]))
continue;
if (nodes[i]->view_number == nodes[j]->view_number) {
fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n",
(unsigned long)nodes[i]->view_number, i, j);
__builtin_trap();
}
}
}
// 6. Committed prefix agreement (State Machine Safety).
// For any two nodes, their logs must agree on all entries up to
// min(commit_index_i, commit_index_j). This is the core safety
// property of VSR: all committed operations are identical across
// replicas.
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
for (int j = i + 1; j < num_nodes; j++) {
if (nodes[j] == NULL)
continue;
int mc = nodes[i]->commit_index;
if (nodes[j]->commit_index < mc)
mc = nodes[j]->commit_index;
for (int k = 0; k < mc; k++) {
if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(MetaOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d "
"between node %d and node %d\n", k, i, j);
__builtin_trap();
}
}
}
}
////////////////////////////////////////////////////////////////////
// Shadow log: external commit tracking
////////////////////////////////////////////////////////////////////
// Phase 1: Find the observed max commit index and a source node.
int observed_max_commit = 0;
int source_node_idx = -1;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->status == STATUS_RECOVERY)
continue;
if (nodes[i]->commit_index > observed_max_commit) {
observed_max_commit = nodes[i]->commit_index;
source_node_idx = i;
}
}
// Phase 2: Append newly committed entries to the shadow log.
if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) {
ServerState *source = nodes[source_node_idx];
assert(source->log.count >= observed_max_commit);
for (int k = ic->shadow_count; k < observed_max_commit; k++) {
MetaOper *source_oper = &source->log.entries[k].oper;
// Cross-validate against other live non-recovering nodes
// that have also committed this entry.
for (int j = 0; j < num_nodes; j++) {
if (j == source_node_idx)
continue;
if (nodes[j] == NULL)
continue;
if (nodes[j]->status == STATUS_RECOVERY)
continue;
if (nodes[j]->commit_index <= k)
continue;
if (nodes[j]->log.count <= k)
continue;
if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(MetaOper)) != 0) {
fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d "
"between source node %d and node %d during shadow log append\n",
k, source_node_idx, j);
__builtin_trap();
}
}
if (shadow_log_append(ic, *source_oper) < 0) {
fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n");
__builtin_trap();
}
}
}
// Phase 3: Verify shadow log against the cluster.
// Sub-check A: Committed entries must match the shadow log.
for (int k = 0; k < ic->shadow_count; k++) {
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL)
continue;
if (nodes[i]->log.count <= k)
continue;
if (nodes[i]->commit_index <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) != 0) {
char shadow_buf[128], node_buf[128];
meta_snprint_oper(shadow_buf, sizeof(shadow_buf), &ic->shadow_log[k]);
meta_snprint_oper(node_buf, sizeof(node_buf), &nodes[i]->log.entries[k].oper);
fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n"
" shadow: %s\n"
" node: %s\n",
k, i, shadow_buf, node_buf);
__builtin_trap();
}
}
}
// Sub-check B: When commit regresses, previously committed entries
// must still be held by a majority of the cluster.
// Recovering nodes are treated like dead nodes: they haven't
// restored their log yet and may still recover the entry through
// the recovery protocol.
if (observed_max_commit < ic->shadow_count) {
for (int k = observed_max_commit; k < ic->shadow_count; k++) {
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
if (nodes[i]->status == STATUS_RECOVERY) {
num_dead++;
continue;
}
if (nodes[i]->log.count <= k)
continue;
if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) == 0)
holders++;
}
if (holders + num_dead <= num_nodes / 2) {
char oper_buf[128];
meta_snprint_oper(oper_buf, sizeof(oper_buf), &ic->shadow_log[k]);
fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d "
"no longer held by majority (holders=%d, dead=%d, total=%d)\n"
" entry: %s\n",
k, holders, num_dead, num_nodes, oper_buf);
__builtin_trap();
}
}
}
////////////////////////////////////////////////////////////////////
// Blob invariants: newly committed PUT entries must have their
// chunks stored on at least one live server's disk.
////////////////////////////////////////////////////////////////////
//
// Only check newly committed entries (since last run) to avoid
// O(total_entries * ticks) cost. Uses quakey_enter_host to access
// each server's mock filesystem.
if (node_handles != NULL && observed_max_commit > 0) {
// Check the most recently committed entries for chunk presence
for (int k = MAX(0, ic->shadow_count - 5); k < ic->shadow_count; k++) {
MetaOper *oper = &ic->shadow_log[k];
if (oper->type != META_OPER_PUT)
continue;
for (uint32_t c = 0; c < oper->num_chunks; c++) {
SHA256 hash = oper->chunks[c].hash;
int holders = 0;
int num_dead = 0;
for (int i = 0; i < num_nodes; i++) {
if (nodes[i] == NULL) {
num_dead++;
continue;
}
// Enter host context to access its mock filesystem
quakey_enter_host(node_handles[i]);
bool has_chunk = chunk_store_exists(&nodes[i]->chunk_store, hash);
quakey_leave_host();
if (has_chunk)
holders++;
}
// Chunk must exist on at least one server, OR all
// non-holders are dead (they may have had it before crash).
if (holders == 0 && num_dead < num_nodes) {
fprintf(stderr, "INVARIANT VIOLATED: committed blob %s/%s "
"chunk %u not found on any live server "
"(holders=%d, dead=%d)\n",
oper->bucket, oper->key, c, holders, num_dead);
__builtin_trap();
}
}
}
}
}
src/log.c
+55
View File
@@ -0,0 +1,55 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include "log.h"
void log_init(Log *log)
{
log->count = 0;
log->capacity = 0;
log->entries = NULL;
}
int log_init_from_network(Log *log, void *src, int num)
{
log->count = num;
log->capacity = num;
log->entries = malloc(num * sizeof(LogEntry));
if (log->entries == NULL)
return -1;
memcpy(log->entries, src, num * sizeof(LogEntry));
return 0;
}
void log_free(Log *log)
{
free(log->entries);
}
void log_move(Log *dst, Log *src)
{
log_free(dst);
*dst = *src;
log_init(src);
}
int log_append(Log *log, LogEntry entry)
{
if (log->count == log->capacity) {
int n= 2 * log->capacity;
if (n < 8)
n = 8;
LogEntry *p = realloc(log->entries, n * sizeof(LogEntry));
if (p == NULL)
return -1;
log->entries = p;
log->capacity = n;
}
log->entries[log->count++] = entry;
return 0;
}
src/log.h
+30
View File
@@ -0,0 +1,30 @@
#ifndef LOG_INCLUDED
#define LOG_INCLUDED
#include "metadata.h"
#include "config.h"
typedef struct {
MetaOper oper;
uint32_t votes;
int view_number;
uint64_t client_id;
uint64_t request_id;
} LogEntry;
_Static_assert(NODE_LIMIT <= 32, "");
typedef struct {
int count;
int capacity;
LogEntry *entries;
} Log;
void log_init(Log *log);
int log_init_from_network(Log *log, void *src, int num);
void log_free(Log *log);
void log_move(Log *dst, Log *src);
int log_append(Log *log, LogEntry entry);
#endif // LOG_INCLUDED
src/main.c
+215 -264
View File
@@ -1,116 +1,14 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#else
#define POLL_CAPACITY 1024
#endif
#include <signal.h>
#include <stdint.h>
#include <quakey.h>
#include <assert.h>
#include "chunk_server.h"
#include "random_client.h"
#include "metadata_server.h"
#ifdef MAIN_METADATA_SERVER
int main(int argc, char **argv)
{
int ret;
MetadataServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = metadata_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = metadata_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
metadata_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_CHUNK_SERVER
int main(int argc, char **argv)
{
int ret;
ChunkServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = chunk_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = chunk_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
chunk_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_SIMULATION
#include <signal.h>
#include "server.h"
#include "client.h"
#include "blob_client.h"
static volatile int simulation_running = 1;
@@ -120,233 +18,286 @@ static void sigint_handler(int sig)
simulation_running = 0;
}
int main(void)
int main(int argc, char **argv)
{
signal(SIGINT, sigint_handler);
QuakeyUInt64 seed = 1;
QuakeyUInt64 time_limit_ns = 0; // 0 means no limit
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) {
seed = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) {
time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL;
}
}
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
int ret = quakey_init(&quakey, seed);
if (ret < 0)
return -1;
QuakeyNode node_1;
QuakeyNode node_2;
QuakeyNode node_3;
// Client 1
{
QuakeySpawn config = {
.name = "cli1",
.state_size = sizeof(RandomClient),
.init_func = random_client_init,
.tick_func = random_client_tick,
.free_func = random_client_free,
.name = "rndcli1",
.state_size = sizeof(ClientState),
.init_func = client_init,
.tick_func = client_tick,
.free_func = client_free,
.addrs = (char*[]) { "127.0.0.2" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4 8080");
(void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Client 2
{
QuakeySpawn config = {
.name = "cli2",
.state_size = sizeof(RandomClient),
.init_func = random_client_init,
.tick_func = random_client_tick,
.free_func = random_client_free,
.name = "rndcli2",
.state_size = sizeof(ClientState),
.init_func = client_init,
.tick_func = client_tick,
.free_func = client_free,
.addrs = (char*[]) { "127.0.0.3" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.4 8080");
(void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
// Metadata Server
// Blob Client
{
QuakeySpawn config = {
.name = "ms",
.state_size = sizeof(MetadataServer),
.init_func = metadata_server_init,
.tick_func = metadata_server_tick,
.free_func = metadata_server_free,
.addrs = (char*[]) { "127.0.0.4" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "ms --addr 127.0.0.4 --port 8080");
}
// Chunk Server 1
{
QuakeySpawn config = {
.name = "cs1",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.addrs = (char*[]) { "127.0.0.5" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.5 --port 8081 --remote-addr 127.0.0.4 --remote-port 8080");
}
// Chunk Server 2
{
QuakeySpawn config = {
.name = "cs2",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.addrs = (char*[]) { "127.0.0.6" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.6 --port 8082 --remote-addr 127.0.0.4 --remote-port 8080");
}
// Chunk Server 3
{
QuakeySpawn config = {
.name = "cs3",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.name = "blobcli",
.state_size = sizeof(BlobClientState),
.init_func = blob_client_init,
.tick_func = blob_client_tick,
.free_func = blob_client_free,
.addrs = (char*[]) { "127.0.0.7" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.7 --port 8083 --remote-addr 127.0.0.4 --remote-port 8080");
(void) quakey_spawn(quakey, config, "blob --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080");
}
while (simulation_running)
quakey_schedule_one(quakey);
quakey_free(quakey);
return 0;
}
#endif // MAIN_SIMULATION
#ifdef MAIN_TEST
#include <signal.h>
#include "test_client.h"
static volatile int simulation_running = 1;
static void sigint_handler(int sig)
{
(void)sig;
simulation_running = 0;
}
int main(void)
{
signal(SIGINT, sigint_handler);
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
if (ret < 0)
return -1;
// Client 1
// Node 1
{
QuakeySpawn config = {
.name = "test_cli_1",
.state_size = sizeof(TestClient),
.init_func = test_client_init,
.tick_func = test_client_tick,
.free_func = test_client_free,
.addrs = (char*[]) { "127.0.0.2" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cli --server 127.0.0.3 8080");
}
// Metadata Server
{
QuakeySpawn config = {
.name = "ms",
.state_size = sizeof(MetadataServer),
.init_func = metadata_server_init,
.tick_func = metadata_server_tick,
.free_func = metadata_server_free,
.addrs = (char*[]) { "127.0.0.3" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "ms --addr 127.0.0.3 --port 8080");
}
// Chunk Server 1
{
QuakeySpawn config = {
.name = "cs1",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.name = "server1",
.state_size = sizeof(ServerState),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.4" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.4 --port 8081 --remote-addr 127.0.0.3 --remote-port 8080");
node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Chunk Server 2
// Node 2
{
QuakeySpawn config = {
.name = "cs2",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.name = "server2",
.state_size = sizeof(ServerState),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.5" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.5 --port 8082 --remote-addr 127.0.0.3 --remote-port 8080");
node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080");
}
// Chunk Server 3
// Node 3
{
QuakeySpawn config = {
.name = "cs3",
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.name = "server3",
.state_size = sizeof(ServerState),
.init_func = server_init,
.tick_func = server_tick,
.free_func = server_free,
.addrs = (char*[]) { "127.0.0.6" },
.num_addrs = 1,
.disk_size = 10<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.6 --port 8083 --remote-addr 127.0.0.3 --remote-port 8080");
node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080");
}
while (simulation_running) {
// Limit crashes to 1 node at a time (within fault tolerance of f=1).
// This is dynamically adjusted below: crashes are disabled while
// any node is still recovering.
quakey_set_max_crashes(quakey, 1);
quakey_network_partitioning(quakey, true);
InvariantChecker invariant_checker;
invariant_checker_init(&invariant_checker);
while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) {
quakey_schedule_one(quakey);
QuakeySignal signal;
while (quakey_get_signal(quakey, &signal)) {
ServerState *arr[] = {
quakey_node_state(node_1),
quakey_node_state(node_2),
quakey_node_state(node_3),
};
unsigned long long handles[] = { node_1, node_2, node_3 };
invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]), handles);
if (!strcmp(signal.name, "exit"))
simulation_running = false;
// VR-Revisited Section 8.2: "a replica is considered failed
// until it has recovered its state." Disable crashes while
// any node is recovering to avoid exceeding f simultaneous
// failures (dead + recovering).
bool any_recovering = false;
for (int i = 0; i < 3; i++) {
if (arr[i] && arr[i]->status == STATUS_RECOVERY)
any_recovering = true;
}
quakey_set_max_crashes(quakey, any_recovering ? 0 : 1);
}
invariant_checker_free(&invariant_checker);
quakey_free(quakey);
return 0;
}
#endif // MAIN_TEST
#endif // MAIN_SIMULATION
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
#ifdef MAIN_CLIENT
#include <poll.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include "client.h"
#define POLL_CAPACITY 1024
int main(int argc, char **argv)
{
srand((unsigned)time(NULL) ^ (unsigned)getpid());
int ret;
ClientState state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = client_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = client_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
client_free(&state);
return 0;
}
#endif // MAIN_CLIENT
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
#ifdef MAIN_SERVER
#include <poll.h>
#include "server.h"
#define POLL_CAPACITY 1024
int main(int argc, char **argv)
{
int ret;
ServerState state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
server_free(&state);
return 0;
}
#endif // MAIN_SERVER
src/message.c
-485
View File
@@ -1,485 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "message.h"
bool binary_read(BinaryReader *reader, void *dst, int len)
{
if (reader->len - reader->cur < len)
return false;
if (dst)
memcpy(dst, reader->src + reader->cur, len);
reader->cur += len;
return true;
}
bool binary_read_addr_ipv4(BinaryReader *reader, Address *addr)
{
if (!binary_read(reader, &addr->ipv4, sizeof(IPv4))) return false;
if (!binary_read(reader, &addr->port, sizeof(uint16_t))) return false;
addr->is_ipv4 = true;
return true;
}
bool binary_read_addr_ipv6(BinaryReader *reader, Address *addr)
{
if (!binary_read(reader, &addr->ipv6, sizeof(IPv6))) return false;
if (!binary_read(reader, &addr->port, sizeof(uint16_t))) return false;
addr->is_ipv4 = false;
return true;
}
int binary_read_addr_list(BinaryReader *reader, Address *addrs, int max_addrs)
{
uint32_t num_ipv4;
uint32_t num_ipv6;
if (!binary_read(reader, &num_ipv4, sizeof(num_ipv4)))
return -1;
if (!binary_read(reader, &num_ipv6, sizeof(num_ipv6)))
return -1;
int num = 0;
for (uint32_t i = 0; i < num_ipv4; i++) {
Address tmp;
if (!binary_read_addr_ipv4(reader, &tmp))
return -1;
if (num < max_addrs)
addrs[num++] = tmp;
}
for (uint32_t i = 0; i < num_ipv6; i++) {
Address tmp;
if (!binary_read_addr_ipv6(reader, &tmp))
return -1;
if (num < max_addrs)
addrs[num++] = tmp;
}
return num;
}
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type)
{
uint16_t version = MESSAGE_VERSION;
uint32_t dummy = 0; // Dummy value
writer->output = output;
writer->start = byte_queue_offset(output);
byte_queue_write(output, &version, sizeof(version));
byte_queue_write(output, &type, sizeof(type));
writer->patch = byte_queue_offset(output);
byte_queue_write(output, &dummy, sizeof(dummy));
}
bool message_writer_free(MessageWriter *writer)
{
uint32_t length = byte_queue_size_from_offset(writer->output, writer->start);
byte_queue_patch(writer->output, writer->patch, &length, sizeof(length));
if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure?
return false;
return true;
}
void message_write(MessageWriter *writer, void *mem, int len)
{
byte_queue_write(writer->output, mem, len);
}
void message_write_u8(MessageWriter *writer, uint8_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_u32(MessageWriter *writer, uint32_t value)
{
message_write(writer, &value, (int) sizeof(value));
}
void message_write_hash(MessageWriter *writer, SHA256 value)
{
message_write(writer, &value, (int) sizeof(value));
}
int message_peek(ByteView msg, uint16_t *type, uint32_t *len)
{
if (msg.len < (int) sizeof(MessageHeader))
return 0;
MessageHeader header;
memcpy(&header, msg.ptr, sizeof(header));
// (We ignore endianess for now)
if (header.version != MESSAGE_VERSION)
return -1;
if (header.length > msg.len)
return 0;
if (type) *type = header.type;
if (len) *len = header.length;
return 1;
}
static char *message_type_to_str(uint16_t type)
{
switch (type) {
// Client -> Metadata server
case MESSAGE_TYPE_CREATE: return "CREATE";
case MESSAGE_TYPE_DELETE: return "DELETE";
case MESSAGE_TYPE_LIST: return "LIST";
case MESSAGE_TYPE_READ: return "READ";
case MESSAGE_TYPE_WRITE: return "WRITE";
// Client -> Chunk server
case MESSAGE_TYPE_CREATE_CHUNK: return "CREATE_CHUNK";
case MESSAGE_TYPE_UPLOAD_CHUNK: return "UPLOAD_CHUNK";
case MESSAGE_TYPE_DOWNLOAD_CHUNK: return "DOWNLOAD_CHUNK";
// Metadata server -> Client
case MESSAGE_TYPE_CREATE_ERROR: return "CREATE_ERROR";
case MESSAGE_TYPE_CREATE_SUCCESS: return "CREATE_SUCCESS";
case MESSAGE_TYPE_DELETE_ERROR: return "DELETE_ERROR";
case MESSAGE_TYPE_DELETE_SUCCESS: return "DELETE_SUCCESS";
case MESSAGE_TYPE_LIST_ERROR: return "LIST_ERROR";
case MESSAGE_TYPE_LIST_SUCCESS: return "LIST_SUCCESS";
case MESSAGE_TYPE_READ_ERROR: return "READ_ERROR";
case MESSAGE_TYPE_READ_SUCCESS: return "READ_SUCCESS";
case MESSAGE_TYPE_WRITE_ERROR: return "WRITE_ERROR";
case MESSAGE_TYPE_WRITE_SUCCESS: return "WRITE_SUCCESS";
// Metadata server -> Chunk server
case MESSAGE_TYPE_SYNC_2: return "SYNC_2";
case MESSAGE_TYPE_SYNC_4: return "SYNC_4";
case MESSAGE_TYPE_DOWNLOAD_LOCATIONS: return "DOWNLOAD_LOCATIONS";
// Chunk server -> Metadata server
case MESSAGE_TYPE_AUTH: return "AUTH";
case MESSAGE_TYPE_SYNC: return "SYNC";
case MESSAGE_TYPE_SYNC_3: return "SYNC_3";
// Chunk server -> Client
case MESSAGE_TYPE_CREATE_CHUNK_ERROR: return "CREATE_CHUNK_ERROR";
case MESSAGE_TYPE_CREATE_CHUNK_SUCCESS: return "CREATE_CHUNK_SUCCESS";
case MESSAGE_TYPE_UPLOAD_CHUNK_ERROR: return "UPLOAD_CHUNK_ERROR";
case MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS: return "UPLOAD_CHUNK_SUCCESS";
case MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR: return "DOWNLOAD_CHUNK_ERROR";
case MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS: return "DOWNLOAD_CHUNK_SUCCESS";
}
return "???";
}
void message_dump(FILE *stream, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
fprintf(stream, "message:\n");
fprintf(stream, " header:\n");
uint16_t version;
if (!binary_read(&reader, &version, sizeof(version))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " version: %d\n", version);
uint16_t type;
if (!binary_read(&reader, &type, sizeof(type))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " type: %s\n", message_type_to_str(type));
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " length: %d\n", length);
fprintf(stream, " body:\n");
switch (type) {
// Client -> Metadata server
case MESSAGE_TYPE_CREATE:
{
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path_len: %d\n", path_len);
char *path = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, path_len)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path: %.*s\n", (int) path_len, path);
uint8_t is_dir;
if (!binary_read(&reader, &is_dir, sizeof(is_dir))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " is_dir: %s\n", is_dir ? "true" : "false");
if (!is_dir) {
uint32_t chunk_size;
if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " chunk_size: %d\n", chunk_size);
}
}
break;
case MESSAGE_TYPE_DELETE:
{
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path_len: %d\n", path_len);
char *path = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, path_len)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path: %.*s\n", (int) path_len, path);
}
break;
case MESSAGE_TYPE_LIST:
{
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path_len: %d\n", path_len);
char *path = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, path_len)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path: %.*s\n", (int) path_len, path);
}
break;
case MESSAGE_TYPE_READ:
{
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path_len: %d\n", path_len);
char *path = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, path_len)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path: %.*s\n", (int) path_len, path);
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " offset: %d\n", offset);
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " length: %d\n", length);
}
break;
case MESSAGE_TYPE_WRITE:
{
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path_len: %d\n", path_len);
char *path = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, path_len)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " path: %.*s\n", (int) path_len, path);
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " offset: %d\n", offset);
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " length: %d\n", length);
uint32_t num_chunks;
if (!binary_read(&reader, &num_chunks, sizeof(num_chunks))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " num_chunks: %d\n", num_chunks);
for (uint32_t i = 0; i < num_chunks; i++) {
char hash_str[64];
SHA256 old_hash;
if (!binary_read(&reader, &old_hash, sizeof(old_hash))) {
fprintf(stream, " (incomplete)\n");
return;
}
append_hex_as_str(hash_str, old_hash);
fprintf(stream, " old_hash: %.64s\n", hash_str);
SHA256 new_hash;
if (!binary_read(&reader, &new_hash, sizeof(new_hash))) {
fprintf(stream, " (incomplete)\n");
return;
}
append_hex_as_str(hash_str, new_hash);
fprintf(stream, " new_hash: %.64s\n", hash_str);
uint32_t num_locations;
if (!binary_read(&reader, &num_locations, sizeof(num_locations))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " num_locations: %d\n", num_locations);
for (uint32_t j = 0; j < num_locations; j++) {
uint8_t is_ipv4;
if (!binary_read(&reader, &is_ipv4, sizeof(is_ipv4))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " is_ipv4: %s (%d)\n", is_ipv4 ? "true" : "false", is_ipv4);
if (is_ipv4) {
IPv4 ipv4;
if (!binary_read(&reader, &ipv4, sizeof(ipv4))) {
fprintf(stream, " (incomplete)\n");
return;
}
char ip_str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ipv4, ip_str, sizeof(ip_str));
fprintf(stream, " ipv4: %s\n", ip_str);
} else {
IPv6 ipv6;
if (!binary_read(&reader, &ipv6, sizeof(ipv6))) {
fprintf(stream, " (incomplete)\n");
return;
}
char ip_str[INET_ADDRSTRLEN];
inet_ntop(AF_INET6, &ipv6, ip_str, sizeof(ip_str));
fprintf(stream, " ipv6: %s\n", ip_str);
}
uint16_t port;
if (!binary_read(&reader, &port, sizeof(port))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " port: %d\n", port);
}
}
}
break;
// Client -> Chunk server
case MESSAGE_TYPE_CREATE_CHUNK:
{
uint32_t chunk_size;
if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " chunk_size: %d\n", chunk_size);
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " offset: %d\n", offset);
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " length: %d\n", length);
if (!binary_read(&reader, NULL, length)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " data: (...)\n");
}
break;
case MESSAGE_TYPE_UPLOAD_CHUNK:
{
SHA256 hash;
if (!binary_read(&reader, &hash, sizeof(hash))) {
fprintf(stream, " (incomplete)\n");
return;
}
char hash_str[64];
append_hex_as_str(hash_str, hash);
fprintf(stream, " hash: %.64s\n", hash_str);
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " offset: %d\n", offset);
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length))) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " length: %d\n", length);
if (!binary_read(&reader, NULL, length)) {
fprintf(stream, " (incomplete)\n");
return;
}
fprintf(stream, " data: (...)\n");
}
break;
default:
printf(" (TODO)\n");
break;
}
if (binary_read(&reader, NULL, 1))
fprintf(stream, " (unexpected bytes)\n");
}
src/message.h
-96
View File
@@ -1,96 +0,0 @@
#ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
#include "byte_queue.h"
enum {
// Client -> Metadata server
MESSAGE_TYPE_CREATE,
MESSAGE_TYPE_DELETE,
MESSAGE_TYPE_LIST,
MESSAGE_TYPE_READ,
MESSAGE_TYPE_WRITE,
// Client -> Chunk server
MESSAGE_TYPE_CREATE_CHUNK,
MESSAGE_TYPE_UPLOAD_CHUNK,
MESSAGE_TYPE_DOWNLOAD_CHUNK,
// Metadata server -> Client
MESSAGE_TYPE_CREATE_ERROR,
MESSAGE_TYPE_CREATE_SUCCESS,
MESSAGE_TYPE_DELETE_ERROR,
MESSAGE_TYPE_DELETE_SUCCESS,
MESSAGE_TYPE_LIST_ERROR,
MESSAGE_TYPE_LIST_SUCCESS,
MESSAGE_TYPE_READ_ERROR,
MESSAGE_TYPE_READ_SUCCESS,
MESSAGE_TYPE_WRITE_ERROR,
MESSAGE_TYPE_WRITE_SUCCESS,
// Metadata server -> Chunk server
MESSAGE_TYPE_AUTH_RESPONSE,
MESSAGE_TYPE_SYNC_2,
MESSAGE_TYPE_SYNC_4,
MESSAGE_TYPE_DOWNLOAD_LOCATIONS,
// Chunk server -> Metadata server
MESSAGE_TYPE_AUTH,
MESSAGE_TYPE_SYNC,
MESSAGE_TYPE_SYNC_3,
// Chunk server -> Client
MESSAGE_TYPE_CREATE_CHUNK_ERROR,
MESSAGE_TYPE_CREATE_CHUNK_SUCCESS,
MESSAGE_TYPE_UPLOAD_CHUNK_ERROR,
MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS,
MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR,
MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS,
};
#define MESSAGE_VERSION 1
typedef struct {
uint8_t *src;
int len;
int cur;
} BinaryReader;
typedef struct {
uint16_t version;
uint16_t type;
uint32_t length;
} MessageHeader;
typedef struct {
ByteQueue *output;
ByteQueueOffset start;
ByteQueueOffset patch;
} MessageWriter;
bool binary_read(BinaryReader *reader, void *dst, int len);
bool binary_read_addr_ipv4(BinaryReader *reader, Address *addr);
bool binary_read_addr_ipv6(BinaryReader *reader, Address *addr);
int binary_read_addr_list(BinaryReader *reader, Address *addrs, int max_addr);
void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type);
bool message_writer_free(MessageWriter *writer);
void message_write(MessageWriter *writer, void *mem, int len);
void message_write_u8(MessageWriter *writer, uint8_t value);
void message_write_u32(MessageWriter *writer, uint32_t value);
void message_write_hash(MessageWriter *writer, SHA256 value);
int message_peek(ByteView msg, uint16_t *type, uint32_t *len);
void message_dump(FILE *stream, ByteView msg);
#endif // MESSAGE_INCLUDED
src/metadata.c
+136
View File
@@ -0,0 +1,136 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include <stdio.h>
#include "metadata.h"
void meta_store_init(MetaStore *ms)
{
memset(ms, 0, sizeof(*ms));
}
void meta_store_free(MetaStore *ms)
{
(void) ms;
}
static int find_entry(MetaStore *ms, const char *bucket, const char *key)
{
for (int i = 0; i < ms->count; i++) {
if (!ms->entries[i].deleted
&& strncmp(ms->entries[i].bucket, bucket, META_BUCKET_MAX) == 0
&& strncmp(ms->entries[i].key, key, META_KEY_MAX) == 0)
return i;
}
return -1;
}
static void copy_oper_to_entry(ObjectMeta *entry, MetaOper *oper)
{
memcpy(entry->bucket, oper->bucket, META_BUCKET_MAX);
memcpy(entry->key, oper->key, META_KEY_MAX);
entry->size = oper->size;
entry->content_hash = oper->content_hash;
entry->num_chunks = oper->num_chunks;
memcpy(entry->chunks, oper->chunks, oper->num_chunks * sizeof(ChunkRef));
entry->deleted = false;
}
MetaResult meta_store_update(MetaStore *ms, MetaOper *oper)
{
MetaResult result;
switch (oper->type) {
case META_OPER_NOOP:
result.type = META_RESULT_OK;
break;
case META_OPER_PUT:
{
// Try to find existing entry with same bucket/key
int i = find_entry(ms, oper->bucket, oper->key);
if (i >= 0) {
// Overwrite in-place
copy_oper_to_entry(&ms->entries[i], oper);
result.type = META_RESULT_OK;
break;
}
// Try to reuse a tombstoned slot
for (i = 0; i < ms->count; i++) {
if (ms->entries[i].deleted) {
copy_oper_to_entry(&ms->entries[i], oper);
result.type = META_RESULT_OK;
goto done;
}
}
// Allocate new slot
if (ms->count >= META_ENTRY_LIMIT) {
result.type = META_RESULT_FULL;
break;
}
copy_oper_to_entry(&ms->entries[ms->count++], oper);
result.type = META_RESULT_OK;
}
break;
case META_OPER_DELETE:
{
int i = find_entry(ms, oper->bucket, oper->key);
if (i < 0) {
result.type = META_RESULT_NOT_FOUND;
} else {
ms->entries[i].deleted = true;
result.type = META_RESULT_OK;
}
}
break;
default:
assert(0);
break;
}
done:
return result;
}
ObjectMeta *meta_store_lookup(MetaStore *ms, const char *bucket, const char *key)
{
int i = find_entry(ms, bucket, key);
if (i < 0)
return NULL;
return &ms->entries[i];
}
int meta_snprint_oper(char *buf, int size, MetaOper *oper)
{
switch (oper->type) {
case META_OPER_NOOP:
return snprintf(buf, size, "NOOP");
case META_OPER_PUT:
return snprintf(buf, size, "PUT(%s/%s, %u chunks)",
oper->bucket, oper->key, oper->num_chunks);
case META_OPER_DELETE:
return snprintf(buf, size, "DELETE(%s/%s)",
oper->bucket, oper->key);
default:
return snprintf(buf, size, "???");
}
}
int meta_snprint_result(char *buf, int size, MetaResult result)
{
switch (result.type) {
case META_RESULT_OK: return snprintf(buf, size, "OK");
case META_RESULT_NOT_FOUND: return snprintf(buf, size, "NOT_FOUND");
case META_RESULT_FULL: return snprintf(buf, size, "FULL");
default: return snprintf(buf, size, "???");
}
}
src/metadata.h
+69
View File
@@ -0,0 +1,69 @@
#ifndef METADATA_INCLUDED
#define METADATA_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#include <lib/basic.h>
typedef struct {
SHA256 hash;
uint32_t size;
} ChunkRef;
#define META_BUCKET_MAX 64
#define META_KEY_MAX 512
#define META_CHUNKS_MAX 256
typedef enum {
META_OPER_NOOP,
META_OPER_PUT,
META_OPER_DELETE,
} MetaOperType;
typedef struct {
MetaOperType type;
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
uint64_t size;
SHA256 content_hash;
uint32_t num_chunks;
ChunkRef chunks[META_CHUNKS_MAX];
} MetaOper;
typedef enum {
META_RESULT_OK,
META_RESULT_NOT_FOUND,
META_RESULT_FULL,
} MetaResultType;
typedef struct {
MetaResultType type;
} MetaResult;
typedef struct {
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
uint64_t size;
SHA256 content_hash;
uint32_t num_chunks;
ChunkRef chunks[META_CHUNKS_MAX];
bool deleted;
} ObjectMeta;
#define META_ENTRY_LIMIT 4096
typedef struct {
int count;
ObjectMeta entries[META_ENTRY_LIMIT];
} MetaStore;
void meta_store_init(MetaStore *ms);
void meta_store_free(MetaStore *ms);
MetaResult meta_store_update(MetaStore *ms, MetaOper *oper);
ObjectMeta *meta_store_lookup(MetaStore *ms, const char *bucket, const char *key);
int meta_snprint_oper(char *buf, int size, MetaOper *oper);
int meta_snprint_result(char *buf, int size, MetaResult result);
#endif // METADATA_INCLUDED
src/metadata_server.c
-1281
View File
File diff suppressed because it is too large Load Diff
src/metadata_server.h
-66
View File
@@ -1,66 +0,0 @@
#ifndef METADATA_SERVER_INCLUDED
#define METADATA_SERVER_INCLUDED
#include "tcp.h"
#include "wal.h"
#include "basic.h"
#include "config.h"
#include "hash_set.h"
#include "file_tree.h"
#define CONNECTION_TAG_CLIENT -2
#define CONNECTION_TAG_UNKNOWN -3
typedef struct {
bool used;
bool auth;
int num_addrs;
Address addrs[MAX_SERVER_ADDRS];
// List of chunks that are known to be held by CS
HashSet ms_old_list; // TODO: rename all *_list symbols to *_set
// List of chunks that should be held by CS
HashSet ms_add_list;
// List of chunks that may be held by CS but should removed from it
HashSet ms_rem_list;
// Time when last STATE_UPDATE was sent
Time last_sync_time;
bool last_sync_done;
// Time when last response was received
Time last_response_time; // TODO: don't init to INVALID_TIME but current_time
} ChunkServerPeer;
typedef struct {
TCP tcp;
WAL wal;
FileTree file_tree;
bool trace;
int replication_factor;
int num_chunk_servers;
ChunkServerPeer chunk_servers[MAX_CHUNK_SERVERS];
} MetadataServer;
struct pollfd;
int metadata_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int metadata_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int metadata_server_free(void *state);
#endif // METADATA_SERVER_INCLUDED
src/random_client.c
-278
View File
@@ -1,278 +0,0 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#include <quakey.h>
#include <assert.h>
#include "tcp.h"
#include "random_client.h"
// Helper function to parse address and port from command line
static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port)
{
// Default to metadata server
*addr = "127.0.0.1";
*port = 8080;
for (int i = 0; i < argc - 1; i++) {
if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) {
*addr = argv[i + 1];
if (i + 2 < argc) {
errno = 0;
char *end;
long val = strtol(argv[i+2], &end, 10);
if (end == argv[i+2] || *end != '\0' || errno == ERANGE)
break;
if (val < 0 || val > UINT16_MAX)
break;
*port = (uint16_t) val;
return true;
}
}
}
return true;
}
int random_client_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
RandomClient *client = state_;
char *addr;
uint16_t port;
parse_server_addr(argc, argv, &addr, &port);
client->toasty = toasty_connect((ToastyString) { addr, strlen(addr) }, port);
if (client->toasty == NULL)
return -1;
client->num_pending = 0;
printf("Client set up (remote=%s:%d)\n", addr, port);
*timeout = 0;
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
return 0;
}
static int random_in_range(int min, int max)
{
uint64_t n = quakey_random();
return min + n % (max - min + 1);
}
int random_client_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
RandomClient *client = state_;
// Process any pending events from the network and get new poll descriptors
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
for (int i = 0; i < client->num_pending; i++) {
ToastyResult result;
if (toasty_get_result(client->toasty, client->pending[i].handle, &result) != 0)
continue;
PendingOperation pending = client->pending[i];
switch (result.type) {
case TOASTY_RESULT_EMPTY:
assert(0);
break;
case TOASTY_RESULT_CREATE_ERROR:
assert(pending.type == PENDING_OPERATION_CREATE);
//printf("[Client] create error\n");
break;
case TOASTY_RESULT_CREATE_SUCCESS:
assert(pending.type == PENDING_OPERATION_CREATE);
//printf("[Client] create success\n");
break;
case TOASTY_RESULT_DELETE_ERROR:
assert(pending.type == PENDING_OPERATION_DELETE);
//printf("[Client] delete error\n");
break;
case TOASTY_RESULT_DELETE_SUCCESS:
assert(pending.type == PENDING_OPERATION_DELETE);
//printf("[Client] delete success\n");
break;
case TOASTY_RESULT_LIST_ERROR:
assert(pending.type == PENDING_OPERATION_LIST);
//printf("[Client] list error\n");
break;
case TOASTY_RESULT_LIST_SUCCESS:
assert(pending.type == PENDING_OPERATION_LIST);
//printf("[Client] list success\n");
break;
case TOASTY_RESULT_READ_ERROR:
assert(pending.type == PENDING_OPERATION_READ);
//printf("[Client] read error\n");
break;
case TOASTY_RESULT_READ_SUCCESS:
assert(pending.type == PENDING_OPERATION_READ);
//printf("[Client] read success\n");
break;
case TOASTY_RESULT_WRITE_ERROR:
assert(pending.type == PENDING_OPERATION_WRITE);
//printf("[Client] write error\n");
break;
case TOASTY_RESULT_WRITE_SUCCESS:
assert(pending.type == PENDING_OPERATION_WRITE);
//printf("[Client] write success\n");
break;
}
free(pending.ptr);
toasty_free_result(&result);
client->pending[i--] = client->pending[--client->num_pending];
}
while (client->num_pending < MAX_PENDING_OPERATION) {
typedef struct {
ToastyString path;
bool is_dir;
} TableEntry;
static const TableEntry table[] = {
{ TOASTY_STR("/f0"), false },
{ TOASTY_STR("/f1"), false },
{ TOASTY_STR("/d0"), true },
{ TOASTY_STR("/d1"), true },
{ TOASTY_STR("/d0/f1"), false },
{ TOASTY_STR("/d0/f2"), false },
{ TOASTY_STR("/d0/d0"), true },
{ TOASTY_STR("/d0/d1"), true },
{ TOASTY_STR("/d1/f1"), false },
{ TOASTY_STR("/d1/f2"), false },
{ TOASTY_STR("/d1/d0"), true },
{ TOASTY_STR("/d1/d1"), true },
};
static const int table_len = sizeof(table) / sizeof(table[0]);
static const PendingOperationType type_table[] = {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
};
static const int type_table_len = sizeof(type_table)/sizeof(type_table[0]);
void *ptr = NULL;
int off = 0;
int len = 0;
ToastyHandle handle;
PendingOperationType type = type_table[random_in_range(0, type_table_len-1)];
switch (type) {
TableEntry entry;
uint32_t chunk_size;
uint32_t flags;
case PENDING_OPERATION_CREATE:
entry = table[random_in_range(0, table_len-1)];
if (entry.is_dir) {
handle = toasty_begin_create_dir(client->toasty, entry.path);
} else {
chunk_size = random_in_range(0, 5000);
handle = toasty_begin_create_file(client->toasty, entry.path, chunk_size);
}
//printf("[Client] submit create (path=%s, is_dir=%s, chunk_size=%d)\n", entry.path, entry.is_dir ? "true" : "false", chunk_size);
break;
case PENDING_OPERATION_DELETE:
entry = table[random_in_range(0, table_len-1)];
handle = toasty_begin_delete(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit delete (path=%s)\n", entry.path);
break;
case PENDING_OPERATION_LIST:
entry = table[random_in_range(0, table_len-1)];
handle = toasty_begin_list(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit list (path=%s)\n", entry.path);
break;
case PENDING_OPERATION_READ:
entry = table[random_in_range(0, table_len-1)];
off = random_in_range(0, 10000);
len = random_in_range(0, 5000);
ptr = malloc(len);
if (ptr == NULL) assert(0);
handle = toasty_begin_read(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit read (path=%s, off=%d, len=%d)\n", entry.path, off, len);
break;
case PENDING_OPERATION_WRITE:
entry = table[random_in_range(0, table_len-1)];
off = random_in_range(0, 10000);
len = random_in_range(0, 5000);
ptr = malloc(len);
if (ptr == NULL) assert(0);
memset(ptr, 'a', len);
flags = 0;
switch (random_in_range(0, 3)) {
case 0:
flags = 0;
break;
case 1:
flags = TOASTY_WRITE_CREATE_IF_MISSING;
break;
case 2:
flags = TOASTY_WRITE_TRUNCATE_AFTER;
break;
case 3:
flags = TOASTY_WRITE_CREATE_IF_MISSING
| TOASTY_WRITE_TRUNCATE_AFTER;
break;
default:
assert(0);
}
handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY, flags);
//printf("[Client] submit write (path=%s, off=%d, len=%d)\n", entry.path, off, len);
break;
}
if (handle == TOASTY_INVALID)
break;
client->pending[client->num_pending++] = (PendingOperation) { .type=type, .handle=handle, .ptr=ptr };
}
if (client->num_pending == 0)
*timeout = 10;
else
*timeout = -1;
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, 0);
return 0;
}
int random_client_free(void *state_)
{
RandomClient *client = state_;
toasty_disconnect(client->toasty);
return 0;
}
#endif // MAIN_SIMULATION
src/random_client.h
-39
View File
@@ -1,39 +0,0 @@
#ifndef RANDOM_CLIENT_INCLUDED
#define RANDOM_CLIENT_INCLUDED
#include "ToastyFS.h"
#define MAX_PENDING_OPERATION 8
typedef enum {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
} PendingOperationType;
typedef struct {
PendingOperationType type;
ToastyHandle handle;
void *ptr;
} PendingOperation;
typedef struct {
ToastyFS *toasty;
int num_pending;
PendingOperation pending[MAX_PENDING_OPERATION];
} RandomClient;
struct pollfd;
int random_client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int random_client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int random_client_free(void *state);
#endif // RANDOM_CLIENT_INCLUDED
src/server.c
+2006
View File
File diff suppressed because it is too large Load Diff
src/server.h
+301
View File
@@ -0,0 +1,301 @@
#ifndef NODE_INCLUDED
#define NODE_INCLUDED
#include <lib/tcp.h>
#include <lib/basic.h>
#include <lib/message.h>
#include "log.h"
#include "config.h"
#include "metadata.h"
#include "chunk_store.h"
#include "client_table.h"
enum {
// Normal Protocol
MESSAGE_TYPE_REQUEST,
MESSAGE_TYPE_REPLY,
MESSAGE_TYPE_PREPARE,
MESSAGE_TYPE_PREPARE_OK,
MESSAGE_TYPE_COMMIT,
// View Change Protocol
MESSAGE_TYPE_BEGIN_VIEW_CHANGE,
MESSAGE_TYPE_DO_VIEW_CHANGE,
MESSAGE_TYPE_BEGIN_VIEW,
// Recovery Protocol
MESSAGE_TYPE_RECOVERY,
MESSAGE_TYPE_RECOVERY_RESPONSE,
// State Transfer Protocol
MESSAGE_TYPE_GET_STATE,
MESSAGE_TYPE_NEW_STATE,
// Client Redirect
MESSAGE_TYPE_REDIRECT,
// Chunk Storage Protocol (bypasses log)
MESSAGE_TYPE_STORE_CHUNK,
MESSAGE_TYPE_STORE_CHUNK_ACK,
MESSAGE_TYPE_FETCH_CHUNK,
MESSAGE_TYPE_FETCH_CHUNK_RESPONSE,
// Blob Client Protocol
MESSAGE_TYPE_COMMIT_PUT,
MESSAGE_TYPE_GET_BLOB,
MESSAGE_TYPE_GET_BLOB_RESPONSE,
};
typedef struct {
MessageHeader base;
MetaOper oper;
uint64_t client_id;
uint64_t request_id;
} RequestMessage;
typedef struct {
MessageHeader base;
MetaOper oper;
int sender_idx;
int log_index;
int commit_index;
uint64_t view_number;
uint64_t client_id;
uint64_t request_id;
} PrepareMessage;
typedef struct {
MessageHeader base;
int sender_idx;
int log_index;
uint64_t view_number;
} PrepareOKMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int sender_idx;
int commit_index;
} CommitMessage;
typedef struct {
MessageHeader base;
bool rejected;
MetaResult result;
uint64_t request_id;
} ReplyMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int sender_idx;
} BeginViewChangeMessage;
typedef struct {
MessageHeader base;
uint64_t view_number; // The new view number
uint64_t old_view_number; // Last view number when replica was in normal status
int op_number; // Number of entries in the log
int commit_index;
int sender_idx;
// Followed by: LogEntry log[op_number]
} DoViewChangeMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int commit_index;
int op_number; // Number of log entries that follow
// Followed by: LogEntry log[op_number]
} BeginViewMessage;
typedef struct {
MessageHeader base;
int sender_idx;
uint64_t nonce;
} RecoveryMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number;
uint64_t nonce;
int commit_index;
int sender_idx;
} RecoveryResponseMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Requester's current log count
int sender_idx;
} GetStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
int op_number; // Number of log entries that follow
int commit_index;
int start_index; // Global log index of the first entry in the suffix
// Followed by: LogEntry log[op_number]
} NewStateMessage;
typedef struct {
MessageHeader base;
uint64_t view_number;
} RedirectMessage;
// StoreChunk: client -> any server. Carries chunk data as trailing bytes.
typedef struct {
MessageHeader base;
SHA256 hash;
uint32_t size;
// Followed by: uint8_t data[size]
} StoreChunkMessage;
// StoreChunkAck: server -> client.
typedef struct {
MessageHeader base;
SHA256 hash;
bool success;
} StoreChunkAckMessage;
// FetchChunk: client/server -> server. Request a chunk by hash.
typedef struct {
MessageHeader base;
SHA256 hash;
int sender_idx; // -1 if from a client
} FetchChunkMessage;
// FetchChunkResponse: server -> client/server. Chunk data as trailing bytes.
typedef struct {
MessageHeader base;
SHA256 hash;
uint32_t size; // 0 if chunk not found
// Followed by: uint8_t data[size]
} FetchChunkResponseMessage;
// CommitPut: blob client -> leader. Commits metadata after chunks uploaded.
// Processed like a REQUEST (goes through VSR log).
typedef struct {
MessageHeader base;
MetaOper oper;
uint64_t client_id;
uint64_t request_id;
} CommitPutMessage;
// GetBlob: client -> any server. Request metadata for a blob.
typedef struct {
MessageHeader base;
char bucket[META_BUCKET_MAX];
char key[META_KEY_MAX];
} GetBlobMessage;
// GetBlobResponse: server -> client. Returns blob metadata.
typedef struct {
MessageHeader base;
bool found;
uint64_t size;
SHA256 content_hash;
uint32_t num_chunks;
ChunkRef chunks[META_CHUNKS_MAX];
} GetBlobResponseMessage;
typedef enum {
STATUS_NORMAL,
STATUS_CHANGE_VIEW,
STATUS_RECOVERY,
} Status;
typedef struct {
TCP tcp;
Address self_addr;
Address node_addrs[NODE_LIMIT];
int num_nodes;
Status status;
ClientTable client_table;
int next_client_tag;
uint64_t view_number;
uint64_t last_normal_view; // Latest view where status was NORMAL
// These fields are used in recovery mode
uint32_t recovery_votes;
uint64_t recovery_nonce;
uint64_t recovery_view;
Log recovery_log;
uint64_t recovery_log_view;
Time recovery_time;
int recovery_commit;
///////////////////////////////////////////////////////////
// VIEW CHANGE
uint32_t view_change_begin_votes;
uint32_t view_change_apply_votes;
Log view_change_log; // Best log seen
uint64_t view_change_old_view; // Best old_view_number seen in DoViewChange
int view_change_commit; // Best commit_index seen
///////////////////////////////////////////////////////////
// If this is the leader, commit_index is the index of the next uncommitted element
// of the log (if no uncommitted elements are present, it's equal to the total number
// of log elements)
int commit_index;
PrepareMessage future[FUTURE_LIMIT];
int num_future;
bool state_transfer_pending;
Time state_transfer_time;
Log log;
Time heartbeat;
MetaStore metastore;
ChunkStore chunk_store;
// Set at each wakeup
Time now;
} ServerState;
struct pollfd;
int server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int server_free(void *state);
typedef struct {
int last_min_commit;
int last_max_commit;
Status prev_status[NODE_LIMIT];
// External shadow log of committed operations (unbounded, dynamically allocated)
MetaOper *shadow_log;
int shadow_count;
int shadow_capacity;
} InvariantChecker;
void invariant_checker_init(InvariantChecker *ic);
void invariant_checker_free(InvariantChecker *ic);
// node_handles: opaque QuakeyNode handles for entering host context.
// Pass NULL when running outside the simulation (real mode).
void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes,
unsigned long long *node_handles);
#endif // NODE_INCLUDED
src/sha256.c
-249
View File
@@ -1,249 +0,0 @@
#include "sha256.h"
//usr/bin/env clang -Ofast -Wall -Wextra -pedantic ${0} -o ${0%%.c*} $* ;exit $?
//
// SHA-256 implementation, Mark 2
//
// Copyright (c) 2010,2014 Literatecode, http://www.literatecode.com
// Copyright (c) 2022 Ilia Levin (ilia@levin.sg)
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
//
#define SHA256_SIZE_BYTES (32)
typedef struct {
uint8_t buf[64];
uint32_t hash[8];
uint32_t bits[2];
uint32_t len;
uint32_t rfu__;
uint32_t W[64];
} sha256_context;
#ifndef _cbmc_
#define __CPROVER_assume(...) do {} while(0)
#endif
#define FN_ static inline __attribute__((const))
static const uint32_t K[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
};
FN_ uint8_t _shb(uint32_t x, uint32_t n)
{
return ((x >> (n & 31)) & 0xff);
}
FN_ uint32_t _shw(uint32_t x, uint32_t n)
{
return ((x << (n & 31)) & 0xffffffff);
}
FN_ uint32_t _r(uint32_t x, uint8_t n)
{
return ((x >> n) | _shw(x, 32 - n));
}
FN_ uint32_t _Ch(uint32_t x, uint32_t y, uint32_t z)
{
return ((x & y) ^ ((~x) & z));
}
FN_ uint32_t _Ma(uint32_t x, uint32_t y, uint32_t z)
{
return ((x & y) ^ (x & z) ^ (y & z));
}
FN_ uint32_t _S0(uint32_t x)
{
return (_r(x, 2) ^ _r(x, 13) ^ _r(x, 22));
}
FN_ uint32_t _S1(uint32_t x)
{
return (_r(x, 6) ^ _r(x, 11) ^ _r(x, 25));
}
FN_ uint32_t _G0(uint32_t x)
{
return (_r(x, 7) ^ _r(x, 18) ^ (x >> 3));
}
FN_ uint32_t _G1(uint32_t x)
{
return (_r(x, 17) ^ _r(x, 19) ^ (x >> 10));
}
FN_ uint32_t _word(uint8_t *c)
{
return (_shw(c[0], 24) | _shw(c[1], 16) | _shw(c[2], 8) | (c[3]));
}
static void _addbits(sha256_context *ctx, uint32_t n)
{
__CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx));
if (ctx->bits[0] > (0xffffffff - n)) {
ctx->bits[1] = (ctx->bits[1] + 1) & 0xFFFFFFFF;
}
ctx->bits[0] = (ctx->bits[0] + n) & 0xFFFFFFFF;
} // _addbits
static void _hash(sha256_context *ctx)
{
__CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx));
register uint32_t a, b, c, d, e, f, g, h;
uint32_t t[2];
a = ctx->hash[0];
b = ctx->hash[1];
c = ctx->hash[2];
d = ctx->hash[3];
e = ctx->hash[4];
f = ctx->hash[5];
g = ctx->hash[6];
h = ctx->hash[7];
for (uint32_t i = 0; i < 64; i++) {
if (i < 16) {
ctx->W[i] = _word(&ctx->buf[_shw(i, 2)]);
} else {
ctx->W[i] = _G1(ctx->W[i - 2]) + ctx->W[i - 7] +
_G0(ctx->W[i - 15]) + ctx->W[i - 16];
}
t[0] = h + _S1(e) + _Ch(e, f, g) + K[i] + ctx->W[i];
t[1] = _S0(a) + _Ma(a, b, c);
h = g;
g = f;
f = e;
e = d + t[0];
d = c;
c = b;
b = a;
a = t[0] + t[1];
}
ctx->hash[0] += a;
ctx->hash[1] += b;
ctx->hash[2] += c;
ctx->hash[3] += d;
ctx->hash[4] += e;
ctx->hash[5] += f;
ctx->hash[6] += g;
ctx->hash[7] += h;
}
static void sha256_init(sha256_context *ctx)
{
if (ctx != NULL) {
ctx->bits[0] = ctx->bits[1] = ctx->len = 0;
ctx->hash[0] = 0x6a09e667;
ctx->hash[1] = 0xbb67ae85;
ctx->hash[2] = 0x3c6ef372;
ctx->hash[3] = 0xa54ff53a;
ctx->hash[4] = 0x510e527f;
ctx->hash[5] = 0x9b05688c;
ctx->hash[6] = 0x1f83d9ab;
ctx->hash[7] = 0x5be0cd19;
}
}
static void sha256_hash(sha256_context *ctx, const void *data, size_t len)
{
const uint8_t *bytes = (const uint8_t *)data;
if ((ctx != NULL) && (bytes != NULL) && (ctx->len < sizeof(ctx->buf))) {
__CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(bytes));
__CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx));
for (size_t i = 0; i < len; i++) {
ctx->buf[ctx->len++] = bytes[i];
if (ctx->len == sizeof(ctx->buf)) {
_hash(ctx);
_addbits(ctx, sizeof(ctx->buf) * 8);
ctx->len = 0;
}
}
}
}
static void sha256_done(sha256_context *ctx, uint8_t *hash)
{
register uint32_t i, j;
if (ctx != NULL) {
j = ctx->len % sizeof(ctx->buf);
ctx->buf[j] = 0x80;
for (i = j + 1; i < sizeof(ctx->buf); i++) {
ctx->buf[i] = 0x00;
}
if (ctx->len > 55) {
_hash(ctx);
for (j = 0; j < sizeof(ctx->buf); j++) {
ctx->buf[j] = 0x00;
}
}
_addbits(ctx, ctx->len * 8);
ctx->buf[63] = _shb(ctx->bits[0], 0);
ctx->buf[62] = _shb(ctx->bits[0], 8);
ctx->buf[61] = _shb(ctx->bits[0], 16);
ctx->buf[60] = _shb(ctx->bits[0], 24);
ctx->buf[59] = _shb(ctx->bits[1], 0);
ctx->buf[58] = _shb(ctx->bits[1], 8);
ctx->buf[57] = _shb(ctx->bits[1], 16);
ctx->buf[56] = _shb(ctx->bits[1], 24);
_hash(ctx);
if (hash != NULL) {
for (i = 0, j = 24; i < 4; i++, j -= 8) {
hash[i + 0] = _shb(ctx->hash[0], j);
hash[i + 4] = _shb(ctx->hash[1], j);
hash[i + 8] = _shb(ctx->hash[2], j);
hash[i + 12] = _shb(ctx->hash[3], j);
hash[i + 16] = _shb(ctx->hash[4], j);
hash[i + 20] = _shb(ctx->hash[5], j);
hash[i + 24] = _shb(ctx->hash[6], j);
hash[i + 28] = _shb(ctx->hash[7], j);
}
}
}
}
void sha256(const void *data, size_t len, uint8_t *hash)
{
sha256_context ctx;
sha256_init(&ctx);
sha256_hash(&ctx, data, len);
sha256_done(&ctx, hash);
}
src/sha256.h
-9
View File
@@ -1,9 +0,0 @@
#ifndef SHA256_INCLUDED
#define SHA256_INCLUDED
#include <stddef.h>
#include <stdint.h>
void sha256(const void *data, size_t len, uint8_t *hash);
#endif // SHA256_INCLUDED
src/test_client.c
-640
View File
@@ -1,640 +0,0 @@
#ifdef MAIN_TEST
#define QUAKEY_ENABLE_MOCKS
#include <quakey.h>
#include <assert.h>
#include "tcp.h"
#include "test_client.h"
// Helper function to parse address and port from command line
static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port)
{
// Default to metadata server
*addr = "127.0.0.1";
*port = 8080;
for (int i = 0; i < argc - 1; i++) {
if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) {
*addr = argv[i + 1];
if (i + 2 < argc) {
errno = 0;
char *end;
long val = strtol(argv[i+2], &end, 10);
if (end == argv[i+2] || *end != '\0' || errno == ERANGE)
break;
if (val < 0 || val > UINT16_MAX)
break;
*port = (uint16_t) val;
return true;
}
}
}
return true;
}
int test_client_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
TestClient *client = state_;
char *addr;
uint16_t port;
parse_server_addr(argc, argv, &addr, &port);
client->toasty = toasty_connect((ToastyString) { addr, strlen(addr) }, port);
if (client->toasty == NULL)
return -1;
client->state = TEST_CLIENT_STATE_INIT;
client->tick = 0;
client->tests_passed = 0;
printf("Client set up (remote=%s:%d)\n", addr, port);
fflush(stdout);
*timeout = -1;
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
return 0;
}
// Static test data
static char lorem_ipsum[] =
"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
"eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut "
"enim ad minim veniam, quis nostrud exercitation ullamco laboris "
"nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor "
"in reprehenderit in voluptate velit esse cillum dolore eu fugiat "
"nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
"sunt in culpa qui officia deserunt mollit anim id est laborum.";
static char short_msg[] = "Hello World!";
static char offset_msg[] = "INSERTED_DATA";
static char truncate_msg[] = "Truncated content";
// Test path constants
static ToastyString file_path = TOASTY_STR("some_file.txt");
static ToastyString dir_path = TOASTY_STR("/testdir");
static ToastyString subdir_path = TOASTY_STR("/testdir/subdir");
static ToastyString file_in_dir_path = TOASTY_STR("/testdir/nested_file.txt");
static ToastyString auto_create_path = TOASTY_STR("/auto_created_file.txt");
static ToastyString truncate_file_path = TOASTY_STR("/truncate_test.txt");
static ToastyString offset_file_path = TOASTY_STR("/offset_test.txt");
#define TEST_PASS(name) do { \
client->tests_passed++; \
printf("Test PASSED: %s\n", name); \
fflush(stdout); \
} while(0)
#define TEST_FAIL(name) do { \
printf("Test FAILED: %s\n", name); \
fflush(stdout); \
exit(1); \
} while(0)
int test_client_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
TestClient *client = state_;
// Process any pending events from the network and get new poll descriptors
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
int ret;
ToastyResult result;
switch (client->state) {
//==========================================================================
// PHASE 1: Basic file create/write/read test
//==========================================================================
case TEST_CLIENT_STATE_INIT:
if (client->tick < 10) {
*timeout = 0;
client->tick++;
return 0;
}
printf("\n=== Phase 1: Basic file operations ===\n");
client->handle = toasty_begin_create_file(client->toasty, file_path, 128);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create file");
}
printf("Create file started\n");
client->state = TEST_CLIENT_STATE_CREATE_FILE_WAIT;
break;
case TEST_CLIENT_STATE_CREATE_FILE_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create file");
if (ret == 1) break; // Still in progress
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create file result");
}
TEST_PASS("Create file");
toasty_free_result(&result);
// Start write
client->handle = toasty_begin_write(client->toasty, file_path, 0,
lorem_ipsum, sizeof(lorem_ipsum)-1, TOASTY_VERSION_TAG_EMPTY, 0);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin write");
}
printf("Write started\n");
client->state = TEST_CLIENT_STATE_WRITE_WAIT;
break;
case TEST_CLIENT_STATE_WRITE_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for write");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_WRITE_SUCCESS) {
TEST_FAIL("write result");
}
TEST_PASS("Write data");
toasty_free_result(&result);
// Start read
client->handle = toasty_begin_read(client->toasty, file_path, 0,
client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin read");
}
printf("Read started\n");
client->state = TEST_CLIENT_STATE_READ_WAIT;
break;
case TEST_CLIENT_STATE_READ_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for read");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_READ_SUCCESS) {
TEST_FAIL("read result type");
}
if (result.bytes_read != sizeof(lorem_ipsum)-1) {
printf("Expected %zu bytes, got %d\n", sizeof(lorem_ipsum)-1, result.bytes_read);
TEST_FAIL("read bytes count");
}
if (memcmp(client->buf, lorem_ipsum, sizeof(lorem_ipsum)-1)) {
TEST_FAIL("read data mismatch");
}
TEST_PASS("Read data matches written data");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_CREATE_DIR;
*timeout = 0; // Trigger immediate next iteration
return 0;
//==========================================================================
// PHASE 2: Directory operations
//==========================================================================
case TEST_CLIENT_STATE_CREATE_DIR:
printf("\n=== Phase 2: Directory operations ===\n");
client->handle = toasty_begin_create_dir(client->toasty, dir_path);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create dir");
}
printf("Create directory started\n");
client->state = TEST_CLIENT_STATE_CREATE_DIR_WAIT;
break;
case TEST_CLIENT_STATE_CREATE_DIR_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create dir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create dir result");
}
TEST_PASS("Create directory");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_CREATE_SUBDIR;
*timeout = 0; return 0;
case TEST_CLIENT_STATE_CREATE_SUBDIR:
client->handle = toasty_begin_create_dir(client->toasty, subdir_path);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create subdir");
}
printf("Create subdirectory started\n");
client->state = TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT;
break;
case TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create subdir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create subdir result");
}
TEST_PASS("Create subdirectory");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_CREATE_FILE_IN_DIR;
*timeout = 0; return 0;
case TEST_CLIENT_STATE_CREATE_FILE_IN_DIR:
client->handle = toasty_begin_create_file(client->toasty, file_in_dir_path, 256);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create file in dir");
}
printf("Create file in directory started\n");
client->state = TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT;
break;
case TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create file in dir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create file in dir result");
}
TEST_PASS("Create file in directory");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_LIST_DIR;
*timeout = 0; return 0;
case TEST_CLIENT_STATE_LIST_DIR:
client->handle = toasty_begin_list(client->toasty, dir_path, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin list dir");
}
printf("List directory started\n");
client->state = TEST_CLIENT_STATE_LIST_DIR_WAIT;
break;
case TEST_CLIENT_STATE_LIST_DIR_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for list dir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_LIST_SUCCESS) {
TEST_FAIL("list dir result type");
}
printf("Directory listing count: %d\n", result.listing.count);
// Directory should contain subdir and nested_file.txt
if (result.listing.count != 2) {
printf("Expected 2 entries, got %d\n", result.listing.count);
TEST_FAIL("list dir count");
}
// Print the entries
for (int i = 0; i < result.listing.count; i++) {
printf(" Entry %d: %s (is_dir=%d)\n", i,
result.listing.items[i].name,
result.listing.items[i].is_dir);
}
TEST_PASS("List directory");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_LIST_ROOT;
*timeout = 0; return 0;
case TEST_CLIENT_STATE_LIST_ROOT:
client->handle = toasty_begin_list(client->toasty, TOASTY_STR("/"), TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin list root");
}
printf("List root directory started\n");
client->state = TEST_CLIENT_STATE_LIST_ROOT_WAIT;
break;
case TEST_CLIENT_STATE_LIST_ROOT_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for list root");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_LIST_SUCCESS) {
TEST_FAIL("list root result type");
}
printf("Root listing count: %d\n", result.listing.count);
for (int i = 0; i < result.listing.count; i++) {
printf(" Entry %d: %s (is_dir=%d)\n", i,
result.listing.items[i].name,
result.listing.items[i].is_dir);
}
// Root should have some entries at this point
if (result.listing.count < 1) {
TEST_FAIL("list root count");
}
TEST_PASS("List root directory");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING;
*timeout = 0; return 0;
//==========================================================================
// PHASE 3: Write flags tests
//==========================================================================
case TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING:
printf("\n=== Phase 3: Additional write tests ===\n");
// Create a file for testing additional writes
client->handle = toasty_begin_create_file(client->toasty, auto_create_path, 256);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create auto file");
}
printf("Create auto file started\n");
client->state = TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT;
break;
case TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create auto file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create auto file result");
}
TEST_PASS("Create auto file");
toasty_free_result(&result);
// Now write to the file
client->handle = toasty_begin_write(client->toasty, auto_create_path, 0,
short_msg, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY, 0);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin write to auto file");
}
printf("Write to auto file started\n");
client->state = TEST_CLIENT_STATE_READ_CREATED_FILE;
break;
case TEST_CLIENT_STATE_READ_CREATED_FILE:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for write to auto file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_WRITE_SUCCESS) {
TEST_FAIL("write to auto file result");
}
TEST_PASS("Write to auto file");
toasty_free_result(&result);
// Read back the file
memset(client->buf, 0, sizeof(client->buf));
client->handle = toasty_begin_read(client->toasty, auto_create_path, 0,
client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin read auto file");
}
printf("Read auto file started\n");
client->state = TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT;
break;
case TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for read auto file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_READ_SUCCESS) {
TEST_FAIL("read auto file result type");
}
if (result.bytes_read != sizeof(short_msg)-1) {
printf("Expected %zu bytes, got %d\n", sizeof(short_msg)-1, result.bytes_read);
TEST_FAIL("read auto file bytes count");
}
if (memcmp(client->buf, short_msg, sizeof(short_msg)-1)) {
TEST_FAIL("read auto file data mismatch");
}
TEST_PASS("Read auto file");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_WRITE_TRUNCATE;
*timeout = 0; return 0;
case TEST_CLIENT_STATE_WRITE_TRUNCATE:
// Create a file for testing multiple writes
client->handle = toasty_begin_create_file(client->toasty, truncate_file_path, 256);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create multi-write file");
}
printf("Create multi-write test file started\n");
client->state = TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT;
break;
case TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create multi-write file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create multi-write file result");
}
TEST_PASS("Create multi-write file");
toasty_free_result(&result);
// Write initial content
client->handle = toasty_begin_write(client->toasty, truncate_file_path, 0,
lorem_ipsum, sizeof(lorem_ipsum)-1, TOASTY_VERSION_TAG_EMPTY, 0);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin write multi-write content");
}
printf("Write multi-write content started\n");
client->state = TEST_CLIENT_STATE_READ_TRUNCATED;
break;
case TEST_CLIENT_STATE_READ_TRUNCATED:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for write multi-write content");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_WRITE_SUCCESS) {
TEST_FAIL("write multi-write content result");
}
TEST_PASS("Write multi-write content");
toasty_free_result(&result);
// Read back the file
memset(client->buf, 0, sizeof(client->buf));
client->handle = toasty_begin_read(client->toasty, truncate_file_path, 0,
client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin read multi-write file");
}
printf("Read multi-write file started\n");
client->state = TEST_CLIENT_STATE_READ_TRUNCATED_WAIT;
break;
case TEST_CLIENT_STATE_READ_TRUNCATED_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for read multi-write file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_READ_SUCCESS) {
TEST_FAIL("read multi-write file result type");
}
if (result.bytes_read != sizeof(lorem_ipsum)-1) {
printf("Expected %zu bytes, got %d\n", sizeof(lorem_ipsum)-1, result.bytes_read);
TEST_FAIL("read multi-write file bytes count");
}
TEST_PASS("Read multi-write file");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_WRITE_AT_OFFSET;
*timeout = 0; return 0;
//==========================================================================
// PHASE 4: Offset read/write tests
//==========================================================================
case TEST_CLIENT_STATE_WRITE_AT_OFFSET:
printf("\n=== Phase 4: Offset read/write tests ===\n");
// Create a new file for offset testing with larger chunk size
client->handle = toasty_begin_create_file(client->toasty, offset_file_path, 256);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin create offset file");
}
printf("Create offset test file started\n");
client->state = TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT;
break;
case TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for create offset file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
TEST_FAIL("create offset file result");
}
TEST_PASS("Create offset file");
toasty_free_result(&result);
// Write initial data (small amount)
client->handle = toasty_begin_write(client->toasty, offset_file_path, 0,
short_msg, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY, 0);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin write initial offset data");
}
printf("Write initial offset data started\n");
client->state = TEST_CLIENT_STATE_READ_AT_OFFSET;
break;
case TEST_CLIENT_STATE_READ_AT_OFFSET:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for write initial offset");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_WRITE_SUCCESS) {
TEST_FAIL("write initial offset result");
}
TEST_PASS("Write initial offset data");
toasty_free_result(&result);
// Read from the beginning to verify
memset(client->buf, 0, sizeof(client->buf));
client->handle = toasty_begin_read(client->toasty, offset_file_path, 0,
client->buf, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin read offset file");
}
printf("Read offset file started\n");
client->state = TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT;
break;
case TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for read offset file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_READ_SUCCESS) {
TEST_FAIL("read offset file result type");
}
if (result.bytes_read != sizeof(short_msg)-1) {
printf("Expected %zu bytes, got %d\n", sizeof(short_msg)-1, result.bytes_read);
TEST_FAIL("read offset file bytes count");
}
TEST_PASS("Read offset file");
toasty_free_result(&result);
client->state = TEST_CLIENT_STATE_DELETE_FILE;
*timeout = 0; return 0;
//==========================================================================
// PHASE 5: Delete operations
//==========================================================================
case TEST_CLIENT_STATE_DELETE_FILE:
printf("\n=== Phase 5: Delete operations ===\n");
// Delete the auto-created file
client->handle = toasty_begin_delete(client->toasty, auto_create_path, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin delete file");
}
printf("Delete auto file started\n");
client->state = TEST_CLIENT_STATE_DELETE_FILE_WAIT;
break;
case TEST_CLIENT_STATE_DELETE_FILE_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for delete file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_DELETE_SUCCESS) {
TEST_FAIL("delete file result");
}
TEST_PASS("Delete file");
toasty_free_result(&result);
// Delete the nested file first (before deleting directory)
client->handle = toasty_begin_delete(client->toasty, file_in_dir_path, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin delete nested file");
}
printf("Delete nested file started\n");
client->state = TEST_CLIENT_STATE_DELETE_DIR;
break;
case TEST_CLIENT_STATE_DELETE_DIR:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for delete nested file");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_DELETE_SUCCESS) {
TEST_FAIL("delete nested file result");
}
TEST_PASS("Delete nested file");
toasty_free_result(&result);
// Delete the subdirectory
client->handle = toasty_begin_delete(client->toasty, subdir_path, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin delete subdir");
}
printf("Delete subdirectory started\n");
client->state = TEST_CLIENT_STATE_DELETE_DIR_WAIT;
break;
case TEST_CLIENT_STATE_DELETE_DIR_WAIT:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for delete subdir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_DELETE_SUCCESS) {
TEST_FAIL("delete subdir result");
}
TEST_PASS("Delete subdirectory");
toasty_free_result(&result);
// Delete the parent directory
client->handle = toasty_begin_delete(client->toasty, dir_path, TOASTY_VERSION_TAG_EMPTY);
if (client->handle == TOASTY_INVALID) {
TEST_FAIL("begin delete dir");
}
printf("Delete directory started\n");
client->state = TEST_CLIENT_STATE_DONE;
break;
case TEST_CLIENT_STATE_DONE:
ret = toasty_get_result(client->toasty, client->handle, &result);
if (ret < 0) TEST_FAIL("get result for delete dir");
if (ret == 1) break;
if (result.type != TOASTY_RESULT_DELETE_SUCCESS) {
TEST_FAIL("delete dir result");
}
TEST_PASS("Delete directory");
toasty_free_result(&result);
printf("\n========================================\n");
printf("All %d tests PASSED!\n", client->tests_passed);
printf("========================================\n");
QUAKEY_SIGNAL("exit");
}
*timeout = -1;
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, 0);
return 0;
}
int test_client_free(void *state_)
{
TestClient *client = state_;
toasty_disconnect(client->toasty);
return 0;
}
#endif // MAIN_TEST
src/test_client.h
-72
View File
@@ -1,72 +0,0 @@
#ifndef TEST_CLIENT_INCLUDED
#define TEST_CLIENT_INCLUDED
#include "ToastyFS.h"
typedef enum {
// Basic file create/write/read test (existing)
TEST_CLIENT_STATE_INIT,
TEST_CLIENT_STATE_CREATE_FILE_WAIT,
TEST_CLIENT_STATE_WRITE_WAIT,
TEST_CLIENT_STATE_READ_WAIT,
// Directory tests
TEST_CLIENT_STATE_CREATE_DIR,
TEST_CLIENT_STATE_CREATE_DIR_WAIT,
TEST_CLIENT_STATE_CREATE_SUBDIR,
TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT,
TEST_CLIENT_STATE_CREATE_FILE_IN_DIR,
TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT,
TEST_CLIENT_STATE_LIST_DIR,
TEST_CLIENT_STATE_LIST_DIR_WAIT,
TEST_CLIENT_STATE_LIST_ROOT,
TEST_CLIENT_STATE_LIST_ROOT_WAIT,
// Delete tests
TEST_CLIENT_STATE_DELETE_FILE,
TEST_CLIENT_STATE_DELETE_FILE_WAIT,
TEST_CLIENT_STATE_DELETE_DIR,
TEST_CLIENT_STATE_DELETE_DIR_WAIT,
// Write flags tests
TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING,
TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT,
TEST_CLIENT_STATE_READ_CREATED_FILE,
TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT,
TEST_CLIENT_STATE_WRITE_TRUNCATE,
TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT,
TEST_CLIENT_STATE_READ_TRUNCATED,
TEST_CLIENT_STATE_READ_TRUNCATED_WAIT,
// Offset read/write tests
TEST_CLIENT_STATE_WRITE_AT_OFFSET,
TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT,
TEST_CLIENT_STATE_READ_AT_OFFSET,
TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT,
// Test done
TEST_CLIENT_STATE_DONE,
} TestClientState;
typedef struct {
ToastyFS* toasty;
ToastyHandle handle;
TestClientState state;
int tick;
int tests_passed;
char buf[1<<10];
char buf2[1<<10]; // Secondary buffer for additional reads
} TestClient;
struct pollfd;
int test_client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int test_client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int test_client_free(void *state);
#endif // TEST_CLIENT_INCLUDED
src/wal.c
-630
View File
@@ -1,630 +0,0 @@
#if defined(MAIN_SIMULATION) || defined(MAIN_TEST)
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include "wal.h"
#include "file_tree.h"
#include "file_system.h"
#define WAL_MAGIC 0xcafebebe
#define WAL_VERSION 1
typedef struct {
uint32_t magic;
uint32_t version;
uint64_t reserved;
} WALHeader;
typedef enum {
WAL_ENTRY_CREATE,
WAL_ENTRY_DELETE,
WAL_ENTRY_WRITE,
} WALEntryType;
typedef struct {
WALEntryType type;
// create, delete, write
string path;
// create, write
uint64_t chunk_size;
// delete, write
uint64_t expect_gen;
// create
bool is_dir;
// write
uint64_t offset;
uint64_t length;
uint32_t num_chunks;
SHA256 *hashes;
} WALEntry;
static int write_exact(Handle handle, char *src, int len)
{
int copied = 0;
while (copied < len) {
int ret = file_write(handle, src + copied, len - copied);
if (ret < 0)
return -1;
copied += ret;
}
return 0;
}
typedef struct {
Handle handle;
} WriteSnapshotContext;
static int
serialize_callback(char *src, int num, void *data)
{
WriteSnapshotContext *wsc = data;
return write_exact(wsc->handle, src, num);
}
static int write_snapshot(FileTree *file_tree, Handle handle)
{
WriteSnapshotContext wsc;
wsc.handle = handle;
if (file_tree_serialize(file_tree, serialize_callback, &wsc) < 0)
return -1;
return 0;
}
typedef struct {
Handle handle;
} ReadSnapshotContext;
static int
deserialize_callback(char *dst, int num, void *data)
{
ReadSnapshotContext *rsc = data;
int copied = 0;
while (copied < num) {
int ret = file_read(rsc->handle, dst + copied, num - copied);
if (ret < 0)
return -1;
if (ret == 0)
break;
copied += ret;
}
return copied;
}
static int read_snapshot(FileTree *file_tree, Handle handle)
{
ReadSnapshotContext rsc;
rsc.handle = handle;
int num = file_tree_deserialize(file_tree, deserialize_callback, &rsc);
if (num < 0)
return -1;
return num;
}
static int swap_file(WAL *wal)
{
// Create a temporary file path
char temp_path_buf[1<<10];
if (wal->file_path.len + 5 > (int) sizeof(temp_path_buf))
return -1;
memcpy(temp_path_buf, wal->file_path.ptr, wal->file_path.len);
memcpy(temp_path_buf + wal->file_path.len, ".tmp", 4);
temp_path_buf[wal->file_path.len + 4] = '\0';
string temp_path = { temp_path_buf, wal->file_path.len + 4 };
// Create and open the temporary file
Handle temp_handle;
if (file_open(temp_path, &temp_handle) < 0)
return -1;
// Write the WAL header
WALHeader header;
header.magic = WAL_MAGIC;
header.version = WAL_VERSION;
header.reserved = 0;
if (write_exact(temp_handle, (char*) &header, sizeof(header)) < 0) {
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
// Serialize the current file tree to the new file
if (write_snapshot(wal->file_tree, temp_handle) < 0) {
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
// Sync the temporary file to ensure it's written to disk
if (file_sync(temp_handle) < 0) {
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
// Lock the new file before switching
if (file_lock(temp_handle) < 0) {
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
// Unlock and close the old file
file_unlock(wal->handle);
file_close(wal->handle);
// Atomically rename the temporary file to replace the old file
// On Unix: rename() atomically replaces the destination
// On Windows: we use MoveFileEx with MOVEFILE_REPLACE_EXISTING for atomicity
#ifdef _WIN32
// On Windows, use MoveFileEx for atomic replace
char old_path_zt[1<<10];
if (wal->file_path.len >= (int) sizeof(old_path_zt)) {
file_unlock(temp_handle);
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
memcpy(old_path_zt, wal->file_path.ptr, wal->file_path.len);
old_path_zt[wal->file_path.len] = '\0';
WCHAR old_path_w[MAX_PATH];
WCHAR temp_path_w[MAX_PATH];
if (!MultiByteToWideChar(CP_UTF8, 0, old_path_zt, -1, old_path_w, MAX_PATH) ||
!MultiByteToWideChar(CP_UTF8, 0, temp_path_buf, -1, temp_path_w, MAX_PATH)) {
file_unlock(temp_handle);
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
// MOVEFILE_REPLACE_EXISTING allows atomic overwrite
if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
file_unlock(temp_handle);
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
#else
// On Unix/Linux, rename() atomically replaces the destination
if (rename_file_or_dir(temp_path, wal->file_path) < 0) {
file_unlock(temp_handle);
file_close(temp_handle);
remove_file_or_dir(temp_path);
return -1;
}
#endif
// Update the WAL to use the new file handle
wal->handle = temp_handle;
wal->entry_count = 0;
return 0;
}
static int read_exact(Handle handle, char *dst, int len)
{
int copied = 0;
while (copied < len) {
int ret = file_read(handle, dst + copied, len - copied);
if (ret < 0)
return -1;
if (ret == 0)
return 0; // EOF
copied += ret;
}
return copied;
}
static int read_u8(Handle handle, uint8_t *value)
{
return read_exact(handle, (char*) value, sizeof(*value));
}
static int read_u16(Handle handle, uint16_t *value)
{
return read_exact(handle, (char*) value, sizeof(*value));
}
static int read_u32(Handle handle, uint32_t *value)
{
return read_exact(handle, (char*) value, sizeof(*value));
}
static int read_u64(Handle handle, uint64_t *value)
{
return read_exact(handle, (char*) value, sizeof(*value));
}
static int next_entry(Handle handle, WALEntry *entry)
{
// Initialize pointers to NULL for cleanup on error
entry->path.ptr = NULL;
entry->hashes = NULL;
uint8_t type;
int ret = read_u8(handle, &type);
if (ret == 0)
return 0; // EOF
if (ret < 0)
return -1;
entry->type = (WALEntryType) type;
uint16_t path_len;
if (read_u16(handle, &path_len) <= 0)
return -1;
// Dynamically allocate path buffer
char *path_buffer = malloc(path_len);
if (!path_buffer)
return -1;
if (read_exact(handle, path_buffer, path_len) <= 0) {
free(path_buffer);
return -1;
}
entry->path.ptr = path_buffer;
entry->path.len = path_len;
switch (entry->type) {
case WAL_ENTRY_CREATE:
{
uint8_t is_dir;
if (read_u8(handle, &is_dir) <= 0)
goto cleanup_error;
entry->is_dir = is_dir;
if (!is_dir) {
if (read_u64(handle, &entry->chunk_size) <= 0)
goto cleanup_error;
} else {
entry->chunk_size = 0;
}
}
break;
case WAL_ENTRY_DELETE:
{
if (read_u64(handle, &entry->expect_gen) <= 0)
goto cleanup_error;
}
break;
case WAL_ENTRY_WRITE:
{
if (read_u64(handle, &entry->expect_gen) <= 0)
goto cleanup_error;
if (read_u64(handle, &entry->offset) <= 0)
goto cleanup_error;
if (read_u64(handle, &entry->length) <= 0)
goto cleanup_error;
if (read_u32(handle, &entry->num_chunks) <= 0)
goto cleanup_error;
// Dynamically allocate hash buffers
SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256));
if (!hashes_buffer) {
goto cleanup_error;
}
if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
free(hashes_buffer);
goto cleanup_error;
}
entry->hashes = hashes_buffer;
}
break;
default:
goto cleanup_error;
}
return 1;
cleanup_error:
if (entry->path.ptr)
free((char*) entry->path.ptr);
if (entry->hashes)
free(entry->hashes);
return -1;
}
int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
{
wal->entry_count = 0;
wal->entry_limit = entry_limit;
wal->file_tree = file_tree;
wal->file_path.ptr = NULL;
// Copy file_path since the passed string may not have the same lifetime as WAL
char *path_copy = malloc(file_path.len);
if (!path_copy)
return -1;
memcpy(path_copy, file_path.ptr, file_path.len);
wal->file_path.ptr = path_copy;
wal->file_path.len = file_path.len;
Handle handle;
if (file_open(file_path, &handle) < 0)
goto error_cleanup_path;
if (file_lock(handle) < 0) {
file_close(handle);
goto error_cleanup_path;
}
// Check if the file is empty (newly created) and initialize it
size_t size;
if (file_size(handle, &size) < 0) {
file_close(handle);
goto error_cleanup_path;
}
if (size == 0) {
// Initialize a new WAL file
WALHeader header;
header.magic = WAL_MAGIC;
header.version = WAL_VERSION;
header.reserved = 0;
if (write_exact(handle, (char*) &header, sizeof(header)) < 0) {
file_close(handle);
goto error_cleanup_path;
}
if (write_snapshot(file_tree, handle) < 0) {
file_close(handle);
goto error_cleanup_path;
}
if (file_sync(handle) < 0) {
file_close(handle);
goto error_cleanup_path;
}
// Reset to beginning after initialization
if (file_set_offset(handle, 0) < 0) {
file_close(handle);
goto error_cleanup_path;
}
}
// Read file header
// NOTE: For now we don't worry about fixing endianess
WALHeader header;
for (int copied = 0; copied < (int) sizeof(header); ) {
int ret = file_read(handle, (char*) &header + copied, (int) sizeof(header) - copied);
if (ret <= 0) {
file_close(handle); // TODO: what happens if I close a file without unlocking it?
goto error_cleanup_path;
}
copied += ret;
}
// Validate header fields
if (header.magic != WAL_MAGIC) {
file_close(handle);
goto error_cleanup_path;
}
if (header.version != WAL_VERSION) {
file_close(handle);
goto error_cleanup_path;
}
// The read_snapshot function may read more
// bytes than necessary from the buffer, so
// we need to save our current position to
// later restore it to this offset plus what
// read_snapshot really consumed.
int saved_offset;
if (file_get_offset(handle, &saved_offset) < 0) {
file_close(handle);
goto error_cleanup_path;
}
int num = read_snapshot(file_tree, handle);
if (num < 0) {
file_close(handle);
goto error_cleanup_path;
}
// Now restore the offset to the correct position.
if (num > INT_MAX - saved_offset) {
file_close(handle);
goto error_cleanup_path;
}
if (file_set_offset(handle, saved_offset + num) < 0) {
file_close(handle);
goto error_cleanup_path;
}
WALEntry entry;
for (;;) {
int ret = next_entry(handle, &entry);
if (ret == 0)
break;
if (ret < 0) {
file_close(handle);
goto error_cleanup_path;
}
assert(ret == 1);
switch (entry.type) {
uint64_t gen;
case WAL_ENTRY_CREATE:
file_tree_create_entity(file_tree, entry.path, entry.is_dir, entry.chunk_size, &gen);
break;
case WAL_ENTRY_DELETE:
file_tree_delete_entity(file_tree, entry.path, entry.expect_gen);
break;
case WAL_ENTRY_WRITE:
// WAL replay: use false for truncate_after since truncation was already handled
file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.expect_gen, &gen, entry.hashes, NULL, NULL, false);
break;
default:
UNREACHABLE;
}
// Free dynamically allocated fields from next_entry
if (entry.path.ptr)
free((char*) entry.path.ptr);
if (entry.hashes)
free(entry.hashes);
wal->entry_count++;
}
wal->handle = handle;
if (wal->entry_count >= wal->entry_limit) {
if (swap_file(wal) < 0) {
goto error_cleanup_path;
}
}
return 0;
error_cleanup_path:
free(path_copy);
return -1;
}
void wal_close(WAL *wal)
{
file_unlock(wal->handle);
file_close(wal->handle);
if (wal->file_path.ptr)
free((char*) wal->file_path.ptr);
}
static int write_u8(Handle handle, uint8_t value)
{
return write_exact(handle, (char*) &value, sizeof(value));
}
static int write_u16(Handle handle, uint16_t value)
{
return write_exact(handle, (char*) &value, sizeof(value));
}
static int write_u32(Handle handle, uint32_t value)
{
return write_exact(handle, (char*) &value, sizeof(value));
}
static int write_u64(Handle handle, uint64_t value)
{
return write_exact(handle, (char*) &value, sizeof(value));
}
static int write_str(Handle handle, string value)
{
return write_exact(handle, value.ptr, value.len);
}
static int append_begin(WAL *wal)
{
if (wal->entry_count >= wal->entry_limit) {
if (swap_file(wal) < 0)
return -1;
}
return 0;
}
static int append_end(WAL *wal)
{
if (file_sync(wal->handle) < 0)
return -1;
wal->entry_count++;
return 0;
}
int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size)
{
if (path.len > UINT16_MAX)
return -1;
if (append_begin(wal) < 0)
return -1;
write_u8(wal->handle, WAL_ENTRY_CREATE);
write_u16(wal->handle, path.len);
write_str(wal->handle, path);
write_u8(wal->handle, is_dir);
if (!is_dir)
write_u64(wal->handle, chunk_size);
if (append_end(wal) < 0)
return -1;
return 0;
}
int wal_append_delete(WAL *wal, string path, uint64_t expect_gen)
{
if (path.len > UINT16_MAX)
return -1;
if (append_begin(wal) < 0)
return -1;
// TODO: check for errors
write_u8(wal->handle, WAL_ENTRY_DELETE);
write_u16(wal->handle, path.len);
write_str(wal->handle, path);
write_u64(wal->handle, expect_gen);
if (append_end(wal) < 0)
return -1;
return 0;
}
int wal_append_write(WAL *wal, string path, uint64_t off,
uint64_t len, uint32_t num_chunks, uint64_t expect_gen,
SHA256 *hashes)
{
if (path.len > UINT16_MAX)
return -1;
if (append_begin(wal) < 0)
return -1;
if (write_u8(wal->handle, WAL_ENTRY_WRITE) < 0)
return -1;
if (write_u16(wal->handle, path.len) < 0)
return -1;
if (write_str(wal->handle, path) < 0)
return -1;
if (write_u64(wal->handle, expect_gen) < 0)
return -1;
if (write_u64(wal->handle, off) < 0)
return -1;
if (write_u64(wal->handle, len) < 0)
return -1;
if (write_u32(wal->handle, num_chunks) < 0)
return -1;
if (write_exact(wal->handle, (char*) hashes, num_chunks * sizeof(SHA256)) < 0)
return -1;
if (append_end(wal) < 0)
return -1;
return 0;
}
src/wal.h
-27
View File
@@ -1,27 +0,0 @@
#ifndef WAL_INCLUDED
#define WAL_INCLUDED
#include "file_tree.h"
#include "file_system.h"
typedef struct {
Handle handle;
FileTree *file_tree;
string file_path;
int entry_count;
int entry_limit;
} WAL;
int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit);
void wal_close(WAL *wal);
int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size);
int wal_append_delete(WAL *wal, string path, uint64_t expect_gen);
int wal_append_write(WAL *wal, string path, uint64_t off,
uint64_t len, uint32_t num_chunks, uint64_t expect_gen,
SHA256 *hashes);
#endif // WAL_INCLUDED