From b64517c20e319871ea903066a49a187d08800732 Mon Sep 17 00:00:00 2001 From: Francesco Cozzuto Date: Fri, 20 Feb 2026 16:55:29 +0100 Subject: [PATCH] Full rewrite with VSR support --- .gitattributes | 2 + .gitignore | 12 +- TODO.txt | 28 - build.sh | 7 +- build_coverage.sh | 37 - build_feature.sh | 36 - include/ToastyFS.h | 263 ---- {src => lib}/basic.c | 104 ++ {src => lib}/basic.h | 7 + {src => lib}/byte_queue.c | 0 {src => lib}/byte_queue.h | 0 {src => lib}/file_system.c | 59 +- {src => lib}/file_system.h | 5 + lib/message.c | 81 ++ lib/message.h | 46 + {src => lib}/tcp.c | 62 +- {src => lib}/tcp.h | 3 +- quakey/include/quakey.h | 34 +- quakey/src/mockfs.c | 57 + quakey/src/mockfs.h | 1 + quakey/src/quakey.c | 809 ++++++++++- spec/design.txt | 233 --- spec/protocol.txt | 281 ---- spec/spec.txt | 482 ------- src/blob_client.c | 675 +++++++++ src/blob_client.h | 86 ++ src/chunk_server.c | 1069 -------------- src/chunk_server.h | 70 - src/chunk_store.c | 107 ++ src/chunk_store.h | 20 + src/client.c | 2766 ++++-------------------------------- src/client.h | 49 + src/client_table.c | 70 + src/client_table.h | 31 + src/config.h | 18 +- src/file_tree.c | 820 ----------- src/file_tree.h | 97 -- src/hash_set.c | 171 --- src/hash_set.h | 38 - src/invariant_checker.c | 445 ++++++ src/log.c | 55 + src/log.h | 30 + src/main.c | 479 +++---- src/message.c | 485 ------- src/message.h | 96 -- src/metadata.c | 136 ++ src/metadata.h | 69 + src/metadata_server.c | 1281 ----------------- src/metadata_server.h | 66 - src/random_client.c | 278 ---- src/random_client.h | 39 - src/server.c | 2006 ++++++++++++++++++++++++++ src/server.h | 301 ++++ src/sha256.c | 249 ---- src/sha256.h | 9 - src/test_client.c | 640 --------- src/test_client.h | 72 - src/wal.c | 630 -------- src/wal.h | 27 - 59 files changed, 5790 insertions(+), 10339 deletions(-) create mode 100644 .gitattributes delete mode 100644 TODO.txt delete mode 100644 build_coverage.sh delete mode 100755 build_feature.sh delete mode 100644 include/ToastyFS.h rename {src => lib}/basic.c (60%) rename {src => lib}/basic.h (83%) rename {src => lib}/byte_queue.c (100%) rename {src => lib}/byte_queue.h (100%) rename {src => lib}/file_system.c (87%) rename {src => lib}/file_system.h (85%) create mode 100644 lib/message.c create mode 100644 lib/message.h rename {src => lib}/tcp.c (91%) rename {src => lib}/tcp.h (96%) delete mode 100644 spec/design.txt delete mode 100644 spec/protocol.txt delete mode 100644 spec/spec.txt create mode 100644 src/blob_client.c create mode 100644 src/blob_client.h delete mode 100644 src/chunk_server.c delete mode 100644 src/chunk_server.h create mode 100644 src/chunk_store.c create mode 100644 src/chunk_store.h create mode 100644 src/client.h create mode 100644 src/client_table.c create mode 100644 src/client_table.h delete mode 100644 src/file_tree.c delete mode 100644 src/file_tree.h delete mode 100644 src/hash_set.c delete mode 100644 src/hash_set.h create mode 100644 src/invariant_checker.c create mode 100644 src/log.c create mode 100644 src/log.h delete mode 100644 src/message.c delete mode 100644 src/message.h create mode 100644 src/metadata.c create mode 100644 src/metadata.h delete mode 100644 src/metadata_server.c delete mode 100644 src/metadata_server.h delete mode 100644 src/random_client.c delete mode 100644 src/random_client.h create mode 100644 src/server.c create mode 100644 src/server.h delete mode 100644 src/sha256.c delete mode 100644 src/sha256.h delete mode 100644 src/test_client.c delete mode 100644 src/test_client.h delete mode 100644 src/wal.c delete mode 100644 src/wal.h diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..09594c7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Enforce LF line endings for all text files +* text eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore index a342c39..436edca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,9 @@ -toasty_metadata_server -toasty_chunk_server -toasty_test -toasty_test_coverage -toasty_simulation -toasty_simulation_coverage +toastyfs +toastyfs_client +toastyfs_simulation *.gcno *.gcda *.gcov *.info -coverage_html/ \ No newline at end of file +coverage_html/ +*.log \ No newline at end of file diff --git a/TODO.txt b/TODO.txt deleted file mode 100644 index 78d24e8..0000000 --- a/TODO.txt +++ /dev/null @@ -1,28 +0,0 @@ -[X] Implement proper operation handles that allow users to wait for specific events -[X] Implement metadata server WAL -[X] Check that the corner case where read and writes go over the length of the file work correctly -[X] Return the number of bytes read or written in the ToastyFS_Result struct -[X] Add a change counter to FileTree. The counter is initialized to 0 and updated any time the tree is changed. When a new file is created, the current change counter is assigned to it. This makes it possible to verify that the file didn't change in between read and write operations without that clumsy previous hash list and previous chunk size. -[X] Add a flag for write operations to create the file if it doesn't exist alraedy -[X] Add failt injections to the simulation -[ ] Test node crash -[ ] Reference count chunks -[ ] Clean up the read operation for clients -[ ] Make parallel uploads/downloads configurable -[ ] Recalculate next write locations whenever a write occurs, not at each read -[ ] add logging (when chunk servers connect and disconnect to the metadata server) -[ ] Should list scenarios that need testing, like those where chunks would be dropped -[ ] Update DESIGN.txt and the code to remove the chunk list message. The information of chunks held by chunk servers is now transmitted to the metadata server during state updates -[ ] clean up the replication factor business -[ ] Add authentication -[ ] Add release build and allow switching between debug, release, coverage, sanitizer builds while calling make -[ ] When an operation is committed to the WAL, it may then fail due to not deterministic reasons (out of memory). When the metadata server restarts and replays the log, the operation may succede and cause the system to diverge from the last instance of the process. Is this not a problem? How do we solve it? -[ ] Add notes on who inspired the testing approach -[ ] Add notes on benchmarks and that we don't do batching -[ ] Fix endianess when writing to network and the WAL -[ ] WAL entry checksum -[ ] Check write() errors when appending WAL entries -[ ] When WAL entries are partially written, remove partial data and fail -[ ] Find a way to ensure listing operations of large directories are handled -[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication? -[ ] Rename generation counters to version numbers \ No newline at end of file diff --git a/build.sh b/build.sh index c9abd40..39ee41e 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,3 @@ -#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c quakey/src/mockfs.c quakey/src/quakey.c -o toasty_simulation -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION -gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c quakey/src/mockfs.c quakey/src/quakey.c -o toasty_test -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_TEST -#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c -o toasty_metadata_server -DMAIN_METADATA_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include -#gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c -o toasty_chunk_server -DMAIN_CHUNK_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/invariant_checker.c src/chunk_store.c src/metadata.c quakey/src/mockfs.c quakey/src/quakey.c -o toastyfs_simulation -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION -DFAULT_INJECTION +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_SERVER +gcc lib/basic.c lib/file_system.c lib/byte_queue.c lib/message.c lib/tcp.c src/server.c src/client.c src/blob_client.c src/main.c src/log.c src/client_table.c src/chunk_store.c src/metadata.c -o toastyfs_client -Iquakey/include -I. -Wall -Wextra -ggdb -O0 -DMAIN_CLIENT diff --git a/build_coverage.sh b/build_coverage.sh deleted file mode 100644 index e7097ed..0000000 --- a/build_coverage.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -SIMULATION_BINARY="toasty_simulation_coverage" -RUN_DURATION=60 -COVERAGE_DIR="coverage_html" - -echo "=== Step 1: Building simulation with coverage instrumentation ===" -gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c \ - src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c \ - src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c \ - quakey/src/mockfs.c quakey/src/quakey.c \ - -o "$SIMULATION_BINARY" \ - -Iquakey/include -Iinclude \ - -Wall -Wextra -ggdb -O0 \ - -DMAIN_SIMULATION \ - --coverage -fprofile-arcs -ftest-coverage -echo "Build complete: $SIMULATION_BINARY" - -echo "" -echo "=== Step 2: Running simulation for ${RUN_DURATION}s ===" -lcov --zerocounters --directory . --quiet -lcov --capture --initial --directory . --output-file coverage_base.info --rc lcov_branch_coverage=1 --quiet -timeout --signal=INT --kill-after=5 "$RUN_DURATION" "./$SIMULATION_BINARY" || true -echo "Simulation finished" - -echo "" -echo "=== Step 3: Generating coverage report ===" -lcov --capture --directory . --output-file coverage_test.info --rc lcov_branch_coverage=1 --quiet -lcov --add-tracefile coverage_base.info --add-tracefile coverage_test.info --output-file coverage.info --rc lcov_branch_coverage=1 --quiet -lcov --remove coverage.info '/usr/*' 'quakey/*' --output-file coverage.info --rc lcov_branch_coverage=1 --quiet -genhtml coverage.info --output-directory "$COVERAGE_DIR" --branch-coverage --quiet -rm -f coverage_base.info coverage_test.info - -echo "" -echo "=== Done ===" -echo "Coverage report: $COVERAGE_DIR/index.html" diff --git a/build_feature.sh b/build_feature.sh deleted file mode 100755 index c059183..0000000 --- a/build_feature.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -TEST_BINARY="toasty_test_coverage" -COVERAGE_DIR="coverage_html" - -echo "=== Step 1: Building test with coverage instrumentation ===" -gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c \ - src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c \ - src/metadata_server.c src/chunk_server.c src/random_client.c src/test_client.c src/main.c \ - quakey/src/mockfs.c quakey/src/quakey.c \ - -o "$TEST_BINARY" \ - -Iquakey/include -Iinclude \ - -Wall -Wextra -ggdb -O0 \ - -DMAIN_TEST \ - --coverage -fprofile-arcs -ftest-coverage -echo "Build complete: $TEST_BINARY" - -echo "" -echo "=== Step 2: Running tests ===" -lcov --zerocounters --directory . --quiet -lcov --capture --initial --directory . --output-file coverage_base.info --rc lcov_branch_coverage=1 --quiet -"./$TEST_BINARY" -echo "Tests finished" - -echo "" -echo "=== Step 3: Generating coverage report ===" -lcov --capture --directory . --output-file coverage_test.info --rc lcov_branch_coverage=1 --quiet -lcov --add-tracefile coverage_base.info --add-tracefile coverage_test.info --output-file coverage.info --rc lcov_branch_coverage=1 --quiet -lcov --remove coverage.info '/usr/*' 'quakey/*' --output-file coverage.info --rc lcov_branch_coverage=1 --quiet -genhtml coverage.info --output-directory "$COVERAGE_DIR" --branch-coverage --quiet -rm -f coverage_base.info coverage_test.info - -echo "" -echo "=== Done ===" -echo "Coverage report: $COVERAGE_DIR/index.html" diff --git a/include/ToastyFS.h b/include/ToastyFS.h deleted file mode 100644 index 0df6557..0000000 --- a/include/ToastyFS.h +++ /dev/null @@ -1,263 +0,0 @@ -#ifndef TOASTY_INCLUDED -#define TOASTY_INCLUDED -////////////////////////////////////////////////////////////////////////////////// -// INCLUDES -////////////////////////////////////////////////////////////////////////////////// - -#include -#include - -////////////////////////////////////////////////////////////////////////////////// -// TYPES & UTILITIES -////////////////////////////////////////////////////////////////////////////////// - -typedef struct ToastyFS ToastyFS; - -// Helper type to avoid zero-terminated strings -typedef struct { - char *ptr; - int len; -} ToastyString; - -// Macro to convert string literals to ToastyStrings -#define TOASTY_STR(X) ((ToastyString) { (X), (int) sizeof(X)-1 }) - -////////////////////////////////////////////////////////////////////////////////// -// PRIMARY -////////////////////////////////////////////////////////////////////////////////// - -// Instanciate a ToastyFS client object. The "addr" and "port" -// arguments refer to the address and port of the cluster's -// metadata server. -ToastyFS *toasty_connect(ToastyString addr, uint16_t port); - -// Release all resources associated to this client -void toasty_disconnect(ToastyFS *toasty); - -// Threads can call this function to wake up a thread blocked -// inside "toasty_wait_result". -// Return 0 on success, -1 on error. -int toasty_wakeup(ToastyFS *toasty); - -////////////////////////////////////////////////////////////////////////////////// -// BLOCKING API -////////////////////////////////////////////////////////////////////////////////// - -// TODO: comment -typedef uint64_t ToastyVersionTag; - -// TODO: comment -#define TOASTY_VERSION_TAG_EMPTY ((ToastyVersionTag) 0) - -// Write operation flags -#define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) // Create file if it doesn't exist -#define TOASTY_WRITE_TRUNCATE_AFTER (1 << 1) // Truncate file after write offset+length - -// Creates a directory at the specified path. -// Returns 0 on success, -1 on error. -// -// If the version tag is not NULL, it's used to -// return the version tag associated to the newly -// created file. -int toasty_create_dir(ToastyFS *toasty, ToastyString path, - ToastyVersionTag *vtag); - -// Creates a file with the given chunk size at -// the specified path. Returns 0 on success, -1 -// on error. The chunk size can't be 0. -// -// If the version tag is not NULL, it's used to -// return the version tag associated to the newly -// created file. -int toasty_create_file(ToastyFS *toasty, ToastyString path, - unsigned int chunk_size, ToastyVersionTag *vtag); - -// Deletes a file or directory at the specified path. -// Returns 0 on success, -1 on error. -// -// If the version tag is not 0, the file/directory -// is only deleted if the tags match. -int toasty_delete(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag); - -typedef struct { - char name[128]; // TODO: Implement a proper name length - bool is_dir; - ToastyVersionTag vtag; -} ToastyListingEntry; - -typedef struct { - int count; - ToastyListingEntry *items; -} ToastyListing; - -// Lists all files and directories within the given -// path. Returns 0 and fills up the listing argument -// on success, returns -1 on error. The listing is -// a dynamic array that needs to be freed using -// "toasy_free_listing". -// -// If the version tag is not 0, the listing only -// succedes if the tag matches the remote one. -// If the operation succedes, the vtag is set to -// the remote tag. -int toasty_list(ToastyFS *toasty, ToastyString path, - ToastyListing *listing, ToastyVersionTag *vtag); - -// Frees a listing created by "toasty_list". -void toasty_free_listing(ToastyListing *listing); - -// Reads "len" bytes at offset "off" from the file at -// the given path. Returns the number of bytes read on -// success, or -1 on error. -// -// If vtag is not NULL, the read only succedes if the -// target version tag matches vtag or vtag was 0. If -// the operation succedes, the version tag is set to -// the remote entity's. -int toasty_read(ToastyFS *toasty, ToastyString path, int off, - void *dst, int len, ToastyVersionTag *vtag); - -// Writes "len" bytes at offset "off" to the file at -// the given path. Returns the number of bytes written -// on success, or -1 on error. -// -// For how vtag works, see toasty_read. -// -// The flags parameter accepts the following values: -// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it -// doesn't exist. A default chunk size of 4096 bytes will be used. -// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len, -// discarding any data beyond the write range. Useful for HTTP PUT semantics. -int toasty_write(ToastyFS *toasty, ToastyString path, int off, - void *src, int len, ToastyVersionTag *vtag, uint32_t flags); - -////////////////////////////////////////////////////////////////////////////////// -// ASYNCHRONOUS API -////////////////////////////////////////////////////////////////////////////////// - -// Handle representing a pending operation -typedef uint32_t ToastyHandle; - -// Invalid value for a ToastyHandle -#define TOASTY_INVALID ((ToastyHandle) 0) - -// Begins a directory creation operation and returns -// a handle to it. On error, TOASTY_INVALID is returned. -ToastyHandle toasty_begin_create_dir(ToastyFS *toasty, ToastyString path); - -// Begins a file creation operation and returns a -// handle to it. On error, TOASTY_INVALID is returned. -ToastyHandle toasty_begin_create_file(ToastyFS *toasty, ToastyString path, - unsigned int chunk_size); - -// Begins a file or directory deletion operation and -// returns a handle to it. On error, TOASTY_INVALID is -// returned. -// -// If vtag is not 0, the operation only succedes if the -// tag matches the remote entity's. -ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path, - ToastyVersionTag vtag); - -// Begins a directory listing operation and returns -// a handle to it. On error, TOASTY_INVALID is returned. -// -// If vtag is not 0, the operation only succedes if the -// tag matches the remote entity's. -ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag); - -// Begins a read operation and returns a handle to it. -// On error, TOASTY_INVALID is returned. -// -// If vtag is not 0, the operation only succedes if the -// tag matches the remote entity's. -ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, - int off, void *dst, int len, ToastyVersionTag vtag); - -// Begins a write operation and returns a handle to it. -// On error, TOASTY_INVALID is returned. Note that the source -// buffer must be valid until the operation completes. -// -// If vtag is not 0, the operation only succedes if the -// tag matches the remote entity's. -// -// The flags parameter accepts the following values: -// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it -// doesn't exist. A default chunk size of 4096 bytes will be used. -// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len, -// discarding any data beyond the write range. Useful for HTTP PUT semantics. -ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, - int off, void *src, int len, ToastyVersionTag vtag, uint32_t flags); - -// Associate the pointer "user" to the handle. The user -// pointer will be returned in the ToastyResult when the -// operation compltes. -void toasty_set_user(ToastyFS *toasty, ToastyHandle handle, void *user); - -typedef enum { - TOASTY_RESULT_EMPTY, - TOASTY_RESULT_CREATE_ERROR, - TOASTY_RESULT_CREATE_SUCCESS, - TOASTY_RESULT_DELETE_ERROR, - TOASTY_RESULT_DELETE_SUCCESS, - TOASTY_RESULT_LIST_ERROR, - TOASTY_RESULT_LIST_SUCCESS, - TOASTY_RESULT_READ_ERROR, - TOASTY_RESULT_READ_SUCCESS, - TOASTY_RESULT_WRITE_ERROR, - TOASTY_RESULT_WRITE_SUCCESS, -} ToastyResultType; - -typedef struct { - ToastyResultType type; - ToastyListing listing; - ToastyVersionTag vtag; - void* user; - int bytes_read; // For read operations: actual number of bytes read -} ToastyResult; - -// If the operation specified by "handle" is complete, -// its result is stored in "result" and 0 is returned. -// If the operation is still in progress, 1 is returned. -// On error, -1 is returned. If the handle is TOASTY_INVALID, -// then the result of the first complete operation is -// returned. -// Note that if a result is returned, handles to that -// operation are invalidated. -// The "result" must be freed using "toasty_free_result". -// -// Note: -// If you call this function in a loop, the state of the -// operation will not progress. You can't wait for completion -// by calling this in a loop. You need to use "toasty_wait_result" -int toasty_get_result(ToastyFS *toasty, ToastyHandle handle, - ToastyResult *result); - -// Blocks execution until an operation is complete. This works -// like "toasty_get_result", except it waits for "timeout" milliseconds -// if the result isn't available. If "timeout" is -1, it waits -// indefinitely. -// The "result" must be freed using "toasty_free_result". -int toasty_wait_result(ToastyFS *toasty, ToastyHandle handle, - ToastyResult *result, int timeout); - -// Frees resources of a "ToastyResult" previously initialized -// by "toasty_get_result" or "toasty_wait_result". -void toasty_free_result(ToastyResult *result); - -////////////////////////////////////////////////////////////////////////////////// -// OTHER -////////////////////////////////////////////////////////////////////////////////// - -#define TOASTY_POLL_CAPACITY 514 - -struct pollfd; - -// TODO: comment -int toasty_process_events(ToastyFS *toasty, void **contexts, - struct pollfd *polled, int num_polled); - -////////////////////////////////////////////////////////////////////////////////// -// THE END -////////////////////////////////////////////////////////////////////////////////// -#endif // TOASTY_INCLUDED diff --git a/src/basic.c b/lib/basic.c similarity index 60% rename from src/basic.c rename to lib/basic.c index 64715db..c15078b 100644 --- a/src/basic.c +++ b/lib/basic.c @@ -126,3 +126,107 @@ void append_hex_as_str(char *out, SHA256 hash) out[(i << 1) + 1] = table[(uint8_t) hash.data[i] & 0xF]; } } + +// TODO: check this function +bool addr_lower(Address a, Address b) +{ + if (a.is_ipv4) { + + if (!b.is_ipv4) + return true; + + if (a.ipv4.data < b.ipv4.data) + return true; + + if (a.ipv4.data == b.ipv4.data && + a.port < b.port) + return true; + + return false; + + } else { + + if (b.is_ipv4) + return false; + + for (int i = 0; i < 8; i++) { + + if (a.ipv6.data[i] < b.ipv6.data[i]) + return true; + + if (a.ipv6.data[i] > b.ipv6.data[i]) + return false; + } + + if (a.port < b.port) + return true; + + return false; + } +} + +bool addr_eql(Address a, Address b) +{ + if (a.is_ipv4 != b.is_ipv4) + return false; + + if (a.port != b.port) + return false; + + if (a.is_ipv4) { + if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4))) + return false; + } else { + if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6))) + return false; + } + + return true; +} + +int parse_addr_arg(char *arg, Address *out) +{ + int len = strlen(arg); + + int i = 0; + while (i < len && arg[i] != ':') + i++; + + if (i == len) + return -1; // No ':' character. + arg[i] = '\0'; + + IPv4 ipv4; + int ret = inet_pton(AF_INET, arg, &ipv4); + arg[i] = ':'; + + if (ret != 1) + return -1; + + errno = 0; + ret = atoi(arg + i + 1); + if (ret == 0 && errno != 0) + return -1; + + out->ipv4 = ipv4; + out->is_ipv4 = true; + out->port = ret; + + return 0; +} + +void addr_sort(Address *addrs, int count) +{ + for (int i = 0; i < count; i++) { + + int k = i; // Index of the lowest address in [i, num_nodes-1] + for (int j = i+1; j < count; j++) { + if (addr_lower(addrs[j], addrs[k])) + k = j; + } + + Address tmp = addrs[i]; + addrs[i] = addrs[k]; + addrs[k] = tmp; + } +} \ No newline at end of file diff --git a/src/basic.h b/lib/basic.h similarity index 83% rename from src/basic.h rename to lib/basic.h index d095f12..dfb641f 100644 --- a/src/basic.h +++ b/lib/basic.h @@ -36,6 +36,7 @@ typedef uint64_t Time; #define S(X) ((string) { (X), (int) sizeof(X)-1 }) #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) #define UNREACHABLE __builtin_trap(); @@ -51,4 +52,10 @@ int getargi(int argc, char **argv, char *name, int fallback); void append_hex_as_str(char *out, SHA256 hash); +bool addr_eql(Address a, Address b); +bool addr_lower(Address a, Address b); + +int parse_addr_arg(char *arg, Address *out); +void addr_sort(Address *addrs, int count); + #endif // BASIC_INCLUDED diff --git a/src/byte_queue.c b/lib/byte_queue.c similarity index 100% rename from src/byte_queue.c rename to lib/byte_queue.c diff --git a/src/byte_queue.h b/lib/byte_queue.h similarity index 100% rename from src/byte_queue.h rename to lib/byte_queue.h diff --git a/src/file_system.c b/lib/file_system.c similarity index 87% rename from src/file_system.c rename to lib/file_system.c index 469566d..5acea6f 100644 --- a/src/file_system.c +++ b/lib/file_system.c @@ -9,6 +9,24 @@ int rename_file_or_dir(string oldpath, string newpath); +bool file_exists(string path) +{ + char zt[1<<10]; + if (path.len >= (int) sizeof(zt)) + return false; + memcpy(zt, path.ptr, path.len); + zt[path.len] = '\0'; + +#ifdef __linux__ + return access(zt, F_OK) == 0; +#endif + +#ifdef _WIN32 + DWORD attrs = GetFileAttributesA(zt); + return attrs != INVALID_FILE_ATTRIBUTES; +#endif +} + int file_open(string path, Handle *fd) { #ifdef __linux__ @@ -18,7 +36,7 @@ int file_open(string path, Handle *fd) memcpy(zt, path.ptr, path.len); zt[path.len] = '\0'; - int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644); + int ret = open(zt, O_RDWR | O_CREAT, 0644); if (ret < 0) return -1; @@ -59,6 +77,19 @@ void file_close(Handle fd) #endif } +int file_truncate(Handle fd, size_t new_size) +{ +#ifdef __linux__ + if (ftruncate((int) fd.data, new_size) < 0) + return -1; + return 0; +#endif + +#ifdef _WIN32 + return -1; // TODO: Not implemented +#endif +} + int file_set_offset(Handle fd, int off) { #ifdef __linux__ @@ -400,3 +431,29 @@ void directory_scanner_free(DirectoryScanner *scanner) } #endif + +int file_read_exact(Handle handle, char *dst, int len) +{ + int copied = 0; + while (copied < len) { + int ret = file_read(handle, dst + copied, len - copied); + if (ret < 0) + return -1; + if (ret == 0) + return 0; // EOF + copied += ret; + } + return copied; +} + +int file_write_exact(Handle handle, char *src, int len) +{ + int copied = 0; + while (copied < len) { + int ret = file_write(handle, src + copied, len - copied); + if (ret < 0) + return -1; + copied += ret; + } + return 0; +} \ No newline at end of file diff --git a/src/file_system.h b/lib/file_system.h similarity index 85% rename from src/file_system.h rename to lib/file_system.h index f987b42..916200e 100644 --- a/src/file_system.h +++ b/lib/file_system.h @@ -29,8 +29,10 @@ typedef struct { } DirectoryScanner; #endif +bool file_exists(string path); int file_open(string path, Handle *fd); void file_close(Handle fd); +int file_truncate(Handle fd, size_t new_size); int file_set_offset(Handle fd, int off); int file_get_offset(Handle fd, int *off); int file_lock(Handle fd); @@ -50,4 +52,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path); int directory_scanner_next(DirectoryScanner *scanner, string *name); void directory_scanner_free(DirectoryScanner *scanner); +int file_read_exact(Handle handle, char *dst, int len); +int file_write_exact(Handle handle, char *src, int len); + #endif // FILE_SYSTEM_INCLUDED \ No newline at end of file diff --git a/lib/message.c b/lib/message.c new file mode 100644 index 0000000..790d120 --- /dev/null +++ b/lib/message.c @@ -0,0 +1,81 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "message.h" + +bool binary_read(BinaryReader *reader, void *dst, int len) +{ + if (reader->len - reader->cur < len) + return false; + if (dst) + memcpy(dst, reader->src + reader->cur, len); + reader->cur += len; + return true; +} + +void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type) +{ + uint16_t version = MESSAGE_VERSION; + uint32_t dummy = 0; // Dummy value + writer->output = output; + writer->start = byte_queue_offset(output); + byte_queue_write(output, &version, sizeof(version)); + byte_queue_write(output, &type, sizeof(type)); + writer->patch = byte_queue_offset(output); + byte_queue_write(output, &dummy, sizeof(dummy)); +} + +bool message_writer_free(MessageWriter *writer) +{ + uint32_t length = byte_queue_size_from_offset(writer->output, writer->start); + byte_queue_patch(writer->output, writer->patch, &length, sizeof(length)); + if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure? + return false; + return true; +} + +void message_write(MessageWriter *writer, void *mem, int len) +{ + byte_queue_write(writer->output, mem, len); +} + +void message_write_u8(MessageWriter *writer, uint8_t value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +void message_write_u32(MessageWriter *writer, uint32_t value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +void message_write_hash(MessageWriter *writer, SHA256 value) +{ + message_write(writer, &value, (int) sizeof(value)); +} + +int message_peek(ByteView msg, uint16_t *type, uint32_t *len) +{ + if (msg.len < (int) sizeof(MessageHeader)) + return 0; + + MessageHeader header; + memcpy(&header, msg.ptr, sizeof(header)); + + // (We ignore endianess for now) + + if (header.version != MESSAGE_VERSION) + return -1; + + if (header.length > msg.len) + return 0; + + if (type) *type = header.type; + if (len) *len = header.length; + + return 1; +} diff --git a/lib/message.h b/lib/message.h new file mode 100644 index 0000000..0c9ec11 --- /dev/null +++ b/lib/message.h @@ -0,0 +1,46 @@ +#ifndef MESSAGE_INCLUDED +#define MESSAGE_INCLUDED + +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "basic.h" +#include "byte_queue.h" + +#define MESSAGE_VERSION 1 + +typedef struct { + uint8_t *src; + int len; + int cur; +} BinaryReader; + +typedef struct { + uint16_t version; + uint16_t type; + uint32_t length; +} MessageHeader; + +typedef struct { + ByteQueue *output; + ByteQueueOffset start; + ByteQueueOffset patch; +} MessageWriter; + +bool binary_read(BinaryReader *reader, void *dst, int len); + +void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type); +bool message_writer_free(MessageWriter *writer); +void message_write(MessageWriter *writer, void *mem, int len); +void message_write_u8(MessageWriter *writer, uint8_t value); +void message_write_u32(MessageWriter *writer, uint32_t value); +void message_write_hash(MessageWriter *writer, SHA256 value); + +int message_peek(ByteView msg, uint16_t *type, uint32_t *len); +void message_dump(FILE *stream, ByteView msg); + +#endif // MESSAGE_INCLUDED diff --git a/src/tcp.c b/lib/tcp.c similarity index 91% rename from src/tcp.c rename to lib/tcp.c index b922b66..938b80f 100644 --- a/src/tcp.c +++ b/lib/tcp.c @@ -8,25 +8,6 @@ #include "tcp.h" #include "message.h" -bool addr_eql(Address a, Address b) -{ - if (a.is_ipv4 != b.is_ipv4) - return false; - - if (a.port != b.port) - return false; - - if (a.is_ipv4) { - if (memcmp(&a.ipv4, &b.ipv4, sizeof(a.ipv4))) - return false; - } else { - if (memcmp(&a.ipv6, &b.ipv6, sizeof(a.ipv6))) - return false; - } - - return true; -} - static int set_socket_blocking(SOCKET sock, bool value) { #ifdef _WIN32 @@ -46,7 +27,7 @@ static int set_socket_blocking(SOCKET sock, bool value) return 0; } -static SOCKET create_listen_socket(string addr, uint16_t port) +static SOCKET create_listen_socket(Address addr) { SOCKET fd = socket(AF_INET, SOCK_STREAM, 0); if (fd == INVALID_SOCKET) @@ -59,22 +40,14 @@ static SOCKET create_listen_socket(string addr, uint16_t port) // TODO: mark address as reusable in debug builds - char tmp[1<<10]; - if (addr.len >= (int) sizeof(tmp)) { - CLOSE_SOCKET(fd); - return INVALID_SOCKET; + if (!addr.is_ipv4) { + assert(0); // TODO } - memcpy(tmp, addr.ptr, addr.len); - tmp[addr.len] = '\0'; struct sockaddr_in bind_buf; bind_buf.sin_family = AF_INET; - bind_buf.sin_port = htons(port); - if (inet_pton(AF_INET, tmp, &bind_buf.sin_addr) != 1) { - CLOSE_SOCKET(fd); - return INVALID_SOCKET; - } - + bind_buf.sin_port = htons(addr.port); + memcpy(&bind_buf.sin_addr, &addr.ipv4, sizeof(IPv4)); if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) { CLOSE_SOCKET(fd); return INVALID_SOCKET; @@ -155,8 +128,8 @@ static void conn_init(Connection *conn, SOCKET fd, bool connecting) conn->connecting = connecting; conn->closing = false; conn->msglen = 0; - byte_queue_init(&conn->input, 1<<20); - byte_queue_init(&conn->output, 1<<20); + byte_queue_init(&conn->input, 1<<30); + byte_queue_init(&conn->output, 1<<30); } static void conn_free(Connection *conn) @@ -225,9 +198,9 @@ int tcp_index_from_tag(TCP *tcp, int tag) return -1; } -int tcp_listen(TCP *tcp, string addr, uint16_t port) +int tcp_listen(TCP *tcp, Address addr) { - SOCKET listen_fd = create_listen_socket(addr, port); + SOCKET listen_fd = create_listen_socket(addr); if (listen_fd == INVALID_SOCKET) return -1; @@ -420,8 +393,22 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd if (removed[i]) { Connection *conn = contexts[i]; assert(conn); + int removed_idx = conn - tcp->conns; conn_free(conn); - *conn = tcp->conns[--tcp->num_conns]; + int last_idx = --tcp->num_conns; + if (removed_idx != last_idx) { + *conn = tcp->conns[last_idx]; + // Update event conn_idx values to reflect the swap + for (int j = 0; j < num_events; j++) { + if (events[j].conn_idx == last_idx) + events[j].conn_idx = removed_idx; + } + // Update contexts pointers for remaining iterations + for (int j = i + 1; j < num_polled; j++) { + if (contexts[j] == &tcp->conns[last_idx]) + contexts[j] = conn; + } + } } } @@ -491,6 +478,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output) void tcp_close(TCP *tcp, int conn_idx) { tcp->conns[conn_idx].closing = true; + tcp->conns[conn_idx].tag = -1; // Clear tag so new sends create a fresh connection // TODO: if no event will be triggered, the connection will not be closed // if the output buffer is empty, the connection should be closed here. } diff --git a/src/tcp.h b/lib/tcp.h similarity index 96% rename from src/tcp.h rename to lib/tcp.h index fa77297..8ccaa84 100644 --- a/src/tcp.h +++ b/lib/tcp.h @@ -69,12 +69,11 @@ typedef struct { Connection conns[TCP_CONNECTION_LIMIT]; } TCP; -bool addr_eql(Address a, Address b); int tcp_context_init(TCP *tcp); void tcp_context_free(TCP *tcp); int tcp_wakeup(TCP *tcp); int tcp_index_from_tag(TCP *tcp, int tag); -int tcp_listen(TCP *tcp, string addr, uint16_t port); +int tcp_listen(TCP *tcp, Address addr); int tcp_next_message(TCP *tcp, int conn_idx, ByteView *msg, uint16_t *type); void tcp_consume_message(TCP *tcp, int conn_idx); int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd *polled, int num_polled); diff --git a/quakey/include/quakey.h b/quakey/include/quakey.h index 24b508e..498ddfb 100644 --- a/quakey/include/quakey.h +++ b/quakey/include/quakey.h @@ -26,6 +26,8 @@ #include #endif +#include + typedef struct {} Quakey; // Function pointers to a simulated program's code @@ -72,12 +74,19 @@ int quakey_init(Quakey **quakey, QuakeyUInt64 seed); // Stop a simulation void quakey_free(Quakey *quakey); +typedef unsigned long long QuakeyNode; + // Add a program to the simulation -void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg); +QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg); + +void *quakey_node_state(QuakeyNode node); // Schedule and executes one program until it would block, then returns int quakey_schedule_one(Quakey *quakey); +// Get the current simulated time in nanoseconds +QuakeyUInt64 quakey_current_time(Quakey *quakey); + // Generate a random u64 QuakeyUInt64 quakey_random(void); @@ -89,6 +98,25 @@ typedef struct { void quakey_signal(char *name); int quakey_get_signal(Quakey *quakey, QuakeySignal *signal); +// Limit the number of hosts that can be crashed at the same time. +// Set to 0 for no limit (default). +void quakey_set_max_crashes(Quakey *quakey, int max_crashes); + +void quakey_network_partitioning(Quakey *quakey, bool enable); + +// Access spawned host information +int quakey_num_hosts(Quakey *quakey); +void *quakey_host_state(Quakey *quakey, int idx); // Returns NULL if host is dead +int quakey_host_is_dead(Quakey *quakey, int idx); +const char *quakey_host_name(Quakey *quakey, int idx); + +// Enter/leave a host's context so that mock_xxx functions (file I/O, +// etc.) operate on that host's resources. Use from code that runs +// outside of the normal init/tick/free callbacks (e.g. invariant +// checkers). +void quakey_enter_host(QuakeyNode node); +void quakey_leave_host(void); + int *mock_errno_ptr(void); #ifdef _WIN32 @@ -125,6 +153,8 @@ int mock_clock_gettime(clockid_t clockid, struct timespec *tp); int mock_open(char *path, int flags, int mode); int mock_fcntl(int fd, int cmd, int flags); int mock_close(int fd); +int mock_access(const char *path, int mode); +int mock_ftruncate(int fd, size_t new_size); int mock_fstat(int fd, struct stat *buf); int mock_read(int fd, char *dst, int len); int mock_write(int fd, char *src, int len); @@ -172,6 +202,8 @@ void mock_free(void *ptr); #define open mock_open #define fcntl mock_fcntl #define close mock_close +#define access mock_access +#define ftruncate mock_ftruncate #define CreateFileW mock_CreateFileW #define CloseHandle mock_CloseHandle #define ReadFile mock_ReadFile diff --git a/quakey/src/mockfs.c b/quakey/src/mockfs.c index 1bc9822..a6cc262 100644 --- a/quakey/src/mockfs.c +++ b/quakey/src/mockfs.c @@ -657,6 +657,63 @@ int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence) return new_offset; } +static void byte_buffer_truncate(ByteBuffer *byte_buffer, int new_size, ByteChunk **free_list) +{ + if (new_size == 0) { + byte_buffer_free(byte_buffer, free_list); + byte_buffer_init(byte_buffer); + return; + } + + // Walk through chunks to find the one containing the new end + int remaining = new_size; + ByteChunk *chunk = byte_buffer->head; + while (chunk) { + int chunk_used = (chunk->next ? BYTE_CHUNK_SIZE : byte_buffer->tail_used); + if (remaining <= chunk_used) { + // This chunk becomes the new tail + // Free all subsequent chunks + ByteChunk *to_free = chunk->next; + if (to_free) { + // Find end of chain to free + ByteChunk *last = to_free; + while (last->next) + last = last->next; + last->next = *free_list; + *free_list = to_free; + } + chunk->next = NULL; + byte_buffer->tail = chunk; + byte_buffer->tail_used = remaining; + return; + } + remaining -= BYTE_CHUNK_SIZE; + chunk = chunk->next; + } +} + +int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size) +{ + if (new_size < 0) + return MOCKFS_ERRNO_INVAL; + + if (!(open_file->flags & (MOCKFS_O_WRONLY | MOCKFS_O_RDWR))) + return MOCKFS_ERRNO_BADF; + + ByteBuffer *bb = &open_file->entity->byte_buffer; + int current_size = byte_buffer_size(bb); + + if (new_size < current_size) { + byte_buffer_truncate(bb, new_size, &open_file->mfs->chunk_free_list); + } else if (new_size > current_size) { + int ret = byte_buffer_extend(bb, new_size, open_file->mfs); + if (ret < 0) + return ret; + } + + return 0; +} + static int remove_inner(MockFS *mfs, MockFS_Entity *entity, bool recursive) { if (entity->parent == NULL) diff --git a/quakey/src/mockfs.h b/quakey/src/mockfs.h index 99bf86a..6953075 100644 --- a/quakey/src/mockfs.h +++ b/quakey/src/mockfs.h @@ -99,6 +99,7 @@ int mockfs_read_dir(MockFS_OpenDir *open_dir, MockFS_Dirent *dirent); int mockfs_sync(MockFS_OpenFile *open_file); int mockfs_lseek(MockFS_OpenFile *open_file, int offset, int whence); +int mockfs_ftruncate(MockFS_OpenFile *open_file, int new_size); int mockfs_remove(MockFS *mfs, char *path, int path_len, bool recursive); diff --git a/quakey/src/quakey.c b/quakey/src/quakey.c index 579044c..4a9ffc9 100644 --- a/quakey/src/quakey.c +++ b/quakey/src/quakey.c @@ -11,6 +11,10 @@ #define TODO __builtin_trap() +#define SIM_TRACE(sim, fmt, ...) \ + fprintf(stderr, "SIM: [%.3fs] " fmt "\n", \ + (double)(sim)->current_time / 1e9, ##__VA_ARGS__) + #ifdef NDEBUG #define UNREACHABLE {} #define ASSERT(X) {} @@ -281,6 +285,7 @@ struct Host { b32 timedout; b32 blocked; + b32 dead; // Current error number set by system call mocks int errno_; @@ -291,6 +296,10 @@ struct Host { // MockFS instance managing the disk bytes MockFS *mfs; + + // Saved spawn configuration for restart after crash + QuakeySpawn spawn_config; + char *spawn_arg; }; typedef struct { @@ -311,6 +320,8 @@ typedef enum { EVENT_TYPE_DISCONNECT, EVENT_TYPE_DATA, EVENT_TYPE_WAKEUP, + EVENT_TYPE_RESTART, + EVENT_TYPE_PARTITION, } TimeEventType; typedef struct { @@ -349,6 +360,17 @@ typedef struct { #define SIM_SIGNAL_LIMIT 8 +typedef struct { + Host *a; + Host *b; +} HostPair; + +typedef struct { + int count; + int capacity; + HostPair *pairs; +} HostPairList; + struct Sim { uint64_t seed; @@ -372,13 +394,30 @@ struct Sim { int num_signals; int head_signal; QuakeySignal signals[SIM_SIGNAL_LIMIT]; + + // Network partition simulation. The partition list holds pairs + // of hosts whose link is currently broken. The target_partition + // holds the desired end state. A periodic timer event gradually + // breaks or repairs links to converge toward the target. Once + // reached, a new random target is generated. + HostPairList partition; + HostPairList target_partition; + + // Maximum number of hosts that can be dead at the same time. + int max_crashes; + + bool network_partitioning; }; static void time_event_wakeup(Sim *sim, Nanos time, Host *host); static void time_event_connect(Sim *sim, Nanos time, Desc *desc); static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst); -static void time_event_send_data(Sim *sim, Nanos time, Desc *desc); +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count); +static void time_event_free(TimeEvent *event); +static void time_event_restart(Sim *sim, Nanos time, Host *host); +static void time_event_partition(Sim *sim, Nanos time); static void remove_events_targeting_desc(Sim *sim, Desc *desc); +static b32 remove_wakeup_event(Sim *sim, Host *host); ///////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////// @@ -706,11 +745,6 @@ static b32 socket_queue_empty(SocketQueue *queue) return queue->used == 0; } -static b32 socket_queue_used(SocketQueue *queue) -{ - return queue->used; -} - ///////////////////////////////////////////////////////////////// // Accept Queue Code @@ -909,6 +943,15 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) host->platform = config.platform; host->next_ephemeral_port = FIRST_EPHEMERAL_PORT; + // Save spawn config for restart after crash + host->spawn_config = config; + int spawn_arg_len = strlen(arg); + host->spawn_arg = malloc(spawn_arg_len+1); + if (host->spawn_arg == NULL) { + TODO; + } + memcpy(host->spawn_arg, arg, spawn_arg_len+1); + int arg_len = strlen(arg); host->arg = malloc(arg_len+1); if (host->arg == NULL) { @@ -932,7 +975,7 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) host->num_addrs = config.num_addrs; host->state_size = config.state_size; - host->state = malloc(config.state_size); + host->state = calloc(1, config.state_size); if (host->state == NULL) { TODO; } @@ -962,6 +1005,7 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) TODO; } + host->dead = false; host->timedout = false; host->blocked = false; host->poll_count = 0; @@ -985,31 +1029,210 @@ static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) TODO; } - if (host->poll_timeout > 0) + if (host->poll_timeout > 0) { + remove_wakeup_event(host->sim, host); time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); - else if (host->poll_timeout == 0) + } else if (host->poll_timeout == 0) host->timedout = true; // Immediate timeout, don't create event at current_time } static void host_free(Host *host) { + if (!host->dead) { + host___ = host; + int ret = host->free_func(host->state); + host___ = NULL; + if (ret < 0) { + TODO; + } + + mockfs_free(host->mfs); + free(host->disk_data); + + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(host->sim, &host->desc[i], true); + } + + free(host->state); + free(host->arg); + } + + free(host->spawn_arg); +} + +// Remove all events associated with a host (WAKEUP events and events +// whose source or destination descriptors belong to this host). +// If skip_restart is true, RESTART events are kept (used during +// restart failure to avoid removing the event currently being +// processed by process_events_at_current_time). +static void remove_events_for_host_ex(Sim *sim, Host *host, bool skip_restart) +{ + int i = 0; + while (i < sim->num_events) { + TimeEvent *event = &sim->events[i]; + bool remove = false; + + if (event->type == EVENT_TYPE_WAKEUP && event->host == host) + remove = true; + if (!skip_restart && event->type == EVENT_TYPE_RESTART && event->host == host) + remove = true; + if (event->src_desc && event->src_desc->host == host) + remove = true; + if (event->dst_desc && event->dst_desc->host == host) + remove = true; + + if (remove) { + time_event_free(event); + sim->events[i] = sim->events[--sim->num_events]; + } else { + i++; + } + } +} + +static void remove_events_for_host(Sim *sim, Host *host) +{ + remove_events_for_host_ex(sim, host, false); +} + +static int count_dead(Sim *sim) +{ + int n = 0; + for (int i = 0; i < sim->num_hosts; i++) + if (sim->hosts[i]->dead) + n++; + return n; +} + +// Simulate a crash: tear down all connections and wipe application +// state. Everything else (MockFS, arg, addresses, etc.) remains +// initialized until restart. +static void host_crash(Host *host) +{ + assert(!host->dead); + Sim *sim = host->sim; + + // Close all descriptors (RST connections to notify peers) + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(sim, &host->desc[i], true); + } + host->num_desc = 0; + + // Remove all pending events for this host + remove_events_for_host(sim, host); + + // Let the application clean up (e.g. log the crash) host___ = host; - int ret = host->free_func(host->state); + host->free_func(host->state); host___ = NULL; - if (ret < 0) { + + // Free application state + free(host->state); + host->state = NULL; + + host->dead = true; + host->timedout = false; + host->blocked = false; + host->poll_count = 0; + host->poll_timeout = -1; + +} + +// Restart a crashed host: re-initialize from saved config, +// preserving the disk contents +static void host_restart(Host *host) +{ + assert(host->dead); + Sim *sim = host->sim; + + // Re-parse arguments (split_args mutates the string with null bytes, + // so we need a fresh copy from spawn_arg) + free(host->arg); + int arg_len = strlen(host->spawn_arg); + host->arg = malloc(arg_len+1); + if (host->arg == NULL) { + TODO; + } + memcpy(host->arg, host->spawn_arg, arg_len+1); + + host->argc = split_args(host->arg, host->argv, HOST_ARGC_LIMIT); + if (host->argc < 0) { TODO; } - mockfs_free(host->mfs); - free(host->disk_data); - - for (int i = 0; i < HOST_DESC_LIMIT; i++) { - if (host->desc[i].type != DESC_EMPTY) - desc_free(host->sim, &host->desc[i], true); + // Re-allocate application state (zeroed, as on a fresh boot) + host->state_size = host->spawn_config.state_size; + host->state = calloc(1, host->state_size); + if (host->state == NULL) { + TODO; } - free(host->state); - free(host->arg); + // Re-initialize descriptors + host->num_desc = 0; + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + host->desc[i].host = host; + host->desc[i].type = DESC_EMPTY; + } + + host->errno_ = 0; + host->next_ephemeral_port = FIRST_EPHEMERAL_PORT; + + // Keep existing MockFS - persistent state (files) survives crash. + // All file descriptors were closed during host_crash via desc_free, + // so entity refcounts are already correct. + + host->dead = false; + host->timedout = false; + host->blocked = false; + host->poll_count = 0; + host->poll_timeout = -1; + + host___ = host; + int ret = host->init_func( + host->state, + host->argc, + host->argv, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + // Clean up any descriptors created during the failed init + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(sim, &host->desc[i], true); + } + host->num_desc = 0; + // Skip removing RESTART events: this function is called from + // time_event_process (EVENT_TYPE_RESTART), so the current + // restart event is still in the array being iterated by + // process_events_at_current_time. Removing it here would cause + // the outer loop's swap-removal to discard an unrelated event. + remove_events_for_host_ex(sim, host, true); + free(host->state); + host->state = NULL; + host->dead = true; + + // Schedule another restart attempt. Init can fail due to + // transient I/O errors from fault injection; permanently + // killing the host would be unrealistic since a real + // process supervisor would keep retrying. + Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL); + time_event_restart(sim, sim->current_time + restart_delay, host); + return; + } + + if (host->poll_timeout > 0) { + remove_wakeup_event(sim, host); + time_event_wakeup(sim, sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); + } else if (host->poll_timeout == 0) + host->timedout = true; + } static b32 host_is_linux(Host *host) @@ -1042,6 +1265,9 @@ static bool is_desc_idx_valid(Host *host, int desc_idx); static bool host_ready(Host *host) { + if (host->dead) + return false; + if (host->blocked) return false; @@ -1176,12 +1402,16 @@ static void host_update(Host *host) ); host___ = NULL; if (ret < 0) { - TODO; + host_crash(host); + Nanos restart_delay = 1000000000ULL + (sim_random(host->sim) % 9000000000ULL); + time_event_restart(host->sim, host->sim->current_time + restart_delay, host); + return; } - if (host->poll_timeout > 0) + if (host->poll_timeout > 0) { + remove_wakeup_event(host->sim, host); time_event_wakeup(host->sim, host->sim->current_time + (Nanos)host->poll_timeout * 1000000ULL, host); - else if (host->poll_timeout == 0) + } else if (host->poll_timeout == 0) host->timedout = true; // Immediate timeout, don't create event at current_time } @@ -1278,9 +1508,15 @@ static int host_create_pipe(Host *host, int *desc_idxs) return HOST_ERROR_FULL; Desc *desc_1 = &host->desc[desc_idx_1]; + // Mark the first slot as in-use before searching for the second, + // otherwise find_empty_desc_struct returns the same index twice. + desc_1->type = DESC_PIPE; + int desc_idx_2 = find_empty_desc_struct(host); - if (desc_idx_2 < 0) + if (desc_idx_2 < 0) { + desc_1->type = DESC_EMPTY; return HOST_ERROR_FULL; + } Desc *desc_2 = &host->desc[desc_idx_2]; desc_1->type = DESC_PIPE; @@ -1727,7 +1963,7 @@ static int send_inner(Desc *desc, char *src, int len) uint64_t rng = sim_random(sim); latency = 1000000 + (rng % 99000000); // between 1ms and 100ms #endif - time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc); + time_event_send_data(desc->host->sim, desc->host->sim->current_time + latency, desc, ret); return ret; } @@ -1749,14 +1985,14 @@ static int host_read(Host *host, int desc_idx, char *dst, int len) desc->type == DESC_PIPE) { num = recv_inner(desc, dst, len); } else if (desc->type == DESC_FILE) { -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS) uint64_t roll = sim_random(host->sim) % 1000; if (roll == 0) return HOST_ERROR_IO; #endif int ret = mockfs_read(&desc->file, dst, len); if (ret < 0) return mockfs_to_quakey_error(ret); -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) if (ret > 0) { // 1 in 10,000 reads gets a bit flip if ((sim_random(host->sim) % 10000) == 0) { @@ -1793,12 +2029,12 @@ static int host_write(Host *host, int desc_idx, char *src, int len) desc->type == DESC_PIPE) { num = send_inner(desc, src, len); } else if (desc->type == DESC_FILE) { -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS) uint64_t roll = sim_random(host->sim) % 1000; if (roll == 0) return HOST_ERROR_IO; if (roll == 1) return HOST_ERROR_NOSPC; #endif -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) int byte_idx = -1; int bit_idx; if (len > 0) { @@ -1811,7 +2047,7 @@ static int host_write(Host *host, int desc_idx, char *src, int len) } #endif int ret = mockfs_write(&desc->file, src, len); -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_DISK_CORRUPTION) if (byte_idx > -1) { src[byte_idx] ^= (1 << bit_idx); } @@ -1833,9 +2069,11 @@ static int host_recv(Host *host, int desc_idx, char *dst, int len) Desc *desc = &host->desc[desc_idx]; #ifdef FAULT_INJECTION - Sim *sim = desc->host->sim; - uint64_t rng = sim_random(sim); - len = 1 + (rng % len); + if (len > 0) { + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); + } #endif int num = 0; @@ -1857,9 +2095,11 @@ static int host_send(Host *host, int desc_idx, char *src, int len) Desc *desc = &host->desc[desc_idx]; #ifdef FAULT_INJECTION - Sim *sim = desc->host->sim; - uint64_t rng = sim_random(sim); - len = 1 + (rng % len); + if (len > 0) { + Sim *sim = desc->host->sim; + uint64_t rng = sim_random(sim); + len = 1 + (rng % len); + } #endif int num = 0; @@ -1965,7 +2205,7 @@ static int host_fsync(Host *host, int desc_idx) if (desc->type != DESC_FILE) return HOST_ERROR_BADIDX; -#ifdef FAULT_INJECTION +#if defined(FAULT_INJECTION) && !defined(DISABLE_SPURIOUS_IO_ERRORS) uint64_t roll = sim_random(host->sim) % 100; if (roll == 0) return HOST_ERROR_IO; @@ -1978,6 +2218,22 @@ static int host_fsync(Host *host, int desc_idx) return 0; } +static int host_ftruncate(Host *host, int desc_idx, int new_size) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + + int ret = mockfs_ftruncate(&desc->file, new_size); + if (ret < 0) + return mockfs_to_quakey_error(ret); + + return 0; +} + static int host_setdescflags(Host *host, int desc_idx, int flags) { if (!is_desc_idx_valid(host, desc_idx)) @@ -2068,6 +2324,19 @@ static b32 remove_connect_event(Sim *sim, Desc *desc) return true; } +static b32 remove_wakeup_event(Sim *sim, Host *host) +{ + int i = 0; + while (i < sim->num_events && (sim->events[i].type != EVENT_TYPE_WAKEUP || sim->events[i].host != host)) + i++; + + if (i == sim->num_events) + return false; + + sim->events[i] = sim->events[--sim->num_events]; + return true; +} + // Remove all events that target a specific descriptor (DISCONNECT and DATA events) static void remove_events_targeting_desc(Sim *sim, Desc *desc) { @@ -2109,7 +2378,7 @@ static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst) append_event(sim, event); } -static void time_event_send_data(Sim *sim, Nanos time, Desc *desc) +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc, int count) { TimeEvent event = { .type = EVENT_TYPE_DATA, @@ -2117,7 +2386,7 @@ static void time_event_send_data(Sim *sim, Nanos time, Desc *desc) .host = NULL, .src_desc = NULL, .dst_desc = desc->peer, - .data_count = socket_queue_used(desc->output), + .data_count = count, .data_queue = socket_queue_ref(desc->output), .rst = false, }; @@ -2130,12 +2399,67 @@ static void time_event_free(TimeEvent *event) socket_queue_unref(event->data_queue); } -static int sim_find_host(Sim *sim, Addr addr); +static void time_event_restart(Sim *sim, Nanos time, Host *host) +{ + TimeEvent event = { + .type = EVENT_TYPE_RESTART, + .time = time, + .host = host, + .src_desc = NULL, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static void time_event_partition(Sim *sim, Nanos time) +{ + TimeEvent event = { + .type = EVENT_TYPE_PARTITION, + .time = time, + }; + append_event(sim, event); +} + +static int sim_find_host(Sim *sim, Addr addr); +static int change_target_partition(Sim *sim); +static bool break_or_repair_link(Sim *sim); +static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b); + +static Nanos pick_partition_delay(Sim *sim, bool longer) +{ + Nanos min_delay = 1000000000; // 1s + Nanos max_delay = 10000000000; // 10s + if (longer) { + min_delay = 10000000000; + max_delay = 20000000000; + } + return min_delay + sim_random(sim) % (max_delay - min_delay); +} static b32 time_event_process(TimeEvent *event, Sim *sim) { b32 consumed = true; switch (event->type) { + case EVENT_TYPE_PARTITION: + { + // Try to move one step toward the target partition. + // If current already matches target, pick a new target. + if (!break_or_repair_link(sim)) { + if (sim->network_partitioning) { + if (change_target_partition(sim) < 0) { + TODO; + } + } + } + + // Reschedule for the next partition step + event->time = sim->current_time + pick_partition_delay(sim, true); + consumed = false; + } + break; case EVENT_TYPE_CONNECT: { Desc *src_desc = event->src_desc; @@ -2148,6 +2472,13 @@ static b32 time_event_process(TimeEvent *event, Sim *sim) } Host *peer_host = sim->hosts[idx]; + if (hosts_are_partitioned(sim, src_desc->host, peer_host)) { + //SIM_TRACE(sim, "PARTITION: blocked connection %s -> %s", + // src_desc->host->name, peer_host->name); + src_desc->connect_status = CONNECT_STATUS_NOHOST; + break; + } + Desc *peer = host_find_desc_bound_to(peer_host, src_desc->connect_addr, src_desc->connect_port); if (peer == NULL) { @@ -2202,6 +2533,9 @@ static b32 time_event_process(TimeEvent *event, Sim *sim) case EVENT_TYPE_WAKEUP: event->host->timedout = true; break; + case EVENT_TYPE_RESTART: + host_restart(event->host); + break; } return consumed; } @@ -2209,6 +2543,242 @@ static b32 time_event_process(TimeEvent *event, Sim *sim) ///////////////////////////////////////////////////////////////// // Sim Code +static void +host_pair_list_init(HostPairList *list) +{ + list->count = 0; + list->capacity = 0; + list->pairs = NULL; +} + +static void +host_pair_list_free(HostPairList *list) +{ + free(list->pairs); +} + +static bool +host_pair_list_exists(HostPairList *list, Host *a, Host *b) +{ + for (int i = 0; i < list->count; i++) + if ((list->pairs[i].a == a && list->pairs[i].b == b) || + (list->pairs[i].a == b && list->pairs[i].b == a)) + return true; + return false; +} + +static int +host_pair_list_add(HostPairList *list, Host *a, Host *b) +{ + if (!host_pair_list_exists(list, a, b)) { + if (list->count == list->capacity) { + int n = 2 * list->count; + if (n < 8) n = 8; + void *p = realloc(list->pairs, n * sizeof(HostPair)); + if (p == NULL) + return -1; + list->capacity = n; + list->pairs = p; + } + list->pairs[list->count++] = (HostPair) { a, b }; + } + return 0; +} + +static void +host_pair_list_remove_at(HostPairList *list, int idx) +{ + list->pairs[idx] = list->pairs[--list->count]; +} + +// Generate a new random target partition by assigning hosts to +// random buckets. Hosts in the same bucket can communicate; hosts +// in different buckets have their link marked as broken. +static int change_target_partition(Sim *sim) +{ + HostPairList list; + host_pair_list_init(&list); + + #define MAX_BUCKETS 3 + #define MAX_NODES_PER_BUCKET 8 + + int num_buckets = 1 + sim_random(sim) % MAX_BUCKETS; + + Host *buckets[MAX_BUCKETS][MAX_NODES_PER_BUCKET]; + int bucket_sizes[MAX_BUCKETS]; + + for (int i = 0; i < num_buckets; i++) + bucket_sizes[i] = 0; + + // Randomly assign each host to a bucket + for (int i = 0; i < sim->num_hosts; i++) { + int bucket_index = sim_random(sim) % num_buckets; + buckets[bucket_index][bucket_sizes[bucket_index]++] = sim->hosts[i]; + } + + // Every cross-bucket pair is a broken link in the target + for (int i = 0; i < num_buckets; i++) { + for (int j = 0; j < bucket_sizes[i]; j++) { + for (int k = 0; k < num_buckets; k++) { + if (k == i) continue; + for (int g = 0; g < bucket_sizes[k]; g++) { + Host *a = buckets[i][j]; + Host *b = buckets[k][g]; + if (host_pair_list_add(&list, a, b) < 0) { + host_pair_list_free(&list); + return -1; + } + } + } + } + } + + host_pair_list_free(&sim->target_partition); + sim->target_partition = list; + SIM_TRACE(sim, "PARTITION: new target partition with %d broken links (%d buckets)", + list.count, num_buckets); + return 0; +} + +// Pick a random pair that exists in 'left' but not in 'right'. +// Returns the index into 'left', or -1 if no such pair exists. +static int pick_random_from_left_not_in_right(Sim *sim, + HostPairList *left, HostPairList *right) +{ + int *arr = malloc(left->count * sizeof(int)); + if (arr == NULL) { + TODO; // Allowing this to return error would make execution not deterministic + } + int num = 0; + + for (int i = 0; i < left->count; i++) { + HostPair pair = left->pairs[i]; + if (!host_pair_list_exists(right, pair.a, pair.b)) { + arr[num++] = i; + } + } + + int idx; + if (num == 0) { + idx = -1; + } else { + idx = arr[sim_random(sim) % num]; + } + + free(arr); + return idx; +} + +// Forcefully reset a connection socket and notify its peer. +// Removes any pending time events that reference either end. +static void reset_conn(Sim *sim, Desc *desc) +{ + assert(desc->type == DESC_SOCKET_C); + + if (desc->connect_status == CONNECT_STATUS_WAIT || + desc->connect_status == CONNECT_STATUS_DONE) { + + remove_events_targeting_desc(sim, desc); + + if (desc->peer) { + assert(desc->peer->type == DESC_SOCKET_C + || desc->peer->type == DESC_SOCKET_L); + + if (desc->peer->type == DESC_SOCKET_C) { + remove_events_targeting_desc(sim, desc->peer); + desc->peer->connect_status = CONNECT_STATUS_RESET; + desc->peer->peer = NULL; + } else { + accept_queue_remove(&desc->peer->accept_queue, desc); + } + desc->peer = NULL; + } + + desc->connect_status = CONNECT_STATUS_RESET; + } +} + +// Reset all active connections between two hosts (both directions) +static void drop_conns_between_hosts(Sim *sim, Host *host, Host *peer) +{ + for (int i = 0, j = 0; j < host->num_desc; i++) { + + Desc *desc = &host->desc[i]; + if (desc->type == DESC_EMPTY) + continue; + j++; + + if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == peer) + reset_conn(sim, desc); + } + + for (int i = 0, j = 0; j < peer->num_desc; i++) { + + Desc *desc = &peer->desc[i]; + if (desc->type == DESC_EMPTY) + continue; + j++; + + if (desc->type == DESC_SOCKET_C && desc->peer && desc->peer->host == host) + reset_conn(sim, desc); + } +} + +// Break a link that is in the target but not yet in current +static bool break_link(Sim *sim) +{ + int idx = pick_random_from_left_not_in_right(sim, + &sim->target_partition, &sim->partition); + if (idx < 0) + return false; + HostPair pair = sim->target_partition.pairs[idx]; + + host_pair_list_add(&sim->partition, pair.a, pair.b); + drop_conns_between_hosts(sim, pair.a, pair.b); + SIM_TRACE(sim, "PARTITION: break link %s <-> %s (%d broken links)", + pair.a->name, pair.b->name, sim->partition.count); + return true; +} + +// Repair a link that is broken in current but not in the target +static bool repair_link(Sim *sim) +{ + int idx = pick_random_from_left_not_in_right(sim, + &sim->partition, &sim->target_partition); + if (idx < 0) + return false; + + HostPair pair = sim->partition.pairs[idx]; + host_pair_list_remove_at(&sim->partition, idx); + SIM_TRACE(sim, "PARTITION: repair link %s <-> %s (%d broken links)", + pair.a->name, pair.b->name, sim->partition.count); + return true; +} + +// Try to move one step toward the target: randomly attempt a +// break or repair first, falling back to the other on failure. +// Returns false when current already matches the target. +static bool break_or_repair_link(Sim *sim) +{ + if (sim_random(sim) & 1) { + if (break_link(sim)) + return true; + if (repair_link(sim)) + return true; + } else { + if (repair_link(sim)) + return true; + if (break_link(sim)) + return true; + } + return false; +} + +static bool hosts_are_partitioned(Sim *sim, Host *a, Host *b) +{ + return host_pair_list_exists(&sim->partition, a, b); +} + static void sim_init(Sim *sim, uint64_t seed) { sim->seed = seed; @@ -2221,10 +2791,27 @@ static void sim_init(Sim *sim, uint64_t seed) sim->events = NULL; sim->num_signals = 0; sim->head_signal = 0; + sim->max_crashes = 0; + sim->network_partitioning = true; + host_pair_list_init(&sim->partition); + host_pair_list_init(&sim->target_partition); + time_event_partition(sim, pick_partition_delay(sim, false)); +} + +static void sim_network_partitioning(Sim *sim, bool enable) +{ + sim->network_partitioning = enable; + if (!enable) { + host_pair_list_free(&sim->target_partition); + host_pair_list_init(&sim->target_partition); + } } static void sim_free(Sim *sim) { + host_pair_list_free(&sim->target_partition); + host_pair_list_free(&sim->partition); + for (int i = 0; i < sim->num_hosts; i++) host_free(sim->hosts[i]); free(sim->hosts); @@ -2234,7 +2821,7 @@ static void sim_free(Sim *sim) free(sim->events); } -static void sim_spawn(Sim *sim, QuakeySpawn config, char *arg) +static Host *sim_spawn(Sim *sim, QuakeySpawn config, char *arg) { if (sim->num_hosts == sim->max_hosts) { int n = 2 * sim->max_hosts; @@ -2255,12 +2842,13 @@ static void sim_spawn(Sim *sim, QuakeySpawn config, char *arg) host_init(host, sim, config, arg); sim->hosts[sim->num_hosts++] = host; + return host; } static int sim_find_host(Sim *sim, Addr addr) { for (int i = 0; i < sim->num_hosts; i++) - if (host_has_addr(sim->hosts[i], addr)) + if (!sim->hosts[i]->dead && host_has_addr(sim->hosts[i], addr)) return i; return -1; } @@ -2301,6 +2889,9 @@ static void process_events_at_current_time(Sim *sim) } } } + + //if (sim->num_events > 10000) + // SIM_TRACE(sim, "WARNING: event queue large: %d events", sim->num_events); } static int find_first_ready_host(Sim *sim) @@ -2355,6 +2946,42 @@ static b32 sim_update(Sim *sim) if (host_ready(host)) host->blocked = true; +#ifdef FAULT_INJECTION + // Crash failure injection: with a small probability, crash a random + // live host. The host will be restarted after a random delay (1-10 + // seconds), simulating a reboot. The disk contents persist across + // crashes, but all volatile state (memory, connections) is lost. + if (sim->num_hosts > 1) { + uint64_t rng = sim_random(sim); + if ((rng % 10000) == 0) { + + int live_count = 0; + for (int i = 0; i < sim->num_hosts; i++) + if (!sim->hosts[i]->dead) + live_count++; + int dead_count = sim->num_hosts - live_count; + + if (dead_count < sim->max_crashes) { + // Pick one of the live hosts at random + int target = sim_random(sim) % live_count; + int j = 0; + for (int i = 0; i < sim->num_hosts; i++) { + if (!sim->hosts[i]->dead) { + if (j == target) { + host_crash(sim->hosts[i]); + // Schedule restart after 1-10 seconds + Nanos restart_delay = 1000000000ULL + (sim_random(sim) % 9000000000ULL); + time_event_restart(sim, sim->current_time + restart_delay, sim->hosts[i]); + break; + } + j++; + } + } + } + } + } +#endif + return true; } @@ -2426,9 +3053,38 @@ void quakey_free(Quakey *quakey) } } -void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg) +void quakey_set_max_crashes(Quakey *quakey, int max_crashes) { - return sim_spawn((Sim*) quakey, config, arg); + Sim *sim = (Sim*) quakey; + sim->max_crashes = max_crashes; +} + +void quakey_network_partitioning(Quakey *quakey, bool enable) +{ + Sim *sim = (Sim*) quakey; + sim_network_partitioning(sim, enable); +} + +QuakeyNode quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg) +{ + return (QuakeyNode) sim_spawn((Sim*) quakey, config, arg); +} + +void *quakey_node_state(QuakeyNode node) +{ + Host *host = (void*) node; + return host->state; +} + +void quakey_enter_host(QuakeyNode node) +{ + Host *host = (void*) node; + host___ = host; +} + +void quakey_leave_host(void) +{ + host___ = NULL; } int quakey_schedule_one(Quakey *quakey) @@ -2436,6 +3092,12 @@ int quakey_schedule_one(Quakey *quakey) return sim_update((Sim*) quakey); } +QuakeyUInt64 quakey_current_time(Quakey *quakey) +{ + Sim *sim = (Sim*) quakey; + return sim->current_time; +} + QuakeyUInt64 quakey_random(void) { Host *host = host___; @@ -2462,6 +3124,39 @@ int quakey_get_signal(Quakey *quakey, QuakeySignal *signal) return sim_get_signal((Sim*) quakey, signal); } +int quakey_num_hosts(Quakey *quakey) +{ + Sim *sim = (Sim*) quakey; + return sim->num_hosts; +} + +void *quakey_host_state(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return NULL; + Host *host = sim->hosts[idx]; + if (host->dead) + return NULL; + return host->state; +} + +int quakey_host_is_dead(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return 1; + return sim->hosts[idx]->dead ? 1 : 0; +} + +const char *quakey_host_name(Quakey *quakey, int idx) +{ + Sim *sim = (Sim*) quakey; + if (idx < 0 || idx >= sim->num_hosts) + return NULL; + return sim->hosts[idx]->name; +} + ///////////////////////////////////////////////////////////////// // Mock System Calls @@ -2546,6 +3241,11 @@ int mock_close(int fd) return 0; } +int mock_access(const char *path, int mode) +{ + assert(0); // TODO +} + static int convert_addr(void *addr, size_t addr_len, Addr *converted_addr, uint16_t *converted_port) { @@ -3126,6 +3826,29 @@ int mock_fsync(int fd) return 0; } +int mock_ftruncate(int fd, size_t new_size) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_ftruncate() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_ftruncate() not from Linux\n"); + + int ret = host_ftruncate(host, fd, (int) new_size); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX || ret == HOST_ERROR_BADF) + *host_errno_ptr(host) = EBADF; + else if (ret == HOST_ERROR_NOSPC) + *host_errno_ptr(host) = ENOSPC; + else + *host_errno_ptr(host) = EINVAL; + return -1; + } + + return 0; +} + off_t mock_lseek(int fd, off_t offset, int whence) { Host *host = host___; diff --git a/spec/design.txt b/spec/design.txt deleted file mode 100644 index 7415df2..0000000 --- a/spec/design.txt +++ /dev/null @@ -1,233 +0,0 @@ -Architecture - A ToastyFS instance is composed by a metadata server, a number - of chunk servers, and a number of clients. - - The metadata server stores the full file system hieararchy, - except instead of storing the file contents, it stores an - array of hashes of the chunks of each file. A "chunk" is a - file range that is fixed for a single file but may vary - between files. Chunk servers hold an array of chunks that - are identified by their hash. The metadata server keeps - track of which chunks each chunk server is holding. - - Clients are users of the file system that can read and - write metadata and files. They are assumed to behave - correctly. - - Any read and write operation that doesn't involve file - contents can be performed by clients by talking to the - metadata server directly. Such operations include creating - an empty file or a directory, deleting a file or directory, - listing files. - - If a client wants to read a range of bytes from a file, - it sends the metadata server the file name and range. - The metadata server responds with the chunk size of that - file, the list of hashes for the chunks involved in the - read, and the IP addresses of the chunk servers that hold - each chunk. The metadata server also adds the IP addresses - of three chunk servers any new chunks should be written - to. The client can then download the chunks from the chunk - servers and reassemble the result. - - If a client wants to write at a range of bytes of a file, - it starts by reading that range from the metadata server, - getting the list of hashes it will modify, their locations, - and locations for any new chunks. The client then modifies - the chunk by sending to each chunk server the hash to modify - and the patch (a range of bytes within a chunk plus the new - data). The chunk server creates a new modified chunk and - keeps the old version, then returns the new hash. If all - modifications are successful, the client holds the set of - old hashes and new hashes for that file range. It completes - the write by telling the metadata server to swap the old - hashes with the new ones (optionally including write flags - such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes - don't match, another write succeded in the mean time and - touched that range, therefore the write fails. If the old - hashes match, the write succeded. If the client fails to - modify any chunks, it doesn't commit the write with the - metadata server. - - If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING - flag is set, the metadata server atomically creates the file - with a default chunk size (4096 bytes) and retries the write. - This operation is logged to the WAL to ensure crash consistency. - - If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is - truncated after the write, setting its size to exactly offset+length - and discarding any data beyond that point. This is useful for HTTP - PUT semantics where the entire file content should be replaced. - - Note that write failures may cause chunks to be orphaned - on chunk servers. This is solved by a garbage collection - algorithm implemented by the synchronization messages - between metadata and chunk server. - - Note that clients may cache chunks and index them by their - hash. When they read a file and receive its hashes, they may - avoid reaching for the chunk servers if they already cached - the chunks with those hashes. This allows reading files with - only one round trip at no cost of correctness. If getting - the up-to-date contents is not a concern, clients may also - cache file metadata. - -Metadata and chunk server exchange: - - The metadata server is only aware of each chunk server - as long as they have a TCP connection. When a chunk server - first connects to the metadata server, it authenticates - itself and sends its own IP addresses. If the server is - authentic, the metadata server requests the full list - of chunks the chunk server is holding. Upon receiving the - state of chunk server, the metadata server adds all useful - chunks to the "old_list" and all useless chunks to the - "rem_list", then sends the rem_list to the chunk server - which removes those chunks. - - When writes are committed to the metadata server involving - new chunks to a chunk server, the metadata server adds those - hashes to an "add_list" and any hashes that are not useful - anymore to the rem_list. - - Periodically, the metadata server sends the add_list and - rem_list to the chunk server. These list tell the chunk - server the ideal state it should have from the point of - view of the metadata server. Elements in the add_list should - already be in the chunk servers, and elements from the - rem_list are to be removed. A chunk server marks any chunk - in the rem_list as to be removed and checks that hashes - in the add list are present. If a chunk in the add list - is marked as to be removed, it is unmarked. When a chunk - is marked as to be removed for a certain amount of time, - it is permanently deleted. When the synchronization is - complete, the metadata server merges the add_list into - the old_list and clears the rem_list. If chunks in the - add_list are not present in the chunk server, it responds - with an error message containing the list of missing chunks. - The metadata server then responds with a list of chunk - server addresses where the chunk server with the missing - chunk can download it from. Each chunk server goes - through its download list one at the time downloading - the missing chunks. - - Note that if the chunk server finds that its holding some - chunks that are not in the hash list of the metadata server, - that does not mean they are orphaned. It's possible that - some writes are being performed by clients that have uploaded - chunks to that chunk server but didn't yet acknowledge it - to the metadata server. If all goes well and the write - succeded, the metadata server will add those hashes to the - hash list. Chunk servers should only drop chunks if they - are not referenced by the metadata server for a period of - time (say, 30 minutes). - -Security - All nodes of the system share a secret key and use it to - authenticate each other and encrypt messages. This allows - the server to accept new chunk servers and clients with - no prior setup - -Reliability - The metadata server is a single point of failure. To reduce - the impact of crashes, the metadata server stores all write - operations into a write-ahead log that is replayed any time - the process goes online. - -Chunk Management: - Chunks are added to the system when: - 1. A chunk server connects - 2. A write operation on metadata occurs (adding chunks) - - They are removed when: - 1. A chunk server disconnects - 2. A write operation on metadata occurs (overwriting old chunks) - 3. A delete operation on metadata occurs - 4. A chunk is corrupted or removed forcefully from the chunk server - - The system must make sure that chunks are not over-replicated - or under-replicated. If they are over-replicated, some chunk - servers need to forget some copies. If they are under-replicated, - some chunk servers need to copy chunks from elsewhere. - - Metadata server variables for a chunk server: - ms_old_list: List of chunks that are known to be held by CS - ms_add_list: List of chunks that should be held by CS - ms_rem_list: List of chunks that may be held by CS but should removed from it - - Chunk server variables: - cs_add_list: List of chunks added since the last update - cs_rem_list: List of chunks marked for removal after a timeout - cs_lst_list: List of chunks that were lost due to errors or forceful removals of chunk files - - Metadata change for write: - When clients commit a write by adding new hashes to the metadata, - MS adds those hashes to the ms_add_list for the CS where the client - uploaded the chunks. If a hash was overwritten and became useless, - that hash is added to the ms_rem_list for all CS holding it. - - Metadata change for delete: - All hashes that are no longer reachable by the file tree are added - to the ms_rem_list of their holders - - Chunk upload: - When a chunk is uploaded to a chunk server, its hash is added to - the cs_add_list. - - Periodically on the chunk server: - Elements in the cs_rem_list have a 30 minute timeout after which they are deleted - permanently. - - Periodically: - CS sends cs_add_list to MS - MS may add a subset of cs_add_list to ms_add_list based on the chunk replication and distribution policy - MS sends ms_add_list and ms_rem_list to CS - CS (1) Adds all elements from ms_rem_list to cs_rem_list - (2) Elements in ms_add_list that are not held by the chunk server are added to - a temporary list tmp_list - (3) Removes elements in ms_add_list from cs_add_list and cs_rem_list, then merges cs_add_list into cs_rem_list and clears cs_add_list. - (4) Elements in cs_lst_list are added to tmp_list, then cs_lst_list is cleared. - (6) tmp_list is sent to MS - (7) cs_add_list is cleared - MS (1) Receives tmp_list and sends download locations to CS for those chunks - (2) Merges ms_add_list into ms_old_list, then removes all items in tmp_list from ms_old_list - (3) Sets ms_add_list equal to tmp_list - - Chunk replication and distribution policy: - During an update, when CS reports a new chunk to MS, MS has to decide whether - to allow the CS to keep it or not. - - There are 4 cases: - - The chunk is useless (not referenced by any file) - - The chunk is under-replicated even counting the new copy - - The chunk is properly replicated with the new copy - - The chunk is over-replicated with the new copy - - If the chunk is not referenced by the file tree, do nothing. - - If the chunk is properly replicated or under-replicated, add it to the ms_add_list. - - If the chunk is over-replicated, either don't add it to the ms_add_list or add it to the ms_rem_list of some other holder. - - TODO: The way chunk servers tell about the chunks they are holding - recently changed. Now instead of sending a full chunk list when - connecting, they send batches of chunks to the metadata server - during state updates. It also changed that now chunk servers - initiate updates. - - When a chunk server connects (and authenticates) - it sends alongside the auth message the hash of - the hash list of all chunks. The metadata server - then replies with a message saying whether it - already cached that chunk list or not. If it didn't, the chunk server sends the entire list - in chunks during state updates, with an increased - update frequency. - -Metadata Persistence & Crash Recovery: - The metadata server uses a write-ahead log (WAL) file to store its state on disk. - - Log files start with a full snapshot of the metadata and continues with operation - log entries. - - When a file gets too big, the metadata server creates a new WAL file by writing - a snapshot to it and continuing logging there. \ No newline at end of file diff --git a/spec/protocol.txt b/spec/protocol.txt deleted file mode 100644 index 4b32e7a..0000000 --- a/spec/protocol.txt +++ /dev/null @@ -1,281 +0,0 @@ -1. Introduction & Notation - -The DESIGN.txt file gives an overview of the system -and how nodes of a cluster interact with each other. -This file documents the specific binary format used -to exchange information between nodes. - -All messages start with a shared header, defined as: - - struct Header { - uint16_t version; - uint16_t type; - uint32_t length; - }; - -1.1 Generation Counter Special Values - -Generation counters (uint64_t) have two special values: - - NO_GENERATION (0): - When used in expect_gen, means "skip generation check entirely". - Files/directories are never assigned this generation value. - - MISSING_FILE_GENERATION (UINT64_MAX): - When used in expect_gen, means "expect file/directory to NOT exist". - - For DELETE: succeeds if file doesn't exist (no-op) - - For WRITE: fails with BADGEN if file exists, fails with NOENT if missing - - For existing operations: causes generation mismatch if file exists - Files/directories are never assigned this generation value. - -2. Client Messages - -2.1 Client to Metadata Server messages - -Let's start from the interactions between a client and -the metadata server: - -[ CREATE | C -> MS ] - Upon file creation, the client sends a "CREATE" message with the following layout: - - struct CreateMessage { - Header header; // type=CREATE - uint16_t path_len; - char path[path_len]; - uint8_t is_dir; - if (is_dir != 0) { - uint32_t chunk_size; - } - }; - - Note that in general only paths up to 65K bytes are - supported and that the layout of fields may depend - on the value of others. - - The server then responds with a CREATE_SUCCESS or CREATE_ERROR message. - -[ DELETE | C -> MS ] - When a client deletes a file, it sends a DELETE - message with the following layout: - - struct DeleteMessage { - Header header; // type=DELETE - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - }; - - The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless. - - The server then responds with a DELETE_SUCCESS or DELETE_ERROR message. - -[ LIST | C -> MS ] - When a client requests a directory listing, it sends a LIST message whith the following format: - - struct ListMessage { - Header header; // type=LIST - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - }; - - If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped. - - The server then responds with a LIST_SUCCESS or LIST_ERROR message. - -[ READ | C -> MS ] - When a client requests to read a file, it sends a READ message with the following format: - - struct ReadMessage { - Header header; // type=READ - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - uint32_t offset; - uint32_t length; - }; - - The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped. - - The offset and length fields determine the region to be read from the file. - -[ WRITE | C -> MS ] - When a client wants to write to a file, it sends a WRITE message with the following layout: - - struct Location { - uint8_t is_ipv4; - if (is_ipv4) { - uint32_t ipv4; - } else { - uint128_t ipv6; - } - uint16_t port; - }; - - struct WriteChunk { - SHA256 hash; - uint32_t num_locations; - Location locations[num_locations]; - }; - - struct WriteMessage { - Header header; // type=WRITE - uint64_t expect_gen; - uint32_t flags; - uint16_t path_len; - char path[path_len]; - uint32_t offset; - uint32_t length; - uint32_t num_chunks; - WriteChunk chunks[num_chunks]; - }; - - If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata. - - The flags field contains write operation flags. Currently defined flags: - - TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist, - the metadata server will atomically create it with a default chunk - size of 4096 bytes and retry the write operation. The creation is - logged to the WAL for crash consistency. - - TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write - operation, setting the file size to exactly offset+length. Any data - beyond this point is discarded. Useful for HTTP PUT semantics where - the entire file content should be replaced. - - The offset and length mark the region that is being written to. - - Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to - - num_chunks == length / chunk_size - - Where chunk_size is the one for the target file at the specified generation. - - Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it. - -2.2 Client to Chunk Server messages - - TODO - -3. Metadata Server messages - -The following is the list of messages the Metadata Server may send to a Client: - -[ CREATE_ERROR | MS -> C ] - When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout: - - struct CreateErrorMessage { - Header header; // type=CREATE_ERROR - uint16_t message_len; - char message[message_len]; - }; - -[ CREATE_SUCCESS | MS -> C ] - When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout: - - struct CreateSuccessMessage { - Header header; // type=CREATE_SUCCESS - uint64_t gen; - }; - - The gen field is the generation counter given to the file. - -[ DELETE_ERROR | MS -> C ] - See CREATE_ERROR - -[ DELETE_SUCCESS | MS -> C ] - When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout: - - struct DeleteSuccessMessage { - Header header; // type=DELETE_SUCCESS - }; - - It does not store any fields other than the header. - -[ LIST_ERROR | MS -> C ] - When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout: - - struct ListErrorMessage { - Header header; // type=LIST_ERROR - uint16_t message_len; - char message[message_len]; - }; - -[ LIST_SUCCESS | MS -> C ] - When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout: - - struct ListItem { - uint64_t gen; - uint8_t is_dir; - uint16_t name_len; - char name[name_len]; - }; - - struct ListSuccessMessage { - Header header; // type=LIST_SUCCESS - uint64_t gen; - uint8_t truncated; - uint32_t item_count; - ListItem items[item_count]; - }; - - The ListSuccessMessage gen field contains the generation counter for the directory, while the gen field in ListItem is the counter for that specific child. - - If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message. - -[ READ_ERROR | MS -> C ] - TODO - -[ READ_SUCCESS | MS -> C ] - - struct IPv4AndPort { - uint32_t ipv4; - uint16_t port; - }; - - struct IPv6AndPort { - uint128_t ipv6; - uint16_t port; - }; - - struct ChunkServerAddrList { - uint32_t num_ipv4; - IPv4AndPort ipv4s[num_ipv4]; - uint32_t num_ipv6; - IPv6AndPort ipv6s[num_ipv6]; - }; - - struct ReadChunk { - SHA256 hash; - uint32_t num_holders; - ChunkServerAddrList holders[num_holders]; - }; - - struct ReadSuccessMessage { - Header header; // type=READ_SUCCESS - uint64_t gen; - uint32_t chunk_size; - uint32_t file_length; - uint32_t num_hashes; - ReadChunk chunks[num_hashes]; - uint32_t num_write_locations; - ChunkServerAddrList write_locations[num_write_locations]; - }; - - The message returns general information about the file such as its generation counter, length in bytes, and chunk size. - - The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them. - - The write locations are a list of chunk servers that clients may write new chunks to. - -[ WRITE_ERROR | MS -> C ] - See CREATE_ERROR - -[ WRITE_SUCCESS | MS -> C ] - When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout: - - struct WriteSuccessMessage { - Header header; // type=WRITE_SUCCESS - uint64_t gen; - }; - - The gen field is the new generation counter for that file. \ No newline at end of file diff --git a/spec/spec.txt b/spec/spec.txt deleted file mode 100644 index 21059b2..0000000 --- a/spec/spec.txt +++ /dev/null @@ -1,482 +0,0 @@ -note that: - MS = metadata server - CS = chunk server - CLI = client - -========================================================= -=== METADATA SERVER PROCESS ============================= -========================================================= - -metadata_server_init: - Start a TCP server at the interface and port specified using options --addr and --port (default 127.0.0.1 and 8080) - - Initialize the file tree as empty - - Initialize the WAL and, if it exists already, replay its contents into the file tree - -metadata_server_free: - Free resources associated to the MS - -metadata_server_tick: - for each network event: - if connection established: - mark the connection as UNKNOWN - - if disconnect: - if the connection was marked as CS: - Free the information associated to it - - if message received: - if the connection is marked as UNKNOWN: - if the message is from a CS: - mark the connection as CS and associated to it a struct for the information associated to it. - else - mark the connection as CLI - - if the peer is a client: - if the message has type CREATE: - Read path_len, path, and is_dir from the message - If is_dir is false: - Read chunk_size from message - Else: - Set chunk_size to 0 - - Append the create operation to the WAL - Attempt to create entity in the file_tree - - If file_tree creation fails: - Send CREATE_ERROR with the error description - Else: - Send CREATE_SUCCESS containing the new generation number - - if the message has type DELETE: - Read expect_gen, path_len, and path from message - Append delete operation to the WAL - Attempt to delete entity in the file_tree - - If file_tree deletion fails: - Send DELETE_ERROR with the error description - Else: - Send DELETE_SUCCESS (no payload) - - if the message has type LIST: - Read expect_gen, path_len, and path from message - Attempt to list directory contents from file_tree - - If listing fails: - Send LIST_ERROR with error description - Else: - Send LIST_SUCCESS containing: - - Directory generation - - Truncated flag (if results exceed MAX_LIST_SIZE) - - Item count - - List of items (generation, is_dir, name_len, name) - - if the message has type READ: - Read expect_gen, path_len, path, offset, and length - Attempt to read file info from file_tree - - If read fails: - Send READ_ERROR containing: - - Error description - - A list of suggested chunk server addresses for writing (load balanced) - NOTE: Providing write locations on read error allows clients to handle - "missing file" scenarios by immediately writing if desired. - Else: - Send READ_SUCCESS containing: - - File generation - - Chunk size - - Actual bytes available (may be less than requested length) - - Number of chunks - - List of chunks, where each contains: - - SHA256 Hash - - List of chunk servers holding this chunk (holders) - - A list of suggested chunk server addresses for writing new chunks - (load balanced via choose_servers_for_write) - - if the message has type WRITE: - Read expect_gen, flags, path_len, path - Read offset, length, num_chunks - - For each chunk in num_chunks: - Read the hash and the list of Chunk Server addresses (locations) that hold it - (This implies the client has already uploaded the data to these servers) - - Append write operation to WAL - - Attempt to apply write to file_tree (returns new_gen and a list of removed_hashes) - - If result is NOENT (File Not Found) AND flags has TOASTY_WRITE_CREATE_IF_MISSING: - - Append create operation (default chunk_size 4096) to WAL - - Create entity in file_tree - - Retry the write operation to file_tree - - If write fails: - Send WRITE_ERROR with error description - Else: - For each new chunk hash provided in the message: - Find the chunk servers associated with that hash in the message - Add the hash to those servers' ms_add_list - - For each removed_hash returned by the file_tree: - Find all chunk servers currently holding this hash - Add the hash to their ms_rem_list (marking it for deletion) - - Send WRITE_SUCCESS containing the new generation number - - if the peer is a CS: - if the message has type AUTH: - Add all IPv4/port and IPv6/port pairs to the data associated to this connection. - Then, respond with a AUTH_RESPONSE message. - if the message has type SYNC: - For each hash received: - If it's not in use by the file tree: - ignore it - If less than REPLICATION_FACTOR CS are holding it: - Add this hash to this CS's add_list - Respond with a SYNC_2 message containing the add_list and the rem_list. - if the message has type SYNC_3: - Create an empty tmp_list - For each hash received: - We expect the hash not to be in the add_list or the old_list. Only hashes that were actually expected to be in on the server should be recovered. - Add the hash to tmp_list. - For each CS holding this hash in their old_list or add_list: - Write its address to the output message - Merge add_list into old_list - Remove elements in tmp_list from old_list - Set old_list equal to tmp_list - Send the message as SYNC_4 - - For each chunk server: - If its last response was more than RESPONSE_TIME_LIMIT seconds ago: - Drop the connection - - Set the current timeout to the next most imminent event - -========================================================= -=== CHUNK SERVER PROCESS ================================ -========================================================= - -chunk_server_init: - Start a TCP server at the interface and port specified using options --addr and --port (default 127.0.0.1 and 8081) - - Create a directory for chunk files at path specified using option --path (default "chunk_server_data") - - Connect to MS using the remote address and port specified using options --remote-addr and --remote-port (default 127.0.0.1, 8080) and send an AUTH message containing the list of local IPv4/port and IPv6/port pairs. - -chunk_server_free: - free all resources - -start_download: - if not downloading already: - get a descriptor from the download_targets array and send DOWNLOAD_CHUNK message with the specified hash. - -chunk_server_tick: - - for each network event: - if connection established: - do nothing - - if disconnect: - if the connection was to MS: - disconnect_time = current_time - - if the connection was to a CS: - downloading = false - - if message received: - if from MS: - if tagged as AUTH_RESPONSE: - Read all chunk file names from the data directory and convert them into hashes by decoding them as hexadecimal. Add all of these hashes to the add_list. - No message is sent back to MS in response. - if tagged as SYNC_2: - For each hash in the first list received by MS: - If no chunk file is associated to it: - add it to a tmp_list - else: - remove it from the rem_list and from the add_list. - For each hash still in the add_list: - Add it to the rem_list - Mark the add_list as empty - - For each hash in the second list received by MS: - Add it to rem_list - - Send SYNC_3 message with a list of hashes made by those in tmp_list and lst_list. - - if tagged as SYNC_4: - At this point MS knows which chunks this CS is missing, so it sent back information for where to download them. - For each element in the list: - Add a download descrptor to download_targets list. - start_download() - - (other cases are not allowed) - - if from CS: - - We may get messages from other CS when trying to download a chunk from it that was lost locally. - - if tagged DOWNLOAD_CHUNK_ERROR: - Start the next download - - if tagged DOWNLOAD_CHUNK_SUCCESS: - Create a chunk file with the received data, then add its hash to the add_list. - start_download() - - if from CLI: - - if tagged CREATE_CHUNK: - Take the received data and make a chunk file for it, then add its hash to the add_list. - Respond with a CREATE_CHUNK_SUCCESS message. - - if tagged UPLOAD_CHUNK: - Load the chunk referred by the client and patch it with the data it sent at the specified offset, then calculate its hash and store it as a new chunk. - Add its hash to the add_list. - Respond with a UPLOAD_CHUNK_SUCCESS message. - - if tagged DOWNLOAD_CHUNK: - Load the chunk referred by the client and respond with its data in a DOWNLOAD_CHUNK_SUCCESS message. - - For each expired item in rem_list: - Delete its associated chunk file - - If the connection to MS is intact and more than SYNC_INTERVAL seconds elapsed since the last sync: - Send a SYNC message containing all add_list items. - - start_download() - - If there is no connection to MS and more than RECONNECT_DELAY seconds passed: - Connect to the metadata server and send a message of type AUTH containing the list of our IPv4s/port pairs and IPv6/port pairs. - - Set the current timeout to the earliest time, considering the next reconnect attempt timeto MS if disconnected and the most imminent rem_list deadline. - -========================================================= -=== CLIENT LIBRARY ====================================== -========================================================= - -# ------------------------------------------------------------------------------ -# 1. DATA STRUCTURES -# ------------------------------------------------------------------------------ - -class ToastyClient: - metadata_server_conn: Connection - chunk_server_pool: Map - pending_operations: Map - -class Operation: - id: Integer - type: Enum(READ, WRITE, CREATE, DELETE, LIST) - status: Enum(PENDING, COMPLETED, FAILED) - - # Context - path: String - user_buffer: Byte[] - offset: Integer - length: Integer - flags: Bitmask - expected_generation: Integer - - # Internal State - chunks_to_process: List - bytes_transferred: Integer - result_data: Any - -class ChunkTask: - # Used for tracking individual chunk uploads/downloads - chunk_index: Integer - hash: String - target_servers: List
- status: Enum(WAITING, IN_PROGRESS, DONE) - final_hash: String # Returned by CS after upload/patch - -# ------------------------------------------------------------------------------ -# 2. PUBLIC BLOCKING API (User Facing) -# ------------------------------------------------------------------------------ - -function toasty_connect(address, port): - client = new ToastyClient() - client.metadata_server_conn = tcp_connect(address, port) - return client - -function toasty_read(client, path, offset, length): - op_handle = begin_read_op(client, path, offset, length) - return wait_for_result(client, op_handle) - -function toasty_write(client, path, data, offset, flags): - op_handle = begin_write_op(client, path, data, offset, flags) - return wait_for_result(client, op_handle) - -# (Create, Delete, and List follow the same pattern: begin_op -> wait_for_result) - -# ------------------------------------------------------------------------------ -# 3. INTERNAL ASYNCHRONOUS LOGIC -# ------------------------------------------------------------------------------ - -# --- READ LOGIC --- - -function begin_read_op(client, path, offset, length): - op = new Operation(type=READ, path, offset, length) - - # Step 1: Ask Metadata Server (MS) for the file layout - send_message(client.metadata_server_conn, - type=READ_REQUEST, - path=path, range=(offset, length)) - - return register_operation(client, op) - -function handle_read_metadata_response(op, msg): - if msg.type == ERROR: - op.status = FAILED - return - - # Step 2: Calculate chunk plan - chunk_size = msg.chunk_size - op.chunks_to_process = calculate_required_chunks(op.offset, op.length, chunk_size) - - # Step 3: Queue downloads from Chunk Servers (CS) - for chunk in op.chunks_to_process: - # MS returns a list of holders for each chunk hash - best_cs = load_balance_select(chunk.holders) - conn = get_cs_connection(best_cs) - - send_message(conn, - type=DOWNLOAD_CHUNK, - hash=chunk.hash) - -function handle_chunk_download_success(op, msg): - # Step 4: Assemble data - copy_bytes(src=msg.data, dest=op.user_buffer, at_offset=msg.chunk_index) - - mark_task_complete(op, msg.chunk_index) - - if all_tasks_complete(op): - op.status = COMPLETED - -# --- WRITE LOGIC (The Complex Part) --- - -function begin_write_op(client, path, data, offset, flags): - op = new Operation(type=WRITE, path, data, offset, flags) - - # Step 1: Send READ to MS first. - # We need the current generation number and existing chunk hashes to perform - # patches or ensure consistency, even if we are overwriting. - send_message(client.metadata_server_conn, - type=READ_REQUEST, - path=path, range=(offset, len(data))) - - return register_operation(client, op) - -function handle_write_metadata_ready(op, msg): - # Step 2: Prepare Upload Tasks - # If file missing & CREATE_IF_MISSING flag is set, we use defaults (size 0, gen 0). - current_generation = msg.generation - chunk_size = msg.chunk_size - write_targets = msg.write_locations # Recommended CS IPs from MS - - # Slice user data into chunks - sliced_chunks = split_into_chunks(op.user_buffer, chunk_size) - - for slice in sliced_chunks: - task = new ChunkTask() - task.target_servers = write_targets # We must replicate to 3 servers - - # Determine if this is a Create (New) or Update (Patch) - if slice.index >= len(msg.existing_hashes): - task.type = NEW_CHUNK - else: - task.type = PATCH_CHUNK - task.base_hash = msg.existing_hashes[slice.index] - - op.chunks_to_process.add(task) - - # Step 3: Start Uploads (Parallel up to limit) - pump_upload_queue(op) - -function pump_upload_queue(op): - while active_uploads(op) < MAX_PARALLEL_UPLOADS: - task = get_next_waiting_task(op) - if not task: break - - # We must upload to N replicas (usually 3) - for replica_addr in task.target_servers: - conn = get_cs_connection(replica_addr) - - if task.type == NEW_CHUNK: - # Full upload - send_message(conn, type=CREATE_CHUNK, data=task.data) - else: - # Differential patch (Rsync-style or offset-based) - send_message(conn, type=UPLOAD_CHUNK, - base_hash=task.base_hash, - patch_data=task.data) - -function handle_upload_ack(op, msg): - # Step 4: Collect new hashes - task = get_task(op, msg.task_id) - - # The CS computes the new SHA256 after applying the patch/data - task.final_hash = msg.new_hash - task.replicas_confirmed++ - - if task.replicas_confirmed >= REQUIRED_REPLICATION: - task.status = DONE - - if all_tasks_complete(op): - commit_write(op) - else: - pump_upload_queue(op) # Start next batch - -function commit_write(op): - # Step 5: Atomic Commit at Metadata Server - commit_msg = new Message(type=WRITE_REQUEST) - commit_msg.path = op.path - commit_msg.expected_generation = op.expected_generation # Optimistic Locking - commit_msg.flags = op.flags - - # Map the linear chunk index to the new Hash + Location list - commit_msg.chunks = map_chunks_to_hashes_and_locations(op.chunks_to_process) - - send_message(client.metadata_server_conn, commit_msg) - -function handle_commit_response(op, msg): - if msg.type == WRITE_SUCCESS: - op.status = COMPLETED - op.result_data = msg.new_generation - elif msg.type == ERROR and msg.code == GENERATION_MISMATCH: - # Optimistic concurrency failed (someone else wrote). - # Retry the whole operation from Step 1 (READ). - reset_and_retry(op) - else: - op.status = FAILED - -# ------------------------------------------------------------------------------ -# 4. EVENT LOOP (Driver) -# ------------------------------------------------------------------------------ - -function wait_for_result(client, op_handle): - while get_op_status(op_handle) == PENDING: - # Wait for network activity (Poll/Select) - events = wait_for_network_events(timeout=DEFAULT_TIMEOUT) - - for event in events: - if event.type == DISCONNECT: - handle_disconnect(client, event.conn) - continue - - # Route message to the specific Operation ID embedded in the packet tag - op = client.pending_operations[event.tag.op_id] - msg = read_message(event.conn) - - # Dispatcher - switch (op.type, msg.type): - case (READ, READ_SUCCESS): handle_read_metadata_response(op, msg) - case (READ, CHUNK_DATA): handle_chunk_download_success(op, msg) - - case (WRITE, READ_SUCCESS): handle_write_metadata_ready(op, msg) - case (WRITE, UPLOAD_ACK): handle_upload_ack(op, msg) - case (WRITE, WRITE_SUCCESS): handle_commit_response(op, msg) - - case (_, ERROR): handle_error(op, msg) - - return get_op_result(op_handle) -} \ No newline at end of file diff --git a/src/blob_client.c b/src/blob_client.c new file mode 100644 index 0000000..6e03794 --- /dev/null +++ b/src/blob_client.c @@ -0,0 +1,675 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "blob_client.h" +#include "server.h" + +#define TIME_FMT "%7.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static uint64_t next_blob_client_id = 100; + +static uint64_t blob_random(void) +{ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) + return quakey_random(); +#else + return (uint64_t)rand(); +#endif +} + +static void blob_log_impl(BlobClientState *state, Time now, const char *event, const char *detail) +{ + printf("[" TIME_FMT "] BLOB %lu | %-20s %s\n", + TIME_VAL(now), + state->client_id, + event, + detail ? detail : ""); +} + +#define blob_log(state, now, event, fmt, ...) do { \ + char _detail[256]; \ + snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \ + blob_log_impl(state, now, event, _detail); \ +} while (0) + +#define blob_log_simple(state, now, event) \ + blob_log_impl(state, now, event, NULL) + +static int leader_idx(BlobClientState *state) +{ + return state->view_number % state->num_servers; +} + +/////////////////////////////////////////////////////////////// +// Chunk ack counting +/////////////////////////////////////////////////////////////// + +static int count_acks(uint32_t mask) +{ + int n = 0; + for (int i = 0; i < 32; i++) + if (mask & (1u << i)) n++; + return n; +} + +static bool all_chunks_acked(BlobClientState *state) +{ + for (int i = 0; i < state->num_chunks; i++) { + if (count_acks(state->chunks[i].ack_mask) < state->f_plus_one) + return false; + } + return true; +} + +static bool all_chunks_fetched(BlobClientState *state) +{ + for (int i = 0; i < state->num_chunks; i++) { + if (!state->chunks[i].fetched) + return false; + } + return true; +} + +/////////////////////////////////////////////////////////////// +// Generate a test blob +/////////////////////////////////////////////////////////////// + +static void generate_test_blob(BlobClientState *state) +{ + // Generate random bucket/key + snprintf(state->bucket, META_BUCKET_MAX, "blob-b%d", (int)(blob_random() % 4)); + snprintf(state->key, META_KEY_MAX, "blob-k%d", (int)(blob_random() % 64)); + + // 1-3 chunks per blob + state->num_chunks = 1 + blob_random() % 3; + state->blob_size = 0; + + for (int i = 0; i < state->num_chunks; i++) { + // Generate random hash for each chunk + for (int j = 0; j < 32; j++) + state->chunks[i].hash.data[j] = blob_random() & 0xFF; + state->chunks[i].size = BLOB_TEST_CHUNK_SIZE; + state->chunks[i].ack_mask = 0; + state->chunks[i].fetched = false; + state->blob_size += BLOB_TEST_CHUNK_SIZE; + } + + // Content hash (random for now) + for (int j = 0; j < 32; j++) + state->content_hash.data[j] = blob_random() & 0xFF; +} + +/////////////////////////////////////////////////////////////// +// Send operations +/////////////////////////////////////////////////////////////// + +// Send StoreChunk for a given chunk to a specific server. +// The chunk data is the hash bytes (32 bytes). +static void send_store_chunk(BlobClientState *state, int chunk_idx, int server_idx) +{ + int conn_idx = tcp_index_from_tag(&state->tcp, server_idx); + if (conn_idx < 0) { + tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL); + return; // Will retry next tick + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) return; + + StoreChunkMessage msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_STORE_CHUNK, + .length = sizeof(StoreChunkMessage) + state->chunks[chunk_idx].size, + }, + .hash = state->chunks[chunk_idx].hash, + .size = state->chunks[chunk_idx].size, + }; + + byte_queue_write(output, &msg, sizeof(msg)); + // Chunk data = hash bytes (BLOB_TEST_CHUNK_SIZE = 32 = sizeof(SHA256)) + byte_queue_write(output, state->chunks[chunk_idx].hash.data, state->chunks[chunk_idx].size); +} + +static void send_commit_put(BlobClientState *state) +{ + int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state)); + if (conn_idx < 0) { + tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL); + return; + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) return; + + CommitPutMessage msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_COMMIT_PUT, + .length = sizeof(CommitPutMessage), + }, + .oper = { + .type = META_OPER_PUT, + .size = state->blob_size, + .content_hash = state->content_hash, + .num_chunks = state->num_chunks, + }, + .client_id = state->client_id, + .request_id = state->request_id, + }; + memcpy(msg.oper.bucket, state->bucket, META_BUCKET_MAX); + memcpy(msg.oper.key, state->key, META_KEY_MAX); + for (int i = 0; i < state->num_chunks; i++) { + msg.oper.chunks[i].hash = state->chunks[i].hash; + msg.oper.chunks[i].size = state->chunks[i].size; + } + + byte_queue_write(output, &msg, sizeof(msg)); +} + +static void send_get_blob(BlobClientState *state, int server_idx) +{ + int conn_idx = tcp_index_from_tag(&state->tcp, server_idx); + if (conn_idx < 0) { + tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL); + return; + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) return; + + GetBlobMessage msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_GET_BLOB, + .length = sizeof(GetBlobMessage), + }, + }; + memcpy(msg.bucket, state->bucket, META_BUCKET_MAX); + memcpy(msg.key, state->key, META_KEY_MAX); + + byte_queue_write(output, &msg, sizeof(msg)); +} + +static void send_fetch_chunk(BlobClientState *state, int chunk_idx, int server_idx) +{ + int conn_idx = tcp_index_from_tag(&state->tcp, server_idx); + if (conn_idx < 0) { + tcp_connect(&state->tcp, state->server_addrs[server_idx], server_idx, NULL); + return; + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) return; + + FetchChunkMessage msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_FETCH_CHUNK, + .length = sizeof(FetchChunkMessage), + }, + .hash = state->chunks[chunk_idx].hash, + .sender_idx = -1, // Client (not a peer server) + }; + + byte_queue_write(output, &msg, sizeof(msg)); +} + +/////////////////////////////////////////////////////////////// +// Message processing +/////////////////////////////////////////////////////////////// + +static int +process_message(BlobClientState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + (void) conn_idx; + Time now = get_current_time(); + + switch (type) { + + case MESSAGE_TYPE_REDIRECT: { + RedirectMessage redirect; + if (msg.len != sizeof(RedirectMessage)) + return -1; + memcpy(&redirect, msg.ptr, sizeof(redirect)); + if (redirect.view_number > state->view_number) { + blob_log(state, now, "RECV REDIRECT", "view=%lu -> %lu", + (unsigned long)state->view_number, + (unsigned long)redirect.view_number); + state->view_number = redirect.view_number; + // Re-send CommitPut to new leader + if (state->phase == BLOB_COMMITTING) { + send_commit_put(state); + state->phase_time = now; + } + } + return 0; + } + + case MESSAGE_TYPE_STORE_CHUNK_ACK: { + if (state->phase != BLOB_UPLOADING) + return 0; + + StoreChunkAckMessage ack; + if (msg.len != sizeof(StoreChunkAckMessage)) + return -1; + memcpy(&ack, msg.ptr, sizeof(ack)); + + // Find which chunk this ack is for + for (int i = 0; i < state->num_chunks; i++) { + if (memcmp(&state->chunks[i].hash, &ack.hash, sizeof(SHA256)) == 0) { + int tag = tcp_get_tag(&state->tcp, conn_idx); + if (tag >= 0 && tag < 32 && ack.success) + state->chunks[i].ack_mask |= (1u << tag); + blob_log(state, now, "RECV STORE_ACK", "chunk=%d server=%d ok=%d acks=%d/%d", + i, tag, ack.success, + count_acks(state->chunks[i].ack_mask), state->f_plus_one); + break; + } + } + + // Check if all chunks have f+1 acks + if (all_chunks_acked(state)) { + blob_log(state, now, "UPLOAD DONE", "%s/%s chunks=%d", + state->bucket, state->key, state->num_chunks); + + // Move to commit phase + state->phase = BLOB_COMMITTING; + state->phase_time = now; + state->request_id++; + send_commit_put(state); + blob_log(state, now, "SEND COMMIT_PUT", "%s/%s req=%lu", + state->bucket, state->key, state->request_id); + } + return 0; + } + + case MESSAGE_TYPE_REPLY: { + if (state->phase != BLOB_COMMITTING) + return 0; + + ReplyMessage reply; + if (msg.len != sizeof(ReplyMessage)) + return -1; + memcpy(&reply, msg.ptr, sizeof(reply)); + + if (reply.request_id != state->request_id) + return 0; + + if (reply.rejected) { + blob_log(state, now, "RECV REPLY", "REJECTED, retrying"); + state->phase_time = now; + send_commit_put(state); + return 0; + } + + state->puts_completed++; + blob_log(state, now, "PUT DONE", "%s/%s puts=%d", + state->bucket, state->key, state->puts_completed); + + // Next: GET this blob back to verify + state->do_get_next = true; + state->phase = BLOB_IDLE; + return 0; + } + + case MESSAGE_TYPE_GET_BLOB_RESPONSE: { + if (state->phase != BLOB_FETCHING_META) + return 0; + + GetBlobResponseMessage resp; + if (msg.len != sizeof(GetBlobResponseMessage)) + return -1; + memcpy(&resp, msg.ptr, sizeof(resp)); + + if (!resp.found) { + // Blob not yet committed on this server, retry another + blob_log(state, now, "RECV GET_BLOB", "NOT_FOUND, retrying"); + state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers; + state->phase_time = now; + send_get_blob(state, state->fetch_server_idx); + return 0; + } + + blob_log(state, now, "RECV GET_BLOB", "%s/%s chunks=%u", + state->bucket, state->key, resp.num_chunks); + + // Verify metadata matches what we uploaded + if (resp.num_chunks != (uint32_t)state->num_chunks) { + fprintf(stderr, "BLOB CLIENT: metadata mismatch! expected %d chunks, got %u\n", + state->num_chunks, resp.num_chunks); + return -1; + } + for (int i = 0; i < state->num_chunks; i++) { + if (memcmp(&resp.chunks[i].hash, &state->chunks[i].hash, sizeof(SHA256)) != 0) { + fprintf(stderr, "BLOB CLIENT: chunk %d hash mismatch!\n", i); + return -1; + } + } + + // Start fetching chunk data + state->phase = BLOB_FETCHING_DATA; + state->phase_time = now; + state->fetch_chunk_idx = 0; + state->fetch_server_idx = 0; + for (int i = 0; i < state->num_chunks; i++) + state->chunks[i].fetched = false; + + // Send FetchChunk for the first chunk + send_fetch_chunk(state, 0, state->fetch_server_idx); + blob_log(state, now, "SEND FETCH_CHUNK", "chunk=0 server=%d", state->fetch_server_idx); + return 0; + } + + case MESSAGE_TYPE_FETCH_CHUNK_RESPONSE: { + if (state->phase != BLOB_FETCHING_DATA) + return 0; + + FetchChunkResponseMessage resp; + if (msg.len < sizeof(FetchChunkResponseMessage)) + return -1; + memcpy(&resp, msg.ptr, sizeof(resp)); + + // Find which chunk this is for + int chunk_idx = -1; + for (int i = 0; i < state->num_chunks; i++) { + if (memcmp(&state->chunks[i].hash, &resp.hash, sizeof(SHA256)) == 0) { + chunk_idx = i; + break; + } + } + if (chunk_idx < 0) + return 0; // Unknown chunk, ignore + + if (resp.size == 0) { + // Server doesn't have the chunk. Try next server. + blob_log(state, now, "RECV FETCH_RESP", "chunk=%d NOT_FOUND, trying next server", chunk_idx); + state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers; + send_fetch_chunk(state, chunk_idx, state->fetch_server_idx); + return 0; + } + + // Verify data size + uint32_t data_size = msg.len - sizeof(FetchChunkResponseMessage); + if (data_size != resp.size || resp.size != state->chunks[chunk_idx].size) { + fprintf(stderr, "BLOB CLIENT: chunk %d size mismatch! expected %u, got %u\n", + chunk_idx, state->chunks[chunk_idx].size, resp.size); + return -1; + } + + // Verify data content (data should equal hash bytes) + uint8_t *data = (uint8_t *)(msg.ptr + sizeof(FetchChunkResponseMessage)); + if (memcmp(data, state->chunks[chunk_idx].hash.data, BLOB_TEST_CHUNK_SIZE) != 0) { + fprintf(stderr, "BLOB CLIENT: chunk %d data verification FAILED!\n", chunk_idx); + return -1; + } + + state->chunks[chunk_idx].fetched = true; + blob_log(state, now, "RECV FETCH_RESP", "chunk=%d VERIFIED", chunk_idx); + + // Check if all chunks fetched + if (all_chunks_fetched(state)) { + state->gets_completed++; + state->gets_verified++; + blob_log(state, now, "GET DONE", "%s/%s gets=%d verified=%d", + state->bucket, state->key, state->gets_completed, state->gets_verified); + + state->do_get_next = false; + state->phase = BLOB_IDLE; + return 0; + } + + // Fetch next unfetched chunk + for (int i = 0; i < state->num_chunks; i++) { + if (!state->chunks[i].fetched) { + state->fetch_chunk_idx = i; + state->fetch_server_idx = 0; + send_fetch_chunk(state, i, 0); + blob_log(state, now, "SEND FETCH_CHUNK", "chunk=%d server=0", i); + break; + } + } + return 0; + } + + default: + break; + } + + return 0; +} + +/////////////////////////////////////////////////////////////// +// Init / Tick / Free +/////////////////////////////////////////////////////////////// + +int blob_client_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + BlobClientState *state = state_; + + state->num_servers = 0; + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--server")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --server missing value\n"); + return -1; + } + if (state->num_servers == NODE_LIMIT) { + fprintf(stderr, "Node limit reached\n"); + return -1; + } + if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) { + fprintf(stderr, "Malformed address\n"); + return -1; + } + } else { + // Ignore unknown options + } + } + + addr_sort(state->server_addrs, state->num_servers); + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Blob client :: Couldn't setup TCP context\n"); + return -1; + } + + // f = (num_servers - 1) / 2, so f+1 = (num_servers + 1) / 2 + state->f_plus_one = (state->num_servers + 1) / 2; + + state->view_number = 0; + state->request_id = 0; + state->client_id = next_blob_client_id++; + state->phase = BLOB_IDLE; + state->phase_time = 0; + state->do_get_next = false; + state->puts_completed = 0; + state->gets_completed = 0; + state->gets_verified = 0; + state->reconnect_time = 0; + state->upload_server_cursor = 0; + + // Connect to all servers + for (int i = 0; i < state->num_servers; i++) { + if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) { + fprintf(stderr, "Blob client :: Couldn't connect to server %d\n", i); + tcp_context_free(&state->tcp); + return -1; + } + } + + { + Time now = get_current_time(); + blob_log(state, now, "INIT", "servers=%d f+1=%d", state->num_servers, state->f_plus_one); + } + + *timeout = 0; + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Blob client :: Not enough poll capacity\n"); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int blob_client_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + BlobClientState *state = state_; + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + + if (events[i].type == EVENT_DISCONNECT) { + int conn_idx = events[i].conn_idx; + tcp_close(&state->tcp, conn_idx); + continue; + } + + if (events[i].type != EVENT_MESSAGE) + continue; + + int conn_idx = events[i].conn_idx; + for (;;) { + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + Time now = get_current_time(); + + // Timeout handling: if we've been in any phase too long, retry + if (state->phase != BLOB_IDLE) { + Time phase_deadline = state->phase_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (phase_deadline <= now) { + blob_log(state, now, "TIMEOUT", "phase=%d, retrying", state->phase); + + switch (state->phase) { + case BLOB_UPLOADING: + // Re-send StoreChunk for chunks that need more acks + for (int c = 0; c < state->num_chunks; c++) { + if (count_acks(state->chunks[c].ack_mask) < state->f_plus_one) { + for (int s = 0; s < state->f_plus_one; s++) { + if (!(state->chunks[c].ack_mask & (1u << s))) + send_store_chunk(state, c, s); + } + } + } + state->phase_time = now; + break; + + case BLOB_COMMITTING: + state->view_number++; + send_commit_put(state); + state->phase_time = now; + break; + + case BLOB_FETCHING_META: + state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers; + send_get_blob(state, state->fetch_server_idx); + state->phase_time = now; + break; + + case BLOB_FETCHING_DATA: + state->fetch_server_idx = (state->fetch_server_idx + 1) % state->num_servers; + send_fetch_chunk(state, state->fetch_chunk_idx, state->fetch_server_idx); + state->phase_time = now; + break; + + default: + break; + } + } + } + + // Ensure connections to all servers (needed during all phases for + // retransmission after network partitions or server restarts) + for (int i = 0; i < state->num_servers; i++) { + int ci = tcp_index_from_tag(&state->tcp, i); + if (ci < 0) + tcp_connect(&state->tcp, state->server_addrs[i], i, NULL); + } + + // Start new operation when idle + if (state->phase == BLOB_IDLE) { + + if (state->do_get_next) { + // GET the blob we just PUT + state->phase = BLOB_FETCHING_META; + state->phase_time = now; + state->fetch_server_idx = 0; + + send_get_blob(state, state->fetch_server_idx); + blob_log(state, now, "SEND GET_BLOB", "%s/%s server=%d", + state->bucket, state->key, state->fetch_server_idx); + + } else { + // PUT a new blob + generate_test_blob(state); + + state->phase = BLOB_UPLOADING; + state->phase_time = now; + + blob_log(state, now, "START PUT", "%s/%s chunks=%d", + state->bucket, state->key, state->num_chunks); + + // Send StoreChunk for each chunk to the first f+1 servers + for (int c = 0; c < state->num_chunks; c++) { + for (int s = 0; s < state->f_plus_one; s++) { + send_store_chunk(state, c, s); + } + } + } + } + + // Set timeout + Time deadline = INVALID_TIME; + if (state->phase != BLOB_IDLE) { + nearest_deadline(&deadline, state->phase_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL); + } + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int blob_client_free(void *state_) +{ + BlobClientState *state = state_; + { + Time now = get_current_time(); + blob_log(state, now, "SHUTDOWN", "puts=%d gets=%d verified=%d", + state->puts_completed, state->gets_completed, state->gets_verified); + } + tcp_context_free(&state->tcp); + return 0; +} diff --git a/src/blob_client.h b/src/blob_client.h new file mode 100644 index 0000000..ff70d49 --- /dev/null +++ b/src/blob_client.h @@ -0,0 +1,86 @@ +#ifndef BLOB_CLIENT_INCLUDED +#define BLOB_CLIENT_INCLUDED + +#include +#include + +#include "config.h" +#include "metadata.h" + +// Maximum number of chunks per blob in the test client. +// Kept small for simulation; real blobs use META_CHUNKS_MAX. +#define BLOB_MAX_CHUNKS 8 + +// Chunk data size for the test client (32 bytes = SHA256 hash length). +// Each chunk's data is its hash bytes, making verification trivial. +#define BLOB_TEST_CHUNK_SIZE 32 + +typedef enum { + BLOB_IDLE, // Ready to start a new operation + BLOB_UPLOADING, // Sending StoreChunk, waiting for acks + BLOB_COMMITTING, // Sent CommitPut, waiting for REPLY + BLOB_FETCHING_META, // Sent GetBlob, waiting for response + BLOB_FETCHING_DATA, // Sending FetchChunk, waiting for responses +} BlobPhase; + +typedef struct { + SHA256 hash; + uint32_t size; + uint32_t ack_mask; // Bitmask: which servers acked this chunk + bool fetched; // GET: whether chunk data was received and verified +} BlobChunkState; + +typedef struct { + + TCP tcp; + + Address server_addrs[NODE_LIMIT]; + int num_servers; + int f_plus_one; // Number of servers to upload each chunk to + + uint64_t view_number; + uint64_t client_id; + uint64_t request_id; + + BlobPhase phase; + Time phase_time; // When we entered this phase + + // Current blob metadata + char bucket[META_BUCKET_MAX]; + char key[META_KEY_MAX]; + uint64_t blob_size; + SHA256 content_hash; + int num_chunks; + BlobChunkState chunks[BLOB_MAX_CHUNKS]; + + // Upload tracking + int upload_server_cursor; // Next server index to try for StoreChunk + + // Download tracking + int fetch_chunk_idx; // Which chunk we're currently fetching + int fetch_server_idx; // Which server to try next + + // Alternation between PUT and GET + bool do_get_next; // If true, next op is GET; else PUT + + // Statistics + int puts_completed; + int gets_completed; + int gets_verified; + + Time reconnect_time; + +} BlobClientState; + +struct pollfd; + +int blob_client_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int blob_client_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int blob_client_free(void *state); + +#endif // BLOB_CLIENT_INCLUDED diff --git a/src/chunk_server.c b/src/chunk_server.c deleted file mode 100644 index 6e2448f..0000000 --- a/src/chunk_server.c +++ /dev/null @@ -1,1069 +0,0 @@ -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include -#include - -#include "tcp.h" -#include "sha256.h" -#include "config.h" -#include "message.h" -#include "hash_set.h" -#include "byte_queue.h" -#include "file_system.h" -#include "chunk_server.h" - -static string hash2path(ChunkServer *state, SHA256 hash, char *out) -{ - strcpy(out, state->path); - strcat(out, "/"); - - size_t tmp = strlen(out); - - append_hex_as_str(out + tmp, hash); - - out[tmp + 64] = '\0'; - - return (string) { out, strlen(out) }; -} - -static int load_chunk(ChunkServer *state, SHA256 hash, string *data) -{ - char buf[PATH_MAX]; - string path = hash2path(state, hash, buf); - - int ret = file_read_all(path, data); - if (ret < 0) - return -1; - - SHA256 fresh_hash; - sha256(data->ptr, data->len, (uint8_t*) &fresh_hash.data); - - if (memcmp(&hash, &fresh_hash, sizeof(SHA256))) { - free(data->ptr); - return -1; - } - - return 0; -} - -static int store_chunk(ChunkServer *state, string data, SHA256 *hash) -{ - sha256(data.ptr, data.len, (uint8_t*) hash->data); - - char buf[PATH_MAX]; - string path = hash2path(state, *hash, buf); - - // Note that this write is not atomic. If we crash - // while writing, we'll get an inconsistent file. - // This is okay as long as we check that the hash - // is correct while reading back the data. - Handle fd; - if (file_open(path, &fd) < 0) // TODO: open in overwrite mode - return -1; - int copied = 0; - while (copied < data.len) { - int ret = file_write(fd, - data.ptr + copied, - data.len - copied); - if (ret < 0) { - file_close(fd); - return -1; - } - copied += ret; - } - file_close(fd); - return 0; -} - -static bool chunk_exists(ChunkServer *state, SHA256 hash) -{ - char buf[PATH_MAX]; - string path = hash2path(state, hash, buf); - - // Try to open the file to check if it exists - // TODO: this isn't right. There should be something like file_exists - Handle fd; - if (file_open(path, &fd) == 0) { - file_close(fd); - return true; - } - return false; -} - -static int patch_chunk(ChunkServer *state, SHA256 target_chunk, - uint64_t patch_off, string patch, SHA256 *new_hash) -{ - string data; - int ret = load_chunk(state, target_chunk, &data); - if (ret < 0) - return -1; - - if (patch_off > SIZE_MAX - patch.len) { - free(data.ptr); - return -1; - } - - if (patch_off + (size_t) patch.len > (size_t) data.len) { - free(data.ptr); - return -1; - } - - memcpy(data.ptr + patch_off, patch.ptr, patch.len); - - ret = store_chunk(state, data, new_hash); - if (ret < 0) { - free(data.ptr); - return -1; - } - - free(data.ptr); - return 0; -} - -static void download_targets_init(DownloadTargets *targets) -{ - targets->items = NULL; - targets->count = 0; - targets->capacity = 0; -} - -static void download_targets_free(DownloadTargets *targets) -{ - free(targets->items); -} - -static void download_targets_remove(DownloadTargets *targets, - SHA256 hash) -{ - // NOTE: This changes the download order! - for (int i = 0; i < targets->count; i++) - if (!memcmp(&targets->items[i].hash, &hash, sizeof(SHA256))) - targets->items[i--] = targets->items[--targets->count]; -} - -static int download_targets_push(DownloadTargets *targets, - Address addr, SHA256 hash) -{ - // Avoid duplicates! This is important as the metadata server may - // tell us to download our missing chunks again while we are still - // going through the previous list of downloads. This check becomes - // relevant as the update period approaches the time the chunk server - // needs to go through a list of downloads. - for (int i = 0; i < targets->count; i++) - if (addr_eql(targets->items[i].addr, addr) && !memcmp(&targets->items[i].hash, &hash, sizeof(SHA256))) - return 0; - - if (targets->count == targets->capacity) { - - int new_capacity; - if (targets->capacity == 0) - new_capacity = 8; - else - new_capacity = 2 * targets->capacity; - - DownloadTarget *new_items = malloc(new_capacity * sizeof(DownloadTarget)); - if (new_items == NULL) - return -1; - - if (targets->capacity > 0) { - memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0])); - free(targets->items); - } - - targets->items = new_items; - targets->capacity = new_capacity; - } - - targets->items[targets->count++] = (DownloadTarget) { addr, hash }; - return 0; -} - -static bool download_targets_pop(DownloadTargets *targets, - DownloadTarget *target) -{ - // Read the head - if (targets->count == 0) - return false; - *target = targets->items[0]; - - // Pop the head - for (int i = 0; i < targets->count-1; i++) - targets->items[i] = targets->items[i+1]; - targets->count--; - - // We expect the download list to be empty most - // of the time, so if this was the last element - // there may not be a new one for a while and we - // can clear the array. - if (targets->count == 0) { - free(targets->items); - targets->items = NULL; - targets->capacity = 0; - } - - return true; -} - -static int send_error(TCP *tcp, int conn_idx, - bool close, uint16_t type, string msg) -{ - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(tcp, conn_idx); - message_writer_init(&writer, output, type); - - uint16_t len = MIN(msg.len, UINT16_MAX); - message_write(&writer, &len, sizeof(len)); - message_write(&writer, msg.ptr, len); - if (!message_writer_free(&writer)) - return -1; - if (close) - return -1; - return 0; -} - -static void start_download(ChunkServer *state) -{ - if (state->downloading) - return; // Already started - - DownloadTarget target; - if (!download_targets_pop(&state->download_targets, &target)) - return; // No more downloads - - ByteQueue *output; - if (tcp_connect(&state->tcp, target.addr, TAG_CHUNK_SERVER, &output) < 0) - return; // Couldn't start connect operation - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_DOWNLOAD_CHUNK); - message_write_hash(&writer, target.hash); - message_write_u8(&writer, 1); - if (!message_writer_free(&writer)) - return; - - state->current_download_target_hash = target.hash; - state->downloading = true; -} - -static int process_metadata_server_sync_2(ChunkServer *state, - int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t add_count; - if (!binary_read(&reader, &add_count, sizeof(add_count))) - return -1; - - Time current_time = get_current_time(); - if (current_time == INVALID_TIME) - return -1; - - HashSet tmp_list; - hash_set_init(&tmp_list); - - for (uint32_t i = 0; i < add_count; i++) { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) - return -1; - - // Elements in ms_add_list that are not held by the - // chunk server are added to a temporary list tmp_list - - if (!chunk_exists(state, hash)) { - if (hash_set_insert(&tmp_list, hash) < 0) { - hash_set_free(&tmp_list); - return -1; - } - continue; - } - - timed_hash_set_remove(&state->cs_rem_list, hash); - hash_set_remove(&state->cs_add_list, hash); - } - - for (int i = 0; i < state->cs_add_list.count; i++) { - if (timed_hash_set_insert(&state->cs_rem_list, state->cs_add_list.items[i], current_time) < 0) { - hash_set_free(&tmp_list); - return -1; - } - } - - hash_set_clear(&state->cs_add_list); - - uint32_t rem_count; - if (!binary_read(&reader, &rem_count, sizeof(rem_count))) { - hash_set_free(&tmp_list); - return -1; - } - - for (uint32_t i = 0; i < rem_count; i++) { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - hash_set_free(&tmp_list); - return -1; - } - - if (timed_hash_set_insert(&state->cs_rem_list, hash, current_time) < 0) { - hash_set_free(&tmp_list); - return -1; - } - } - - if (binary_read(&reader, NULL, 1)) { - hash_set_free(&tmp_list); - return -1; - } - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_SYNC_3); - - if ((uint32_t) state->cs_lst_list.count > UINT32_MAX - tmp_list.count) { - hash_set_free(&tmp_list); - return -1; - } - message_write_u32(&writer, tmp_list.count + state->cs_lst_list.count); - - for (int i = 0; i < tmp_list.count; i++) - message_write_hash(&writer, tmp_list.items[i]); - - for (int i = 0; i < state->cs_lst_list.count; i++) { - message_write_hash(&writer, state->cs_lst_list.items[i]); - } - - hash_set_free(&tmp_list); - - if (!message_writer_free(&writer)) - return -1; - - return 0; -} - -static int -process_metadata_server_sync_4(ChunkServer *state, int conn_idx, ByteView msg) -{ - (void) conn_idx; - - // The metadata server wants us to download chunks from other chunk servers - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t num_missing; - if (!binary_read(&reader, &num_missing, sizeof(num_missing))) - return -1; - - for (uint32_t i = 0; i < num_missing; i++) { - - uint32_t num_holders; - if (!binary_read(&reader, &num_holders, sizeof(num_holders))) - return -1; - - // Temporary storage for all addresses from all holders - Address addrs[REPLICATION_FACTOR]; - int num_addrs = 0; - - // Read addresses from each holder - for (uint32_t j = 0; j < num_holders; j++) { - int num = binary_read_addr_list(&reader, addrs + num_addrs, REPLICATION_FACTOR - num_addrs); - if (num < 0) - return -1; - num_addrs += num; - } - - // Read the hash - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) - return -1; - - for (int j = 0; j < num_addrs; j++) - download_targets_push(&state->download_targets, addrs[j], hash); - } - - if (binary_read(&reader, NULL, 1)) - return -1; - - start_download(state); - - // There is no need to respond here - return 0; -} - -static int -process_metadata_server_auth_response(ChunkServer *state, int conn_idx, ByteView msg) -{ - (void) conn_idx; - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - // TODO: Read whether the metadata server already - // holds our list of chunks. If it doesn't, - // add all held chunks to the add list. - bool avoid_full_scan = false; - - if (binary_read(&reader, NULL, 1)) - return -1; - - if (avoid_full_scan) - return 0; - - string path = { state->path, strlen(state->path) }; - - DirectoryScanner scanner; - if (directory_scanner_init(&scanner, path) < 0) { - return -1; - } - - for (;;) { - - string name; - int ret = directory_scanner_next(&scanner, &name); - if (ret < 0) { - directory_scanner_free(&scanner); - return -1; - } - if (ret == 1) - break; - assert(ret == 0); - - SHA256 hash; - - // Hash length as a string is 64 bytes - if (name.len != 64) - continue; - - bool invalid = false; - for (int i = 0; i < 64; i += 2) { - - uint8_t h = name.ptr[i+0]; - uint8_t l = name.ptr[i+1]; - - if (0) {} - else if (h >= '0' && h <= '9') h = h - '0'; - else if (h >= 'a' && h <= 'f') h = h - 'a' + 10; - else if (h >= 'A' && h <= 'F') h = h - 'A' + 10; - else { invalid = true; break; } - - if (0) {} - else if (l >= '0' && l <= '9') l = l - '0'; - else if (l >= 'a' && l <= 'f') l = l - 'a' + 10; - else if (l >= 'A' && l <= 'F') l = l - 'A' + 10; - else { invalid = true; break; } - - hash.data[i >> 1] = (h << 4) | l; - } - if (invalid) continue; - - if (hash_set_insert(&state->cs_add_list, hash) < 0) { - directory_scanner_free(&scanner); - return -1; - } - } - - directory_scanner_free(&scanner); - return 0; -} - -static int -process_chunk_server_download_error(ChunkServer *state, int conn_idx, ByteView msg) -{ - (void) msg; - (void) conn_idx; - - state->downloading = false; - - start_download(state); - return 0; -} - -static int -process_chunk_server_download_success(ChunkServer *state, int conn_idx, ByteView msg) -{ - (void) conn_idx; - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t data_len; - if (!binary_read(&reader, &data_len, sizeof(data_len))) - return -1; - - if (data_len > (uint32_t) (reader.len - reader.cur)) - return -1; - string data = { (char*) reader.src + reader.cur, data_len }; - - if (!binary_read(&reader, NULL, data_len)) - return -1; - - if (binary_read(&reader, NULL, 1)) - return -1; - - // Store the downloaded chunk - SHA256 dummy; - if (store_chunk(state, data, &dummy) < 0) - return -1; - - // The download succeded! - - // Mark that we are not downloading anymore - state->downloading = false; - - // Since we managed to acquire this chunk, we can - // remove any other downloads to it. - download_targets_remove(&state->download_targets, state->current_download_target_hash); - - // Add the newly acquired chunk to the add list - if (hash_set_insert(&state->cs_add_list, state->current_download_target_hash) < 0) - return -1; - - start_download(state); - return 0; -} - -static int -process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - uint32_t chunk_size; - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - uint32_t target_off; - if (!binary_read(&reader, &target_off, sizeof(target_off))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - uint32_t target_len; - if (!binary_read(&reader, &target_len, sizeof(target_len))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - string data = { (char*) reader.src + reader.cur, target_len }; - if (!binary_read(&reader, NULL, target_len)) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - - char *mem = malloc(chunk_size); - if (mem == NULL) - return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory")); - - assert((uint32_t) data.len <= chunk_size); - assert(target_off + data.len <= chunk_size); - - memset(mem, 0, chunk_size); - memcpy(mem + target_off, data.ptr, data.len); - - SHA256 new_hash; - sha256(mem, chunk_size, (uint8_t*) new_hash.data); - - SHA256 dummy; - int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy); - - free(mem); - - if (ret < 0) - return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error")); - - if (hash_set_insert(&state->cs_add_list, new_hash) < 0) - return -1; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_CHUNK_SUCCESS); - - message_write(&writer, &new_hash, sizeof(new_hash)); - - if (!message_writer_free(&writer)) - return -1; - - return 0; -} - -static int -process_client_upload_chunk(ChunkServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - SHA256 target_hash; - if (!binary_read(&reader, &target_hash, sizeof(target_hash))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - uint32_t target_off; - if (!binary_read(&reader, &target_off, sizeof(target_off))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - uint32_t data_len; - if (!binary_read(&reader, &data_len, sizeof(data_len))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - string data = { (char*) reader.src + reader.cur, data_len }; - if (!binary_read(&reader, NULL, data_len)) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("Invalid message")); - - SHA256 new_hash; - int ret = patch_chunk(state, target_hash, target_off, data, &new_hash); - - if (ret < 0) { - // TODO: if the chunk is missing add it to the missing list - return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, S("I/O error")); - } - - if (hash_set_insert(&state->cs_add_list, new_hash) < 0) - return -1; - - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - message_writer_init(&writer, output, MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS); - - message_write(&writer, &new_hash, sizeof(new_hash)); - - if (!message_writer_free(&writer)) - return -1; - return 0; -} - -static int -process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - - SHA256 target_hash; - if (!binary_read(&reader, &target_hash, sizeof(target_hash))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - - uint8_t full; - if (!binary_read(&reader, &full, sizeof(full))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - - uint32_t target_off; - uint32_t target_len; - if (full == 0) { - if (!binary_read(&reader, &target_off, sizeof(target_off))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - if (!binary_read(&reader, &target_len, sizeof(target_len))) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - } - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid message")); - - string data; - int ret = load_chunk(state, target_hash, &data); - - if (ret < 0) { - // TODO: if the chunk is missing add it to the missing list - return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("I/O error")); - } - - string slice; - if (full == 0) { - if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) { - free(data.ptr); - return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range")); - } - slice = (string) { data.ptr + target_off, target_len }; - } else { - slice = data; - } - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS); - uint32_t data_len = slice.len; - message_write(&writer, &data_len, sizeof(data_len)); - message_write(&writer, slice.ptr, slice.len); - if (!message_writer_free(&writer)) { - free(data.ptr); - return -1; - } - - free(data.ptr); - return 0; -} - -static int -process_message(ChunkServer *state, int conn_idx, uint16_t type, ByteView msg) -{ - switch (tcp_get_tag(&state->tcp, conn_idx)) { - case TAG_METADATA_SERVER: - switch (type) { - case MESSAGE_TYPE_AUTH_RESPONSE: - return process_metadata_server_auth_response(state, conn_idx, msg); - case MESSAGE_TYPE_SYNC_2: - return process_metadata_server_sync_2(state, conn_idx, msg); - case MESSAGE_TYPE_SYNC_4: - return process_metadata_server_sync_4(state, conn_idx, msg); - } - break; - case TAG_CHUNK_SERVER: - switch (type) { - case MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR: - return process_chunk_server_download_error(state, conn_idx, msg); - case MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS: - return process_chunk_server_download_success(state, conn_idx, msg); - } - break; - default: - switch (type) { - case MESSAGE_TYPE_CREATE_CHUNK: - return process_client_create_chunk(state, conn_idx, msg); - case MESSAGE_TYPE_UPLOAD_CHUNK: - return process_client_upload_chunk(state, conn_idx, msg); - case MESSAGE_TYPE_DOWNLOAD_CHUNK: - return process_client_download_chunk(state, conn_idx, msg); - } - break; - } - return -1; -} - -static int -start_connecting_to_metadata_server(ChunkServer *state) -{ - ByteQueue *output; - if (tcp_connect(&state->tcp, state->remote_addr, TAG_METADATA_SERVER, &output) < 0) - return -1; - - // Send AUTH message to authenticate with metadata server - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_AUTH); - - // Send our listening address(es) - // For now, we only support IPv4 (as noted in program_init) - uint32_t num_ipv4 = 1; - message_write(&writer, &num_ipv4, sizeof(num_ipv4)); - - // Write our IPv4 address and port - message_write(&writer, &state->local_addr.ipv4, sizeof(state->local_addr.ipv4)); - message_write(&writer, &state->local_addr.port, sizeof(state->local_addr.port)); - - // No IPv6 addresses for now - uint32_t num_ipv6 = 0; - message_write(&writer, &num_ipv6, sizeof(num_ipv6)); - - if (!message_writer_free(&writer)) - return -1; - return 0; -} - -static int send_sync_message(ChunkServer *state) -{ - assert(state->disconnect_time == INVALID_TIME); - - int conn_idx = tcp_index_from_tag(&state->tcp, TAG_METADATA_SERVER); - assert(conn_idx > -1); - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_SYNC); - - // TODO: May be worth it to add a limit to how many - // items from the add list are sent every update - // to keep messages under 4GB. - uint32_t count = state->cs_add_list.count; - message_write(&writer, &count, sizeof(count)); - - for (uint32_t i = 0; i < count; i++) { - SHA256 hash = state->cs_add_list.items[i]; - message_write(&writer, &hash, sizeof(hash)); - } - - if (!message_writer_free(&writer)) - return -1; - return 0; -} - -int chunk_server_init(void *state_, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - ChunkServer *state = state_; - - string addr = getargs(argc, argv, "--addr", "127.0.0.1"); - int port = getargi(argc, argv, "--port", 8081); - string path = getargs(argc, argv, "--path", "chunk_server_data/"); - bool trace = getargb(argc, argv, "--trace"); - string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1"); - int remote_port = getargi(argc, argv, "--remote-port", 8080); - - if (port <= 0 || port >= 1<<16) { - fprintf(stderr, "chunk server :: Invalid port\n"); - return -1; - } - - if (remote_port <= 0 || remote_port >= 1<<16) { - fprintf(stderr, "chunk server :: Invalid remote port\n"); - return -1; - } - - Time current_time = get_current_time(); - if (current_time == INVALID_TIME) { - fprintf(stderr, "chunk server :: Couldn't read the time\n"); - return -1; - } - - state->trace = trace; - state->reconnect_delay = 1; // 1 second - - if (tcp_context_init(&state->tcp) < 0) { - fprintf(stderr, "chunk server :: Couldn't setup the TCP context\n"); - return -1; - } - - int ret = tcp_listen(&state->tcp, addr, port); - if (ret < 0) { - fprintf(stderr, "chunk server :: Couldn't setup the TCP listener\n"); - tcp_context_free(&state->tcp); - return -1; - } - - if (create_dir(path) && errno != EEXIST) { - fprintf(stderr, "chunk server :: Couldn't create chunk folder\n"); - tcp_context_free(&state->tcp); - return -1; - } - - if (get_full_path(path, state->path) < 0) { - fprintf(stderr, "chunk server :: Couldn't convert path to absolute\n"); - tcp_context_free(&state->tcp); - return -1; - } - - state->downloading = false; - download_targets_init(&state->download_targets); - hash_set_init(&state->cs_add_list); - hash_set_init(&state->cs_lst_list); - timed_hash_set_init(&state->cs_rem_list); - - char tmp[1<<10]; - if (addr.len >= (int) sizeof(tmp)) { - fprintf(stderr, "chunk server :: Address is too long\n"); - tcp_context_free(&state->tcp); - return -1; - } - memcpy(tmp, addr.ptr, addr.len); - tmp[addr.len] = '\0'; - state->local_addr.is_ipv4 = true; - if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) { - fprintf(stderr, "chunk server :: Couldn't parse address\n"); - tcp_context_free(&state->tcp); - return -1; - } - state->local_addr.port = port; - - // Initialize metadata server address - // // TODO: This should also support IPv6 - if (remote_addr.len >= (int) sizeof(tmp)) { - fprintf(stderr, "chunk server :: Remote address is too long\n"); - tcp_context_free(&state->tcp); - return -1; - } - memcpy(tmp, remote_addr.ptr, remote_addr.len); - tmp[remote_addr.len] = '\0'; - state->remote_addr.is_ipv4 = true; - if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) { - fprintf(stderr, "chunk server :: Couldn't parse remote address\n"); - tcp_context_free(&state->tcp); - return -1; - } - state->remote_addr.port = remote_port; - state->disconnect_time = INVALID_TIME; - state->last_sync_time = current_time; - - start_connecting_to_metadata_server(state); - - printf("Chunk server set up (local=%.*s:%d, remote=%.*s:%d, path=%.*s)\n", - addr.len, - addr.ptr, - port, - remote_addr.len, - remote_addr.ptr, - remote_port, - path.len, - path.ptr - ); - - *timeout = 0; - if (pcap < TCP_POLL_CAPACITY) { - fprintf(stderr, "chunk server :: Capacity isn't large enough\n"); - return -1; - } - *pnum = tcp_register_events(&state->tcp, ctxs, pdata); - return 0; -} - -int chunk_server_tick(void *state_, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, int *timeout) -{ - ChunkServer *state = state_; - Event events[TCP_EVENT_CAPACITY]; - int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); - - Time current_time = get_current_time(); - if (current_time == INVALID_TIME) - return -1; - - for (int i = 0; i < num_events; i++) { - int conn_idx = events[i].conn_idx; - switch (events[i].type) { - - case EVENT_WAKEUP: - // Do nothing - break; - - case EVENT_CONNECT: - if (tcp_get_tag(&state->tcp, conn_idx) == TAG_METADATA_SERVER) { - assert(state->disconnect_time == INVALID_TIME); - } - break; - - case EVENT_DISCONNECT: - switch (events[i].tag) { - - case TAG_METADATA_SERVER: - assert(state->disconnect_time == INVALID_TIME); - state->disconnect_time = current_time; - break; - - case TAG_CHUNK_SERVER: - state->downloading = false; - break; - } - break; - - case EVENT_MESSAGE: - { - for (;;) { - - ByteView msg; - uint16_t msg_type; - int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); - if (ret == 0) - break; - if (ret < 0) { - tcp_close(&state->tcp, conn_idx); - break; - } - - if (state->trace) - message_dump(stdout, msg); - - ret = process_message(state, conn_idx, msg_type, msg); - if (ret < 0) { - tcp_close(&state->tcp, conn_idx); - break; - } - - tcp_consume_message(&state->tcp, conn_idx); - } - } - break; - } - } - - assert((tcp_index_from_tag(&state->tcp, TAG_METADATA_SERVER) < 0) == (state->disconnect_time != INVALID_TIME)); - - Time deadline = INVALID_TIME; - - // Remove items from the remove list that got too old - for (int i = 0; i < state->cs_rem_list.count; i++) { - TimedHash *removal = &state->cs_rem_list.items[i]; - Time removal_time = removal->time + (Time) DELETION_TIMEOUT * 1000000000; - if (removal_time < current_time) { - char buf[PATH_MAX]; - string path = hash2path(state, removal->hash, buf); - if (remove_file_or_dir(path) == 0) - *removal = state->cs_rem_list.items[--state->cs_rem_list.count]; - } else { - nearest_deadline(&deadline, removal_time); - } - } - - if (state->disconnect_time == INVALID_TIME) { - Time next_sync_time = state->last_sync_time + (Time) SYNC_INTERVAL * 1000000000; - if (current_time >= next_sync_time) { - if (send_sync_message(state) < 0) { - assert(0); // TODO - } - state->last_sync_time = current_time; - } else { - nearest_deadline(&deadline, next_sync_time); - } - } - - // TODO: periodically look for chunks that have their hashes messed up and delete them - - // Periodically retry pending downloads - start_download(state); - - if (state->disconnect_time != INVALID_TIME) { - Time reconnect_time = state->disconnect_time + (Time) state->reconnect_delay * 1000000000; - if (reconnect_time <= current_time) { - state->disconnect_time = INVALID_TIME; - if (start_connecting_to_metadata_server(state) < 0) - state->disconnect_time = current_time; - } - if (state->disconnect_time != INVALID_TIME) - nearest_deadline(&deadline, reconnect_time); - } - - *timeout = deadline_to_timeout(deadline, current_time); - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = tcp_register_events(&state->tcp, ctxs, pdata); - return 0; -} - -int chunk_server_free(void *state_) -{ - ChunkServer *state = state_; - download_targets_free(&state->download_targets); - timed_hash_set_free(&state->cs_rem_list); - hash_set_free(&state->cs_lst_list); - hash_set_free(&state->cs_add_list); - tcp_context_free(&state->tcp); - return 0; -} diff --git a/src/chunk_server.h b/src/chunk_server.h deleted file mode 100644 index 8d374b4..0000000 --- a/src/chunk_server.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef CHUNK_SERVER_INCLUDED -#define CHUNK_SERVER_INCLUDED - -#include "tcp.h" -#include "basic.h" -#include "hash_set.h" - -#define TAG_METADATA_SERVER 1 -#define TAG_CHUNK_SERVER 2 - -typedef struct { - Address addr; - SHA256 hash; -} DownloadTarget; - -typedef struct { - DownloadTarget *items; - int count; - int capacity; -} DownloadTargets; - -typedef struct { - - char path[PATH_MAX]; - - bool trace; - - Address local_addr; - Address remote_addr; - - Time disconnect_time; - Time last_sync_time; - int reconnect_delay; // In seconds - - // --- Subsystems --- - - TCP tcp; - - // --- Download Management --- - - bool downloading; - SHA256 current_download_target_hash; - DownloadTargets download_targets; - - // --- Chunk Management --- - - // List of chunks added since the last update - HashSet cs_add_list; - - // List of chunks marked for removal after a timeout - TimedHashSet cs_rem_list; - - // List of chunks that were lost due to errors or forceful removals of chunk files - HashSet cs_lst_list; - -} ChunkServer; - -struct pollfd; - -int chunk_server_init(void *state, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout); - -int chunk_server_tick(void *state, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, - int *timeout); - -int chunk_server_free(void *state); - -#endif // CHUNK_SERVER_INCLUDED \ No newline at end of file diff --git a/src/chunk_store.c b/src/chunk_store.c new file mode 100644 index 0000000..8b4ccd4 --- /dev/null +++ b/src/chunk_store.c @@ -0,0 +1,107 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "chunk_store.h" +#include + +// Build the full path for a chunk: "base_path/HEX_HASH" +// SHA256 hex = 64 chars. Returns string pointing into buf. +static string chunk_path(ChunkStore *cs, SHA256 hash, char *buf, int buf_size) +{ + int base_len = strlen(cs->base_path); + if (base_len + 1 + 64 + 1 > buf_size) + return (string){ buf, 0 }; + + memcpy(buf, cs->base_path, base_len); + buf[base_len] = '/'; + append_hex_as_str(buf + base_len + 1, hash); + buf[base_len + 1 + 64] = '\0'; + + return (string){ buf, base_len + 1 + 64 }; +} + +int chunk_store_init(ChunkStore *cs, const char *base_path) +{ + int len = strlen(base_path); + if (len >= (int) sizeof(cs->base_path)) + return -1; + + memcpy(cs->base_path, base_path, len + 1); + + string path = { cs->base_path, len }; + create_dir(path); // Ignore error if already exists + + return 0; +} + +void chunk_store_free(ChunkStore *cs) +{ + (void) cs; +} + +int chunk_store_write(ChunkStore *cs, SHA256 hash, char *data, uint32_t size) +{ + char buf[512]; + string path = chunk_path(cs, hash, buf, sizeof(buf)); + if (path.len == 0) + return -1; + + Handle fd; + if (file_open(path, &fd) < 0) + return -1; + + if (file_truncate(fd, 0) < 0) { + file_close(fd); + return -1; + } + + int ret = file_write_exact(fd, data, size); + file_close(fd); + return ret; +} + +int chunk_store_read(ChunkStore *cs, SHA256 hash, char *dst, uint32_t size) +{ + char buf[512]; + string path = chunk_path(cs, hash, buf, sizeof(buf)); + if (path.len == 0) + return -1; + + Handle fd; + if (file_open(path, &fd) < 0) + return -1; + + int ret = file_read_exact(fd, dst, size); + file_close(fd); + return ret; +} + +bool chunk_store_exists(ChunkStore *cs, SHA256 hash) +{ + char buf[512]; + string path = chunk_path(cs, hash, buf, sizeof(buf)); + if (path.len == 0) + return false; + + // Use file_open instead of file_exists (access) because + // mock_access is not implemented in the Quakey simulation. + Handle fd; + if (file_open(path, &fd) < 0) + return false; + file_close(fd); + return true; +} + +int chunk_store_delete(ChunkStore *cs, SHA256 hash) +{ + char buf[512]; + string path = chunk_path(cs, hash, buf, sizeof(buf)); + if (path.len == 0) + return -1; + + return remove_file_or_dir(path); +} diff --git a/src/chunk_store.h b/src/chunk_store.h new file mode 100644 index 0000000..4a45d08 --- /dev/null +++ b/src/chunk_store.h @@ -0,0 +1,20 @@ +#ifndef CHUNK_STORE_INCLUDED +#define CHUNK_STORE_INCLUDED + +#include +#include + +#include + +typedef struct { + char base_path[256]; +} ChunkStore; + +int chunk_store_init(ChunkStore *cs, const char *base_path); +void chunk_store_free(ChunkStore *cs); +int chunk_store_write(ChunkStore *cs, SHA256 hash, char *data, uint32_t size); +int chunk_store_read(ChunkStore *cs, SHA256 hash, char *dst, uint32_t size); +bool chunk_store_exists(ChunkStore *cs, SHA256 hash); +int chunk_store_delete(ChunkStore *cs, SHA256 hash); + +#endif // CHUNK_STORE_INCLUDED diff --git a/src/client.c b/src/client.c index 237d837..f3ab5ab 100644 --- a/src/client.c +++ b/src/client.c @@ -6,2536 +6,354 @@ #include #include -#ifdef _WIN32 -# define POLL WSAPoll -#else -# define POLL poll -#endif +#include "client.h" +#include "server.h" -#ifdef _WIN32 -typedef CRITICAL_SECTION Mutex; -#else -typedef pthread_mutex_t Mutex; -#endif +//#define CLIENT_TRACE(fmt, ...) {} +#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__); -#include "tcp.h" -#include "config.h" -#include "message.h" -#include "file_tree.h" +#define KEY_POOL_SIZE 128 +#define BUCKET_POOL_SIZE 4 -#include +static uint64_t next_client_id = 1; -#define TAG_METADATA_SERVER -2 - -#define TAG_RETRIEVE_METADATA_FOR_READ -1 -#define TAG_RETRIEVE_METADATA_FOR_WRITE -3 -#define TAG_COMMIT_WRITE -4 - -#define TAG_UPLOAD_CHUNK_MIN 1000 -#define TAG_UPLOAD_CHUNK_MAX 100000 - -#define PARALLEL_LIMIT 5 - -#define CLIENT_TRACE(fmt, ...) {} -//#define CLIENT_TRACE(fmt, ...) fprintf(stderr, "CLIENT: " fmt "\n", ##__VA_ARGS__); - -typedef struct { - SHA256 hash; - char* dst; - uint32_t offset_within_chunk; - uint32_t length_within_chunk; - Address server_addr; // Chunk server address for this chunk - int chunk_server_idx; // Index in toasty->chunk_servers array -} Range; - -typedef enum { - - // This upload wasn't started yet - UPLOAD_WAITING, - - // This upload started - UPLOAD_PENDING, - - // This upload was WAITING but then - // was marked as IGNORED - UPLOAD_IGNORED, - - // Upload was PENDING and FAILED - UPLOAD_FAILED, - - // Upload was PENDING, then COMPLETED - // successfully - UPLOAD_COMPLETED, - -} UploadScheduleStatus; - -typedef struct { - UploadScheduleStatus status; - - // Location of the chunk to be patched. - // The server local ID is used to indicate - // that different addresses refer to the - // same server. - int server_lid; - Address address; - int chunk_index; - - // The patch offset and data - char *src; - int off; - int len; - - // When the upload is successfull, this will - // hold the hash of the newly created or modified - // patch. - SHA256 final_hash; - -} UploadSchedule; - -typedef enum { - OPERATION_TYPE_FREE, - OPERATION_TYPE_CREATE, - OPERATION_TYPE_DELETE, - OPERATION_TYPE_LIST, - OPERATION_TYPE_READ, - OPERATION_TYPE_WRITE, -} OperationType; - -typedef struct { - - OperationType type; - - // Generation counter. Values 0 and 2^16-1 are - // skipped to make sure ToastyHandle values 0 - // and -1 are always invalid. - uint16_t generation; - - // Opaque pointer set by the user - void *user; - - ToastyString path; // Only set for writes - void *ptr; - int off; - int len; - - Range *ranges; - int ranges_head; - int ranges_count; - int num_pending; - int actual_bytes; // For reads: actual number of bytes that can be read - - // Write fields - SHA256 *hashes; - int num_hashes; - uint32_t num_chunks; - uint32_t chunk_size; - uint64_t expect_gen; - uint32_t flags; // Write operation flags (e.g., TOASTY_WRITE_CREATE_IF_MISSING) - UploadSchedule *uploads; - int num_uploads; - int cap_uploads; - - ToastyResult result; -} Operation; - -typedef struct { - int tag; - int opidx; -} Request; - -typedef struct { - int head; - int count; - Request items[MAX_REQUESTS_PER_QUEUE]; -} RequestQueue; - -typedef struct { - bool used; // TODO: should be more like "connected" - Address addr; - RequestQueue reqs; -} MetadataServer; - -typedef struct { - - bool used; - - // List of addresses associated to this chunk server - int num_addrs; - Address addrs[MAX_SERVER_ADDRS]; - - // Index of the address currently in use - int current_addr_idx; - - // If the connection was established - bool connected; - - RequestQueue reqs; - -} ChunkServer; - -struct ToastyFS { - - TCP tcp; - Mutex mutex; - - MetadataServer metadata_server; - - int num_chunk_servers; - ChunkServer chunk_servers[MAX_CHUNK_SERVERS]; - - int num_operations; - Operation operations[MAX_OPERATIONS]; -}; - -static int mutex_init(Mutex *mutex) +static uint64_t client_random(void) { -#ifdef _WIN32 - InitializeCriticalSection(mutex); // TODO: mock? - return 0; +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) + return quakey_random(); #else - if (pthread_mutex_init(mutex, NULL)) // TODO: mock - return -1; - return 0; + return (uint64_t)rand(); #endif } -static int mutex_free(Mutex *mutex) +static MetaOper random_oper(void) { -#ifdef _WIN32 - DeleteCriticalSection(mutex); // TODO: mock? - return 0; -#else - if (pthread_mutex_destroy(mutex)) // TODO: mock - return -1; - return 0; -#endif + MetaOper oper = {0}; + snprintf(oper.bucket, META_BUCKET_MAX, "b%d", (int)(client_random() % BUCKET_POOL_SIZE)); + snprintf(oper.key, META_KEY_MAX, "k%d", (int)(client_random() % KEY_POOL_SIZE)); + + switch (client_random() % 2) { + case 0: + oper.type = META_OPER_PUT; + oper.size = client_random() % (1 << 20); + oper.num_chunks = 1 + client_random() % 3; + for (uint32_t i = 0; i < oper.num_chunks; i++) { + for (int j = 0; j < 32; j++) + oper.chunks[i].hash.data[j] = client_random() & 0xFF; + oper.chunks[i].size = client_random() % (4 << 20); + } + for (int j = 0; j < 32; j++) + oper.content_hash.data[j] = client_random() & 0xFF; + break; + case 1: + oper.type = META_OPER_DELETE; + break; + } + return oper; } -static int mutex_lock(Mutex *mutex) +// Format time as seconds with 3 decimal places for trace output +#define TIME_FMT "%7.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static void client_log_impl(ClientState *state, Time now, const char *event, const char *detail) { -#ifdef _WIN32 - EnterCriticalSection(mutex); // TODO: mock? - return 0; -#else - if (pthread_mutex_lock(mutex)) // TODO: mock - return -1; - return 0; -#endif + printf("[" TIME_FMT "] CLIENT %lu | V%-3lu | %-20s %s\n", + TIME_VAL(now), + state->client_id, + state->view_number, + event, + detail ? detail : ""); } -static int mutex_unlock(Mutex *mutex) -{ -#ifdef _WIN32 - LeaveCriticalSection(mutex); // TODO: mock? - return 0; -#else - if (pthread_mutex_unlock(mutex)) // TODO: mock - return -1; - return 0; -#endif -} +#define client_log(state, now, event, fmt, ...) do { \ + char _detail[256]; \ + snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \ + client_log_impl(state, now, event, _detail); \ +} while (0) -static void request_queue_init(RequestQueue *reqs) -{ - reqs->head = 0; - reqs->count = 0; -} +#define client_log_simple(state, now, event) \ + client_log_impl(state, now, event, NULL) -static int request_queue_push(RequestQueue *reqs, Request req) +static int leader_idx(ClientState *state) { - if (reqs->count == MAX_REQUESTS_PER_QUEUE) - return -1; - int tail = (reqs->head + reqs->count) % MAX_REQUESTS_PER_QUEUE; - reqs->items[tail] = req; - reqs->count++; - return 0; -} - -static int request_queue_pop(RequestQueue *reqs, Request *req) -{ - if (reqs->count == 0) - return -1; - if (req) *req = reqs->items[reqs->head]; - reqs->head = (reqs->head + 1) % MAX_REQUESTS_PER_QUEUE; - reqs->count--; - return 0; + return state->view_number % state->num_servers; } static int -alloc_operation(ToastyFS *toasty, OperationType type, int off, void *ptr, int len) +process_message(ClientState *state, + int conn_idx, uint8_t type, ByteView msg) { - if (toasty->num_operations == MAX_OPERATIONS) - return -1; - Operation *o = toasty->operations; - while (o->type != OPERATION_TYPE_FREE) { - o++; - assert(o < toasty->operations + MAX_OPERATIONS); + (void) conn_idx; + + if (type == MESSAGE_TYPE_REDIRECT) { + RedirectMessage redirect_message; + if (msg.len != sizeof(RedirectMessage)) + return -1; + memcpy(&redirect_message, msg.ptr, sizeof(redirect_message)); + + if (redirect_message.view_number > state->view_number) { + Time now = get_current_time(); + client_log(state, now, "RECV REDIRECT", "view=%lu -> %lu leader=%d", + (unsigned long)state->view_number, + (unsigned long)redirect_message.view_number, + (int)(redirect_message.view_number % state->num_servers)); + state->view_number = redirect_message.view_number; + state->last_was_rejected = true; + state->last_was_timeout = false; + state->pending = false; + } + return 0; } - o->type = type; - o->user = NULL; - o->ptr = ptr; - o->off = off; - o->len = len; - o->result = (ToastyResult) { .type=TOASTY_RESULT_EMPTY }; - toasty->num_operations++; - return o - toasty->operations; -} - -static void free_operation(ToastyFS *toasty, int opidx) -{ - toasty->operations[opidx].type = OPERATION_TYPE_FREE; - toasty->operations[opidx].generation++; - if (toasty->operations[opidx].generation == 0 || - toasty->operations[opidx].generation == UINT16_MAX) - toasty->operations[opidx].generation = 1; - toasty->num_operations--; -} - -static ToastyHandle operation_to_handle(ToastyFS *toasty, int opidx) -{ - assert(opidx >= 0); - assert(opidx <= UINT16_MAX); - - uint16_t generation = toasty->operations[opidx].generation; - assert(generation != 0); - assert(generation != UINT16_MAX); - - return ((uint16_t) opidx << 16) | generation; -} - -static int handle_to_operation(ToastyFS *toasty, ToastyHandle handle) -{ - if (handle == 0 || handle == UINT32_MAX) + if (!state->pending) return -1; - int opidx = handle >> 16; - assert(opidx >= 0); - assert(opidx <= UINT16_MAX); - - if (opidx >= MAX_OPERATIONS) + if (type != MESSAGE_TYPE_REPLY) return -1; - uint16_t expected_generation = handle & 0xFFFF; - if (toasty->operations[opidx].generation != expected_generation) + ReplyMessage message; + if (msg.len != sizeof(ReplyMessage)) return -1; + memcpy(&message, msg.ptr, sizeof(message)); - return opidx; -} + // Ignore stale replies from previous requests. After a timeout + // the client moves to a new view and sends a new request, but + // the old leader may still deliver a reply for the old request + // on the previous connection. Without this check the client + // would accept the stale result for the wrong operation. + if (message.request_id != state->request_id) + return 0; -ToastyFS *toasty_connect(ToastyString addr, uint16_t port) -{ - ToastyFS *toasty = malloc(sizeof(ToastyFS)); - if (toasty == NULL) - return NULL; - - Address addr2; { - char tmp[128]; - if (addr.len >= (int) sizeof(tmp)) { - free(toasty); - return NULL; - } - memcpy(tmp, addr.ptr, addr.len); - tmp[addr.len] = '\0'; - - addr2.is_ipv4 = true; - addr2.port = port; - if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) { - free(toasty); - return NULL; - } - } - - if (mutex_init(&toasty->mutex) < 0) { - free(toasty); - return NULL; - } - - if (tcp_context_init(&toasty->tcp) < 0) { - mutex_free(&toasty->mutex); - free(toasty); - return NULL; - } - - if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) { - tcp_context_free(&toasty->tcp); - mutex_free(&toasty->mutex); - free(toasty); - return NULL; - } - - toasty->num_operations = 0; - - for (int i = 0; i < MAX_OPERATIONS; i++) { - toasty->operations[i].type = OPERATION_TYPE_FREE; - toasty->operations[i].generation = 1; // Important not to start from 0 !!! - } - - // Initialize metadata server (connected during init) - toasty->metadata_server.used = true; - toasty->metadata_server.addr = addr2; - request_queue_init(&toasty->metadata_server.reqs); - - // Initialize chunk servers array (connections created on demand) - toasty->num_chunk_servers = 0; - for (int i = 0; i < MAX_CHUNK_SERVERS; i++) { - toasty->chunk_servers[i].used = false; - } - - return toasty; -} - -int toasty_wakeup(ToastyFS *toasty) -{ - if (mutex_lock(&toasty->mutex) < 0) - return -1; - - int ret; - if (tcp_wakeup(&toasty->tcp) < 0) - ret = -1; - - if (mutex_unlock(&toasty->mutex) < 0) - return -1; - - return ret; -} - -static void -metadata_server_request_start(ToastyFS *toasty, MessageWriter *writer, uint16_t type) -{ - ByteQueue *output; - if (toasty->metadata_server.used) { - - int conn_idx = tcp_index_from_tag(&toasty->tcp, TAG_METADATA_SERVER); - assert(conn_idx > -1); - - output = tcp_output_buffer(&toasty->tcp, conn_idx); - } else { - if (tcp_connect(&toasty->tcp, toasty->metadata_server.addr, TAG_METADATA_SERVER, &output) < 0) { - assert(0); // TODO - } - toasty->metadata_server.used = true; - } - - message_writer_init(writer, output, type); -} - -static int -metadata_server_request_end(ToastyFS *toasty, MessageWriter *writer, int opidx, int tag) -{ - if (!message_writer_free(writer)) - return -1; - - RequestQueue *reqs = &toasty->metadata_server.reqs; - if (request_queue_push(reqs, (Request) { tag, opidx }) < 0) - return -1; - - return 0; -} - -void toasty_disconnect(ToastyFS *toasty) -{ - tcp_context_free(&toasty->tcp); - mutex_free(&toasty->mutex); - free(toasty); -} - -static bool -have_insertection(Address *a, int a_num, Address *b, int b_num) -{ - for (int i = 0; i < a_num; i++) - for (int j = 0; j < b_num; j++) - if (addr_eql(a[i], b[j])) - return true; - return false; -} - -// Get or create connection to a chunk server -static int get_chunk_server(ToastyFS *toasty, Address *addrs, int num_addrs, ByteQueue **output) -{ - // Check if already connected - - int found = -1; - for (int i = 0; i < toasty->num_chunk_servers; i++) { - - if (!toasty->chunk_servers[i].used) - continue; - - if (!have_insertection(addrs, num_addrs, toasty->chunk_servers[i].addrs, toasty->chunk_servers[i].num_addrs)) - continue; - - int conn_idx = tcp_index_from_tag(&toasty->tcp, i); - assert(conn_idx > -1); - - if (output) - *output = tcp_output_buffer(&toasty->tcp, conn_idx); - - found = i; - break; - } - - if (found == -1) { - - if (toasty->num_chunk_servers == MAX_CHUNK_SERVERS) - return -1; - - // Find free slot - found = 0; - while (toasty->chunk_servers[found].used) { - found++; - assert(found < MAX_CHUNK_SERVERS); - } - - if (tcp_connect(&toasty->tcp, addrs[0], found, output) < 0) - return -1; - - if (num_addrs > MAX_SERVER_ADDRS) - num_addrs = MAX_SERVER_ADDRS; - toasty->chunk_servers[found].num_addrs = num_addrs; - memcpy(toasty->chunk_servers[found].addrs, addrs, num_addrs * sizeof(Address)); - - toasty->chunk_servers[found].used = true; - toasty->chunk_servers[found].current_addr_idx = 0; - toasty->chunk_servers[found].connected = false; - - request_queue_init(&toasty->chunk_servers[found].reqs); - - toasty->num_chunk_servers++; - } - - return found; -} - -// TODO: is this used anywhere? -static void close_chunk_server(ToastyFS *toasty, int chunk_server_idx) -{ - int conn_idx = tcp_index_from_tag(&toasty->tcp, chunk_server_idx); - tcp_close(&toasty->tcp, conn_idx); -} - -// Send download request for a chunk -static int send_download_chunk(ToastyFS *toasty, int chunk_server_idx, - SHA256 hash, uint32_t offset, uint32_t length, int opidx, int range_idx) -{ - int conn_idx = tcp_index_from_tag(&toasty->tcp, chunk_server_idx); - if (conn_idx < 0) - return -1; - - ByteQueue *output = tcp_output_buffer(&toasty->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_DOWNLOAD_CHUNK); - message_write_hash(&writer, hash); - message_write_u8 (&writer, 0); - message_write_u32 (&writer, offset); - message_write_u32 (&writer, length); - if (!message_writer_free(&writer)) - return -1; - - RequestQueue *reqs = &toasty->chunk_servers[chunk_server_idx].reqs; - return request_queue_push(reqs, (Request) { range_idx, opidx }); -} - -static ToastyHandle begin_create(ToastyFS *toasty, - ToastyString path, bool is_dir, uint32_t chunk_size) -{ - int opidx = -1; - ToastyHandle handle = TOASTY_INVALID; - - if (mutex_lock(&toasty->mutex) < 0) - return TOASTY_INVALID; - - OperationType type = OPERATION_TYPE_CREATE; - - opidx = alloc_operation(toasty, type, 0, NULL, 0); - if (opidx < 0) - goto unlock_and_exit; - - if (path.len > UINT16_MAX) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - uint16_t tmp = path.len; - uint8_t tmp_u8 = is_dir; - - MessageWriter writer; - metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_CREATE); - - message_write(&writer, &tmp, sizeof(tmp)); - message_write(&writer, path.ptr, path.len); - message_write(&writer, &tmp_u8, sizeof(tmp_u8)); - - if (!is_dir) { - if (chunk_size == 0 || chunk_size > UINT32_MAX) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - uint32_t tmp_u32 = chunk_size; - message_write(&writer, &tmp_u32, sizeof(tmp_u32)); - } - - if (metadata_server_request_end(toasty, &writer, opidx, 0) < 0) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - - handle = operation_to_handle(toasty, opidx); - -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) { - if (opidx > -1) - free_operation(toasty, opidx); - return TOASTY_INVALID; - } - return handle; -} - -ToastyHandle toasty_begin_create_dir(ToastyFS *toasty, ToastyString path) -{ - return begin_create(toasty, path, true, 0); -} - -ToastyHandle toasty_begin_create_file(ToastyFS *toasty, ToastyString path, - unsigned int chunk_size) -{ - return begin_create(toasty, path, false, chunk_size); -} - -ToastyHandle toasty_begin_delete(ToastyFS *toasty, - ToastyString path, ToastyVersionTag vtag) -{ - int opidx = -1; - ToastyHandle handle = TOASTY_INVALID; - - if (mutex_lock(&toasty->mutex) < 0) - return TOASTY_INVALID; - - OperationType type = OPERATION_TYPE_DELETE; - - opidx = alloc_operation(toasty, type, 0, NULL, 0); - if (opidx < 0) - goto unlock_and_exit; - - if (path.len > UINT16_MAX) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - uint16_t path_len = path.len; - uint64_t expect_gen = vtag; - - MessageWriter writer; - metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_DELETE); - message_write(&writer, &expect_gen, sizeof(expect_gen)); - message_write(&writer, &path_len, sizeof(path_len)); - message_write(&writer, path.ptr, path.len); - if (metadata_server_request_end(toasty, &writer, opidx, 0) < 0) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - - handle = operation_to_handle(toasty, opidx); - -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) { - if (opidx > -1) - free_operation(toasty, opidx); - return TOASTY_INVALID; - } - return handle; -} - -ToastyHandle toasty_begin_list(ToastyFS *toasty, - ToastyString path, ToastyVersionTag vtag) -{ - int opidx = -1; - ToastyHandle handle = TOASTY_INVALID; - - if (mutex_lock(&toasty->mutex) < 0) - return TOASTY_INVALID; - - OperationType type = OPERATION_TYPE_LIST; - - opidx = alloc_operation(toasty, type, 0, NULL, 0); - if (opidx < 0) - goto unlock_and_exit; - - if (path.len > UINT16_MAX) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - uint16_t path_len = path.len; - uint64_t expect_gen = vtag; - - MessageWriter writer; - metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_LIST); - message_write(&writer, &expect_gen, sizeof(expect_gen)); - message_write(&writer, &path_len, sizeof(path_len)); - message_write(&writer, path.ptr, path.len); - if (metadata_server_request_end(toasty, &writer, opidx, 0) < 0) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - - handle = operation_to_handle(toasty, opidx); - -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) { - if (opidx > -1) - free_operation(toasty, opidx); - return TOASTY_INVALID; - } - return handle; -} - -static int send_read_message(ToastyFS *toasty, int opidx, - int tag, ToastyString path, uint32_t offset, - uint32_t length, ToastyVersionTag vtag) -{ - if (path.len > UINT16_MAX) - return -1; - uint16_t path_len = path.len; - uint64_t expect_gen = vtag; - - MessageWriter writer; - metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_READ); - message_write(&writer, &expect_gen, sizeof(expect_gen)); - message_write(&writer, &path_len, sizeof(path_len)); - message_write(&writer, path.ptr, path.len); - message_write(&writer, &offset, sizeof(offset)); - message_write(&writer, &length, sizeof(length)); - if (metadata_server_request_end(toasty, &writer, opidx, tag) < 0) - return -1; - return 0; -} - -ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, - int off, void *dst, int len, ToastyVersionTag vtag) -{ - int opidx = -1; - ToastyHandle handle = TOASTY_INVALID; - - if (mutex_lock(&toasty->mutex) < 0) - goto unlock_and_exit; - - OperationType type = OPERATION_TYPE_READ; - - opidx = alloc_operation(toasty, type, off, dst, len); - if (opidx < 0) - goto unlock_and_exit; - - if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_READ, path, off, len, vtag) < 0) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - - handle = operation_to_handle(toasty, opidx); - -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) { - if (opidx > -1) - free_operation(toasty, opidx); - return TOASTY_INVALID; - } - return handle; -} - -ToastyHandle toasty_begin_write(ToastyFS *toasty, - ToastyString path, int off, void *src, int len, - ToastyVersionTag vtag, uint32_t flags) -{ - int opidx = -1; - ToastyHandle handle = TOASTY_INVALID; - - if (mutex_lock(&toasty->mutex) < 0) - goto unlock_and_exit; - - OperationType type = OPERATION_TYPE_WRITE; - - opidx = alloc_operation(toasty, type, off, src, len); - if (opidx < 0) - goto unlock_and_exit; - - toasty->operations[opidx].path = path; // TODO: must be a copy - toasty->operations[opidx].flags = flags; - - if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_WRITE, path, off, len, vtag) < 0) { - free_operation(toasty, opidx); - opidx = -1; - goto unlock_and_exit; - } - - handle = operation_to_handle(toasty, opidx); - -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) { - if (opidx > -1) - free_operation(toasty, opidx); - return TOASTY_INVALID; - } - return handle; -} - -void toasty_free_result(ToastyResult *result) -{ - if (result->type == TOASTY_RESULT_LIST_SUCCESS) - toasty_free_listing(&result->listing); -} - -void toasty_set_user(ToastyFS *toasty, ToastyHandle handle, void *user) -{ - int opidx = handle_to_operation(toasty, handle); - if (opidx < 0) - return; - - toasty->operations[opidx].user = user; -} - -static void process_event_for_create(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - (void) request_tag; - - if (msg.len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_CREATE_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read generation counter - uint64_t gen; - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Check there is nothing else to read - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_CREATE_SUCCESS, .user=toasty->operations[opidx].user }; -} - -static void process_event_for_delete(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - (void) request_tag; - - if (msg.len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_DELETE_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Check there is nothing else to read - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_DELETE_SUCCESS, .user=toasty->operations[opidx].user }; -} - -static void process_event_for_list(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - (void) request_tag; - - if (msg.len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_LIST_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read generation counter - uint64_t gen; - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read and validate the list data - uint8_t truncated; - if (!binary_read(&reader, &truncated, sizeof(truncated))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint32_t item_count; - if (!binary_read(&reader, &item_count, sizeof(item_count))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - ToastyListingEntry *entities = malloc(item_count * sizeof(ToastyListingEntry)); - if (entities == NULL) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Parse each list item - for (uint32_t i = 0; i < item_count; i++) { - - uint64_t gen; - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - - uint8_t is_dir; - if (!binary_read(&reader, &is_dir, sizeof(is_dir))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - - uint16_t name_len; - if (!binary_read(&reader, &name_len, sizeof(name_len))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - - char *name = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, name_len)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - - entities[i].vtag = gen; - entities[i].is_dir = is_dir; - - if (name_len > sizeof(entities[i].name)-1) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - memcpy(entities[i].name, name, name_len); - entities[i].name[name_len] = '\0'; - } - - // Check there is nothing else to read - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - free(entities); - return; - } - - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_SUCCESS, .user=toasty->operations[opidx].user, .listing=(ToastyListing) { item_count, entities } }; -} - -static void process_event_for_read(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - if (msg.len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (request_tag == TAG_RETRIEVE_METADATA_FOR_READ) { - // Handle metadata response from metadata server - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Skip version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Check message type - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_READ_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Skip message length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read generation counter - uint64_t gen; - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read chunk size - uint32_t chunk_size; - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read actual bytes that can be read - uint32_t actual_bytes; - if (!binary_read(&reader, &actual_bytes, sizeof(actual_bytes))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - // Store actual_bytes for later use - toasty->operations[opidx].actual_bytes = actual_bytes; - - // Calculate which chunks we need - int off = toasty->operations[opidx].off; - int len = toasty->operations[opidx].len; - - if (len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=0 }; - return; - } - - uint32_t first_byte = off; - uint32_t last_byte = off + len - 1; - uint32_t first_chunk = first_byte / chunk_size; - uint32_t last_chunk = last_byte / chunk_size; - uint32_t num_chunks_needed = last_chunk - first_chunk + 1; - - // Read number of hashes - uint32_t num_hashes; - if (!binary_read(&reader, &num_hashes, sizeof(num_hashes))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Allocate ranges - Range *ranges = malloc(num_chunks_needed * sizeof(Range)); - if (ranges == NULL) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - char *ptr = toasty->operations[opidx].ptr; - int num_ranges_with_data = 0; - - // Parse each chunk's hash and server locations - for (uint32_t i = 0; i < num_hashes; i++) { - - // Read hash - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read number of servers - uint32_t num_servers; - if (!binary_read(&reader, &num_servers, sizeof(num_servers))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - Address server_addr = {0}; - bool found = false; - - // Parse each server's address list (use first one, skip the rest) - for (uint32_t s = 0; s < num_servers; s++) { - // Parse IPv4 addresses - uint32_t num_ipv4; - if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Get first IPv4 address from first server - for (uint32_t j = 0; j < num_ipv4; j++) { - IPv4 ipv4; - uint16_t port; - if (!binary_read(&reader, &ipv4, sizeof(ipv4)) || - !binary_read(&reader, &port, sizeof(port))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - if (!found) { - server_addr.is_ipv4 = true; - server_addr.ipv4 = ipv4; - server_addr.port = port; - found = true; - } - } - - // Skip IPv6 addresses - uint32_t num_ipv6; - if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - for (uint32_t j = 0; j < num_ipv6; j++) { - if (!binary_read(&reader, NULL, sizeof(IPv6)) || - !binary_read(&reader, NULL, sizeof(uint16_t))) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - } - - if (!found) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Calculate byte range for this chunk - uint32_t chunk_idx = first_chunk + i; - uint32_t first_in_chunk = (chunk_idx == first_chunk) ? (first_byte % chunk_size) : 0; - uint32_t last_in_chunk = (chunk_idx == last_chunk) ? (last_byte % chunk_size) : (chunk_size - 1); - uint32_t len_in_chunk = 1 + last_in_chunk - first_in_chunk; - - // Fill in range info - ranges[i].hash = hash; - ranges[i].dst = ptr; - ranges[i].offset_within_chunk = first_in_chunk; - ranges[i].length_within_chunk = len_in_chunk; - ranges[i].server_addr = server_addr; - ranges[i].chunk_server_idx = -1; - - ptr += len_in_chunk; - num_ranges_with_data++; - } - - // Fill remaining chunks with zeros (sparse file) - for (uint32_t i = num_hashes; i < num_chunks_needed; i++) { - uint32_t chunk_idx = first_chunk + i; - uint32_t first_in_chunk = (chunk_idx == first_chunk) ? (first_byte % chunk_size) : 0; - uint32_t last_in_chunk = (chunk_idx == last_chunk) ? (last_byte % chunk_size) : (chunk_size - 1); - uint32_t len_in_chunk = 1 + last_in_chunk - first_in_chunk; - - memset(ptr, 0, len_in_chunk); - ptr += len_in_chunk; - } - - // Store range info - toasty->operations[opidx].ranges = ranges; - toasty->operations[opidx].ranges_head = 0; - toasty->operations[opidx].ranges_count = num_ranges_with_data; - toasty->operations[opidx].num_pending = 0; - - // Start first download - if (num_ranges_with_data > 0) { - Range *r = &ranges[0]; - int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL); - if (cs_idx < 0) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - r->chunk_server_idx = cs_idx; - - if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk, - r->length_within_chunk, opidx, 0) < 0) { - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].num_pending = 1; - toasty->operations[opidx].ranges_head = 1; + Time now = get_current_time(); + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &state->last_oper); + if (message.rejected) { + client_log(state, now, "RECV REPLY", "%s -> REJECTED", oper_buf); } else { - // No chunks to download - free(ranges); - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; - } - - } else { - - // Handle chunk download response - int range_idx = request_tag; - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Parse response - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint32_t data_len; - if (!binary_read(&reader, &data_len, sizeof(data_len))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint8_t *data = reader.src + reader.cur; - if (!binary_read(&reader, NULL, data_len)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Copy data to destination - if (range_idx >= 0 && range_idx < toasty->operations[opidx].ranges_count) { - memcpy(toasty->operations[opidx].ranges[range_idx].dst, data, data_len); - } - - toasty->operations[opidx].num_pending--; - - // Start next download (sequential) - int next_idx = toasty->operations[opidx].ranges_head; - if (next_idx < toasty->operations[opidx].ranges_count) { - Range *r = &toasty->operations[opidx].ranges[next_idx]; - - int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL); - if (cs_idx >= 0) { - r->chunk_server_idx = cs_idx; - if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk, - r->length_within_chunk, opidx, next_idx) == 0) { - toasty->operations[opidx].num_pending++; - toasty->operations[opidx].ranges_head++; - } - } - } - - // Check if done - if (toasty->operations[opidx].num_pending == 0) { - free(toasty->operations[opidx].ranges); - toasty->operations[opidx].ranges = NULL; - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; - } - } -} - -static int start_upload(ToastyFS *toasty, int opidx) -{ - Operation *o = &toasty->operations[opidx]; - - int found = -1; - - // Find a WAITING operation that can be started - for (int i = 0; i < o->num_uploads; i++) { - - if (o->uploads[i].status != UPLOAD_WAITING) - continue; - - // Can't start uploads of a chunk to the - // same server twice. - bool invalid = false; - for (int j = 0; j < o->num_uploads; j++) { - - if (j == i) - continue; - - if (o->uploads[j].status != UPLOAD_PENDING) - continue; - - if (o->uploads[i].server_lid == o->uploads[j].server_lid || - addr_eql(o->uploads[i].address, o->uploads[j].address)) { - invalid = true; - break; - } - } - - if (invalid) - continue; - - found = i; - break; - } - - if (found < 0) - return -1; // No upload can be started at this time - - int tag = TAG_UPLOAD_CHUNK_MIN + found; - assert(tag <= TAG_UPLOAD_CHUNK_MAX); - - ByteQueue *output; - int chunk_server_idx = get_chunk_server(toasty, &o->uploads[found].address, 1, &output); - if (chunk_server_idx < 0) - return -1; - - RequestQueue *reqs = &toasty->chunk_servers[chunk_server_idx].reqs; - if (request_queue_push(reqs, (Request) { tag, opidx }) < 0) { - close_chunk_server(toasty, chunk_server_idx); - return -1; - } - - if (o->uploads[found].chunk_index >= o->num_hashes) { - - char *data_ptr = o->uploads[found].src; - uint32_t chunk_size = o->chunk_size; - uint32_t target_off = o->uploads[found].off; - uint32_t target_len = o->uploads[found].len; - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_CHUNK); - message_write(&writer, &chunk_size, sizeof(chunk_size)); - message_write(&writer, &target_off, sizeof(target_off)); - message_write(&writer, &target_len, sizeof(target_len)); - message_write(&writer, data_ptr, target_len); - if (!message_writer_free(&writer)) { - close_chunk_server(toasty, chunk_server_idx); - request_queue_pop(reqs, NULL); - return -1; - } - - } else { - - char *data_ptr = o->uploads[found].src; - SHA256 target_hash = o->hashes[o->uploads[found].chunk_index]; - uint32_t target_off = o->uploads[found].off; - uint32_t target_len = o->uploads[found].len; - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_UPLOAD_CHUNK); - message_write(&writer, &target_hash, sizeof(target_hash)); - message_write(&writer, &target_off, sizeof(target_off)); - message_write(&writer, &target_len, sizeof(target_len)); - message_write(&writer, data_ptr, target_len); - if (!message_writer_free(&writer)) { - close_chunk_server(toasty, chunk_server_idx); - request_queue_pop(reqs, NULL); - return -1; + char result_buf[64]; + meta_snprint_result(result_buf, sizeof(result_buf), message.result); + client_log(state, now, "RECV REPLY", "%s -> %s", oper_buf, result_buf); } } - o->uploads[found].status = UPLOAD_PENDING; + state->last_result = message.result; + state->last_was_timeout = false; + state->last_was_rejected = message.rejected; + state->pending = false; return 0; } -static int count_pending_uploads(ToastyFS *toasty, int opidx) +int client_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) { - int n = 0; - for (int i = 0; i < toasty->operations[opidx].num_uploads; i++) - if (toasty->operations[opidx].uploads[i].status == UPLOAD_PENDING) - n++; - return n; -} + ClientState *state = state_; -static int schedule_upload(ToastyFS *toasty, int opidx, UploadSchedule upload) -{ - Operation *o = &toasty->operations[opidx]; + state->num_servers = 0; - if (o->num_uploads == o->cap_uploads) { - - int new_cap_uploads; - if (o->uploads == NULL) - new_cap_uploads = 8; - else - new_cap_uploads = 2 * o->cap_uploads; - - UploadSchedule *uploads = malloc(new_cap_uploads * sizeof(UploadSchedule)); - if (uploads == NULL) - return -1; - - if (o->num_uploads > 0) { - memcpy( - uploads, - o->uploads, - o->num_uploads * sizeof(UploadSchedule) - ); - free(o->uploads); - } - - o->uploads = uploads; - o->cap_uploads = new_cap_uploads; - } - - o->uploads[o->num_uploads++] = upload; - return 0; -} - -static void process_event_for_write(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - if (msg.len == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (request_tag == TAG_RETRIEVE_METADATA_FOR_WRITE) { - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint64_t gen; - uint32_t chunk_size; - uint32_t num_hashes; - - if (type != MESSAGE_TYPE_READ_SUCCESS) { - - if ((toasty->operations[opidx].flags & TOASTY_WRITE_CREATE_IF_MISSING) == 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t message_len; - if (!binary_read(&reader, &message_len, sizeof(message_len))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (!binary_read(&reader, NULL, message_len)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - gen = MISSING_FILE_GENERATION; // TODO: is setting the generation to 0 right? - chunk_size = 4096; // The creation flag defaults to a chunk size of 4K - num_hashes = 0; - - } else { - - // Read generation counter - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - assert(gen != NO_GENERATION); // TODO: should this be an assertion? - - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Skip file_length field that comes after chunk_size - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (!binary_read(&reader, &num_hashes, sizeof(num_hashes))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - - toasty->operations[opidx].expect_gen = gen; - toasty->operations[opidx].chunk_size = chunk_size; - - uint32_t num_all_hasehs = (toasty->operations[opidx].len + chunk_size - 1) / chunk_size; - uint32_t num_new_hashes = num_all_hasehs - num_hashes; - assert(num_all_hasehs >= num_hashes); - - toasty->operations[opidx].num_chunks = num_all_hasehs; - toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow - toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256)); - if (toasty->operations[opidx].hashes == NULL && num_hashes > 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].uploads = NULL; - toasty->operations[opidx].num_uploads = 0; - toasty->operations[opidx].cap_uploads = 0; - - char *full_ptr = toasty->operations[opidx].ptr; - int full_off = toasty->operations[opidx].off; - int full_len = toasty->operations[opidx].len; - - int relative_off = 0; - - int next_server_lid = 0; - toasty->operations[opidx].num_uploads = 0; - - // This loop is for chunks that already exist in the file - for (uint32_t i = 0; i < num_hashes; i++) { - - char *src = full_ptr + relative_off; - - uint32_t off = 0; - if (i == 0) - off = full_off % chunk_size; - - uint32_t len = full_len - relative_off; - if (len > chunk_size - off) - len = chunk_size - off; - - assert(len <= chunk_size); - assert(off <= chunk_size); - assert(off + len <= chunk_size); - - relative_off += len; - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].hashes[i] = hash; - - uint32_t num_holders; - if (!binary_read(&reader, &num_holders, sizeof(num_holders))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t j = 0; j < num_holders; j++) { - - int server_lid = next_server_lid; - next_server_lid++; - - uint32_t num_ipv4; - if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t k = 0; k < num_ipv4; k++) { - - IPv4 ipv4; - if (!binary_read(&reader, &ipv4, sizeof(ipv4))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - UploadSchedule upload; - upload.status = UPLOAD_WAITING; - upload.server_lid = server_lid; - upload.address.is_ipv4 = true; - upload.address.ipv4 = ipv4; - upload.address.port = port; - upload.chunk_index = i; - upload.src = src; - upload.off = off; - upload.len = len; - if (schedule_upload(toasty, opidx, upload) < 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - - uint32_t num_ipv6; - if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t k = 0; k < num_ipv6; k++) { - - IPv6 ipv6; - if (!binary_read(&reader, &ipv6, sizeof(ipv6))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - UploadSchedule upload; - upload.status = UPLOAD_WAITING; - upload.server_lid = server_lid; - upload.address.is_ipv4 = false; - upload.address.ipv6 = ipv6; - upload.address.port = port; - upload.chunk_index = i; - upload.src = src; - upload.off = off; - upload.len = len; - if (schedule_upload(toasty, opidx, upload) < 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - } - } - - uint32_t num_locations; - if (!binary_read(&reader, &num_locations, sizeof(num_locations))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t i = 0; i < num_locations; i++) { - - int server_lid = next_server_lid; - next_server_lid++; - - uint32_t num_ipv4; - if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t k = 0; k < num_ipv4; k++) { - - IPv4 ipv4; - if (!binary_read(&reader, &ipv4, sizeof(ipv4))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } -#if 0 - { - char ip_str[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &ipv4, ip_str, sizeof(ip_str)); - printf("write location %s:%d\n", ip_str, port); - } -#endif - int old_relative_off = relative_off; - - for (uint32_t w = 0; w < num_new_hashes; w++) { - - char *src = full_ptr + relative_off; - - uint32_t off = 0; - if (num_hashes == 0 && w == 0) - off = full_off % chunk_size; - - uint32_t len = full_len - relative_off; - if (len > chunk_size - off) - len = chunk_size - off; - - assert(len <= chunk_size); - assert(off <= chunk_size); - assert(off + len <= chunk_size); - - relative_off += len; - - UploadSchedule upload; - upload.status = UPLOAD_WAITING; - upload.server_lid = server_lid; - upload.address.is_ipv4 = true; - upload.address.ipv4 = ipv4; - upload.address.port = port; - upload.chunk_index = num_hashes + w; - upload.src = src; - upload.off = off; - upload.len = len; - if (schedule_upload(toasty, opidx, upload) < 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - - relative_off = old_relative_off; - } - - uint32_t num_ipv6; - if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (uint32_t k = 0; k < num_ipv6; k++) { - - char *src = full_ptr + relative_off; - - IPv6 ipv6; - if (!binary_read(&reader, &ipv6, sizeof(ipv6))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } -#if 0 - { - char ip_str[INET_ADDRSTRLEN]; - inet_ntop(AF_INET6, &ipv6, ip_str, sizeof(ip_str)); - printf("write location %s:%d\n", ip_str, port); - } -#endif - int old_relative_off = relative_off; - - for (uint32_t w = 0; w < num_new_hashes; w++) { - - uint32_t off = 0; - if (num_hashes == 0 && w == 0) - off = full_off % chunk_size; - - uint32_t len = full_len - relative_off; - if (len > chunk_size - off) - len = chunk_size - off; - - assert(len <= chunk_size); - assert(off <= chunk_size); - assert(off + len <= chunk_size); - - relative_off += len; - - UploadSchedule upload; - upload.status = UPLOAD_WAITING; - upload.server_lid = server_lid; - upload.address.is_ipv4 = false; - upload.address.ipv6 = ipv6; - upload.address.port = port; - upload.chunk_index = num_hashes + w; - upload.src = src; - upload.off = off; - upload.len = len; - if (schedule_upload(toasty, opidx, upload) < 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - - relative_off = old_relative_off; - } - } - - // Now start the first batch of uploads - int started = 0; - for (int i = 0; i < PARALLEL_LIMIT; i++) { - if (start_upload(toasty, opidx) == 0) - started++; - } - - if (started == 0) { - // We already failed - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // TODO: Now we need to upload the patches to N of the - // chunk servers that are holding each old chunk - // All new chunks need to be written to the specified - // locations at least N times. If any upload fails, - // the write fails. If all writes succede, the client - // sends the metadata server a WRITE operation - // swapping the old hashes with the new ones. - // - // The algorithm should go like this: - // - Iterate over each chunk - // - Pick the first N holders of the chunk. If less than N - // are available, pick M. - // - For each pick, take the first address and start the - // chunk upload - // - // If an upload fails, - // - // - // - // example upload schedule: - // chunk_A server_A addr_0 - // chunk_A server_A addr_1 - // chunk_A server_B addr_0 - // chunk_A server_B addr_1 - // chunk_A server_B addr_2 - // chunk_A server_C addr_0 - // chunk_B server_D addr_0 - // chunk_B server_E addr_0 - // chunk_B server_E addr_1 - // chunk_B server_F addr_0 - // - // If an upload succedes, all uploads of the chunk to the same server - // are removed and if this was the N-th successful upload of a chunk, - // all uploads of the same chunk are removed. - // - // Uploads to the same chunk server with different addresses can't - // be parallelized, so - - // The client should not try any random N chunk servers - // for upload. It must try all chunk servers until N respond - - } else if (request_tag >= TAG_UPLOAD_CHUNK_MIN && request_tag <= TAG_UPLOAD_CHUNK_MAX) { - - int found = request_tag - TAG_UPLOAD_CHUNK_MIN; - - // Upload complete - // - // TODO: - // - Mark upload as complete or failed - // - If successful, ignore other uploads that don't - // need performing anymore, then start new uploads - // - On error, return an overall error - - // TODO: Should differentiate between chunk creation - // and chunk update. - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t expected_type; - if (toasty->operations[opidx].uploads[found].chunk_index >= toasty->operations[opidx].num_hashes) { - expected_type = MESSAGE_TYPE_CREATE_CHUNK_SUCCESS; - } else { - expected_type = MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS; - } - - if (type != expected_type) - toasty->operations[opidx].uploads[found].status = UPLOAD_FAILED; - else { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Check that there is nothing else to read - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].uploads[found].status = UPLOAD_COMPLETED; - toasty->operations[opidx].uploads[found].final_hash = hash; - for (int i = 0; i < toasty->operations[opidx].num_uploads; i++) { - - if (toasty->operations[opidx].uploads[i].status == UPLOAD_WAITING - && toasty->operations[opidx].uploads[i].chunk_index == toasty->operations[opidx].uploads[found].chunk_index - && (addr_eql(toasty->operations[opidx].uploads[i].address, toasty->operations[opidx].uploads[found].address) - || toasty->operations[opidx].uploads[i].server_lid == toasty->operations[opidx].uploads[found].server_lid)) - toasty->operations[opidx].uploads[i].status = UPLOAD_IGNORED; - } - - // TODO: the new chunk hash should be stored in - // the upload struct here - } - - // Count the number of PENDING uploads and - // start uploads until N are pending or an - // error occurs - int num_pending = count_pending_uploads(toasty, opidx); - while (num_pending < PARALLEL_LIMIT) { - if (start_upload(toasty, opidx) < 0) - break; - num_pending++; - } - - if (num_pending == 0) { - - // TODO: Check whether we managed to replicate - // all chunks. - // - // We need to make sure that every chunk was - // uploaded to at least N different servers - - typedef struct { - SHA256 old_hash; - SHA256 new_hash; - int num_locations; - Address locations[REPLICATION_FACTOR]; - } ChunkUploadResult; - - int num_upload_results = toasty->operations[opidx].num_chunks; - ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult)); - if (upload_results == NULL && num_upload_results > 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - for (int i = 0; i < num_upload_results; i++) { - if (i < toasty->operations[opidx].num_hashes) - upload_results[i].old_hash = toasty->operations[opidx].hashes[i]; - else - memset(&upload_results[i].old_hash, 0, sizeof(SHA256)); - upload_results[i].num_locations = 0; - } - - for (int i = 0; i < toasty->operations[opidx].num_uploads; i++) { - UploadSchedule *u = &toasty->operations[opidx].uploads[i]; - if (u->status == UPLOAD_COMPLETED) { - int n = upload_results[u->chunk_index].num_locations++; - upload_results[u->chunk_index].locations[n] = u->address; - upload_results[u->chunk_index].new_hash = u->final_hash; - } - } - - // Now check that each chunk is replicated - // at least N times - - bool ok = true; - for (int i = 0; i < num_upload_results; i++) { - if (upload_results[i].num_locations < REPLICATION_FACTOR) { - ok = false; - break; - } - } - - if (!ok) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - free(upload_results); - return; - } - - MessageWriter writer; - metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_WRITE); - - ToastyString path = toasty->operations[opidx].path; - uint32_t offset = toasty->operations[opidx].off; - uint32_t length = toasty->operations[opidx].len; - uint32_t flags = toasty->operations[opidx].flags; - - if (path.len > UINT16_MAX) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - free(upload_results); - return; - } - uint16_t path_len = path.len; - uint64_t expect_gen = toasty->operations[opidx].expect_gen; - assert(expect_gen != NO_GENERATION); - - uint32_t num_chunks = num_upload_results; - - message_write(&writer, &expect_gen, sizeof(expect_gen)); - message_write(&writer, &flags, sizeof(flags)); - message_write(&writer, &path_len, sizeof(path_len)); - message_write(&writer, path.ptr, path.len); - message_write(&writer, &offset, sizeof(offset)); - message_write(&writer, &length, sizeof(length)); - message_write(&writer, &num_chunks, sizeof(num_chunks)); - - for (int i = 0; i < num_upload_results; i++) { - - message_write(&writer, &upload_results[i].new_hash, sizeof(upload_results[i].new_hash)); - - uint32_t tmp = upload_results[i].num_locations; - message_write(&writer, &tmp, sizeof(tmp)); - - for (int j = 0; j < upload_results[i].num_locations; j++) { - - Address addr = upload_results[i].locations[j]; - - uint8_t is_ipv4 = addr.is_ipv4; - message_write(&writer, &is_ipv4, sizeof(is_ipv4)); - if (addr.is_ipv4) message_write(&writer, &addr.ipv4, sizeof(addr.ipv4)); - else message_write(&writer, &addr.ipv6, sizeof(addr.ipv6)); - message_write(&writer, &addr.port, sizeof(addr.port)); - } - } - - free(upload_results); - - if (metadata_server_request_end(toasty, &writer, opidx, TAG_COMMIT_WRITE) < 0) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - } - - } else { - - assert(request_tag == TAG_COMMIT_WRITE); - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // version - if (!binary_read(&reader, NULL, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(uint16_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // length - if (!binary_read(&reader, NULL, sizeof(uint32_t))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (type != MESSAGE_TYPE_WRITE_SUCCESS) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - // Read generation counter - uint64_t gen; - if (!binary_read(&reader, &gen, sizeof(gen))) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - if (binary_read(&reader, NULL, 1)) { - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user }; - return; - } - - toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_SUCCESS, .user=toasty->operations[opidx].user }; - } -} - -static void process_event(ToastyFS *toasty, - int opidx, int request_tag, ByteView msg) -{ - switch (toasty->operations[opidx].type) { - - case OPERATION_TYPE_CREATE: - process_event_for_create(toasty, opidx, request_tag, msg); - break; - - case OPERATION_TYPE_DELETE: - process_event_for_delete(toasty, opidx, request_tag, msg); - break; - - case OPERATION_TYPE_LIST: - process_event_for_list(toasty, opidx, request_tag, msg); - break; - - case OPERATION_TYPE_READ: - process_event_for_read(toasty, opidx, request_tag, msg); - break; - - case OPERATION_TYPE_WRITE: - process_event_for_write(toasty, opidx, request_tag, msg); - break; - - default: - UNREACHABLE; - } -} - -static bool -translate_operation_into_result(ToastyFS *toasty, int opidx, ToastyResult *result) -{ - if (toasty->operations[opidx].result.type == TOASTY_RESULT_EMPTY) - return false; - *result = toasty->operations[opidx].result; - toasty->operations[opidx].type = OPERATION_TYPE_FREE; - toasty->num_operations--; - return true; -} - -int toasty_get_result(ToastyFS *toasty, ToastyHandle handle, ToastyResult *result) -{ - int ret; - - if (mutex_lock(&toasty->mutex) < 0) { - ret = -1; - goto unlock_and_exit; - } - - int opidx; - if (handle == TOASTY_INVALID) - opidx = -1; - else { - opidx = handle_to_operation(toasty, handle); - if (opidx < 0) { - ret = -1; - goto unlock_and_exit; - } - } - - if (opidx < 0) { - for (int i = 0, j = 0; j < toasty->num_operations; i++) { - - if (toasty->operations[i].type == OPERATION_TYPE_FREE) - continue; - j++; - - if (translate_operation_into_result(toasty, i, result)) { - ret = 0; - goto unlock_and_exit; - } - } - } else { - if (translate_operation_into_result(toasty, opidx, result)) { - ret = 0; - goto unlock_and_exit; - } - } - - ret = 1; -unlock_and_exit: - if (mutex_unlock(&toasty->mutex) < 0) - return -1; - return ret; -} - -_Static_assert(TCP_POLL_CAPACITY == TOASTY_POLL_CAPACITY); - -int toasty_process_events(ToastyFS *toasty, void **contexts, struct pollfd *polled, int num_polled) -{ - if (mutex_lock(&toasty->mutex) < 0) - return -1; - - int num_events; - Event events[TCP_EVENT_CAPACITY]; - - num_events = tcp_translate_events(&toasty->tcp, events, contexts, polled, num_polled); - for (int i = 0; i < num_events; i++) { - int conn_idx = events[i].conn_idx; - switch (events[i].type) { - - case EVENT_WAKEUP: - CLIENT_TRACE("TCP EVENT: wakeup"); - // Do nothing - break; - - case EVENT_CONNECT: - { - int tag = tcp_get_tag(&toasty->tcp, conn_idx); - if (tag != TAG_METADATA_SERVER) - toasty->chunk_servers[tag].connected = true; - - CLIENT_TRACE("TCP EVENT: CONNECT (tag=%d)", tag); - } - break; - - case EVENT_DISCONNECT: - { - // A TCP connection was just dropped. - // For clients, connections can be: - // 1. To the metadata server - // 2. or to a chunk server - // If requests were buffered for the metadata - // or chunk server, they are considered as failed - // and their failure event is processed. - // - // If a chunk server was never connected, - // then it's possible that using a different - // address will allow connecting succesfully - // and send the buffered messages. Therefore, - // if a chunk server wasn't connected and - // there are addresses to try, the messages - // are not dropped and a new connect process - // is started. - - RequestQueue *reqs = NULL; - - int tag = events[i].tag; - CLIENT_TRACE("TCP EVENT: DISCONNECT (tag=%d)", tag); - -#ifdef MAIN_SIMULATION - if (toasty->metadata_server.used) { - int conn_idx = tcp_index_from_tag(&toasty->tcp, TAG_METADATA_SERVER); - if (tag == TAG_METADATA_SERVER) { - assert(conn_idx == -1); - } else { - assert(conn_idx > -1); - } - } -#endif - - if (tag == TAG_METADATA_SERVER) { - reqs = &toasty->metadata_server.reqs; - toasty->metadata_server.used = false; - CLIENT_TRACE("MS disconnect"); - } else { - - assert(tag > -1); - CLIENT_TRACE("CS disconnect"); - - if (toasty->chunk_servers[tag].connected) - reqs = &toasty->chunk_servers[tag].reqs; - else { - - toasty->chunk_servers[tag].current_addr_idx++; - - bool started = false; - while (toasty->chunk_servers[tag].current_addr_idx < toasty->chunk_servers[tag].num_addrs) { - - if (tcp_connect(&toasty->tcp, toasty->chunk_servers[tag].addrs[toasty->chunk_servers[tag].current_addr_idx], tag, NULL) == 0) { - started = true; - break; - } - - toasty->chunk_servers[tag].current_addr_idx++; - } - - if (!started) { - reqs = &toasty->chunk_servers[tag].reqs; - toasty->chunk_servers[tag].used = false; - } - } - } - - if (reqs) { - for (Request req; request_queue_pop(reqs, &req) == 0; ) - process_event(toasty, req.opidx, req.tag, (ByteView) { NULL, 0 }); - } - } - break; - - case EVENT_MESSAGE: - { - CLIENT_TRACE("TCP EVENT: message"); - - for (;;) { - - ByteView msg; - uint16_t msg_type; - int ret = tcp_next_message(&toasty->tcp, conn_idx, &msg, &msg_type); - if (ret == 0) { - CLIENT_TRACE("Incomplete message"); - break; - } - if (ret < 0) { - CLIENT_TRACE("Invalid message"); - tcp_close(&toasty->tcp, conn_idx); - break; - } - - RequestQueue *reqs; - - int tag = tcp_get_tag(&toasty->tcp, conn_idx); - if (tag == TAG_METADATA_SERVER) { - CLIENT_TRACE("Message is from MS"); - reqs = &toasty->metadata_server.reqs; - } else { - CLIENT_TRACE("Message is from CS"); - reqs = &toasty->chunk_servers[tag].reqs; - } - - Request req; - if (request_queue_pop(reqs, &req) < 0) { - // Unexpected message - CLIENT_TRACE("Message was unexpected"); - tcp_consume_message(&toasty->tcp, conn_idx); - continue; - } - process_event(toasty, req.opidx, req.tag, msg); - - tcp_consume_message(&toasty->tcp, conn_idx); - } - } - break; - } - } - - int ret = tcp_register_events(&toasty->tcp, contexts, polled); - - if (mutex_unlock(&toasty->mutex) < 0) - return -1; - - return ret; -} - -int toasty_wait_result(ToastyFS *toasty, ToastyHandle handle, ToastyResult *result, int timeout) -{ - Time start_time = INVALID_TIME; - if (timeout > -1) { - start_time = get_current_time(); - if (start_time == INVALID_TIME) - return -1; - } - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled; - - num_polled = toasty_process_events(toasty, contexts, polled, 0); - - for (;;) { - - int ret = toasty_get_result(toasty, handle, result); - - if (ret == -1) - return -1; // Error - - if (ret == 0) - break; // Completed - - assert(ret == 1); - - int remaining_timeout = -1; - if (timeout > -1) { - - Time current_time = get_current_time(); - if (current_time == INVALID_TIME) + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--server")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --server missing value. Usage is --server :\n"); return -1; + } + if (state->num_servers == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + // TODO: Check address is not duplicated + if (parse_addr_arg(argv[i], &state->server_addrs[state->num_servers++]) < 0) { + fprintf(stderr, "Malformed : pair for --server option\n"); + return -1; + } + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } - int elapsed = (current_time - start_time) / 1000000; - if (elapsed > timeout) - return 1; // Timed out + // Now sort the addresses + addr_sort(state->server_addrs, state->num_servers); - remaining_timeout = timeout - elapsed; + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Client :: Couldn't setup TCP context\n"); + return -1; + } + + state->pending = false; + + state->view_number = 0; + state->request_id = 0; + state->reconnect_time = 0; + + state->client_id = next_client_id++; + + // Connect to all known servers + for (int i = 0; i < state->num_servers; i++) { + if (tcp_connect(&state->tcp, state->server_addrs[i], i, NULL) < 0) { + fprintf(stderr, "Client :: Couldn't connect to server %d\n", i); + tcp_context_free(&state->tcp); + return -1; + } + } + + { + Time now = get_current_time(); + client_log(state, now, "INIT", "servers=%d leader=%d", state->num_servers, leader_idx(state)); + } + + *timeout = 0; + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Client :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int client_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + ClientState *state = state_; + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + + if (events[i].type == EVENT_DISCONNECT) { + int conn_idx = events[i].conn_idx; + int tag = tcp_get_tag(&state->tcp, conn_idx); + if (tag == leader_idx(state) && state->pending) { + Time now = get_current_time(); + client_log(state, now, "DISCONNECT", "%s/%s lost leader (node %d)", + state->last_oper.bucket, state->last_oper.key, leader_idx(state)); + state->last_was_timeout = true; + state->last_was_rejected = false; + state->pending = false; + } + tcp_close(&state->tcp, conn_idx); + continue; } - ret = POLL(polled, num_polled, remaining_timeout); - if (ret < 0) - return -1; + if (events[i].type != EVENT_MESSAGE) + continue; - // TODO: Detect when this returns WAKEUP - num_polled = toasty_process_events(toasty, contexts, polled, num_polled); - if (num_polled < 0) - return -1; + int conn_idx = events[i].conn_idx; + + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + ret = process_message(state, conn_idx, msg_type, msg); + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + tcp_consume_message(&state->tcp, conn_idx); + } } + Time now = get_current_time(); + + // If we've been waiting too long for a response, give up and + // try the next server (the current leader may have crashed and + // a view change may have happened) + if (state->pending) { + Time request_deadline = state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (request_deadline <= now) { + + { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &state->last_oper); + client_log(state, now, "TIMEOUT", "%s", oper_buf); + } + + state->view_number++; + state->last_was_timeout = true; + state->last_was_rejected = false; + state->pending = false; + } + } + + if (!state->pending) { + + int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state)); + if (conn_idx < 0) { + if (state->reconnect_time <= now) { + tcp_connect(&state->tcp, state->server_addrs[leader_idx(state)], leader_idx(state), NULL); + state->reconnect_time = now + HEARTBEAT_INTERVAL_SEC * 1000000000ULL; + } + } else { + + // Now start a new operation + state->request_id++; + state->last_oper = random_oper(); + + RequestMessage request_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REQUEST, + .length = sizeof(RequestMessage), + }, + .oper = state->last_oper, + .client_id = state->client_id, + .request_id = state->request_id, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &request_message, request_message.base.length); + + { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &state->last_oper); + client_log(state, now, "SEND REQUEST", "%s", oper_buf); + } + + state->pending = true; + state->request_time = now; + } + } + + // Set timeout based on pending request deadline or reconnection delay + Time deadline = INVALID_TIME; + if (state->pending) { + nearest_deadline(&deadline, state->request_time + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL); + } else { + int conn_idx = tcp_index_from_tag(&state->tcp, leader_idx(state)); + if (conn_idx < 0 && state->reconnect_time > now) { + nearest_deadline(&deadline, state->reconnect_time); + } + } + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); return 0; } -int toasty_create_dir(ToastyFS *toasty, ToastyString path, - ToastyVersionTag *vtag) +int client_free(void *state_) { - ToastyHandle handle = toasty_begin_create_dir(toasty, path); - if (handle == TOASTY_INVALID) - return -1; + ClientState *state = state_; - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - toasty_free_result(&result); - return -1; + { + Time now = get_current_time(); + client_log_simple(state, now, "CRASHED"); } - if (vtag) *vtag = result.vtag; - toasty_free_result(&result); + tcp_context_free(&state->tcp); return 0; } - -int toasty_create_file(ToastyFS *toasty, ToastyString path, - unsigned int chunk_size, ToastyVersionTag *vtag) -{ - ToastyHandle handle = toasty_begin_create_file(toasty, path, chunk_size); - if (handle == TOASTY_INVALID) - return -1; - - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - toasty_free_result(&result); - return -1; - } - - if (vtag) *vtag = result.vtag; - toasty_free_result(&result); - return 0; -} - -int toasty_delete(ToastyFS *toasty, ToastyString path, - ToastyVersionTag vtag) -{ - ToastyHandle handle = toasty_begin_delete(toasty, path, vtag); - if (handle == TOASTY_INVALID) - return -1; - - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_DELETE_SUCCESS) { - toasty_free_result(&result); - return -1; - } - - toasty_free_result(&result); - return 0; -} - -int toasty_list(ToastyFS *toasty, ToastyString path, - ToastyListing *listing, ToastyVersionTag *vtag) -{ - ToastyHandle handle = toasty_begin_list(toasty, path, vtag ? *vtag : 0); - if (handle == TOASTY_INVALID) - return -1; - - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_LIST_SUCCESS) { - toasty_free_result(&result); - return -1; - } - - if (vtag) - *vtag = result.vtag; - - if (listing) - *listing = result.listing; - else - toasty_free_result(&result); - return 0; -} - -void toasty_free_listing(ToastyListing *listing) -{ - free(listing->items); -} - -int toasty_read(ToastyFS *toasty, ToastyString path, - int off, void *dst, int len, ToastyVersionTag *vtag) -{ - ToastyHandle handle = toasty_begin_read(toasty, path, off, dst, len, vtag ? *vtag : 0); - if (handle == TOASTY_INVALID) - return -1; - - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_READ_SUCCESS) { - toasty_free_result(&result); - return -1; - } - - if (vtag) *vtag = result.vtag; - int bytes_read = result.bytes_read; - toasty_free_result(&result); - return bytes_read; -} - -int toasty_write(ToastyFS *toasty, ToastyString path, - int off, void *src, int len, ToastyVersionTag *vtag, uint32_t flags) -{ - ToastyHandle handle = toasty_begin_write(toasty, path, off, src, len, vtag ? *vtag : 0, flags); - if (handle == TOASTY_INVALID) - return -1; - - ToastyResult result; - if (toasty_wait_result(toasty, handle, &result, -1) < 0) - return -1; - - if (result.type != TOASTY_RESULT_WRITE_SUCCESS) { - toasty_free_result(&result); - return -1; - } - - if (vtag) *vtag = result.vtag; - toasty_free_result(&result); - return 0; // TODO: return the number of bytes written? -} diff --git a/src/client.h b/src/client.h new file mode 100644 index 0000000..bb79428 --- /dev/null +++ b/src/client.h @@ -0,0 +1,49 @@ +#ifndef CLIENT_INCLUDED +#define CLIENT_INCLUDED + +#include +#include + +#include "config.h" +#include "metadata.h" + +typedef struct { + + TCP tcp; + + // True if we are waiting for a response + bool pending; + Time request_time; // When the current request was sent + + // The operation sent in the current pending request (for logging) + MetaOper last_oper; + + // Checker support + MetaResult last_result; + bool last_was_timeout; + bool last_was_rejected; + + Address server_addrs[NODE_LIMIT]; + int num_servers; + + uint64_t view_number; + + uint64_t client_id; + uint64_t request_id; + + Time reconnect_time; // Earliest time to retry connecting to leader + +} ClientState; + +struct pollfd; + +int client_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int client_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int client_free(void *state); + +#endif // CLIENT_INCLUDED \ No newline at end of file diff --git a/src/client_table.c b/src/client_table.c new file mode 100644 index 0000000..0ac8415 --- /dev/null +++ b/src/client_table.c @@ -0,0 +1,70 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include + +#include "client_table.h" + +void client_table_init(ClientTable *client_table) +{ + client_table->count = 0; + client_table->capacity = 0; + client_table->entries = NULL; +} + +void client_table_free(ClientTable *client_table) +{ + free(client_table->entries); +} + +ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id) +{ + for (int i = 0; i < client_table->count; i++) { + if (client_table->entries[i].client_id == client_id) + return &client_table->entries[i]; + } + return NULL; +} + +int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag) +{ + if (client_table->count == client_table->capacity) { + int n = 2 * client_table->capacity; + if (n == 0) n = 8; + void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry)); + if (p == NULL) + return -1; + client_table->capacity = n; + client_table->entries = p; + } + + client_table->entries[client_table->count++] = (ClientTableEntry) { + .client_id = client_id, + .last_request_id = request_id, + .pending = true, + .conn_tag = conn_tag, + }; + return 0; +} + +int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id) +{ + if (client_table->count == client_table->capacity) { + int n = 2 * client_table->capacity; + if (n == 0) n = 8; + void *p = realloc(client_table->entries, n * sizeof(ClientTableEntry)); + if (p == NULL) + return -1; + client_table->capacity = n; + client_table->entries = p; + } + + client_table->entries[client_table->count++] = (ClientTableEntry) { + .client_id=client_id, + .last_request_id=request_id, + .pending=true, + }; + return 0; +} diff --git a/src/client_table.h b/src/client_table.h new file mode 100644 index 0000000..8513040 --- /dev/null +++ b/src/client_table.h @@ -0,0 +1,31 @@ +#ifndef CLIENT_TABLE_INCLUDED +#define CLIENT_TABLE_INCLUDED + +#include + +#include "metadata.h" + +typedef struct { + uint64_t client_id; + uint64_t last_request_id; + MetaResult last_result; + bool pending; // Only meaningful on the leader that received + // the REQUEST. After a view change, a new leader + // may find stale entries with pending=false from + // a previous view when it was leader before. + int conn_tag; +} ClientTableEntry; + +typedef struct { + int count; + int capacity; + ClientTableEntry *entries; +} ClientTable; + +void client_table_init(ClientTable *client_table); +void client_table_free(ClientTable *client_table); +ClientTableEntry *client_table_find(ClientTable *client_table, uint64_t client_id); +int client_table_add(ClientTable *client_table, uint64_t client_id, uint64_t request_id, int conn_tag); +int client_table_insert(ClientTable *client_table, uint64_t client_id, uint64_t request_id); + +#endif // CLIENT_TABLE_INCLUDED \ No newline at end of file diff --git a/src/config.h b/src/config.h index b170d3a..c5a517d 100644 --- a/src/config.h +++ b/src/config.h @@ -1,13 +1,13 @@ #ifndef CONFIG_INCLUDED #define CONFIG_INCLUDED -#define MAX_SERVER_ADDRS 4 -#define MAX_CHUNK_SERVERS 128 -#define MAX_OPERATIONS 128 -#define MAX_REQUESTS_PER_QUEUE 128 -#define REPLICATION_FACTOR 3 -#define SYNC_INTERVAL 10 -#define RESPONSE_TIME_LIMIT 90 -#define DELETION_TIMEOUT 30 +#define NODE_LIMIT 32 +#define FUTURE_LIMIT 32 +#define HEARTBEAT_INTERVAL_SEC 1 +#define PRIMARY_DEATH_TIMEOUT_SEC 5 +#define RECOVERY_TIMEOUT_SEC 3 +#define RECOVERY_ATTEMPT_LIMIT 10 +#define STATE_TRANSFER_TIMEOUT_SEC 2 +#define VIEW_CHANGE_TIMEOUT_SEC 20 -#endif // CONFIG_INCLUDED +#endif // CONFIG_INCLUDED \ No newline at end of file diff --git a/src/file_tree.c b/src/file_tree.c deleted file mode 100644 index 4e5ea8b..0000000 --- a/src/file_tree.c +++ /dev/null @@ -1,820 +0,0 @@ -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include -#include - -#include "basic.h" -#include "file_tree.h" - -static int parse_path(string path, string *comps, int max) -{ - bool is_absolute = false; - if (path.len > 0 && path.ptr[0] == '/') { - is_absolute = true; - path.ptr++; - path.len--; - if (path.len == 0) - return 0; // Absolute paths with no components are allowed - } - - int num = 0; - uint32_t i = 0; - for (;;) { - - uint32_t off = i; - while (i < (uint32_t) path.len && path.ptr[i] != '/') - i++; - uint32_t len = i - off; - - if (len == 0) - return -1; // Empty component - - string comp = { path.ptr + off, len }; - if (comp.len == 2 && comp.ptr[0] == '.' && comp.ptr[1] == '.') { - if (num == 0) { - // For absolute paths, ".." at root is ignored (stays at root) - // For relative paths, ".." with no components references parent, which is invalid - if (!is_absolute) - return -1; - // Otherwise, ignore the ".." (absolute path, already at root) - } else { - num--; - } - } else if (comp.len != 1 || comp.ptr[0] != '.') { - if (num == max) - return -1; // To many components - comps[num++] = comp; - } - - if (i == (uint32_t) path.len) - break; - - assert(path.ptr[i] == '/'); - i++; - - if (i == (uint32_t) path.len) - break; - } - - return num; -} - -static int dir_find(Dir *parent, string name) -{ - for (uint64_t i = 0; i < parent->num_children; i++) - if (streq((string) { parent->children[i].name, parent->children[i].name_len }, name)) - return i; - return -1; -} - -static Entity *resolve_path(Entity *root, string *comps, int num_comps) -{ - assert(root->is_dir); - - Entity *current = root; - for (int i = 0; i < num_comps; i++) { - - if (!current->is_dir) - return NULL; - - int j = dir_find(¤t->d, comps[i]); - if (j == -1) - return NULL; - - current = ¤t->d.children[j]; - } - - return current; -} - -static void entity_free(Entity *e); -static bool entity_uses_hash(Entity *e, SHA256 hash); - -static void dir_init(Dir *d) -{ - d->num_children = 0; - d->max_children = 0; - d->children = NULL; -} - -static void dir_free(Dir *d) -{ - for (uint64_t i = 0; i < d->num_children; i++) - entity_free(&d->children[i]); - free(d->children); -} - -static bool gen_match(uint64_t expected_gen, uint64_t entity_gen) -{ - assert(entity_gen != NO_GENERATION); - assert(entity_gen != MISSING_FILE_GENERATION); - - // NO_GENERATION means "skip generation check" - if (expected_gen == NO_GENERATION) - return true; - - // MISSING_FILE_GENERATION means "expect file to NOT exist" - // Since we're checking against an existing entity, this is a mismatch - if (expected_gen == MISSING_FILE_GENERATION) - return false; - - return expected_gen == entity_gen; -} - -static uint64_t create_generation(uint64_t *next_gen) -{ - (*next_gen)++; - if (*next_gen == 0 || *next_gen == UINT64_MAX) - *next_gen = 1; - return *next_gen; -} - -static int dir_remove(Dir *d, int idx, uint64_t expected_gen) -{ - if (!gen_match(expected_gen, d->children[idx].gen)) - return -1; - - // TODO: pretty sure this leaks memory - d->children[idx] = d->children[--d->num_children]; - return 0; -} - -static bool dir_uses_hash(Dir *d, SHA256 hash) -{ - for (uint64_t i = 0; i < d->num_children; i++) - if (entity_uses_hash(&d->children[i], hash)) - return true; - return false; -} - -static void file_init(File *f, uint64_t chunk_size) -{ - f->chunk_size = chunk_size; - f->num_chunks = 0; - f->file_size = 0; - f->chunks = NULL; -} - -static void file_free(File *f) -{ - free(f->chunks); - f->chunks = NULL; -} - -static bool file_uses_hash(File *f, SHA256 hash) -{ - for (uint64_t i = 0; i < f->num_chunks; i++) - if (!memcmp(&f->chunks[i], &hash, sizeof(SHA256))) - return true; - return false; -} - -// Fails when the name is too long -static int entity_init(Entity *e, char *name, int name_len, - bool is_dir, uint64_t chunk_size, uint64_t *next_gen) -{ - if (name_len >= (int) sizeof(e->name)) - return -1; - - e->gen = create_generation(next_gen); - assert(e->gen != NO_GENERATION); - - memcpy(e->name, name, name_len); - e->name[name_len] = '\0'; - e->name_len = (uint16_t) name_len; - - e->is_dir = is_dir; - if (is_dir) - dir_init(&e->d); - else - file_init(&e->f, chunk_size); - - return 0; -} - -static void entity_free(Entity *e) -{ - if (e->is_dir) - dir_free(&e->d); - else - file_free(&e->f); -} - -static bool entity_uses_hash(Entity *e, SHA256 hash) -{ - if (e->is_dir) - return dir_uses_hash(&e->d, hash); - else - return file_uses_hash(&e->f, hash); -} - -int file_tree_init(FileTree *ft) -{ - ft->next_gen = 1; - - int ret = entity_init(&ft->root, "", 0, true, 0, &ft->next_gen); - if (ret < 0) return -1; - - return 0; -} - -void file_tree_free(FileTree *ft) -{ - entity_free(&ft->root); -} - -bool file_tree_uses_hash(FileTree *ft, SHA256 hash) -{ - return entity_uses_hash(&ft->root, hash); -} - -int file_tree_list(FileTree *ft, string path, - ListItem *items, int max_items, uint64_t *gen) -{ - int num_comps; - string comps[MAX_COMPS]; - - num_comps = parse_path(path, comps, MAX_COMPS); - if (num_comps < 0) - return FILETREE_BADPATH; - - Entity *e = resolve_path(&ft->root, comps, num_comps); - - if (e == NULL) - return FILETREE_NOENT; - - if (!e->is_dir) - return FILETREE_NOTDIR; - - Dir *d = &e->d; - - int num_items = d->num_children; - if (num_items > max_items) num_items = max_items; - for (int i = 0; i < num_items; i++) { - - Entity *c = &d->children[i]; - - int name_cpy = c->name_len; - if (name_cpy > (int) sizeof(items[i].name)-1) - name_cpy = (int) sizeof(items[i].name)-1; - - memcpy(items[i].name, c->name, name_cpy); - items[i].name[name_cpy] = '\0'; - - items[i].name_len = name_cpy; - items[i].is_dir = c->is_dir; - items[i].gen = c->gen; - } - - assert(e->gen != NO_GENERATION); - *gen = e->gen; - - return d->num_children; -} - -int file_tree_create_entity(FileTree *ft, string path, - bool is_dir, uint64_t chunk_size, uint64_t *gen) -{ - int num_comps; - string comps[MAX_COMPS]; - - num_comps = parse_path(path, comps, MAX_COMPS); - - if (num_comps < 0) - // Couldn't parse path - return FILETREE_BADPATH; - - if (num_comps == 0) - // Path is empty, which means the caller is referencing the root, - // which exists already. - return FILETREE_EXISTS; - - // Resolve the path up to the second last component - Entity *e = resolve_path(&ft->root, comps, num_comps-1); - - if (e == NULL) - // Parent directory doesn't exist - return FILETREE_NOENT; - - if (!e->is_dir) - // Parent entity is not a directory - return FILETREE_NOTDIR; - - string name = comps[num_comps-1]; - if (dir_find(&e->d, name) != -1) - return FILETREE_EXISTS; - - Dir *d = &e->d; - if (d->num_children == d->max_children) { - - int new_max = 2 * d->max_children; - if (new_max == 0) - new_max = 8; - - Entity *p = malloc(sizeof(Entity) * new_max); - if (p == NULL) - return FILETREE_NOMEM; - - for (uint64_t i = 0; i < d->num_children; i++) - p[i] = d->children[i]; - - free(d->children); - d->children = p; - d->max_children = new_max; - } - Entity *c = &d->children[d->num_children]; - - int ret = entity_init(c, (char*) name.ptr, name.len, is_dir, chunk_size, &ft->next_gen); - if (ret < 0) - // Invalid name for the new file - return FILETREE_BADPATH; - - assert(e->gen != NO_GENERATION); - *gen = e->gen; - - d->num_children++; - return 0; -} - -// TODO: this should return the list of unreferenced hashes -int file_tree_delete_entity(FileTree *ft, string path, - uint64_t expected_gen) -{ - int num_comps; - string comps[MAX_COMPS]; - - num_comps = parse_path(path, comps, MAX_COMPS); - if (num_comps < 0) - return FILETREE_BADPATH; - if (num_comps == 0) - return FILETREE_BADOP; - - Entity *e = resolve_path(&ft->root, comps, num_comps-1); - if (e == NULL) - return FILETREE_NOENT; - if (!e->is_dir) - return FILETREE_NOTDIR; - - int i = dir_find(&e->d, comps[num_comps-1]); - if (i == -1) { - // File doesn't exist - // If caller expected it not to exist (MISSING_FILE_GENERATION), succeed - if (expected_gen == MISSING_FILE_GENERATION) - return 0; - return FILETREE_NOENT; - } - - // File exists - check generation - if (dir_remove(&e->d, i, expected_gen) < 0) - return FILETREE_BADGEN; - - return 0; -} - -int file_tree_write( - FileTree* ft, - string path, - uint64_t off, - uint64_t len, - uint32_t num_chunks, - uint64_t expect_gen, - uint64_t* new_gen, - SHA256* hashes, - SHA256* removed_hashes, - int* num_removed, - bool truncate_after) -{ - // WRITE operations cannot use expect_gen=0 - if (expect_gen == NO_GENERATION) - return FILETREE_BADGEN; - - int num_comps; - string comps[MAX_COMPS]; - - num_comps = parse_path(path, comps, MAX_COMPS); - if (num_comps < 0) - return FILETREE_BADPATH; - - Entity *e = resolve_path(&ft->root, comps, num_comps); - - if (e == NULL) { - // File doesn't exist - // If caller expected it not to exist (MISSING_FILE_GENERATION), that's correct - // but we still can't write to a non-existent file (need CREATE_IF_MISSING flag in client layer) - if (expect_gen == MISSING_FILE_GENERATION) - return FILETREE_NOENT; // Expected behavior: file missing as expected, but can't write - return FILETREE_NOENT; - } - - if (e->is_dir) - return FILETREE_ISDIR; - - // Check generation - will fail if expect_gen is MISSING_FILE_GENERATION (expects missing but file exists) - if (!gen_match(expect_gen, e->gen)) - return FILETREE_BADGEN; - - File *f = &e->f; - - uint64_t first_chunk_index = off / f->chunk_size; - uint64_t last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size; - - assert(last_chunk_index - first_chunk_index + 1 == num_chunks); - - if (last_chunk_index >= f->num_chunks) { - uint64_t old_num_chunks = f->num_chunks; - SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256)); - if (new_chunks == NULL) - return FILETREE_NOMEM; - if (f->chunks) { - if (f->num_chunks > 0) - memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256)); - free(f->chunks); - } - f->chunks = new_chunks; - f->num_chunks = last_chunk_index+1; - for (uint64_t i = old_num_chunks; i < last_chunk_index+1; i++) - memset(&f->chunks[i], 0, sizeof(SHA256)); - } - - int num_overwritten_hashes = 0; - SHA256 overwritten_hashes[100]; // TODO: fix this limit - if (num_chunks > 100) { - assert(0); // TODO - } - - // Update chunks - for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++) { - overwritten_hashes[num_overwritten_hashes++] = f->chunks[i]; - f->chunks[i] = hashes[i - first_chunk_index]; - } - - // Update file size (last byte written + 1) - uint64_t new_size = off + len; - if (truncate_after) { - // With truncation, set file size to exactly new_size and remove chunks beyond - uint64_t new_num_chunks = last_chunk_index + 1; - - // Add any chunks beyond the write to the overwritten list (they'll be removed) - for (uint64_t i = new_num_chunks; i < f->num_chunks; i++) { - if (num_overwritten_hashes < 100) { // Respect the limit - overwritten_hashes[num_overwritten_hashes++] = f->chunks[i]; - } - } - - f->num_chunks = new_num_chunks; - f->file_size = new_size; - } else { - // Without truncation, only grow the file - if (new_size > f->file_size) - f->file_size = new_size; - } - - // Now check which old hashes are no longer used - // anywhere in the tree - // - // NOTE: If removed_hashes is NULL, the caller isn't - // interested in which hashes are no longer reachable. - if (removed_hashes != NULL) { - *num_removed = 0; - for (int i = 0; i < num_overwritten_hashes; i++) { - - SHA256 hash = overwritten_hashes[i]; - - // Skip zero hashes - bool is_zero = true; - for (int j = 0; j < (int) sizeof(SHA256); j++) { - if (hash.data[j] != 0) { - is_zero = false; - break; - } - } - if (is_zero) - continue; - - // Check if this hash is still used anywhere in the tree - if (!entity_uses_hash(&ft->root, hash)) { - removed_hashes[*num_removed] = hash; - (*num_removed)++; - } - } - } - - e->gen = create_generation(&ft->next_gen); - assert(e->gen != NO_GENERATION); - - *new_gen = e->gen; - return 0; -} - -int file_tree_read(FileTree *ft, string path, - uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size, - SHA256 *hashes, int max_hashes, uint64_t *actual_bytes) -{ - // Initialize gen to NO_GENERATION so error paths have a well-defined value - *gen = NO_GENERATION; - - int num_comps; - string comps[MAX_COMPS]; - - num_comps = parse_path(path, comps, MAX_COMPS); - if (num_comps < 0) - return FILETREE_BADPATH; - - Entity *e = resolve_path(&ft->root, comps, num_comps); - - if (e == NULL) - return FILETREE_NOENT; - - if (e->is_dir) - return FILETREE_ISDIR; - - File *f = &e->f; - - *chunk_size = f->chunk_size; - - // Calculate actual bytes that can be read based on actual file size - if (off >= f->file_size) { - *actual_bytes = 0; - } else if (off + len > f->file_size) { - *actual_bytes = f->file_size - off; - } else { - *actual_bytes = len; - } - - if (len == 0) { - assert(e->gen != NO_GENERATION); - *gen = e->gen; - return 0; - } - - uint64_t first_chunk_index = off / f->chunk_size; - uint64_t last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size; - - if (first_chunk_index >= f->num_chunks) { - *gen = e->gen; - return 0; - } - - if (last_chunk_index >= f->num_chunks) { - if (f->num_chunks == 0) { - assert(e->gen != NO_GENERATION); - *gen = e->gen; - return 0; - } - last_chunk_index = f->num_chunks-1; - } - - int num_hashes = 0; - for (uint32_t i = first_chunk_index; i <= last_chunk_index; i++) { - - SHA256 hash = f->chunks[i]; - - if (num_hashes < max_hashes) - hashes[num_hashes] = hash; - num_hashes++; - } - - assert(e->gen != NO_GENERATION); - *gen = e->gen; - - return num_hashes; -} - -string file_tree_strerror(int code) -{ - switch (code) { - case FILETREE_NOMEM : return S("Out of memory"); - case FILETREE_NOENT : return S("No such file or directory"); - case FILETREE_NOTDIR : return S("Entity is not a directory"); - case FILETREE_ISDIR : return S("Entity is a directory"); - case FILETREE_EXISTS : return S("File or directory already exists"); - case FILETREE_BADPATH: return S("Invalid path"); - case FILETREE_BADOP : return S("Invalid operation"); - case FILETREE_BADGEN : return S("Generation counter mismatch or invalid value"); - default:break; - } - return S("Unknown error"); -} - -typedef struct { - int (*write_fn)(char*,int,void*); - void *write_data; - char *buffer; - int buffer_size; - int buffer_used; - bool error; -} SerializeContext; - -static void sc_flush(SerializeContext *sc) -{ - if (sc->error) - return; - - int ret = sc->write_fn(sc->buffer, sc->buffer_used, sc->write_data); - if (ret < 0) { - sc->error = true; - return; - } - - sc->buffer_used = 0; -} - -static void sc_write_mem(SerializeContext *sc, char *src, int len) -{ - if (sc->error) - return; - - if (sc->buffer_size - sc->buffer_used < len) { - - if (len > sc->buffer_size) { - sc->error = true; - return; - } - - sc_flush(sc); - if (sc->error) - return; - } - - memcpy(sc->buffer + sc->buffer_used, src, len); - sc->buffer_used += len; -} -static void sc_write_u8 (SerializeContext *sc, uint8_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } -static void sc_write_u16 (SerializeContext *sc, uint16_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } -static void sc_write_u64 (SerializeContext *sc, uint64_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } -static void sc_write_hash(SerializeContext *sc, SHA256 value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } - -static void file_serialize(SerializeContext *sc, File *f) -{ - sc_write_u64(sc, f->chunk_size); - sc_write_u64(sc, f->num_chunks); - sc_write_u64(sc, f->file_size); - for (uint64_t i = 0; i < f->num_chunks; i++) - sc_write_hash(sc, f->chunks[i]); -} - -static void entity_serialize(SerializeContext *sc, Entity *e); - -static void dir_serialize(SerializeContext *sc, Dir *d) -{ - sc_write_u64(sc, d->num_children); - for (uint64_t i = 0; i < d->num_children; i++) - entity_serialize(sc, &d->children[i]); -} - -static void entity_serialize(SerializeContext *sc, Entity *e) -{ - sc_write_u16(sc, e->name_len); - sc_write_mem(sc, e->name, e->name_len); - sc_write_u8(sc, e->is_dir); - if (e->is_dir) - dir_serialize(sc, &e->d); - else - file_serialize(sc, &e->f); -} - -int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *write_data) -{ - SerializeContext sc; - sc.write_fn = write_fn; - sc.write_data = write_data; - sc.buffer_used = 0; - sc.buffer_size = 1<<10; - sc.buffer = malloc(sc.buffer_size); - sc.error = false; - if (sc.buffer == NULL) - sc.error = true; - entity_serialize(&sc, &ft->root); - sc_flush(&sc); - free(sc.buffer); - if (sc.error) - return -1; - return 0; -} - -typedef struct { - int (*read_fn)(char*,int,void*); - void *read_data; - char *buffer; - int buffer_size; - int buffer_used; - int buffer_head; - bool error; - uint64_t total_read; -} DeserializeContext; - -static void dc_read_mem(DeserializeContext *dc, void *dst, int len) -{ - if (dc->error) - return; - - if (dc->buffer_used < len) { - - if (dc->buffer_size < len) { - dc->error = true; - return; - } - - memmove(dc->buffer, dc->buffer + dc->buffer_head, dc->buffer_used); - dc->buffer_head = 0; - - int ret = dc->read_fn( - dc->buffer + dc->buffer_used, - dc->buffer_size - dc->buffer_used, - dc->read_data); - if (ret < 0) { - dc->error = true; - return; - } - dc->buffer_used += ret; - - if (dc->buffer_used < len) { - dc->error = true; - return; - } - } - - memcpy(dst, dc->buffer + dc->buffer_head, len); - dc->buffer_head += len; - dc->buffer_used -= len; - dc->total_read += len; -} -static void dc_read_u8 (DeserializeContext *dc, uint8_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } -static void dc_read_u16(DeserializeContext *dc, uint16_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } -static void dc_read_u64(DeserializeContext *dc, uint64_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } -static void dc_read_hash(DeserializeContext *dc, SHA256 *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } - -static void file_deserialize(DeserializeContext *dc, File *f) -{ - dc_read_u64(dc, &f->chunk_size); - dc_read_u64(dc, &f->num_chunks); - dc_read_u64(dc, &f->file_size); - - f->chunks = malloc(f->num_chunks * sizeof(SHA256)); - if (f->chunks == NULL) { - assert(0); // TODO - } - - for (uint64_t i = 0; i < f->num_chunks; i++) - dc_read_hash(dc, &f->chunks[i]); -} - -static void entity_deserialize(DeserializeContext *dc, Entity *e); - -static void dir_deserialize(DeserializeContext *dc, Dir *d) -{ - dc_read_u64(dc, &d->num_children); - - d->max_children = d->num_children; - d->children = malloc(d->num_children * sizeof(Entity)); - if (d->children == NULL) { - assert(0); // TODO - } - - // TODO: not checking for errors is not okay as - // the code will branch based on garbage - // values. - for (uint64_t i = 0; i < d->num_children; i++) - entity_deserialize(dc, &d->children[i]); -} - -static void entity_deserialize(DeserializeContext *dc, Entity *e) -{ - dc_read_u16(dc, &e->name_len); // TODO: make sure this doesn't go over the static buffer - dc_read_mem(dc, e->name, e->name_len); - - uint8_t is_dir; - dc_read_u8 (dc, &is_dir); - e->is_dir = (is_dir != 0); - - if (e->is_dir) - dir_deserialize(dc, &e->d); - else - file_deserialize(dc, &e->f); -} - -int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data) -{ - DeserializeContext dc; - dc.read_fn = read_fn; - dc.read_data = read_data; - dc.buffer_head = 0; - dc.buffer_used = 0; - dc.buffer_size = 1<<10; - dc.buffer = malloc(dc.buffer_size); - dc.error = false; - if (dc.buffer == NULL) - dc.error = true; - dc.total_read = 0; - entity_deserialize(&dc, &ft->root); - free(dc.buffer); - if (dc.error) - return -1; - if (dc.total_read > INT_MAX) { - assert(0); // TODO - } - return dc.total_read; -} diff --git a/src/file_tree.h b/src/file_tree.h deleted file mode 100644 index 30a2305..0000000 --- a/src/file_tree.h +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef FILE_TREE_INCLUDED -#define FILE_TREE_INCLUDED - -#include "basic.h" - -// Special generation counter values: -// NO_GENERATION (0): Skip generation check (accept any generation) -// MISSING_FILE_GENERATION (UINT64_MAX): Expect file/directory to NOT exist -#define NO_GENERATION ((uint64_t) 0) -#define MISSING_FILE_GENERATION ((uint64_t) UINT64_MAX) - -enum { - FILETREE_NOMEM = -1, - FILETREE_NOENT = -2, - FILETREE_NOTDIR = -3, - FILETREE_ISDIR = -4, - FILETREE_EXISTS = -5, - FILETREE_BADPATH = -6, - FILETREE_BADOP = -7, - FILETREE_BADGEN = -8, -}; - -typedef struct Entity Entity; - -typedef struct { - uint64_t chunk_size; // TODO: this should be an u32 - uint64_t num_chunks; // TODO: and this too - uint64_t file_size; // Offset of last byte written + 1 - SHA256 *chunks; -} File; - -typedef struct { - uint64_t max_children; - uint64_t num_children; - Entity *children; -} Dir; - -struct Entity { - uint64_t gen; - char name[1<<8]; - uint16_t name_len; - bool is_dir; - union { - Dir d; - File f; - }; -}; - -typedef struct { - Entity root; - uint64_t next_gen; -} FileTree; - -typedef struct { - char name[1<<8]; - int name_len; - bool is_dir; - uint64_t gen; -} ListItem; - -#define MAX_COMPS 32 - -int file_tree_init(FileTree *ft); - -void file_tree_free(FileTree *ft); - -bool file_tree_uses_hash(FileTree *ft, SHA256 hash); - -int file_tree_list(FileTree *ft, string path, - ListItem *items, int max_items, uint64_t *gen); - -int file_tree_create_entity(FileTree *ft, string path, - bool is_dir, uint64_t chunk_size, uint64_t *gen); - -int file_tree_delete_entity(FileTree *ft, string path, - uint64_t expected_gen); - -int file_tree_write(FileTree *ft, string path, - uint64_t off, uint64_t len, uint32_t num_chunks, - uint64_t expect_gen, - uint64_t *new_gen, - SHA256 *hashes, - SHA256 *removed_hashes, - int *num_removed, - bool truncate_after); - -int file_tree_read(FileTree *ft, string path, - uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size, - SHA256 *hashes, int max_hashes, uint64_t *actual_bytes); - -string file_tree_strerror(int code); - -int file_tree_serialize(FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data); - -int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data); - -#endif // FILE_TREE_INCLUDED diff --git a/src/hash_set.c b/src/hash_set.c deleted file mode 100644 index c10f085..0000000 --- a/src/hash_set.c +++ /dev/null @@ -1,171 +0,0 @@ -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include - -#include "hash_set.h" - -void hash_set_init(HashSet *set) -{ - set->items = NULL; - set->count = 0; - set->capacity = 0; -} - -void hash_set_free(HashSet *set) -{ - free(set->items); - set->items = NULL; -} - -void hash_set_clear(HashSet *set) -{ - free(set->items); - set->items = NULL; - set->count = 0; - set->capacity = 0; -} - -int hash_set_insert(HashSet *set, SHA256 hash) -{ - // Avoid duplicates - for (int i = 0; i < set->count; i++) - if (!memcmp(&set->items[i], &hash, sizeof(SHA256))) - return 0; // Already present - - if (set->count == set->capacity) { - - int new_capacity; - if (set->items == NULL) - new_capacity = 16; - else - new_capacity = 2 * set->capacity; - - SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256)); - if (new_items == NULL) - return -1; - - set->items = new_items; - set->capacity = new_capacity; - } - - set->items[set->count++] = hash; - return 0; -} - -bool hash_set_remove(HashSet *set, SHA256 hash) -{ - for (int i = 0; i < set->count; i++) - if (!memcmp(&hash, &set->items[i], sizeof(SHA256))) { - set->items[i] = set->items[--set->count]; - return true; - } - return false; -} - -bool hash_set_contains(HashSet *set, SHA256 hash) -{ - for (int i = 0; i < set->count; i++) - if (!memcmp(&hash, &set->items[i], sizeof(SHA256))) - return true; - return false; -} - -int hash_set_merge(HashSet *dst, HashSet src) -{ - HashSet ret; - hash_set_init(&ret); - - for (int i = 0; i < dst->count; i++) { - if (hash_set_insert(&ret, dst->items[i]) < 0) - goto error; - } - - for (int i = 0; i < src.count; i++) { - if (hash_set_insert(&ret, src.items[i]) < 0) - goto error; - } - - hash_set_free(dst); - *dst = ret; - return 0; - -error: - hash_set_free(&ret); - return -1; -} - -void hash_set_remove_set(HashSet *dst, HashSet src) -{ - for (int i = 0; i < src.count; i++) - hash_set_remove(dst, src.items[i]); -} - -void timed_hash_set_init(TimedHashSet *set) -{ - set->items = NULL; - set->count = 0; - set->capacity = 0; -} - -void timed_hash_set_free(TimedHashSet *set) -{ - free(set->items); - set->items = NULL; -} - -int timed_hash_set_find(TimedHashSet *set, SHA256 hash) -{ - for (int i = 0; i < set->count; i++) - if (!memcmp(&set->items[i].hash, &hash, sizeof(SHA256))) - return i; - return -1; -} - -int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time) -{ - // Check if already in set - int idx = timed_hash_set_find(set, hash); - if (idx >= 0) { - // Already marked, keep the original time - return 0; - } - - if (set->count == set->capacity) { - int new_capacity; - if (set->capacity == 0) - new_capacity = 8; - else - new_capacity = 2 * set->capacity; - - TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash)); - if (new_items == NULL) - return -1; - - if (set->capacity > 0) { - memcpy(new_items, set->items, set->count * sizeof(set->items[0])); - free(set->items); - } - - set->items = new_items; - set->capacity = new_capacity; - } - - set->items[set->count++] = (TimedHash) { hash, time }; - return 0; -} - -void timed_hash_set_remove(TimedHashSet *set, SHA256 hash) -{ - int idx = timed_hash_set_find(set, hash); - if (idx >= 0) { - // Remove by shifting remaining items - if (idx < set->count - 1) { - memmove(&set->items[idx], &set->items[idx + 1], - (set->count - idx - 1) * sizeof(set->items[0])); - } - set->count--; - } -} diff --git a/src/hash_set.h b/src/hash_set.h deleted file mode 100644 index 31834d5..0000000 --- a/src/hash_set.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef HASH_SET_INCLUDED -#define HASH_SET_INCLUDED - -#include "basic.h" - -typedef struct { - SHA256 *items; - int count; - int capacity; -} HashSet; - -typedef struct { - SHA256 hash; - Time time; -} TimedHash; - -typedef struct { - TimedHash *items; - int count; - int capacity; -} TimedHashSet; - -void hash_set_init (HashSet *set); -void hash_set_free (HashSet *set); -void hash_set_clear (HashSet *set); -int hash_set_insert (HashSet *set, SHA256 hash); -bool hash_set_remove (HashSet *set, SHA256 hash); -int hash_set_merge (HashSet *dst, HashSet src); -void hash_set_remove_set(HashSet *dst, HashSet src); -bool hash_set_contains (HashSet *set, SHA256 hash); - -void timed_hash_set_init (TimedHashSet *set); -void timed_hash_set_free (TimedHashSet *set); -int timed_hash_set_find (TimedHashSet *set, SHA256 hash); -int timed_hash_set_insert (TimedHashSet *set, SHA256 hash, Time time); -void timed_hash_set_remove (TimedHashSet *set, SHA256 hash); - -#endif // HASH_SET_INCLUDED diff --git a/src/invariant_checker.c b/src/invariant_checker.c new file mode 100644 index 0000000..4d004dd --- /dev/null +++ b/src/invariant_checker.c @@ -0,0 +1,445 @@ +// VSR invariant checker with external shadow log tracking. +// +// This file runs in the main simulation loop outside Quakey-scheduled +// processes. It includes node.h for struct definitions, then restores +// real allocators since mock_malloc/realloc/free abort outside process +// context. + +#include "server.h" +#include "chunk_store.h" + +#include + +// Restore real allocators (see checker/linearizability.c for precedent). +#undef malloc +#undef realloc +#undef free +#include + +// These helpers are static in node.c; duplicated here for the checker. + +static int self_idx(ServerState *state) +{ + for (int i = 0; i < state->num_nodes; i++) + if (addr_eql(state->node_addrs[i], state->self_addr)) + return i; + UNREACHABLE; +} + +static int leader_idx(ServerState *state) +{ + return state->view_number % state->num_nodes; +} + +static bool is_leader(ServerState *state) +{ + if (state->status == STATUS_RECOVERY) + return false; + return self_idx(state) == leader_idx(state); +} + +static int shadow_log_append(InvariantChecker *ic, MetaOper oper) +{ + if (ic->shadow_count == ic->shadow_capacity) { + int n = 2 * ic->shadow_capacity; + if (n < 8) + n = 8; + MetaOper *p = realloc(ic->shadow_log, n * sizeof(MetaOper)); + if (p == NULL) + return -1; + ic->shadow_log = p; + ic->shadow_capacity = n; + } + ic->shadow_log[ic->shadow_count++] = oper; + return 0; +} + +void invariant_checker_init(InvariantChecker *ic) +{ + ic->last_min_commit = -1; + ic->last_max_commit = -1; + for (int i = 0; i < NODE_LIMIT; i++) + ic->prev_status[i] = STATUS_NORMAL; + ic->shadow_log = NULL; + ic->shadow_count = 0; + ic->shadow_capacity = 0; +} + +void invariant_checker_free(InvariantChecker *ic) +{ + fprintf(stderr, "INVARIANT CHECKER: shadow log tracked %d committed entries\n", + ic->shadow_count); + free(ic->shadow_log); +} + +void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes, + unsigned long long *node_handles) +{ + int min_commit = -1; + int max_commit = -1; + + bool primary = false; + uint64_t primary_view_number = 0; + + bool min_commit_just_recovered = false; + + uint64_t max_view_number = 0; + for (int i = 0; i < num_nodes; i++) { + if (nodes[i]) + max_view_number = MAX(max_view_number, nodes[i]->view_number); + } + + for (int i = 0; i < num_nodes; i++) { + ServerState *n = nodes[i]; + if (n == NULL || n->status == STATUS_RECOVERY) + continue; + + if (min_commit < 0 || min_commit > n->commit_index) { + min_commit = n->commit_index; + min_commit_just_recovered = (ic->prev_status[i] == STATUS_RECOVERY); + } + + if (max_commit < 0 || max_commit < n->commit_index) { + max_commit = n->commit_index; + } + + if (is_leader(n) && n->view_number > primary_view_number) { + primary = true; + primary_view_number = n->view_number; + } + } + + // If the primary isn't up to date, it's not the + // real primary. + if (primary_view_number < max_view_number) + primary = false; + + if (min_commit < 0) { + assert(ic->last_min_commit == -1); + } else { + // The minimum number of committed entries should + // only increase, but there are some corner-cases + // when this is not true. + // When a node completes the recovery state, its + // log is technically outdated as it was sent some + // point in the past. If operations are committed + // while the recovery response is in transit over + // the network, the minimum number of committed + // entries will decrease. + if (!min_commit_just_recovered) { + assert(ic->last_min_commit <= min_commit); + } + } + + if (max_commit < 0) { + assert(ic->last_max_commit == -1); + } else { + // The maximum number of committed entries + // should only increase. The primary generally + // has more committed entries than replicas. If + // the primary dies, the maximum commit number + // of the live nodes may decrease. This is still + // okay since the new primary will commit up to + // that point as the view change completes. This + // implies that the maximum commit number should + // always increase unless there is no primary. + if (!primary) { + max_commit = ic->last_max_commit; + } else { + //assert(ic->last_max_commit <= max_commit); + } + } + + ic->last_min_commit = min_commit; + ic->last_max_commit = max_commit; + for (int i = 0; i < num_nodes; i++) { + if (nodes[i]) + ic->prev_status[i] = nodes[i]->status; + } + + for (int i = 0; i < num_nodes; i++) { + ServerState *s = nodes[i]; + if (s == NULL) + continue; + + // 1. commit_index <= log.count + // A node cannot have committed more entries than it has in its log. + if (s->commit_index > s->log.count) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) > log.count (%d)\n", + i, s->commit_index, s->log.count); + __builtin_trap(); + } + + // 2. commit_index >= 0 + if (s->commit_index < 0) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: commit_index (%d) < 0\n", + i, s->commit_index); + __builtin_trap(); + } + + // 4. Future buffer count is in valid range. + if (s->num_future < 0 || s->num_future > FUTURE_LIMIT) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: num_future (%d) out of range [0, %d]\n", + i, s->num_future, FUTURE_LIMIT); + __builtin_trap(); + } + + // 7. last_normal_view <= view_number + // The most recent view in which this node was in NORMAL status + // cannot exceed its current view number. + if (s->last_normal_view > s->view_number) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: last_normal_view (%lu) > view_number (%lu)\n", + i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number); + __builtin_trap(); + } + + // 8. When status is NORMAL, last_normal_view must equal view_number. + // Every transition to NORMAL status sets last_normal_view = view_number. + // If they diverge while in NORMAL, a transition forgot to update it. + if (s->status == STATUS_NORMAL && s->last_normal_view != s->view_number) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: status is NORMAL but " + "last_normal_view (%lu) != view_number (%lu)\n", + i, (unsigned long)s->last_normal_view, (unsigned long)s->view_number); + __builtin_trap(); + } + + // 9. Log entry view numbers must not exceed the node's current view. + // Entries are created in the view they were proposed. No entry + // should carry a view number from the future. + for (int k = 0; k < s->log.count; k++) { + if ((uint64_t)s->log.entries[k].view_number > s->view_number) { + fprintf(stderr, "INVARIANT VIOLATED: node %d: log[%d].view_number (%d) " + "> view_number (%lu)\n", + i, k, s->log.entries[k].view_number, + (unsigned long)s->view_number); + __builtin_trap(); + } + } + + // 10. For a leader in NORMAL status, every uncommitted log entry + // must have the leader's own vote bit set. The leader always + // votes for its own entries. + if (s->status == STATUS_NORMAL && is_leader(s)) { + int idx = self_idx(s); + for (int k = s->commit_index; k < s->log.count; k++) { + if (!(s->log.entries[k].votes & (1 << idx))) { + fprintf(stderr, "INVARIANT VIOLATED: node %d (leader): " + "uncommitted log[%d] missing leader's own vote bit\n", + i, k); + __builtin_trap(); + } + } + } + } + + // Cross-node invariants + + // 5. At most one leader in normal status per view. + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL || nodes[i]->status != STATUS_NORMAL || !is_leader(nodes[i])) + continue; + for (int j = i + 1; j < num_nodes; j++) { + if (nodes[j] == NULL || nodes[j]->status != STATUS_NORMAL || !is_leader(nodes[j])) + continue; + if (nodes[i]->view_number == nodes[j]->view_number) { + fprintf(stderr, "INVARIANT VIOLATED: two normal leaders in view %lu: node %d and node %d\n", + (unsigned long)nodes[i]->view_number, i, j); + __builtin_trap(); + } + } + } + + // 6. Committed prefix agreement (State Machine Safety). + // For any two nodes, their logs must agree on all entries up to + // min(commit_index_i, commit_index_j). This is the core safety + // property of VSR: all committed operations are identical across + // replicas. + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) + continue; + for (int j = i + 1; j < num_nodes; j++) { + if (nodes[j] == NULL) + continue; + + int mc = nodes[i]->commit_index; + if (nodes[j]->commit_index < mc) + mc = nodes[j]->commit_index; + + for (int k = 0; k < mc; k++) { + if (memcmp(&nodes[i]->log.entries[k].oper, &nodes[j]->log.entries[k].oper, sizeof(MetaOper)) != 0) { + fprintf(stderr, "INVARIANT VIOLATED: committed log operation mismatch at index %d " + "between node %d and node %d\n", k, i, j); + __builtin_trap(); + } + } + } + } + + //////////////////////////////////////////////////////////////////// + // Shadow log: external commit tracking + //////////////////////////////////////////////////////////////////// + + // Phase 1: Find the observed max commit index and a source node. + int observed_max_commit = 0; + int source_node_idx = -1; + + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) + continue; + if (nodes[i]->status == STATUS_RECOVERY) + continue; + if (nodes[i]->commit_index > observed_max_commit) { + observed_max_commit = nodes[i]->commit_index; + source_node_idx = i; + } + } + + // Phase 2: Append newly committed entries to the shadow log. + if (source_node_idx >= 0 && observed_max_commit > ic->shadow_count) { + + ServerState *source = nodes[source_node_idx]; + assert(source->log.count >= observed_max_commit); + + for (int k = ic->shadow_count; k < observed_max_commit; k++) { + + MetaOper *source_oper = &source->log.entries[k].oper; + + // Cross-validate against other live non-recovering nodes + // that have also committed this entry. + for (int j = 0; j < num_nodes; j++) { + if (j == source_node_idx) + continue; + if (nodes[j] == NULL) + continue; + if (nodes[j]->status == STATUS_RECOVERY) + continue; + if (nodes[j]->commit_index <= k) + continue; + if (nodes[j]->log.count <= k) + continue; + + if (memcmp(&nodes[j]->log.entries[k].oper, source_oper, sizeof(MetaOper)) != 0) { + fprintf(stderr, "INVARIANT VIOLATED: committed entry mismatch at index %d " + "between source node %d and node %d during shadow log append\n", + k, source_node_idx, j); + __builtin_trap(); + } + } + + if (shadow_log_append(ic, *source_oper) < 0) { + fprintf(stderr, "INVARIANT CHECKER: shadow log allocation failed\n"); + __builtin_trap(); + } + } + } + + // Phase 3: Verify shadow log against the cluster. + + // Sub-check A: Committed entries must match the shadow log. + for (int k = 0; k < ic->shadow_count; k++) { + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) + continue; + if (nodes[i]->log.count <= k) + continue; + if (nodes[i]->commit_index <= k) + continue; + + if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) != 0) { + char shadow_buf[128], node_buf[128]; + meta_snprint_oper(shadow_buf, sizeof(shadow_buf), &ic->shadow_log[k]); + meta_snprint_oper(node_buf, sizeof(node_buf), &nodes[i]->log.entries[k].oper); + fprintf(stderr, "INVARIANT VIOLATED: shadow log mismatch at index %d on node %d\n" + " shadow: %s\n" + " node: %s\n", + k, i, shadow_buf, node_buf); + __builtin_trap(); + } + } + } + + // Sub-check B: When commit regresses, previously committed entries + // must still be held by a majority of the cluster. + // Recovering nodes are treated like dead nodes: they haven't + // restored their log yet and may still recover the entry through + // the recovery protocol. + if (observed_max_commit < ic->shadow_count) { + for (int k = observed_max_commit; k < ic->shadow_count; k++) { + int holders = 0; + int num_dead = 0; + + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) { + num_dead++; + continue; + } + if (nodes[i]->status == STATUS_RECOVERY) { + num_dead++; + continue; + } + if (nodes[i]->log.count <= k) + continue; + if (memcmp(&nodes[i]->log.entries[k].oper, &ic->shadow_log[k], sizeof(MetaOper)) == 0) + holders++; + } + + if (holders + num_dead <= num_nodes / 2) { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &ic->shadow_log[k]); + fprintf(stderr, "INVARIANT VIOLATED: previously committed entry at index %d " + "no longer held by majority (holders=%d, dead=%d, total=%d)\n" + " entry: %s\n", + k, holders, num_dead, num_nodes, oper_buf); + __builtin_trap(); + } + } + } + + //////////////////////////////////////////////////////////////////// + // Blob invariants: newly committed PUT entries must have their + // chunks stored on at least one live server's disk. + //////////////////////////////////////////////////////////////////// + // + // Only check newly committed entries (since last run) to avoid + // O(total_entries * ticks) cost. Uses quakey_enter_host to access + // each server's mock filesystem. + if (node_handles != NULL && observed_max_commit > 0) { + + // Check the most recently committed entries for chunk presence + for (int k = MAX(0, ic->shadow_count - 5); k < ic->shadow_count; k++) { + MetaOper *oper = &ic->shadow_log[k]; + if (oper->type != META_OPER_PUT) + continue; + + for (uint32_t c = 0; c < oper->num_chunks; c++) { + SHA256 hash = oper->chunks[c].hash; + + int holders = 0; + int num_dead = 0; + for (int i = 0; i < num_nodes; i++) { + if (nodes[i] == NULL) { + num_dead++; + continue; + } + // Enter host context to access its mock filesystem + quakey_enter_host(node_handles[i]); + bool has_chunk = chunk_store_exists(&nodes[i]->chunk_store, hash); + quakey_leave_host(); + + if (has_chunk) + holders++; + } + + // Chunk must exist on at least one server, OR all + // non-holders are dead (they may have had it before crash). + if (holders == 0 && num_dead < num_nodes) { + fprintf(stderr, "INVARIANT VIOLATED: committed blob %s/%s " + "chunk %u not found on any live server " + "(holders=%d, dead=%d)\n", + oper->bucket, oper->key, c, holders, num_dead); + __builtin_trap(); + } + } + } + } +} diff --git a/src/log.c b/src/log.c new file mode 100644 index 0000000..36f9aa2 --- /dev/null +++ b/src/log.c @@ -0,0 +1,55 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include + +#include "log.h" + +void log_init(Log *log) +{ + log->count = 0; + log->capacity = 0; + log->entries = NULL; +} + +int log_init_from_network(Log *log, void *src, int num) +{ + log->count = num; + log->capacity = num; + log->entries = malloc(num * sizeof(LogEntry)); + if (log->entries == NULL) + return -1; + memcpy(log->entries, src, num * sizeof(LogEntry)); + return 0; +} + +void log_free(Log *log) +{ + free(log->entries); +} + +void log_move(Log *dst, Log *src) +{ + log_free(dst); + *dst = *src; + log_init(src); +} + +int log_append(Log *log, LogEntry entry) +{ + if (log->count == log->capacity) { + int n= 2 * log->capacity; + if (n < 8) + n = 8; + LogEntry *p = realloc(log->entries, n * sizeof(LogEntry)); + if (p == NULL) + return -1; + + log->entries = p; + log->capacity = n; + } + + log->entries[log->count++] = entry; + return 0; +} \ No newline at end of file diff --git a/src/log.h b/src/log.h new file mode 100644 index 0000000..04dc1b5 --- /dev/null +++ b/src/log.h @@ -0,0 +1,30 @@ +#ifndef LOG_INCLUDED +#define LOG_INCLUDED + +#include "metadata.h" + +#include "config.h" + +typedef struct { + MetaOper oper; + uint32_t votes; + int view_number; + uint64_t client_id; + uint64_t request_id; +} LogEntry; + +_Static_assert(NODE_LIMIT <= 32, ""); + +typedef struct { + int count; + int capacity; + LogEntry *entries; +} Log; + +void log_init(Log *log); +int log_init_from_network(Log *log, void *src, int num); +void log_free(Log *log); +void log_move(Log *dst, Log *src); +int log_append(Log *log, LogEntry entry); + +#endif // LOG_INCLUDED \ No newline at end of file diff --git a/src/main.c b/src/main.c index 037bb75..8a6ee36 100644 --- a/src/main.c +++ b/src/main.c @@ -1,116 +1,14 @@ #ifdef MAIN_SIMULATION #define QUAKEY_ENABLE_MOCKS -#else -#define POLL_CAPACITY 1024 -#endif +#include #include #include +#include -#include "chunk_server.h" -#include "random_client.h" -#include "metadata_server.h" - -#ifdef MAIN_METADATA_SERVER -int main(int argc, char **argv) -{ - int ret; - MetadataServer state; - - void* poll_ctxs[POLL_CAPACITY]; - struct pollfd poll_array[POLL_CAPACITY]; - int poll_count; - int poll_timeout; - - ret = metadata_server_init( - &state, - argc, - argv, - poll_ctxs, - poll_array, - POLL_CAPACITY, - &poll_count, - &poll_timeout - ); - if (ret < 0) - return -1; - - for (;;) { - -#ifdef _WIN32 - WSAPoll(poll_array, poll_count, poll_timeout); -#else - poll(poll_array, poll_count, poll_timeout); -#endif - - ret = metadata_server_tick( - &state, - poll_ctxs, - poll_array, - POLL_CAPACITY, - &poll_count, - &poll_timeout - ); - if (ret < 0) - return -1; - } - - metadata_server_free(&state); - return 0; -} -#endif - -#ifdef MAIN_CHUNK_SERVER -int main(int argc, char **argv) -{ - int ret; - ChunkServer state; - - void* poll_ctxs[POLL_CAPACITY]; - struct pollfd poll_array[POLL_CAPACITY]; - int poll_count; - int poll_timeout; - - ret = chunk_server_init( - &state, - argc, - argv, - poll_ctxs, - poll_array, - POLL_CAPACITY, - &poll_count, - &poll_timeout - ); - if (ret < 0) - return -1; - - for (;;) { - -#ifdef _WIN32 - WSAPoll(poll_array, poll_count, poll_timeout); -#else - poll(poll_array, poll_count, poll_timeout); -#endif - - ret = chunk_server_tick( - &state, - poll_ctxs, - poll_array, - POLL_CAPACITY, - &poll_count, - &poll_timeout - ); - if (ret < 0) - return -1; - } - - chunk_server_free(&state); - return 0; -} -#endif - -#ifdef MAIN_SIMULATION -#include +#include "server.h" +#include "client.h" +#include "blob_client.h" static volatile int simulation_running = 1; @@ -120,233 +18,286 @@ static void sigint_handler(int sig) simulation_running = 0; } -int main(void) +int main(int argc, char **argv) { signal(SIGINT, sigint_handler); + QuakeyUInt64 seed = 1; + QuakeyUInt64 time_limit_ns = 0; // 0 means no limit + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--seed") == 0 && i + 1 < argc) { + seed = strtoull(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--time") == 0 && i + 1 < argc) { + time_limit_ns = strtoull(argv[++i], NULL, 10) * 1000000000ULL; + } + } + Quakey *quakey; - int ret = quakey_init(&quakey, 1); + int ret = quakey_init(&quakey, seed); if (ret < 0) return -1; + QuakeyNode node_1; + QuakeyNode node_2; + QuakeyNode node_3; + // Client 1 { QuakeySpawn config = { - .name = "cli1", - .state_size = sizeof(RandomClient), - .init_func = random_client_init, - .tick_func = random_client_tick, - .free_func = random_client_free, + .name = "rndcli1", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, .addrs = (char*[]) { "127.0.0.2" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cli --server 127.0.0.4 8080"); + (void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); } // Client 2 { QuakeySpawn config = { - .name = "cli2", - .state_size = sizeof(RandomClient), - .init_func = random_client_init, - .tick_func = random_client_tick, - .free_func = random_client_free, + .name = "rndcli2", + .state_size = sizeof(ClientState), + .init_func = client_init, + .tick_func = client_tick, + .free_func = client_free, .addrs = (char*[]) { "127.0.0.3" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cli --server 127.0.0.4 8080"); + (void) quakey_spawn(quakey, config, "cli --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); } - // Metadata Server + // Blob Client { QuakeySpawn config = { - .name = "ms", - .state_size = sizeof(MetadataServer), - .init_func = metadata_server_init, - .tick_func = metadata_server_tick, - .free_func = metadata_server_free, - .addrs = (char*[]) { "127.0.0.4" }, - .num_addrs = 1, - .disk_size = 10<<20, - .platform = QUAKEY_LINUX, - }; - quakey_spawn(quakey, config, "ms --addr 127.0.0.4 --port 8080"); - } - - // Chunk Server 1 - { - QuakeySpawn config = { - .name = "cs1", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, - .addrs = (char*[]) { "127.0.0.5" }, - .num_addrs = 1, - .disk_size = 10<<20, - .platform = QUAKEY_LINUX, - }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.5 --port 8081 --remote-addr 127.0.0.4 --remote-port 8080"); - } - - // Chunk Server 2 - { - QuakeySpawn config = { - .name = "cs2", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, - .addrs = (char*[]) { "127.0.0.6" }, - .num_addrs = 1, - .disk_size = 10<<20, - .platform = QUAKEY_LINUX, - }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.6 --port 8082 --remote-addr 127.0.0.4 --remote-port 8080"); - } - - // Chunk Server 3 - { - QuakeySpawn config = { - .name = "cs3", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, + .name = "blobcli", + .state_size = sizeof(BlobClientState), + .init_func = blob_client_init, + .tick_func = blob_client_tick, + .free_func = blob_client_free, .addrs = (char*[]) { "127.0.0.7" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.7 --port 8083 --remote-addr 127.0.0.4 --remote-port 8080"); + (void) quakey_spawn(quakey, config, "blob --server 127.0.0.4:8080 --server 127.0.0.5:8080 --server 127.0.0.6:8080"); } - while (simulation_running) - quakey_schedule_one(quakey); - - quakey_free(quakey); - return 0; -} -#endif // MAIN_SIMULATION - -#ifdef MAIN_TEST -#include -#include "test_client.h" - -static volatile int simulation_running = 1; - -static void sigint_handler(int sig) -{ - (void)sig; - simulation_running = 0; -} - -int main(void) -{ - signal(SIGINT, sigint_handler); - - Quakey *quakey; - int ret = quakey_init(&quakey, 1); - if (ret < 0) - return -1; - - // Client 1 + // Node 1 { QuakeySpawn config = { - .name = "test_cli_1", - .state_size = sizeof(TestClient), - .init_func = test_client_init, - .tick_func = test_client_tick, - .free_func = test_client_free, - .addrs = (char*[]) { "127.0.0.2" }, - .num_addrs = 1, - .disk_size = 10<<20, - .platform = QUAKEY_LINUX, - }; - quakey_spawn(quakey, config, "cli --server 127.0.0.3 8080"); - } - - // Metadata Server - { - QuakeySpawn config = { - .name = "ms", - .state_size = sizeof(MetadataServer), - .init_func = metadata_server_init, - .tick_func = metadata_server_tick, - .free_func = metadata_server_free, - .addrs = (char*[]) { "127.0.0.3" }, - .num_addrs = 1, - .disk_size = 10<<20, - .platform = QUAKEY_LINUX, - }; - quakey_spawn(quakey, config, "ms --addr 127.0.0.3 --port 8080"); - } - - // Chunk Server 1 - { - QuakeySpawn config = { - .name = "cs1", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, + .name = "server1", + .state_size = sizeof(ServerState), + .init_func = server_init, + .tick_func = server_tick, + .free_func = server_free, .addrs = (char*[]) { "127.0.0.4" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.4 --port 8081 --remote-addr 127.0.0.3 --remote-port 8080"); + node_1 = quakey_spawn(quakey, config, "nd --addr 127.0.0.4:8080 --peer 127.0.0.5:8080 --peer 127.0.0.6:8080"); } - // Chunk Server 2 + // Node 2 { QuakeySpawn config = { - .name = "cs2", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, + .name = "server2", + .state_size = sizeof(ServerState), + .init_func = server_init, + .tick_func = server_tick, + .free_func = server_free, .addrs = (char*[]) { "127.0.0.5" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.5 --port 8082 --remote-addr 127.0.0.3 --remote-port 8080"); + node_2 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --addr 127.0.0.5:8080 --peer 127.0.0.6:8080"); } - // Chunk Server 3 + // Node 3 { QuakeySpawn config = { - .name = "cs3", - .state_size = sizeof(ChunkServer), - .init_func = chunk_server_init, - .tick_func = chunk_server_tick, - .free_func = chunk_server_free, + .name = "server3", + .state_size = sizeof(ServerState), + .init_func = server_init, + .tick_func = server_tick, + .free_func = server_free, .addrs = (char*[]) { "127.0.0.6" }, .num_addrs = 1, .disk_size = 10<<20, .platform = QUAKEY_LINUX, }; - quakey_spawn(quakey, config, "cs --addr 127.0.0.6 --port 8083 --remote-addr 127.0.0.3 --remote-port 8080"); + node_3 = quakey_spawn(quakey, config, "nd --peer 127.0.0.4:8080 --peer 127.0.0.5:8080 --addr 127.0.0.6:8080"); } - while (simulation_running) { + // Limit crashes to 1 node at a time (within fault tolerance of f=1). + // This is dynamically adjusted below: crashes are disabled while + // any node is still recovering. + quakey_set_max_crashes(quakey, 1); + + quakey_network_partitioning(quakey, true); + + InvariantChecker invariant_checker; + invariant_checker_init(&invariant_checker); + + while (simulation_running && (time_limit_ns == 0 || quakey_current_time(quakey) < time_limit_ns)) { quakey_schedule_one(quakey); - QuakeySignal signal; - while (quakey_get_signal(quakey, &signal)) { + ServerState *arr[] = { + quakey_node_state(node_1), + quakey_node_state(node_2), + quakey_node_state(node_3), + }; + unsigned long long handles[] = { node_1, node_2, node_3 }; + invariant_checker_run(&invariant_checker, arr, sizeof(arr)/sizeof(arr[0]), handles); - if (!strcmp(signal.name, "exit")) - simulation_running = false; + // VR-Revisited Section 8.2: "a replica is considered failed + // until it has recovered its state." Disable crashes while + // any node is recovering to avoid exceeding f simultaneous + // failures (dead + recovering). + bool any_recovering = false; + for (int i = 0; i < 3; i++) { + if (arr[i] && arr[i]->status == STATUS_RECOVERY) + any_recovering = true; } + quakey_set_max_crashes(quakey, any_recovering ? 0 : 1); } + invariant_checker_free(&invariant_checker); quakey_free(quakey); return 0; } -#endif // MAIN_TEST + +#endif // MAIN_SIMULATION +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_CLIENT + +#include +#include +#include +#include + +#include "client.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + srand((unsigned)time(NULL) ^ (unsigned)getpid()); + + int ret; + ClientState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = client_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = client_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + client_free(&state); + return 0; +} + +#endif // MAIN_CLIENT +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +#ifdef MAIN_SERVER + +#include + +#include "server.h" + +#define POLL_CAPACITY 1024 + +int main(int argc, char **argv) +{ + int ret; + ServerState state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = server_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = server_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + server_free(&state); + return 0; +} + +#endif // MAIN_SERVER \ No newline at end of file diff --git a/src/message.c b/src/message.c deleted file mode 100644 index b1d2346..0000000 --- a/src/message.c +++ /dev/null @@ -1,485 +0,0 @@ -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include - -#include "message.h" - -bool binary_read(BinaryReader *reader, void *dst, int len) -{ - if (reader->len - reader->cur < len) - return false; - if (dst) - memcpy(dst, reader->src + reader->cur, len); - reader->cur += len; - return true; -} - -bool binary_read_addr_ipv4(BinaryReader *reader, Address *addr) -{ - if (!binary_read(reader, &addr->ipv4, sizeof(IPv4))) return false; - if (!binary_read(reader, &addr->port, sizeof(uint16_t))) return false; - addr->is_ipv4 = true; - return true; -} - -bool binary_read_addr_ipv6(BinaryReader *reader, Address *addr) -{ - if (!binary_read(reader, &addr->ipv6, sizeof(IPv6))) return false; - if (!binary_read(reader, &addr->port, sizeof(uint16_t))) return false; - addr->is_ipv4 = false; - return true; -} - -int binary_read_addr_list(BinaryReader *reader, Address *addrs, int max_addrs) -{ - uint32_t num_ipv4; - uint32_t num_ipv6; - if (!binary_read(reader, &num_ipv4, sizeof(num_ipv4))) - return -1; - if (!binary_read(reader, &num_ipv6, sizeof(num_ipv6))) - return -1; - int num = 0; - for (uint32_t i = 0; i < num_ipv4; i++) { - Address tmp; - if (!binary_read_addr_ipv4(reader, &tmp)) - return -1; - if (num < max_addrs) - addrs[num++] = tmp; - } - for (uint32_t i = 0; i < num_ipv6; i++) { - Address tmp; - if (!binary_read_addr_ipv6(reader, &tmp)) - return -1; - if (num < max_addrs) - addrs[num++] = tmp; - } - return num; -} - -void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type) -{ - uint16_t version = MESSAGE_VERSION; - uint32_t dummy = 0; // Dummy value - writer->output = output; - writer->start = byte_queue_offset(output); - byte_queue_write(output, &version, sizeof(version)); - byte_queue_write(output, &type, sizeof(type)); - writer->patch = byte_queue_offset(output); - byte_queue_write(output, &dummy, sizeof(dummy)); -} - -bool message_writer_free(MessageWriter *writer) -{ - uint32_t length = byte_queue_size_from_offset(writer->output, writer->start); - byte_queue_patch(writer->output, writer->patch, &length, sizeof(length)); - if (byte_queue_error(writer->output)) // TODO: is it possible to restore the state of the queue to before the failure? - return false; - return true; -} - -void message_write(MessageWriter *writer, void *mem, int len) -{ - byte_queue_write(writer->output, mem, len); -} - -void message_write_u8(MessageWriter *writer, uint8_t value) -{ - message_write(writer, &value, (int) sizeof(value)); -} - -void message_write_u32(MessageWriter *writer, uint32_t value) -{ - message_write(writer, &value, (int) sizeof(value)); -} - -void message_write_hash(MessageWriter *writer, SHA256 value) -{ - message_write(writer, &value, (int) sizeof(value)); -} - -int message_peek(ByteView msg, uint16_t *type, uint32_t *len) -{ - if (msg.len < (int) sizeof(MessageHeader)) - return 0; - - MessageHeader header; - memcpy(&header, msg.ptr, sizeof(header)); - - // (We ignore endianess for now) - - if (header.version != MESSAGE_VERSION) - return -1; - - if (header.length > msg.len) - return 0; - - if (type) *type = header.type; - if (len) *len = header.length; - - return 1; -} - -static char *message_type_to_str(uint16_t type) -{ - switch (type) { - // Client -> Metadata server - case MESSAGE_TYPE_CREATE: return "CREATE"; - case MESSAGE_TYPE_DELETE: return "DELETE"; - case MESSAGE_TYPE_LIST: return "LIST"; - case MESSAGE_TYPE_READ: return "READ"; - case MESSAGE_TYPE_WRITE: return "WRITE"; - - // Client -> Chunk server - case MESSAGE_TYPE_CREATE_CHUNK: return "CREATE_CHUNK"; - case MESSAGE_TYPE_UPLOAD_CHUNK: return "UPLOAD_CHUNK"; - case MESSAGE_TYPE_DOWNLOAD_CHUNK: return "DOWNLOAD_CHUNK"; - - // Metadata server -> Client - case MESSAGE_TYPE_CREATE_ERROR: return "CREATE_ERROR"; - case MESSAGE_TYPE_CREATE_SUCCESS: return "CREATE_SUCCESS"; - case MESSAGE_TYPE_DELETE_ERROR: return "DELETE_ERROR"; - case MESSAGE_TYPE_DELETE_SUCCESS: return "DELETE_SUCCESS"; - case MESSAGE_TYPE_LIST_ERROR: return "LIST_ERROR"; - case MESSAGE_TYPE_LIST_SUCCESS: return "LIST_SUCCESS"; - case MESSAGE_TYPE_READ_ERROR: return "READ_ERROR"; - case MESSAGE_TYPE_READ_SUCCESS: return "READ_SUCCESS"; - case MESSAGE_TYPE_WRITE_ERROR: return "WRITE_ERROR"; - case MESSAGE_TYPE_WRITE_SUCCESS: return "WRITE_SUCCESS"; - - // Metadata server -> Chunk server - case MESSAGE_TYPE_SYNC_2: return "SYNC_2"; - case MESSAGE_TYPE_SYNC_4: return "SYNC_4"; - case MESSAGE_TYPE_DOWNLOAD_LOCATIONS: return "DOWNLOAD_LOCATIONS"; - - // Chunk server -> Metadata server - case MESSAGE_TYPE_AUTH: return "AUTH"; - case MESSAGE_TYPE_SYNC: return "SYNC"; - case MESSAGE_TYPE_SYNC_3: return "SYNC_3"; - - // Chunk server -> Client - case MESSAGE_TYPE_CREATE_CHUNK_ERROR: return "CREATE_CHUNK_ERROR"; - case MESSAGE_TYPE_CREATE_CHUNK_SUCCESS: return "CREATE_CHUNK_SUCCESS"; - case MESSAGE_TYPE_UPLOAD_CHUNK_ERROR: return "UPLOAD_CHUNK_ERROR"; - case MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS: return "UPLOAD_CHUNK_SUCCESS"; - case MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR: return "DOWNLOAD_CHUNK_ERROR"; - case MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS: return "DOWNLOAD_CHUNK_SUCCESS"; - } - - return "???"; -} - -void message_dump(FILE *stream, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - fprintf(stream, "message:\n"); - - fprintf(stream, " header:\n"); - uint16_t version; - if (!binary_read(&reader, &version, sizeof(version))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " version: %d\n", version); - - uint16_t type; - if (!binary_read(&reader, &type, sizeof(type))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " type: %s\n", message_type_to_str(type)); - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " length: %d\n", length); - - fprintf(stream, " body:\n"); - switch (type) { - // Client -> Metadata server - - case MESSAGE_TYPE_CREATE: - { - uint16_t path_len; - if (!binary_read(&reader, &path_len, sizeof(path_len))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path_len: %d\n", path_len); - - char *path = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, path_len)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path: %.*s\n", (int) path_len, path); - - uint8_t is_dir; - if (!binary_read(&reader, &is_dir, sizeof(is_dir))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " is_dir: %s\n", is_dir ? "true" : "false"); - - if (!is_dir) { - uint32_t chunk_size; - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " chunk_size: %d\n", chunk_size); - } - } - break; - - case MESSAGE_TYPE_DELETE: - { - uint16_t path_len; - if (!binary_read(&reader, &path_len, sizeof(path_len))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path_len: %d\n", path_len); - - char *path = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, path_len)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path: %.*s\n", (int) path_len, path); - } - break; - - case MESSAGE_TYPE_LIST: - { - uint16_t path_len; - if (!binary_read(&reader, &path_len, sizeof(path_len))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path_len: %d\n", path_len); - - char *path = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, path_len)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path: %.*s\n", (int) path_len, path); - } - break; - - case MESSAGE_TYPE_READ: - { - uint16_t path_len; - if (!binary_read(&reader, &path_len, sizeof(path_len))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path_len: %d\n", path_len); - - char *path = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, path_len)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path: %.*s\n", (int) path_len, path); - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " offset: %d\n", offset); - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " length: %d\n", length); - } - break; - - case MESSAGE_TYPE_WRITE: - { - uint16_t path_len; - if (!binary_read(&reader, &path_len, sizeof(path_len))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path_len: %d\n", path_len); - - char *path = (char*) reader.src + reader.cur; - if (!binary_read(&reader, NULL, path_len)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " path: %.*s\n", (int) path_len, path); - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " offset: %d\n", offset); - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " length: %d\n", length); - - uint32_t num_chunks; - if (!binary_read(&reader, &num_chunks, sizeof(num_chunks))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " num_chunks: %d\n", num_chunks); - - for (uint32_t i = 0; i < num_chunks; i++) { - - char hash_str[64]; - - SHA256 old_hash; - if (!binary_read(&reader, &old_hash, sizeof(old_hash))) { - fprintf(stream, " (incomplete)\n"); - return; - } - append_hex_as_str(hash_str, old_hash); - fprintf(stream, " old_hash: %.64s\n", hash_str); - - SHA256 new_hash; - if (!binary_read(&reader, &new_hash, sizeof(new_hash))) { - fprintf(stream, " (incomplete)\n"); - return; - } - append_hex_as_str(hash_str, new_hash); - fprintf(stream, " new_hash: %.64s\n", hash_str); - - uint32_t num_locations; - if (!binary_read(&reader, &num_locations, sizeof(num_locations))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " num_locations: %d\n", num_locations); - - for (uint32_t j = 0; j < num_locations; j++) { - - uint8_t is_ipv4; - if (!binary_read(&reader, &is_ipv4, sizeof(is_ipv4))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " is_ipv4: %s (%d)\n", is_ipv4 ? "true" : "false", is_ipv4); - - if (is_ipv4) { - IPv4 ipv4; - if (!binary_read(&reader, &ipv4, sizeof(ipv4))) { - fprintf(stream, " (incomplete)\n"); - return; - } - char ip_str[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &ipv4, ip_str, sizeof(ip_str)); - fprintf(stream, " ipv4: %s\n", ip_str); - } else { - IPv6 ipv6; - if (!binary_read(&reader, &ipv6, sizeof(ipv6))) { - fprintf(stream, " (incomplete)\n"); - return; - } - char ip_str[INET_ADDRSTRLEN]; - inet_ntop(AF_INET6, &ipv6, ip_str, sizeof(ip_str)); - fprintf(stream, " ipv6: %s\n", ip_str); - } - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " port: %d\n", port); - } - } - } - break; - - // Client -> Chunk server - - case MESSAGE_TYPE_CREATE_CHUNK: - { - uint32_t chunk_size; - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " chunk_size: %d\n", chunk_size); - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " offset: %d\n", offset); - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " length: %d\n", length); - - if (!binary_read(&reader, NULL, length)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " data: (...)\n"); - } - break; - - case MESSAGE_TYPE_UPLOAD_CHUNK: - { - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - fprintf(stream, " (incomplete)\n"); - return; - } - char hash_str[64]; - append_hex_as_str(hash_str, hash); - fprintf(stream, " hash: %.64s\n", hash_str); - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " offset: %d\n", offset); - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " length: %d\n", length); - - if (!binary_read(&reader, NULL, length)) { - fprintf(stream, " (incomplete)\n"); - return; - } - fprintf(stream, " data: (...)\n"); - } - break; - - default: - printf(" (TODO)\n"); - break; - } - - if (binary_read(&reader, NULL, 1)) - fprintf(stream, " (unexpected bytes)\n"); -} diff --git a/src/message.h b/src/message.h deleted file mode 100644 index 5c4851f..0000000 --- a/src/message.h +++ /dev/null @@ -1,96 +0,0 @@ -#ifndef MESSAGE_INCLUDED -#define MESSAGE_INCLUDED - -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include - -#include "basic.h" -#include "byte_queue.h" - -enum { - - // Client -> Metadata server - MESSAGE_TYPE_CREATE, - MESSAGE_TYPE_DELETE, - MESSAGE_TYPE_LIST, - MESSAGE_TYPE_READ, - MESSAGE_TYPE_WRITE, - - // Client -> Chunk server - MESSAGE_TYPE_CREATE_CHUNK, - MESSAGE_TYPE_UPLOAD_CHUNK, - MESSAGE_TYPE_DOWNLOAD_CHUNK, - - // Metadata server -> Client - MESSAGE_TYPE_CREATE_ERROR, - MESSAGE_TYPE_CREATE_SUCCESS, - MESSAGE_TYPE_DELETE_ERROR, - MESSAGE_TYPE_DELETE_SUCCESS, - MESSAGE_TYPE_LIST_ERROR, - MESSAGE_TYPE_LIST_SUCCESS, - MESSAGE_TYPE_READ_ERROR, - MESSAGE_TYPE_READ_SUCCESS, - MESSAGE_TYPE_WRITE_ERROR, - MESSAGE_TYPE_WRITE_SUCCESS, - - // Metadata server -> Chunk server - MESSAGE_TYPE_AUTH_RESPONSE, - MESSAGE_TYPE_SYNC_2, - MESSAGE_TYPE_SYNC_4, - MESSAGE_TYPE_DOWNLOAD_LOCATIONS, - - // Chunk server -> Metadata server - MESSAGE_TYPE_AUTH, - MESSAGE_TYPE_SYNC, - MESSAGE_TYPE_SYNC_3, - - // Chunk server -> Client - MESSAGE_TYPE_CREATE_CHUNK_ERROR, - MESSAGE_TYPE_CREATE_CHUNK_SUCCESS, - MESSAGE_TYPE_UPLOAD_CHUNK_ERROR, - MESSAGE_TYPE_UPLOAD_CHUNK_SUCCESS, - MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, - MESSAGE_TYPE_DOWNLOAD_CHUNK_SUCCESS, -}; - -#define MESSAGE_VERSION 1 - -typedef struct { - uint8_t *src; - int len; - int cur; -} BinaryReader; - -typedef struct { - uint16_t version; - uint16_t type; - uint32_t length; -} MessageHeader; - -typedef struct { - ByteQueue *output; - ByteQueueOffset start; - ByteQueueOffset patch; -} MessageWriter; - -bool binary_read(BinaryReader *reader, void *dst, int len); - -bool binary_read_addr_ipv4(BinaryReader *reader, Address *addr); -bool binary_read_addr_ipv6(BinaryReader *reader, Address *addr); -int binary_read_addr_list(BinaryReader *reader, Address *addrs, int max_addr); - -void message_writer_init(MessageWriter *writer, ByteQueue *output, uint16_t type); -bool message_writer_free(MessageWriter *writer); -void message_write(MessageWriter *writer, void *mem, int len); -void message_write_u8(MessageWriter *writer, uint8_t value); -void message_write_u32(MessageWriter *writer, uint32_t value); -void message_write_hash(MessageWriter *writer, SHA256 value); - -int message_peek(ByteView msg, uint16_t *type, uint32_t *len); -void message_dump(FILE *stream, ByteView msg); - -#endif // MESSAGE_INCLUDED diff --git a/src/metadata.c b/src/metadata.c new file mode 100644 index 0000000..54193a9 --- /dev/null +++ b/src/metadata.c @@ -0,0 +1,136 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "metadata.h" + +void meta_store_init(MetaStore *ms) +{ + memset(ms, 0, sizeof(*ms)); +} + +void meta_store_free(MetaStore *ms) +{ + (void) ms; +} + +static int find_entry(MetaStore *ms, const char *bucket, const char *key) +{ + for (int i = 0; i < ms->count; i++) { + if (!ms->entries[i].deleted + && strncmp(ms->entries[i].bucket, bucket, META_BUCKET_MAX) == 0 + && strncmp(ms->entries[i].key, key, META_KEY_MAX) == 0) + return i; + } + return -1; +} + +static void copy_oper_to_entry(ObjectMeta *entry, MetaOper *oper) +{ + memcpy(entry->bucket, oper->bucket, META_BUCKET_MAX); + memcpy(entry->key, oper->key, META_KEY_MAX); + entry->size = oper->size; + entry->content_hash = oper->content_hash; + entry->num_chunks = oper->num_chunks; + memcpy(entry->chunks, oper->chunks, oper->num_chunks * sizeof(ChunkRef)); + entry->deleted = false; +} + +MetaResult meta_store_update(MetaStore *ms, MetaOper *oper) +{ + MetaResult result; + + switch (oper->type) { + + case META_OPER_NOOP: + result.type = META_RESULT_OK; + break; + + case META_OPER_PUT: + { + // Try to find existing entry with same bucket/key + int i = find_entry(ms, oper->bucket, oper->key); + if (i >= 0) { + // Overwrite in-place + copy_oper_to_entry(&ms->entries[i], oper); + result.type = META_RESULT_OK; + break; + } + + // Try to reuse a tombstoned slot + for (i = 0; i < ms->count; i++) { + if (ms->entries[i].deleted) { + copy_oper_to_entry(&ms->entries[i], oper); + result.type = META_RESULT_OK; + goto done; + } + } + + // Allocate new slot + if (ms->count >= META_ENTRY_LIMIT) { + result.type = META_RESULT_FULL; + break; + } + copy_oper_to_entry(&ms->entries[ms->count++], oper); + result.type = META_RESULT_OK; + } + break; + + case META_OPER_DELETE: + { + int i = find_entry(ms, oper->bucket, oper->key); + if (i < 0) { + result.type = META_RESULT_NOT_FOUND; + } else { + ms->entries[i].deleted = true; + result.type = META_RESULT_OK; + } + } + break; + + default: + assert(0); + break; + } + +done: + return result; +} + +ObjectMeta *meta_store_lookup(MetaStore *ms, const char *bucket, const char *key) +{ + int i = find_entry(ms, bucket, key); + if (i < 0) + return NULL; + return &ms->entries[i]; +} + +int meta_snprint_oper(char *buf, int size, MetaOper *oper) +{ + switch (oper->type) { + case META_OPER_NOOP: + return snprintf(buf, size, "NOOP"); + case META_OPER_PUT: + return snprintf(buf, size, "PUT(%s/%s, %u chunks)", + oper->bucket, oper->key, oper->num_chunks); + case META_OPER_DELETE: + return snprintf(buf, size, "DELETE(%s/%s)", + oper->bucket, oper->key); + default: + return snprintf(buf, size, "???"); + } +} + +int meta_snprint_result(char *buf, int size, MetaResult result) +{ + switch (result.type) { + case META_RESULT_OK: return snprintf(buf, size, "OK"); + case META_RESULT_NOT_FOUND: return snprintf(buf, size, "NOT_FOUND"); + case META_RESULT_FULL: return snprintf(buf, size, "FULL"); + default: return snprintf(buf, size, "???"); + } +} diff --git a/src/metadata.h b/src/metadata.h new file mode 100644 index 0000000..4f37b99 --- /dev/null +++ b/src/metadata.h @@ -0,0 +1,69 @@ +#ifndef METADATA_INCLUDED +#define METADATA_INCLUDED + +#include +#include + +#include + +typedef struct { + SHA256 hash; + uint32_t size; +} ChunkRef; + +#define META_BUCKET_MAX 64 +#define META_KEY_MAX 512 +#define META_CHUNKS_MAX 256 + +typedef enum { + META_OPER_NOOP, + META_OPER_PUT, + META_OPER_DELETE, +} MetaOperType; + +typedef struct { + MetaOperType type; + char bucket[META_BUCKET_MAX]; + char key[META_KEY_MAX]; + uint64_t size; + SHA256 content_hash; + uint32_t num_chunks; + ChunkRef chunks[META_CHUNKS_MAX]; +} MetaOper; + +typedef enum { + META_RESULT_OK, + META_RESULT_NOT_FOUND, + META_RESULT_FULL, +} MetaResultType; + +typedef struct { + MetaResultType type; +} MetaResult; + +typedef struct { + char bucket[META_BUCKET_MAX]; + char key[META_KEY_MAX]; + uint64_t size; + SHA256 content_hash; + uint32_t num_chunks; + ChunkRef chunks[META_CHUNKS_MAX]; + bool deleted; +} ObjectMeta; + +#define META_ENTRY_LIMIT 4096 + +typedef struct { + int count; + ObjectMeta entries[META_ENTRY_LIMIT]; +} MetaStore; + +void meta_store_init(MetaStore *ms); +void meta_store_free(MetaStore *ms); +MetaResult meta_store_update(MetaStore *ms, MetaOper *oper); +ObjectMeta *meta_store_lookup(MetaStore *ms, const char *bucket, const char *key); + +int meta_snprint_oper(char *buf, int size, MetaOper *oper); +int meta_snprint_result(char *buf, int size, MetaResult result); + +#endif // METADATA_INCLUDED diff --git a/src/metadata_server.c b/src/metadata_server.c deleted file mode 100644 index 156a112..0000000 --- a/src/metadata_server.c +++ /dev/null @@ -1,1281 +0,0 @@ -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include -#include - -#include "message.h" -#include "metadata_server.h" - -#define MS_TRACE(fmt, ...) {} -//#define MS_TRACE(fmt, ...) fprintf(stderr, "MS: " fmt "\n", ##__VA_ARGS__); - -static void chunk_server_peer_init(ChunkServerPeer *chunk_server, Time current_time) -{ - chunk_server->used = true; - chunk_server->auth = false; - chunk_server->num_addrs = 0; - hash_set_init(&chunk_server->ms_old_list); - hash_set_init(&chunk_server->ms_add_list); - hash_set_init(&chunk_server->ms_rem_list); - chunk_server->last_sync_time = current_time; - chunk_server->last_response_time = current_time; -} - -static void chunk_server_peer_free(ChunkServerPeer *chunk_server) -{ - hash_set_free(&chunk_server->ms_rem_list); - hash_set_free(&chunk_server->ms_add_list); - hash_set_free(&chunk_server->ms_old_list); - chunk_server->used = false; -} - -static bool chunk_server_peer_contains(ChunkServerPeer *chunk_server, SHA256 hash) -{ - return hash_set_contains(&chunk_server->ms_old_list, hash) - || hash_set_contains(&chunk_server->ms_add_list, hash); -} - -static bool chunk_server_peer_load(ChunkServerPeer *chunk_server) -{ - return chunk_server->ms_old_list.count - + chunk_server->ms_add_list.count; -} - -// Returns all chunk servers holding the given chunk -// -// The indices of the chunk servers is stored into "out", but at -// most "max" indices are written. The return value is the number -// of indices that would be written if "max" were large enough to -// hold all indices. -static int -all_chunk_servers_holding_chunk(MetadataServer *state, SHA256 hash, int *out, int max) -{ - int num = 0; - for (int i = 0; i < state->num_chunk_servers; i++) { - if (chunk_server_peer_contains(&state->chunk_servers[i], hash)) { - if (num < max) - out[num] = i; - num++; - } - } - return num; -} - -#ifdef _WIN32 -static int compare_chunk_servers(void *data, const void *p1, const void *p2) -#else -static int compare_chunk_servers(const void *p1, const void *p2, void *data) -#endif -{ - int a = *(int*) p1; - int b = *(int*) p2; - MetadataServer *state = data; - int l1 = chunk_server_peer_load(&state->chunk_servers[a]); - int l2 = chunk_server_peer_load(&state->chunk_servers[b]); - return l1 - l2; -} - -// Returns the indices of chunk servers with lowest load in -// the "out" array. The return value is the number of indices -// written, but no more than "max" are written. -static int -choose_servers_for_write(MetadataServer *state, int *out, int max) -{ - int num = state->num_chunk_servers; - - int indices[MAX_CHUNK_SERVERS]; - assert(num <= MAX_CHUNK_SERVERS); - - for (int i = 0; i < num; i++) - indices[i] = i; - -#ifdef _WIN32 - qsort_s(indices, num, sizeof(*indices), compare_chunk_servers, state); -#else - qsort_r(indices, num, sizeof(*indices), compare_chunk_servers, state); -#endif - - for (int i = 0; i < num; i++) { - if (i < max) - out[i] = indices[i]; // Or maybe the other way around? indices[max - i - 1]? - } - - return num; -} - -static int find_chunk_server_by_addr(MetadataServer *state, Address addr) -{ - for (int i = 0; i < state->num_chunk_servers; i++) - for (int j = 0; j < state->chunk_servers[i].num_addrs; j++) - if (addr_eql(state->chunk_servers[i].addrs[j], addr)) - return i; - return -1; -} - -// Serialize the list of addresses for the specified -// chunk server. -static void -message_write_server_addr(MessageWriter *writer, ChunkServerPeer *server) -{ - uint32_t num_ipv4 = 0; - for (int i = 0; i < server->num_addrs; i++) - if (server->addrs[i].is_ipv4) - num_ipv4++; - - message_write(writer, &num_ipv4, sizeof(num_ipv4)); - for (int i = 0; i < server->num_addrs; i++) - if (server->addrs[i].is_ipv4) { - message_write(writer, &server->addrs[i].ipv4, sizeof(server->addrs[i].ipv4)); - message_write(writer, &server->addrs[i].port, sizeof(server->addrs[i].port)); - } - - uint32_t num_ipv6 = 0; - for (int i = 0; i < server->num_addrs; i++) - if (!server->addrs[i].is_ipv4) - num_ipv6++; - - message_write(writer, &num_ipv6, sizeof(num_ipv6)); - for (int i = 0; i < server->num_addrs; i++) - if (!server->addrs[i].is_ipv4) { - message_write(writer, &server->addrs[i].ipv6, sizeof(server->addrs[i].ipv6)); - message_write(writer, &server->addrs[i].port, sizeof(server->addrs[i].port)); - } -} - -static int -process_client_create(MetadataServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - char path_mem[1<<10]; - uint16_t path_len; - - if (!binary_read(&reader, &path_len, sizeof(path_len))) - return -1; - - if (path_len > sizeof(path_mem)) - return -2; - - if (!binary_read(&reader, &path_mem, path_len)) - return -1; - - string path = { path_mem, path_len }; - - uint8_t is_dir; - if (!binary_read(&reader, &is_dir, sizeof(is_dir))) - return -1; - - uint32_t chunk_size; - if (is_dir) - chunk_size = 0; - else { - if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) - return -1; - } - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - if (wal_append_create(&state->wal, path, is_dir, chunk_size) < 0) { - return -1; - } - - uint64_t gen; - int ret = file_tree_create_entity(&state->file_tree, path, is_dir, chunk_size, &gen); - - if (ret < 0) { - - string desc = file_tree_strerror(ret); - - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_ERROR); - - uint16_t len = desc.len; - message_write(&writer, &len, sizeof(len)); - message_write(&writer, desc.ptr, desc.len); - - if (!message_writer_free(&writer)) - return -1; - - } else { - - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_SUCCESS); - message_write(&writer, &gen, sizeof(gen)); - if (!message_writer_free(&writer)) - return -1; - } - - return 0; -} - -static int -process_client_delete(MetadataServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint64_t expect_gen; - if (!binary_read(&reader, &expect_gen, sizeof(expect_gen))) - return -1; - - char path_mem[1<<10]; - uint16_t path_len; - - if (!binary_read(&reader, &path_len, sizeof(path_len))) - return -1; - - if (path_len > sizeof(path_mem)) - return -2; - - if (!binary_read(&reader, &path_mem, path_len)) - return -1; - - string path = { path_mem, path_len }; - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - if (wal_append_delete(&state->wal, path, expect_gen) < 0) { - return -1; - } - - // TODO: return unused hashes and add them to the ms_rem_list of holder chunk servers - int ret = file_tree_delete_entity(&state->file_tree, path, expect_gen); - - if (ret < 0) { - - string desc = file_tree_strerror(ret); - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_DELETE_ERROR); - - uint16_t len = desc.len; - message_write(&writer, &len, sizeof(len)); - message_write(&writer, desc.ptr, desc.len); - - if (!message_writer_free(&writer)) - return -1; - - } else { - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_DELETE_SUCCESS); - if (!message_writer_free(&writer)) - return -1; - } - - return 0; -} - -static int -process_client_list(MetadataServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint64_t expect_gen; - if (!binary_read(&reader, &expect_gen, sizeof(expect_gen))) - return -1; - - char path_mem[1<<10]; - uint16_t path_len; - - if (!binary_read(&reader, &path_len, sizeof(path_len))) - return -1; - - if (path_len > sizeof(path_mem)) - return -2; - - if (!binary_read(&reader, &path_mem, path_len)) - return -1; - - string path = { path_mem, path_len }; - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - #define MAX_LIST_SIZE 128 - - uint64_t gen; - ListItem items[MAX_LIST_SIZE]; - int ret = file_tree_list(&state->file_tree, path, items, MAX_LIST_SIZE, &gen); - - if (ret < 0) { - - string desc = file_tree_strerror(ret); - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_LIST_ERROR); - - uint16_t len = desc.len; - message_write(&writer, &len, sizeof(len)); - message_write(&writer, desc.ptr, desc.len); - - if (!message_writer_free(&writer)) - return -1; - - } else { - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_LIST_SUCCESS); - - message_write(&writer, &gen, sizeof(gen)); - - uint32_t item_count = ret; - uint8_t truncated = 0; - - if (ret > MAX_LIST_SIZE) { - truncated = 1; - item_count = MAX_LIST_SIZE; - } - - message_write(&writer, &truncated, sizeof(truncated)); - message_write(&writer, &item_count, sizeof(item_count)); - - for (int i = 0; i < ret && i < MAX_LIST_SIZE; i++) { - - message_write(&writer, &items[i].gen, sizeof(items[i].gen)); - - uint8_t is_dir = items[i].is_dir; - message_write(&writer, &is_dir, sizeof(is_dir)); - - if (items[i].name_len > UINT16_MAX) - return -1; - uint16_t name_len = items[i].name_len; - message_write(&writer, &name_len, sizeof(name_len)); - - message_write(&writer, items[i].name, name_len); - } - - if (!message_writer_free(&writer)) - return -1; - } - - return 0; -} - -static int -process_client_read(MetadataServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint64_t expect_gen; - if (!binary_read(&reader, &expect_gen, sizeof(expect_gen))) - return -1; - - char path_mem[1<<10]; - uint16_t path_len; - - if (!binary_read(&reader, &path_len, sizeof(path_len))) - return -1; - - if (path_len > sizeof(path_mem)) - return -2; - - if (!binary_read(&reader, &path_mem, path_len)) - return -1; - - string path = { path_mem, path_len }; - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) - return -1; - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) - return -1; - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - #define MAX_READ_HASHES 128 - - uint64_t gen; - uint64_t chunk_size; - uint64_t actual_bytes; - SHA256 hashes[MAX_READ_HASHES]; - int ret = file_tree_read(&state->file_tree, path, offset, length, &gen, &chunk_size, hashes, MAX_READ_HASHES, &actual_bytes); - - if (ret < 0) { - - string desc = file_tree_strerror(ret); - - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - message_writer_init(&writer, output, MESSAGE_TYPE_READ_ERROR); - - uint16_t len = desc.len; - message_write(&writer, &len, sizeof(len)); - message_write(&writer, desc.ptr, desc.len); - - int locations[MAX_CHUNK_SERVERS]; - int num_locations = choose_servers_for_write(state, locations, state->replication_factor); - - assert(num_locations > -1 && num_locations < MAX_CHUNK_SERVERS); - if (num_locations > state->replication_factor) - num_locations = state->replication_factor; - - uint32_t tmp_u32 = num_locations; - message_write(&writer, &tmp_u32, sizeof(tmp_u32)); - - for (int j = 0; j < num_locations; j++) { - - int k = locations[j]; - - assert(k > -1); - assert(k < state->num_chunk_servers); - assert(state->chunk_servers[k].auth == true); - assert(state->chunk_servers[k].num_addrs > 0); - - message_write_server_addr(&writer, &state->chunk_servers[k]); - } - - if (!message_writer_free(&writer)) - return -1; - - } else { - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_READ_SUCCESS); - - assert(gen != NO_GENERATION); - message_write(&writer, &gen, sizeof(gen)); - - if (chunk_size > UINT32_MAX) { - message_writer_free(&writer); - return -1; - } - uint32_t tmp = chunk_size; - message_write(&writer, &tmp, sizeof(tmp)); - - // Send the actual number of bytes that can be read - if (actual_bytes > UINT32_MAX) { - message_writer_free(&writer); - return -1; - } - uint32_t tmp_actual = actual_bytes; - message_write(&writer, &tmp_actual, sizeof(tmp_actual)); - - uint32_t num_hashes = ret; - message_write(&writer, &num_hashes, sizeof(num_hashes)); - - for (uint32_t i = 0; i < num_hashes; i++) { - - int holders[MAX_CHUNK_SERVERS]; - int num_holders = all_chunk_servers_holding_chunk(state, hashes[i], holders, state->replication_factor); - assert(num_holders > -1 && num_holders < MAX_CHUNK_SERVERS); - - message_write(&writer, &hashes[i], sizeof(hashes[i])); - - uint32_t tmp = num_holders; - message_write(&writer, &tmp, sizeof(tmp)); - - for (int j = 0; j < num_holders; j++) { - - int k = holders[j]; - - assert(k > -1 && k < state->num_chunk_servers); - assert(state->chunk_servers[k].auth == true); - assert(state->chunk_servers[k].num_addrs > 0); - - message_write_server_addr(&writer, &state->chunk_servers[k]); - } - } - - int locations[MAX_CHUNK_SERVERS]; - int num_locations = choose_servers_for_write(state, locations, state->replication_factor); - - assert(num_locations > -1 && num_locations < MAX_CHUNK_SERVERS); - if (num_locations > state->replication_factor) - num_locations = state->replication_factor; - - uint32_t tmp_u32 = num_locations; - message_write(&writer, &tmp_u32, sizeof(tmp_u32)); - - for (int j = 0; j < num_locations; j++) { - - int k = locations[j]; - - assert(k > -1); - assert(k < state->num_chunk_servers); - assert(state->chunk_servers[k].auth == true); - assert(state->chunk_servers[k].num_addrs > 0); - - message_write_server_addr(&writer, &state->chunk_servers[k]); - } - - if (!message_writer_free(&writer)) - return -1; - } - - return 0; -} - -static int -process_client_write(MetadataServer *state, int conn_idx, ByteView msg) -{ - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint64_t expect_gen; - if (!binary_read(&reader, &expect_gen, sizeof(expect_gen))) - return -1; - - uint32_t flags; - if (!binary_read(&reader, &flags, sizeof(flags))) - return -1; - - char path_mem[1<<10]; - uint16_t path_len; - - if (!binary_read(&reader, &path_len, sizeof(path_len))) - return -1; - - if (path_len > sizeof(path_mem)) - return -2; // TODO: what is this -2 business? - - if (!binary_read(&reader, &path_mem, path_len)) - return -1; - - string path = { path_mem, path_len }; - - uint32_t offset; - if (!binary_read(&reader, &offset, sizeof(offset))) - return -1; - - uint32_t length; - if (!binary_read(&reader, &length, sizeof(length))) - return -1; - - uint32_t num_chunks; - if (!binary_read(&reader, &num_chunks, sizeof(num_chunks))) - return -1; - - #define MAX_CHUNKS_PER_WRITE 32 - - typedef struct { - SHA256 hash; - int num_addrs; - Address addrs[REPLICATION_FACTOR]; - } ChunkWriteResult; - - ChunkWriteResult results[MAX_CHUNKS_PER_WRITE]; - - if (num_chunks > MAX_CHUNKS_PER_WRITE) - return -1; // TODO - - for (uint32_t i = 0; i < num_chunks; i++) { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) - return -1; - - results[i].hash = hash; - results[i].num_addrs = 0; - - uint32_t num_locations; - if (!binary_read(&reader, &num_locations, sizeof(num_locations))) - return -1; - - for (uint32_t j = 0; j < num_locations; j++) { - - uint8_t is_ipv4; - if (!binary_read(&reader, &is_ipv4, sizeof(is_ipv4))) - return -1; - - Address addr = {0}; - addr.is_ipv4 = is_ipv4; - - if (is_ipv4) { - if (!binary_read(&reader, &addr.ipv4, sizeof(addr.ipv4))) - return -1; - } else { - if (!binary_read(&reader, &addr.ipv6, sizeof(addr.ipv6))) - return -1; - } - - if (!binary_read(&reader, &addr.port, sizeof(addr.port))) - return -1; - - if (results[i].num_addrs < REPLICATION_FACTOR) - results[i].addrs[results[i].num_addrs++] = addr; - } - } - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - // Array to collect hashes that are no longer used anywhere in the file tree - SHA256 removed_hashes[MAX_CHUNKS_PER_WRITE]; - int num_removed = 0; - - SHA256 new_hashes[MAX_CHUNKS_PER_WRITE]; - for (uint32_t i = 0; i < num_chunks; i++) - new_hashes[i] = results[i].hash; - - if (wal_append_write(&state->wal, path, offset, length, num_chunks, expect_gen, new_hashes) < 0) { - return -1; - } - - // Extract flag values - #define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) - #define TOASTY_WRITE_TRUNCATE_AFTER (1 << 1) - bool truncate_after = (flags & TOASTY_WRITE_TRUNCATE_AFTER) != 0; - - uint64_t new_gen; - int ret = file_tree_write(&state->file_tree, path, offset, length, - num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed, truncate_after); - - // If write failed because file doesn't exist and CREATE_IF_MISSING flag is set, - // create the file and retry the write. - // Note: MISSING_FILE_GENERATION works WITH CREATE_IF_MISSING to implement - // atomic "create-only-if-not-exists" semantics: creates if missing (NOENT), - // but fails if file already exists (BADGEN from gen_match). - if (ret == FILETREE_NOENT && (flags & TOASTY_WRITE_CREATE_IF_MISSING)) { - // Create the file with default chunk size of 4096 bytes - uint64_t chunk_size = 4096; - - // Log the creation in the WAL - if (wal_append_create(&state->wal, path, false, chunk_size) < 0) { - return -1; - } - - uint64_t create_gen; - int create_ret = file_tree_create_entity(&state->file_tree, path, false, chunk_size, &create_gen); - - if (create_ret == 0) { - // File created successfully, retry the write with the new generation - ret = file_tree_write(&state->file_tree, path, offset, length, - num_chunks, create_gen, &new_gen, new_hashes, removed_hashes, &num_removed, truncate_after); - } - } - - if (ret < 0) { - - string desc = file_tree_strerror(ret); - - MessageWriter writer; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - message_writer_init(&writer, output, MESSAGE_TYPE_WRITE_ERROR); - - uint16_t len = desc.len; - message_write(&writer, &len, sizeof(len)); - message_write(&writer, desc.ptr, desc.len); - - if (!message_writer_free(&writer)) - return -1; - - } else { - - // Add new chunks to add_list - for (uint32_t i = 0; i < num_chunks; i++) { - - for (int j = 0; j < results[i].num_addrs; j++) { - - int k = find_chunk_server_by_addr(state, results[i].addrs[j]); - if (k == -1) return -1; - - if (hash_set_insert(&state->chunk_servers[k].ms_add_list, new_hashes[i]) < 0) - return -1; - } - } - - // Mark removed chunks for deletion on all chunk servers that have them - // These are chunks that were overwritten and are no longer referenced anywhere - for (int i = 0; i < num_removed; i++) { - SHA256 removed_hash = removed_hashes[i]; - - // Add to rem_list for all chunk servers that have this chunk - for (int j = 0; j < state->num_chunk_servers; j++) { - if (chunk_server_peer_contains(&state->chunk_servers[j], removed_hash)) { - if (!hash_set_insert(&state->chunk_servers[j].ms_rem_list, removed_hash)) - return -1; - } - } - } - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_WRITE_SUCCESS); - - message_write(&writer, &new_gen, sizeof(new_gen)); - - if (!message_writer_free(&writer)) - return -1; - } - - return 0; -} - -static int -process_client_message(MetadataServer *state, - int conn_idx, uint8_t type, ByteView msg) -{ - switch (type) { - case MESSAGE_TYPE_CREATE: return process_client_create(state, conn_idx, msg); - case MESSAGE_TYPE_DELETE: return process_client_delete(state, conn_idx, msg); - case MESSAGE_TYPE_LIST : return process_client_list (state, conn_idx, msg); - case MESSAGE_TYPE_READ : return process_client_read (state, conn_idx, msg); - case MESSAGE_TYPE_WRITE : return process_client_write (state, conn_idx, msg); - default:break; - } - return -1; -} - -static ChunkServerPeer* -chunk_server_from_conn(MetadataServer *state, int conn_idx) -{ - int tag = tcp_get_tag(&state->tcp, conn_idx); - assert(tag >= 0); - - return &state->chunk_servers[tag]; -} - -static int process_chunk_server_auth(MetadataServer *state, - int conn_idx, ByteView msg) -{ - ChunkServerPeer *chunk_server = chunk_server_from_conn(state, conn_idx); - chunk_server->num_addrs = 0; - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - // Read header - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t num_ipv4; - if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) - return -1; - - for (uint32_t i = 0; i < num_ipv4; i++) { - - IPv4 ipv4; - if (!binary_read(&reader, &ipv4, sizeof(ipv4))) - return -1; - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) - return -1; - - if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { - Address addr = {0}; - addr.ipv4 = ipv4; - addr.is_ipv4 = true; - addr.port = port; - chunk_server->addrs[chunk_server->num_addrs++] = addr; - } - } - - uint32_t num_ipv6; - if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) - return -1; - - for (uint32_t i = 0; i < num_ipv6; i++) { - - IPv6 ipv6; - if (!binary_read(&reader, &ipv6, sizeof(ipv6))) - return -1; - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) - return -1; - - if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { - Address addr = {0}; - addr.ipv6 = ipv6; - addr.is_ipv4 = false; - addr.port = port; - chunk_server->addrs[chunk_server->num_addrs++] = addr; - } - } - - // No addresses were wpecified - if (chunk_server->num_addrs == 0) - return -1; - - // Check that there are no more bytes to read - if (binary_read(&reader, NULL, 1)) - return -1; - - // NOTE: In a production system, this should verify the authentication - // using the shared secret key mentioned in the architecture. For now, - // we accept all connections that provide valid address information. - chunk_server->auth = true; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_AUTH_RESPONSE); - - // TODO: Check whether we already hold the chunk list - // of this chunk server. If we do, tell it. - - if (!message_writer_free(&writer)) { - return -1; - } - return 0; -} - -static int process_chunk_server_sync(MetadataServer *state, - int conn_idx, ByteView msg) -{ - int chunk_server_idx = tcp_get_tag(&state->tcp, conn_idx); - assert(chunk_server_idx > -1); - assert(chunk_server_idx <= MAX_CHUNK_SERVERS); - - ChunkServerPeer *chunk_server = &state->chunk_servers[chunk_server_idx]; - assert(chunk_server->used); - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t count; - if (!binary_read(&reader, &count, sizeof(count))) - return -1; - - for (uint32_t i = 0; i < count; i++) { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) - return -1; - - // If the chunk is not referenced by the file tree, do - // nothing. - - if (!file_tree_uses_hash(&state->file_tree, hash)) - continue; - - // If the chunk is properly replicated or under-replicated, - // add it to the ms_add_list. - - int holders[MAX_CHUNK_SERVERS]; - int num_holders = all_chunk_servers_holding_chunk(state, hash, holders, MAX_CHUNK_SERVERS); - assert(num_holders > -1); - assert(num_holders <= MAX_CHUNK_SERVERS); - - if (num_holders <= state->replication_factor) { - if (hash_set_insert(&chunk_server->ms_add_list, hash) < 0) { - return -1; - } - continue; - } - - // If the chunk is over-replicated, either don't add - // it to the ms_add_list or add it to the ms_rem_list - // of some other holder. - // - // TODO: For now we don't add it to the ms_add_list, - // but there may be a better solution. - } - - if (binary_read(&reader, NULL, 1)) // TODO: this should probably be an assertion - return -1; - - // Respond with ms_add_list and ms_rem_list - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_SYNC_2); - - uint32_t add_count = chunk_server->ms_add_list.count; // TODO: check implicit casts - message_write(&writer, &add_count, sizeof(add_count)); - - for (uint32_t i = 0; i < add_count; i++) { - SHA256 hash = chunk_server->ms_add_list.items[i]; - message_write(&writer, &hash, sizeof(hash)); - } - - uint32_t rem_count = chunk_server->ms_rem_list.count; // TODO: check implicit casts - message_write(&writer, &rem_count, sizeof(rem_count)); - - for (uint32_t i = 0; i < rem_count; i++) { - SHA256 hash = chunk_server->ms_rem_list.items[i]; - message_write(&writer, &hash, sizeof(hash)); - } - - if (!message_writer_free(&writer)) - return -1; - return 0; -} - -static int process_chunk_server_sync_3(MetadataServer *state, - int conn_idx, ByteView msg) -{ - int chunk_server_idx = tcp_get_tag(&state->tcp, conn_idx); - assert(chunk_server_idx > -1); - assert(chunk_server_idx <= MAX_CHUNK_SERVERS); - - ChunkServerPeer *chunk_server = &state->chunk_servers[chunk_server_idx]; - assert(chunk_server->used); - - BinaryReader reader = { msg.ptr, msg.len, 0 }; - - if (!binary_read(&reader, NULL, sizeof(MessageHeader))) - return -1; - - uint32_t count; - if (!binary_read(&reader, &count, sizeof(count))) - return -1; - - ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); - assert(output); - - MessageWriter writer; - message_writer_init(&writer, output, MESSAGE_TYPE_SYNC_4); - - HashSet tmp_list; - hash_set_init(&tmp_list); - - message_write(&writer, &count, sizeof(count)); - - for (uint32_t i = 0; i < count; i++) { - - SHA256 hash; - if (!binary_read(&reader, &hash, sizeof(hash))) { - hash_set_free(&tmp_list); - return -1; - } - - // Only hashes that were actually expected to be - // in on the server should be recovered. - assert(hash_set_contains(&chunk_server->ms_add_list, hash) - || hash_set_contains(&chunk_server->ms_old_list, hash)); - - if (hash_set_insert(&tmp_list, hash) < 0) { - hash_set_free(&tmp_list); - return -1; - } - - int holders[MAX_CHUNK_SERVERS]; - int num_holders = all_chunk_servers_holding_chunk(state, hash, holders, MAX_CHUNK_SERVERS); - assert(num_holders > -1); - assert(num_holders <= MAX_CHUNK_SERVERS); - - uint32_t tmp = num_holders; - message_write(&writer, &tmp, sizeof(tmp)); - - for (int j = 0; j < num_holders; j++) { - int k = holders[j]; - message_write_server_addr(&writer, &state->chunk_servers[k]); - } - } - - if (binary_read(&reader, NULL, 1)) { // TODO: this should probably be an assertion - hash_set_free(&tmp_list); - return -1; - } - - if (hash_set_merge(&chunk_server->ms_old_list, chunk_server->ms_add_list) < 0) { - hash_set_free(&tmp_list); - return -1; - } - - hash_set_remove_set(&chunk_server->ms_old_list, tmp_list); - - hash_set_free(&chunk_server->ms_add_list); - chunk_server->ms_add_list = tmp_list; - - if (!message_writer_free(&writer)) - return -1; - return 0; -} - -static int -process_chunk_server_message(MetadataServer *state, - int conn_idx, uint8_t type, ByteView msg) -{ - switch (type) { - case MESSAGE_TYPE_AUTH: - return process_chunk_server_auth(state, conn_idx, msg); - - case MESSAGE_TYPE_SYNC: - return process_chunk_server_sync(state, conn_idx, msg); - - case MESSAGE_TYPE_SYNC_3: - return process_chunk_server_sync_3(state, conn_idx, msg); - } - return -1; -} - -static bool is_chunk_server_message_type(uint16_t type) -{ - switch (type) { - case MESSAGE_TYPE_AUTH: - case MESSAGE_TYPE_SYNC: - case MESSAGE_TYPE_SYNC_3: - return true; - - default: - break; - } - return false; -} - -int metadata_server_init(void *state_, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - MetadataServer *state = state_; - - string addr = getargs(argc, argv, "--addr", "127.0.0.1"); - int port = getargi(argc, argv, "--port", 8080); - bool trace = getargb(argc, argv, "--trace"); - string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal"); - int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit - - if (port <= 0 || port >= 1<<16) { - fprintf(stderr, "metadata server :: Invalid port\n"); - return -1; - } - - if (wal_limit < 0) { - fprintf(stderr, "metadata server :: Invalid WAL limit\n"); - return -1; - } - - state->trace = trace; - state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro? - if (state->replication_factor > MAX_CHUNK_SERVERS) - return -1; - - state->num_chunk_servers = 0; - for (int i = 0; i < MAX_CHUNK_SERVERS; i++) - state->chunk_servers[i].used = false; - - if (tcp_context_init(&state->tcp) < 0) { - fprintf(stderr, "metadata server :: Couldn't setup TCP context\n"); - return -1; - } - - int ret = tcp_listen(&state->tcp, addr, port); - if (ret < 0) { - fprintf(stderr, "metadata server :: Couldn't setup TCP listener\n"); - tcp_context_free(&state->tcp); - return -1; - } - - ret = file_tree_init(&state->file_tree); - if (ret < 0) { - fprintf(stderr, "metadata server :: Couldn't setup file tree\n"); - tcp_context_free(&state->tcp); - return -1; - } - - if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) { - fprintf(stderr, "metadata server :: Couldn't setup WAL\n"); - tcp_context_free(&state->tcp); - return -1; - } - - printf("Metadata server set up (local=%.*s:%d)\n", - addr.len, - addr.ptr, - port - ); - - *timeout = -1; // No timeout until we have chunk servers - if (pcap < TCP_POLL_CAPACITY) { - fprintf(stderr, "metadata server :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); - return -1; - } - *pnum = tcp_register_events(&state->tcp, ctxs, pdata); - return 0; -} - -int metadata_server_tick(void *state_, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, int *timeout) -{ - MetadataServer *state = state_; - - Event events[TCP_EVENT_CAPACITY]; - int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); - - Time current_time = get_current_time(); - if (current_time == INVALID_TIME) - return -1; - - for (int i = 0; i < num_events; i++) { - int conn_idx = events[i].conn_idx; - switch (events[i].type) { - - case EVENT_WAKEUP: - MS_TRACE("TCP EVENT: wakeup"); - // Do nothing - break; - - case EVENT_CONNECT: - MS_TRACE("TCP EVENT: connect"); - tcp_set_tag(&state->tcp, conn_idx, CONNECTION_TAG_UNKNOWN, false); - break; - - case EVENT_DISCONNECT: - { - MS_TRACE("TCP EVENT: disconnect"); - if (events[i].tag >= 0) { - MS_TRACE("Chunk server disconnected"); - chunk_server_peer_free(&state->chunk_servers[events[i].tag]); - assert(state->num_chunk_servers > 0); - state->num_chunk_servers--; - } - } - break; - - case EVENT_MESSAGE: - { - MS_TRACE("TCP EVENT: message"); - - // We don't trace message events from chunk servers - // as it would become very verbose - for (;;) { - - ByteView msg; - uint16_t msg_type; - int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); - if (ret == 0) { - MS_TRACE("Incomplete message"); - break; - } - if (ret < 0) { - MS_TRACE("Invalid message"); - tcp_close(&state->tcp, conn_idx); - break; - } - - if (state->trace) - message_dump(stdout, msg); - - if (tcp_get_tag(&state->tcp, conn_idx) == CONNECTION_TAG_UNKNOWN) { - if (is_chunk_server_message_type(msg_type)) { - - if (state->num_chunk_servers == MAX_CHUNK_SERVERS) { - tcp_close(&state->tcp, conn_idx); - break; - } - - int j = 0; - while (state->chunk_servers[j].used) { - j++; - assert(j < MAX_CHUNK_SERVERS); - } - - chunk_server_peer_init(&state->chunk_servers[j], current_time); - state->num_chunk_servers++; - - tcp_set_tag(&state->tcp, conn_idx, j, true); - - } else { - - tcp_set_tag(&state->tcp, conn_idx, CONNECTION_TAG_CLIENT, false); - } - } - - int tag = tcp_get_tag(&state->tcp, conn_idx); - if (tag == CONNECTION_TAG_CLIENT) { - MS_TRACE("Message from client"); - ret = process_client_message(state, conn_idx, msg_type, msg); - } else { - MS_TRACE("Message from CS (conn_idx=%d)", conn_idx); - state->chunk_servers[tag].last_response_time = current_time; - ret = process_chunk_server_message(state, conn_idx, msg_type, msg); - } - if (ret < 0) { - MS_TRACE("Message processing failure"); - tcp_close(&state->tcp, conn_idx); - break; - } - - tcp_consume_message(&state->tcp, conn_idx); - } - } - break; - } - } - - Time next_wakeup = INVALID_TIME; - - // Trigger chunk server timing events - for (int i = 0, j = 0; j < state->num_chunk_servers; i++) { - - ChunkServerPeer *chunk_server = &state->chunk_servers[i]; - if (!chunk_server->used) - continue; - j++; - - Time response_timeout = chunk_server->last_response_time + (Time) RESPONSE_TIME_LIMIT * 1000000000; - if (current_time > response_timeout) { - // Drop unresponsive chunk server - chunk_server->used = false; - state->num_chunk_servers--; - continue; - } - nearest_deadline(&next_wakeup, response_timeout); - } - - *timeout = deadline_to_timeout(next_wakeup, current_time); - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = tcp_register_events(&state->tcp, ctxs, pdata); - return 0; -} - -int metadata_server_free(void *state_) -{ - MetadataServer *state = state_; - - wal_close(&state->wal); - file_tree_free(&state->file_tree); - tcp_context_free(&state->tcp); - return 0; -} diff --git a/src/metadata_server.h b/src/metadata_server.h deleted file mode 100644 index 60aa9e3..0000000 --- a/src/metadata_server.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef METADATA_SERVER_INCLUDED -#define METADATA_SERVER_INCLUDED - -#include "tcp.h" -#include "wal.h" -#include "basic.h" -#include "config.h" -#include "hash_set.h" -#include "file_tree.h" - -#define CONNECTION_TAG_CLIENT -2 -#define CONNECTION_TAG_UNKNOWN -3 - -typedef struct { - - bool used; - bool auth; - - int num_addrs; - Address addrs[MAX_SERVER_ADDRS]; - - // List of chunks that are known to be held by CS - HashSet ms_old_list; // TODO: rename all *_list symbols to *_set - - // List of chunks that should be held by CS - HashSet ms_add_list; - - // List of chunks that may be held by CS but should removed from it - HashSet ms_rem_list; - - // Time when last STATE_UPDATE was sent - Time last_sync_time; - bool last_sync_done; - - // Time when last response was received - Time last_response_time; // TODO: don't init to INVALID_TIME but current_time - -} ChunkServerPeer; - -typedef struct { - - TCP tcp; - WAL wal; - - FileTree file_tree; - - bool trace; - int replication_factor; - - int num_chunk_servers; - ChunkServerPeer chunk_servers[MAX_CHUNK_SERVERS]; - -} MetadataServer; - -struct pollfd; - -int metadata_server_init(void *state, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout); - -int metadata_server_tick(void *state, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, int *timeout); - -int metadata_server_free(void *state); - -#endif // METADATA_SERVER_INCLUDED \ No newline at end of file diff --git a/src/random_client.c b/src/random_client.c deleted file mode 100644 index 4e1d549..0000000 --- a/src/random_client.c +++ /dev/null @@ -1,278 +0,0 @@ -#ifdef MAIN_SIMULATION - -#define QUAKEY_ENABLE_MOCKS -#include -#include - -#include "tcp.h" -#include "random_client.h" - -// Helper function to parse address and port from command line -static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port) -{ - // Default to metadata server - *addr = "127.0.0.1"; - *port = 8080; - - for (int i = 0; i < argc - 1; i++) { - if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) { - *addr = argv[i + 1]; - if (i + 2 < argc) { - - errno = 0; - char *end; - long val = strtol(argv[i+2], &end, 10); - - if (end == argv[i+2] || *end != '\0' || errno == ERANGE) - break; - - if (val < 0 || val > UINT16_MAX) - break; - - *port = (uint16_t) val; - return true; - } - } - } - return true; -} - -int random_client_init(void *state_, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - RandomClient *client = state_; - - char *addr; - uint16_t port; - parse_server_addr(argc, argv, &addr, &port); - - client->toasty = toasty_connect((ToastyString) { addr, strlen(addr) }, port); - if (client->toasty == NULL) - return -1; - - client->num_pending = 0; - - printf("Client set up (remote=%s:%d)\n", addr, port); - - *timeout = 0; - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); - return 0; -} - -static int random_in_range(int min, int max) -{ - uint64_t n = quakey_random(); - return min + n % (max - min + 1); -} - -int random_client_tick(void *state_, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - RandomClient *client = state_; - - // Process any pending events from the network and get new poll descriptors - *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); - - for (int i = 0; i < client->num_pending; i++) { - - ToastyResult result; - if (toasty_get_result(client->toasty, client->pending[i].handle, &result) != 0) - continue; - - PendingOperation pending = client->pending[i]; - switch (result.type) { - - case TOASTY_RESULT_EMPTY: - assert(0); - break; - - case TOASTY_RESULT_CREATE_ERROR: - assert(pending.type == PENDING_OPERATION_CREATE); - //printf("[Client] create error\n"); - break; - - case TOASTY_RESULT_CREATE_SUCCESS: - assert(pending.type == PENDING_OPERATION_CREATE); - //printf("[Client] create success\n"); - break; - - case TOASTY_RESULT_DELETE_ERROR: - assert(pending.type == PENDING_OPERATION_DELETE); - //printf("[Client] delete error\n"); - break; - - case TOASTY_RESULT_DELETE_SUCCESS: - assert(pending.type == PENDING_OPERATION_DELETE); - //printf("[Client] delete success\n"); - break; - - case TOASTY_RESULT_LIST_ERROR: - assert(pending.type == PENDING_OPERATION_LIST); - //printf("[Client] list error\n"); - break; - - case TOASTY_RESULT_LIST_SUCCESS: - assert(pending.type == PENDING_OPERATION_LIST); - //printf("[Client] list success\n"); - break; - - case TOASTY_RESULT_READ_ERROR: - assert(pending.type == PENDING_OPERATION_READ); - //printf("[Client] read error\n"); - break; - - case TOASTY_RESULT_READ_SUCCESS: - assert(pending.type == PENDING_OPERATION_READ); - //printf("[Client] read success\n"); - break; - - case TOASTY_RESULT_WRITE_ERROR: - assert(pending.type == PENDING_OPERATION_WRITE); - //printf("[Client] write error\n"); - break; - - case TOASTY_RESULT_WRITE_SUCCESS: - assert(pending.type == PENDING_OPERATION_WRITE); - //printf("[Client] write success\n"); - break; - } - free(pending.ptr); - toasty_free_result(&result); - client->pending[i--] = client->pending[--client->num_pending]; - } - - while (client->num_pending < MAX_PENDING_OPERATION) { - - typedef struct { - ToastyString path; - bool is_dir; - } TableEntry; - - static const TableEntry table[] = { - { TOASTY_STR("/f0"), false }, - { TOASTY_STR("/f1"), false }, - { TOASTY_STR("/d0"), true }, - { TOASTY_STR("/d1"), true }, - { TOASTY_STR("/d0/f1"), false }, - { TOASTY_STR("/d0/f2"), false }, - { TOASTY_STR("/d0/d0"), true }, - { TOASTY_STR("/d0/d1"), true }, - { TOASTY_STR("/d1/f1"), false }, - { TOASTY_STR("/d1/f2"), false }, - { TOASTY_STR("/d1/d0"), true }, - { TOASTY_STR("/d1/d1"), true }, - }; - static const int table_len = sizeof(table) / sizeof(table[0]); - - static const PendingOperationType type_table[] = { - PENDING_OPERATION_CREATE, - PENDING_OPERATION_DELETE, - PENDING_OPERATION_LIST, - PENDING_OPERATION_READ, - PENDING_OPERATION_WRITE, - }; - static const int type_table_len = sizeof(type_table)/sizeof(type_table[0]); - - void *ptr = NULL; - int off = 0; - int len = 0; - ToastyHandle handle; - - PendingOperationType type = type_table[random_in_range(0, type_table_len-1)]; - switch (type) { - - TableEntry entry; - uint32_t chunk_size; - uint32_t flags; - - case PENDING_OPERATION_CREATE: - entry = table[random_in_range(0, table_len-1)]; - if (entry.is_dir) { - handle = toasty_begin_create_dir(client->toasty, entry.path); - } else { - chunk_size = random_in_range(0, 5000); - handle = toasty_begin_create_file(client->toasty, entry.path, chunk_size); - } - //printf("[Client] submit create (path=%s, is_dir=%s, chunk_size=%d)\n", entry.path, entry.is_dir ? "true" : "false", chunk_size); - break; - - case PENDING_OPERATION_DELETE: - entry = table[random_in_range(0, table_len-1)]; - handle = toasty_begin_delete(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY); - //printf("[Client] submit delete (path=%s)\n", entry.path); - break; - - case PENDING_OPERATION_LIST: - entry = table[random_in_range(0, table_len-1)]; - handle = toasty_begin_list(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY); - //printf("[Client] submit list (path=%s)\n", entry.path); - break; - - case PENDING_OPERATION_READ: - entry = table[random_in_range(0, table_len-1)]; - off = random_in_range(0, 10000); - len = random_in_range(0, 5000); - ptr = malloc(len); - if (ptr == NULL) assert(0); - handle = toasty_begin_read(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY); - //printf("[Client] submit read (path=%s, off=%d, len=%d)\n", entry.path, off, len); - break; - - case PENDING_OPERATION_WRITE: - entry = table[random_in_range(0, table_len-1)]; - off = random_in_range(0, 10000); - len = random_in_range(0, 5000); - ptr = malloc(len); - if (ptr == NULL) assert(0); - memset(ptr, 'a', len); - flags = 0; - switch (random_in_range(0, 3)) { - case 0: - flags = 0; - break; - case 1: - flags = TOASTY_WRITE_CREATE_IF_MISSING; - break; - case 2: - flags = TOASTY_WRITE_TRUNCATE_AFTER; - break; - case 3: - flags = TOASTY_WRITE_CREATE_IF_MISSING - | TOASTY_WRITE_TRUNCATE_AFTER; - break; - default: - assert(0); - } - handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY, flags); - //printf("[Client] submit write (path=%s, off=%d, len=%d)\n", entry.path, off, len); - break; - } - if (handle == TOASTY_INVALID) - break; - client->pending[client->num_pending++] = (PendingOperation) { .type=type, .handle=handle, .ptr=ptr }; - } - - if (client->num_pending == 0) - *timeout = 10; - else - *timeout = -1; - - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = toasty_process_events(client->toasty, ctxs, pdata, 0); - return 0; -} - -int random_client_free(void *state_) -{ - RandomClient *client = state_; - - toasty_disconnect(client->toasty); - return 0; -} - -#endif // MAIN_SIMULATION \ No newline at end of file diff --git a/src/random_client.h b/src/random_client.h deleted file mode 100644 index 157af23..0000000 --- a/src/random_client.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef RANDOM_CLIENT_INCLUDED -#define RANDOM_CLIENT_INCLUDED - -#include "ToastyFS.h" - -#define MAX_PENDING_OPERATION 8 - -typedef enum { - PENDING_OPERATION_CREATE, - PENDING_OPERATION_DELETE, - PENDING_OPERATION_LIST, - PENDING_OPERATION_READ, - PENDING_OPERATION_WRITE, -} PendingOperationType; - -typedef struct { - PendingOperationType type; - ToastyHandle handle; - void *ptr; -} PendingOperation; - -typedef struct { - ToastyFS *toasty; - int num_pending; - PendingOperation pending[MAX_PENDING_OPERATION]; -} RandomClient; - -struct pollfd; - -int random_client_init(void *state, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout); - -int random_client_tick(void *state, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, int *timeout); - -int random_client_free(void *state); - -#endif // RANDOM_CLIENT_INCLUDED \ No newline at end of file diff --git a/src/server.c b/src/server.c new file mode 100644 index 0000000..311abad --- /dev/null +++ b/src/server.c @@ -0,0 +1,2006 @@ +#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include + +#include "server.h" + +typedef enum { + HR_OK, + HR_OUT_OF_MEMORY, + HR_INVALID_MESSAGE, +} HandlerResult; + +static int self_idx(ServerState *state) +{ + for (int i = 0; i < state->num_nodes; i++) + if (addr_eql(state->node_addrs[i], state->self_addr)) + return i; + UNREACHABLE; +} + +static int leader_idx(ServerState *state) +{ + return state->view_number % state->num_nodes; +} + +static bool is_leader(ServerState *state) +{ + if (state->status == STATUS_RECOVERY) + return false; + return self_idx(state) == leader_idx(state); +} + +// ---- Logging infrastructure ---- + +#define TIME_FMT "%7.3fs" +#define TIME_VAL(t) ((double)(t) / 1000000000.0) + +static const char *status_name(Status s) +{ + switch (s) { + case STATUS_NORMAL: return "NR"; + case STATUS_CHANGE_VIEW: return "CV"; + case STATUS_RECOVERY: return "RC"; + } + return "??"; +} + +static void node_log_impl(ServerState *state, const char *event, const char *detail) +{ + printf("[" TIME_FMT "] NODE %d (%s) %s | V%-3lu C%-3d L%-3d | %-20s %s\n", + TIME_VAL(state->now), + self_idx(state), + is_leader(state) ? "PR" : "RE", + status_name(state->status), + state->view_number, + state->commit_index, + state->log.count, + event, + detail ? detail : ""); +} + +#define node_log(state, event, fmt, ...) do { \ + char _detail[256]; \ + snprintf(_detail, sizeof(_detail), fmt, ##__VA_ARGS__); \ + node_log_impl(state, event, _detail); \ +} while (0) + +#define node_log_simple(state, event) \ + node_log_impl(state, event, NULL) + +static int count_set(uint32_t word) +{ + int n = 0; + for (int i = 0; i < (int) sizeof(word) * 8; i++) + if (word & (1 << i)) + n++; + return n; +} + +static bool reached_quorum(ServerState *state, uint32_t votes) +{ + return count_set(votes) > state->num_nodes/2; +} + +static bool already_voted(uint32_t votes, int idx) +{ + uint32_t mask = 1 << idx; + return (votes & mask) == mask; +} + +static void add_vote(uint32_t *votes, int idx) +{ + *votes |= 1 << idx; +} + +static void send_to_peer_ex(ServerState *state, int peer_idx, MessageHeader *msg, void *extra, int extra_len) +{ + ByteQueue *output; + int conn_idx = tcp_index_from_tag(&state->tcp, peer_idx); + if (conn_idx < 0) { + int ret = tcp_connect(&state->tcp, state->node_addrs[peer_idx], peer_idx, &output); + if (ret < 0) + return; + } else { + output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) + return; + } + byte_queue_write(output, msg, msg->length - extra_len); + byte_queue_write(output, extra, extra_len); +} + +static void send_to_peer(ServerState *state, int peer_idx, MessageHeader *msg) +{ + send_to_peer_ex(state, peer_idx, msg, NULL, 0); +} + +static void broadcast_to_peers_ex(ServerState *state, MessageHeader *msg, void *extra, int extra_len) +{ + for (int i = 0; i < state->num_nodes; i++) { + if (i != self_idx(state)) + send_to_peer_ex(state, i, msg, extra, extra_len); + } +} + +static void broadcast_to_peers(ServerState *state, MessageHeader *msg) +{ + broadcast_to_peers_ex(state, msg, NULL, 0); +} + +static void begin_state_transfer(ServerState *state, int sender_idx) +{ + if (state->state_transfer_pending) + return; + state->state_transfer_pending = true; + + GetStateMessage message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_GET_STATE, + .length = sizeof(GetStateMessage), + }, + .view_number = state->view_number, + .op_number = state->log.count, + .sender_idx = self_idx(state), + }; + node_log(state, "SEND GET_STATE", "to=%d op=%d", sender_idx, state->log.count); + send_to_peer(state, sender_idx, &message.base); + + state->state_transfer_time = state->now; +} + +static HandlerResult +process_request(ServerState *state, int conn_idx, ByteView msg) +{ + RequestMessage request_message; + if (msg.len != sizeof(request_message)) + return HR_INVALID_MESSAGE; + memcpy(&request_message, msg.ptr, sizeof(request_message)); + + { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &request_message.oper); + node_log(state, "RECV REQUEST", "client=%lu req=%lu %s", + request_message.client_id, request_message.request_id, + oper_buf); + } + + // Ensure a tag is associated to this connection + int conn_tag = tcp_get_tag(&state->tcp, conn_idx); + if (conn_tag == -1) { + conn_tag = state->next_client_tag++; + tcp_set_tag(&state->tcp, conn_idx, conn_tag, true); + } + + // We must first add or update the client table to + // invalidate the request ID. This makes it so any + // subsequent requests with the same ID are rejected + // while the first one is in progress. + // + // If the request ID is lower than the one stored in + // the table, the request is rejected. + // + // If the request ID is the same as the one in the table + // but no result was saved as the original one is still + // in progress, the request is rejected. + // + // If the request ID is the same and a result is available, + // it is returned immediately. + { + ClientTableEntry *entry = client_table_find(&state->client_table, request_message.client_id); + if (entry == NULL) { + + int ret = client_table_add( + &state->client_table, + request_message.client_id, + request_message.request_id, + conn_tag); + + if (ret < 0) { + + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .rejected = true, + .request_id = request_message.request_id, + }; + + int conn_idx = tcp_index_from_tag(&state->tcp, conn_tag); + assert(conn_idx > -1); + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + node_log(state, "SEND REPLY", "client=%lu REJECTED (table full)", request_message.client_id); + byte_queue_write(output, &reply_message, sizeof(reply_message)); + return HR_OK; + } + + } else { + + if (entry->pending) + return HR_OK; // Only one pending operation per client is allowed. Ignore the message. + + if (entry->last_request_id > request_message.request_id) + return HR_OK; // Request is old. Ignore. + + if (entry->last_request_id == request_message.request_id) { + + // This request was already processed and its value was cached. + // Respond with the cached value. + + ReplyMessage reply_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .rejected = false, + .result = entry->last_result, + .request_id = request_message.request_id, + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + { + char result_buf[64]; + meta_snprint_result(result_buf, sizeof(result_buf), entry->last_result); + node_log(state, "SEND REPLY", "client=%lu cached %s", request_message.client_id, result_buf); + } + byte_queue_write(output, &reply_message, sizeof(reply_message)); + return HR_OK; + } + + entry->last_request_id = request_message.request_id; + entry->conn_tag = conn_tag; + entry->pending = true; + } + } + + LogEntry log_entry = { + .oper = request_message.oper, + .votes = 1 << self_idx(state), + .view_number = state->view_number, + .client_id = request_message.client_id, + .request_id = request_message.request_id, + }; + if (log_append(&state->log, log_entry) < 0) + return HR_OUT_OF_MEMORY; + + // We forwarded the message to all peers. As soon as + // we get enough PREPARE_OK responses, we'll commit + // and reply to the client. + PrepareMessage prepare_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE, + .length = sizeof(PrepareMessage), + }, + .oper = request_message.oper, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .commit_index = state->commit_index, + .view_number = state->view_number, + .client_id = request_message.client_id, + .request_id = request_message.request_id, + }; + { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &request_message.oper); + node_log(state, "SEND PREPARE", "to=* idx=%d %s", state->log.count-1, oper_buf); + } + broadcast_to_peers(state, &prepare_message.base); + return HR_OK; +} + +static void reply_to_client(ServerState *state, ClientTableEntry *table_entry, + uint64_t request_id, MetaOper *oper, MetaResult result) +{ + int conn_idx = tcp_index_from_tag(&state->tcp, table_entry->conn_tag); + if (conn_idx < 0) + return; + + ReplyMessage message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REPLY, + .length = sizeof(ReplyMessage), + }, + .result = result, + .request_id = request_id, + }; + + { + char oper_buf[128], result_buf[64]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), oper); + meta_snprint_result(result_buf, sizeof(result_buf), result); + node_log(state, "SEND REPLY", "client=%lu req=%lu %s -> %s", + table_entry->client_id, request_id, oper_buf, result_buf); + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &message, sizeof(message)); +} + +static void advance_commit_index(ServerState *state, int target_index, bool send_replies) +{ + target_index = MIN(target_index, state->log.count); + + while (state->commit_index < target_index) { + + LogEntry *entry = &state->log.entries[state->commit_index++]; + + MetaResult result = meta_store_update(&state->metastore, &entry->oper); + { + char oper_buf[128], result_buf[64]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &entry->oper); + meta_snprint_result(result_buf, sizeof(result_buf), result); + node_log(state, "APPLY", "idx=%d %s -> %s", state->commit_index-1, oper_buf, result_buf); + } + + // After a view change, the new leader inherits log + // entries but not the client table's pending state. + // A client table entry may exist from a previous view + // (when this node was leader before) with pending=false. + // Only reply if this leader received the original REQUEST. + ClientTableEntry *table_entry = client_table_find(&state->client_table, entry->client_id); + if (table_entry == NULL) + continue; + + if (!table_entry->pending) + continue; + + // Verify this reply is for the request the client is actually + // waiting for. After a view change, the new leader inherits + // uncommitted log entries from the old view carrying a stale + // request_id. If the client has since sent a newer request, + // we must not confuse the old result with the new request. + if (table_entry->last_request_id != entry->request_id) + continue; + + table_entry->pending = false; + table_entry->last_result = result; + + if (send_replies) + reply_to_client(state, table_entry, entry->request_id, &entry->oper, result); + } +} + +// When the primary appends an entry to its log, it sends a +// PREPARE message to all backups. Backups add the entry to +// their own logs and reply with PREPARE_OK messages. When +// the primary receives a quorum of PREPARE_OKs, it commits +// the entry. +// +// In a reliable network and with no node crashes, we would +// expect entries to be committed linearly in the log. If +// log entries A and B are added to the log in that order, +// we expect A to reach PREPARE_OK messages before B. +// +// Unfortunately, we must assume messages will be lost (*). +// If that happens, instead of worrying about resending the +// PREPARE_OK for that entry, we rely on the fact that OK +// messages for future messages imply OK messages for the +// previous ones. The first message for which a quorum of +// OK messages is reached can work as an OK for all previous +// entries. +// +// For this reason, we allow "holes" in the log and if an +// entry reached quorum we advance the commit index to it. +// +// If the log index is lower than the log, it means we +// received an OK message that was not necessary anymore +// so we can ignore it. +// +// (*) This implementation uses TCP as a transport protocol, +// which means messages will be retransmitted if lost on +// the network. Nevertheless, if a node crashes while receiving +// a node or we crash before sending it, the message will +// be lost. +static HandlerResult +process_prepare_ok(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + PrepareOKMessage message; + if (msg.len != sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV PREPARE_OK", "from=%d idx=%d view=%lu", + message.sender_idx, message.log_index, message.view_number); + + if (message.view_number < state->view_number) + return HR_OK; // Drop + + if (message.view_number > state->view_number) { + state->view_number = message.view_number; + begin_state_transfer(state, message.sender_idx); + return HR_OK; + } + + assert(message.log_index > -1); + assert(message.log_index < state->log.count); + + if (message.log_index < state->commit_index) + return HR_OK; // Already processed + + LogEntry *entry = &state->log.entries[message.log_index]; + add_vote(&entry->votes, message.sender_idx); + if (reached_quorum(state, entry->votes)) { + node_log(state, "QUORUM", "idx=%d %s/%s", message.log_index, entry->oper.bucket, entry->oper.key); + advance_commit_index(state, message.log_index+1, true); + } + return HR_OK; +} + +static bool +should_store_view_change_log(ServerState *state, DoViewChangeMessage message) +{ + if (message.old_view_number > state->view_change_old_view) + return true; + + if (message.old_view_number < state->view_change_old_view) + return false; + + if (message.op_number <= state->view_change_log.count) + return false; + + return true; +} + +static void +clear_view_change_fields(ServerState *state) +{ + state->view_change_begin_votes = 0; + state->view_change_apply_votes = 0; + state->view_change_old_view = 0; + state->view_change_commit = 0; + log_free(&state->view_change_log); + log_init(&state->view_change_log); +} + +static HandlerResult +complete_view_change_and_become_primary(ServerState *state) +{ + assert(state->commit_index <= state->view_change_commit); + + log_move(&state->log, &state->view_change_log); + + state->status = STATUS_NORMAL; + state->last_normal_view = state->view_number; + node_log(state, "STATUS NORMAL", "became primary (view change complete)"); + + // Apply committed entries that haven't been executed yet. + // The state machine has only been updated up to the old + // commit_index. The best log may contain additional + // committed entries (up to view_change_commit) + // that must be executed before we start processing new + // requests, otherwise the state machine will be stale. + advance_commit_index(state, state->view_change_commit, false); + + // Reset vote tracking for uncommitted entries. The log + // entries inherited from DO_VIEW_CHANGE have stale + // votes from the previous view. The new leader starts + // with its own vote for each entry. + for (int i = state->commit_index; i < state->log.count; i++) { + + LogEntry *entry = &state->log.entries[i]; + + entry->votes = 0; + add_vote(&entry->votes, self_idx(state)); + } + + BeginViewMessage begin_view_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW, + .length = sizeof(BeginViewMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .commit_index = state->commit_index, + .op_number = state->log.count, + }; + node_log(state, "SEND BEGIN_VIEW", "to=* view=%lu log=%d commit=%d", + state->view_number, state->log.count, state->commit_index); + broadcast_to_peers_ex(state, &begin_view_message.base, state->log.entries, state->log.count * sizeof(LogEntry)); + + clear_view_change_fields(state); + return HR_OK; +} + +static HandlerResult +process_do_view_change(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + DoViewChangeMessage message; + if (msg.len < sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV DO_VIEW_CHANGE", "from=%d view=%lu old_view=%lu ops=%d commit=%d", + message.sender_idx, message.view_number, message.old_view_number, + message.op_number, message.commit_index); + + // TODO: This should trigger a view change in replicas running + // under normal operation + // + // VRR 4.2: A replica notices the need for a view change either based + // on its own timer, or because it receives a STARTVIEWCHANGE + // or DOVIEWCHANGE message for a view with a larger number than + // its own view-number. + + // Only process if the view matches what we're transitioning to + if (message.view_number != state->view_number) + return HR_OK; + + if (!already_voted(state->view_change_apply_votes, message.sender_idx)) { + + if (should_store_view_change_log(state, message)) { + + state->view_change_old_view = message.old_view_number; + + LogEntry *entries = (LogEntry*) (msg.ptr + sizeof(DoViewChangeMessage)); + + // Parse the variable-sized log from the message + int num_entries = (msg.len - sizeof(DoViewChangeMessage)) / sizeof(LogEntry); + if (num_entries != message.op_number) + return HR_INVALID_MESSAGE; // Message size mismatch + + log_free(&state->view_change_log); + if (log_init_from_network(&state->view_change_log, entries, num_entries) < 0) + return HR_OUT_OF_MEMORY; + } + + state->view_change_commit = MAX(state->view_change_commit, message.commit_index); + add_vote(&state->view_change_apply_votes, message.sender_idx); + } + + if (reached_quorum(state, state->view_change_apply_votes)) { + HandlerResult ret = complete_view_change_and_become_primary(state); + if (ret != HR_OK) + return ret; + } + + return HR_OK; +} + +static HandlerResult +process_recovery(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + RecoveryMessage recovery_message; + if (msg.len != sizeof(RecoveryMessage)) + return HR_INVALID_MESSAGE; + memcpy(&recovery_message, msg.ptr, sizeof(recovery_message)); + + node_log(state, "RECV RECOVERY", "from=%d nonce=%lu", recovery_message.sender_idx, recovery_message.nonce); + + node_log(state, "SEND RECOVERY_RESP", "to=%d view=%lu is_primary=%s", + recovery_message.sender_idx, state->view_number, is_leader(state) ? "yes" : "no"); + + RecoveryResponseMessage recovery_response_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY_RESPONSE, + .length = sizeof(RecoveryResponseMessage), + }, + .view_number = state->view_number, + .op_number = state->log.count-1, // TODO: What if the log is empty? + .nonce = recovery_message.nonce, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + if (is_leader(state)) { + recovery_response_message.base.length += state->log.count * sizeof(LogEntry); + send_to_peer_ex(state, recovery_message.sender_idx, &recovery_response_message.base, + state->log.entries, state->log.count * sizeof(LogEntry)); + } else { + send_to_peer(state, recovery_message.sender_idx, &recovery_response_message.base); + } + return HR_OK; +} + +static HandlerResult +perform_log_transfer_for_view_change(ServerState *state) +{ + if (is_leader(state)) { + // We are the new leader: count our own vote directly + // since send_to_peer_ex skips self-sends. + + add_vote(&state->view_change_apply_votes, self_idx(state)); + + state->view_change_old_view = state->last_normal_view; + state->view_change_commit = state->commit_index; + + // TODO: This should use copy-on-write + if (log_init_from_network(&state->view_change_log, state->log.entries, state->log.count) < 0) + return HR_OUT_OF_MEMORY; + + } else { + DoViewChangeMessage do_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_DO_VIEW_CHANGE, + .length = sizeof(DoViewChangeMessage) + state->log.count * sizeof(LogEntry), + }, + .view_number = state->view_number, + .old_view_number = state->last_normal_view, + .op_number = state->log.count, + .commit_index = state->commit_index, + .sender_idx = self_idx(state), + }; + send_to_peer_ex(state, leader_idx(state), &do_view_change_message.base, state->log.entries, state->log.count * sizeof(LogEntry)); + node_log(state, "SEND DO_VIEW_CHANGE", "to=%d view=%lu old_view=%lu log=%d commit=%d", + leader_idx(state), state->view_number, state->last_normal_view, + state->log.count, state->commit_index); + } + + // Clear the future array since we're changing views + state->num_future = 0; + state->state_transfer_pending = false; + return HR_OK; +} + +static HandlerResult +process_begin_view_change(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + BeginViewChangeMessage message; + if (msg.len != sizeof(BeginViewChangeMessage)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV BEGIN_VIEW_CHG", "from=%d view=%lu", message.sender_idx, message.view_number); + + // Ignore old messages + if (message.view_number < state->view_number) + return HR_OK; + + // BeginViewChange messages hold the view number of the view + // they are transitioning into. If this node is in NORMAL + // state and has the same view as the BeginViewChange message, + // it means the transition already happened and the message + // is stale. + if (state->status == STATUS_NORMAL) { + if (state->view_number == message.view_number) + return HR_OK; + } + + // If the peer's view number is larger, we need to transition + // to the view change state. + if (message.view_number > state->view_number) { + + BeginViewChangeMessage message_2 = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = message.view_number, + .sender_idx = self_idx(state), + }; + broadcast_to_peers(state, &message_2.base); + node_log(state, "SEND BEGIN_VIEW_CHG", "to=* view=%lu", message.view_number); + + clear_view_change_fields(state); + state->view_number = message.view_number; + state->heartbeat = state->now; + state->status = STATUS_CHANGE_VIEW; + node_log(state, "STATUS CHANGE_VIEW", "view=%lu", state->view_number); + } + + bool before = reached_quorum(state, state->view_change_begin_votes); + + add_vote(&state->view_change_begin_votes, self_idx(state)); + add_vote(&state->view_change_begin_votes, message.sender_idx); + + if (!before && reached_quorum(state, state->view_change_begin_votes)) { + HandlerResult ret = perform_log_transfer_for_view_change(state); + if (ret != HR_OK) + return ret; + } + + return HR_OK; +} + +static HandlerResult +process_begin_view(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + BeginViewMessage message; + if (msg.len < sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV BEGIN_VIEW", "view=%lu commit=%d ops=%d", + message.view_number, message.commit_index, message.op_number); + + if (message.view_number < state->view_number) + return HR_OK; + + state->view_number = message.view_number; + + state->status = STATUS_NORMAL; + state->last_normal_view = state->view_number; + node_log(state, "STATUS NORMAL", "new view=%lu (follower)", state->view_number); + + int num_entries = (msg.len - sizeof(BeginViewMessage)) / sizeof(LogEntry); + assert(num_entries >= state->commit_index); + + // Replace the local log with the authoritative log from the primary + log_free(&state->log); + if (log_init_from_network(&state->log, msg.ptr + sizeof(BeginViewMessage), num_entries) < 0) + return HR_OUT_OF_MEMORY; + + state->num_future = 0; + state->state_transfer_pending = false; + + // If there are non-committed operations in the log, + // send a PREPAREOK to the new primary + if (state->log.count > message.commit_index) { + PrepareOKMessage ok_msg = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count - 1, + .view_number = state->view_number, + }; + send_to_peer(state, leader_idx(state), &ok_msg.base); + node_log(state, "SEND PREPARE_OK", "to=%d idx=%d %s/%s", leader_idx(state), state->log.count - 1, + state->log.entries[state->log.count - 1].oper.bucket, state->log.entries[state->log.count - 1].oper.key); + } + + advance_commit_index(state, message.commit_index, false); + + clear_view_change_fields(state); + state->heartbeat = state->now; + return HR_OK; +} + +static HandlerResult +process_get_state(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + GetStateMessage get_state_message; + if (msg.len != sizeof(GetStateMessage)) + return HR_INVALID_MESSAGE; + memcpy(&get_state_message, msg.ptr, sizeof(get_state_message)); + + node_log(state, "RECV GET_STATE", "from=%d op=%d view=%lu", + get_state_message.sender_idx, get_state_message.op_number, get_state_message.view_number); + + if (state->status != STATUS_NORMAL) + return HR_OK; + + // Only respond if the requester is in the same view + if (get_state_message.view_number != state->view_number) + return HR_OK; + + // Compute the suffix of log entries the requester is missing + int start = get_state_message.op_number; + if (start < 0 || start >= state->log.count) + return HR_OK; // Nothing to send + + int num_entries = state->log.count - start; + + NewStateMessage new_state_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_NEW_STATE, + .length = sizeof(NewStateMessage) + num_entries * sizeof(LogEntry), + }, + .view_number = state->view_number, + .op_number = num_entries, + .commit_index = state->commit_index, + .start_index = start, + }; + node_log(state, "SEND NEW_STATE", "to=%d entries=%d commit=%d", + get_state_message.sender_idx, num_entries, state->commit_index); + send_to_peer_ex(state, get_state_message.sender_idx, &new_state_message.base, + state->log.entries + start, num_entries * sizeof(LogEntry)); + return HR_OK; +} + +/////////////////////////////////////////////////////////////// +// Chunk and Blob message handlers (bypass VSR log) +/////////////////////////////////////////////////////////////// + +static HandlerResult +process_store_chunk(ServerState *state, int conn_idx, ByteView msg) +{ + StoreChunkMessage message; + if (msg.len < sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + uint32_t data_size = msg.len - sizeof(StoreChunkMessage); + if (data_size != message.size) + return HR_INVALID_MESSAGE; + + char *data = (char *)(msg.ptr + sizeof(StoreChunkMessage)); + + int ret = chunk_store_write(&state->chunk_store, message.hash, data, message.size); + + StoreChunkAckMessage ack = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_STORE_CHUNK_ACK, + .length = sizeof(StoreChunkAckMessage), + }, + .hash = message.hash, + .success = (ret == 0), + }; + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output == NULL) + return HR_OK; + + node_log(state, "RECV STORE_CHUNK", "size=%u ok=%d", message.size, ret == 0); + byte_queue_write(output, &ack, sizeof(ack)); + return HR_OK; +} + +static HandlerResult +process_fetch_chunk(ServerState *state, int conn_idx, ByteView msg) +{ + FetchChunkMessage message; + if (msg.len != sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + // Check if we have the chunk + bool exists = chunk_store_exists(&state->chunk_store, message.hash); + + if (!exists) { + // Respond with size=0 (not found) + FetchChunkResponseMessage response = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_FETCH_CHUNK_RESPONSE, + .length = sizeof(FetchChunkResponseMessage), + }, + .hash = message.hash, + .size = 0, + }; + + // If sender_idx >= 0, it's a peer server; otherwise it's a client + if (message.sender_idx >= 0 && message.sender_idx < state->num_nodes) { + send_to_peer(state, message.sender_idx, &response.base); + } else { + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &response, sizeof(response)); + } + node_log(state, "RECV FETCH_CHUNK", "NOT_FOUND"); + return HR_OK; + } + + // Read the chunk. We need to know the size. Look it up from metadata + // or read the file. Since chunk_store doesn't expose file size, + // we'll try reading up to a reasonable max and use the file size. + // For simplicity, allocate a buffer and try to read. + // We know chunk sizes from StoreChunk messages, but don't have + // a size lookup. Use a fixed max and read what's available. + // + // Actually, chunk_store_read needs the exact size. We don't know it + // from just the hash. We need to figure out the size from disk. + // For now, let the FetchChunk message also carry the expected size, + // or we can use file_size. Let me just try reading with a large buffer. + // The mock filesystem tracks file size, so file_read_exact will fail + // if we give the wrong size. + // + // Better approach: iterate through committed metadata to find the + // chunk size. All servers have all metadata via VSR. + uint32_t chunk_size = 0; + for (int i = 0; i < state->metastore.count; i++) { + ObjectMeta *meta = &state->metastore.entries[i]; + if (meta->deleted) continue; + for (uint32_t j = 0; j < meta->num_chunks; j++) { + if (memcmp(&meta->chunks[j].hash, &message.hash, sizeof(SHA256)) == 0) { + chunk_size = meta->chunks[j].size; + goto found_size; + } + } + } + // Chunk hash not found in any metadata entry + { + FetchChunkResponseMessage response = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_FETCH_CHUNK_RESPONSE, + .length = sizeof(FetchChunkResponseMessage), + }, + .hash = message.hash, + .size = 0, + }; + if (message.sender_idx >= 0 && message.sender_idx < state->num_nodes) + send_to_peer(state, message.sender_idx, &response.base); + else { + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &response, sizeof(response)); + } + node_log(state, "RECV FETCH_CHUNK", "NO_META"); + return HR_OK; + } +found_size:; + char *chunk_data = malloc(chunk_size); + if (chunk_data == NULL) + return HR_OUT_OF_MEMORY; + + int ret = chunk_store_read(&state->chunk_store, message.hash, chunk_data, chunk_size); + if (ret < 0) { + free(chunk_data); + FetchChunkResponseMessage response = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_FETCH_CHUNK_RESPONSE, + .length = sizeof(FetchChunkResponseMessage), + }, + .hash = message.hash, + .size = 0, + }; + if (message.sender_idx >= 0 && message.sender_idx < state->num_nodes) + send_to_peer(state, message.sender_idx, &response.base); + else { + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &response, sizeof(response)); + } + node_log(state, "RECV FETCH_CHUNK", "READ_ERR size=%u", chunk_size); + return HR_OK; + } + + FetchChunkResponseMessage response = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_FETCH_CHUNK_RESPONSE, + .length = sizeof(FetchChunkResponseMessage) + chunk_size, + }, + .hash = message.hash, + .size = chunk_size, + }; + + node_log(state, "RECV FETCH_CHUNK", "size=%u", chunk_size); + + if (message.sender_idx >= 0 && message.sender_idx < state->num_nodes) { + send_to_peer_ex(state, message.sender_idx, &response.base, chunk_data, chunk_size); + } else { + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) { + byte_queue_write(output, &response, sizeof(response)); + byte_queue_write(output, chunk_data, chunk_size); + } + } + + free(chunk_data); + return HR_OK; +} + +static HandlerResult +process_get_blob(ServerState *state, int conn_idx, ByteView msg) +{ + GetBlobMessage message; + if (msg.len != sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV GET_BLOB", "%s/%s", message.bucket, message.key); + + ObjectMeta *meta = meta_store_lookup(&state->metastore, message.bucket, message.key); + + GetBlobResponseMessage response = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_GET_BLOB_RESPONSE, + .length = sizeof(GetBlobResponseMessage), + }, + }; + + if (meta != NULL) { + response.found = true; + response.size = meta->size; + response.content_hash = meta->content_hash; + response.num_chunks = meta->num_chunks; + memcpy(response.chunks, meta->chunks, meta->num_chunks * sizeof(ChunkRef)); + } else { + response.found = false; + } + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + if (output) + byte_queue_write(output, &response, sizeof(response)); + + return HR_OK; +} + +static HandlerResult +process_message_as_leader(ServerState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST: + if (state->status == STATUS_NORMAL) + return process_request(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_COMMIT_PUT: + // CommitPut is structurally identical to REQUEST and follows + // the same VSR path: client table check, log_append, PREPARE. + if (state->status == STATUS_NORMAL) + return process_request(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_PREPARE_OK: + if (state->status == STATUS_NORMAL) + return process_prepare_ok(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_DO_VIEW_CHANGE: + if (state->status == STATUS_CHANGE_VIEW) + return process_do_view_change(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_RECOVERY: + if (state->status == STATUS_NORMAL) + return process_recovery(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + if (state->status != STATUS_RECOVERY) + return process_begin_view_change(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_BEGIN_VIEW: + if (state->status != STATUS_RECOVERY) + return process_begin_view(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_RECOVERY_RESPONSE: + return HR_OK; + + case MESSAGE_TYPE_GET_STATE: + return process_get_state(state, conn_idx, msg); + + default: + break; + } + + return HR_OK; +} + +static HandlerResult +complete_recovery(ServerState *state) +{ + assert(state->commit_index <= state->recovery_commit); + + state->view_number = state->recovery_view; + log_move(&state->log, &state->recovery_log); + advance_commit_index(state, state->recovery_commit, false); + + state->status = STATUS_NORMAL; + state->last_normal_view = state->view_number; + node_log(state, "STATUS NORMAL", "recovery complete view=%lu commit=%d", + state->view_number, state->commit_index); + + // Reset stale votes + if (is_leader(state)) { + for (int i = state->commit_index; i < state->log.count; i++) { + LogEntry *entry = &state->log.entries[i]; + entry->votes = 0; + add_vote(&entry->votes, self_idx(state)); + } + } + + // Update heartbeat to avoid immediate view change timeout + state->heartbeat = state->now; + return HR_OK; +} + +static bool +received_recovery_primary(ServerState *state) +{ + assert(state->status == STATUS_RECOVERY); + + int primary_idx = state->recovery_view % state->num_nodes; + uint32_t primary_mask = 1 << primary_idx; + return (state->recovery_votes & primary_mask) != 0; +} + +static bool +sender_thinks_he_is_primary(ServerState *state, RecoveryResponseMessage message) +{ + return message.sender_idx == (int) (message.view_number % state->num_nodes); +} + +// When a node is trying to recover state, it expects +// a response from each replica and one from the primary +// containing the log. +// +// The recovering node is not aware of what the current +// view number is, and therefore doesn't know who is the +// primary. +// +// The only way a recovering node can infer whether a +// node is the primary or not, is by inferring it from +// the message itself. +// +// This function determines whether the message from a +// node contains a log and if it does, whether it should +// be stored. +// +// If no log was previously received, it is stored +// unconditionally. If a log is already stored, the +// newly received one is only stored if its view number +// is greater than the one associated to the stored +// log. +static bool +should_store_recovery_log(ServerState *state, RecoveryResponseMessage message) +{ + return sender_thinks_he_is_primary(state, message) + && (!received_recovery_primary(state) || state->recovery_log_view < message.view_number); +} + +static HandlerResult +process_recovery_response(ServerState *state, ByteView msg) +{ + RecoveryResponseMessage message; + if (msg.len < sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV RECOVERY_RESP", "from=%d view=%lu commit=%d nonce=%lu", + message.sender_idx, message.view_number, message.commit_index, message.nonce); + + if (message.nonce != state->recovery_nonce) + return HR_OK; + + state->recovery_view = MAX(state->recovery_view, message.view_number); + + if (should_store_recovery_log(state, message)) { + + LogEntry *entries = (LogEntry*) (msg.ptr + sizeof(RecoveryResponseMessage)); + int num_entries = message.op_number + 1; + + assert(num_entries == (int) ((msg.len - sizeof(RecoveryResponseMessage)) / sizeof(LogEntry))); + + log_free(&state->recovery_log); + if (log_init_from_network(&state->recovery_log, entries, num_entries) < 0) + return HR_OUT_OF_MEMORY; + + state->recovery_log_view = message.view_number; + state->recovery_commit = message.commit_index; + } + + add_vote(&state->recovery_votes, message.sender_idx); + if (reached_quorum(state, state->recovery_votes) && received_recovery_primary(state)) { + HandlerResult ret = complete_recovery(state); + if (ret != HR_OK) + return ret; + } + return HR_OK; +} + +static int +process_single_future_list_entry(ServerState *state) +{ + // Look for an entry with the current log index + int i = 0; + while (i < state->num_future && state->future[i].log_index != state->log.count) + i++; + + if (i == state->num_future) + return 0; // No entry + + LogEntry entry = { + .oper = state->future[i].oper, + .votes = 0, + .view_number = state->view_number, + .client_id = state->future[i].client_id, + .request_id = state->future[i].request_id, + }; + if (log_append(&state->log, entry) < 0) + return -1; + + PrepareOKMessage message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .view_number = state->view_number, + }; + send_to_peer(state, state->future[i].sender_idx, &message.base); + return 1; +} + +static void +remove_old_future_list_entries(ServerState *state) +{ + for (int i = 0; i < state->num_future; i++) { + if (state->future[i].log_index < state->log.count) { + state->future[i--] = state->future[--state->num_future]; + } + } +} + +static int process_future_list(ServerState *state) +{ + for (;;) { + int ret = process_single_future_list_entry(state); + if (ret != 1) + break; + } + remove_old_future_list_entries(state); + return 0; +} + +static HandlerResult +process_prepare(ServerState *state, ByteView msg) +{ + PrepareMessage message; + if (msg.len != sizeof(message)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + { + char oper_buf[128]; + meta_snprint_oper(oper_buf, sizeof(oper_buf), &message.oper); + node_log(state, "RECV PREPARE", "from=%d idx=%d commit=%d view=%lu %s", + message.sender_idx, message.log_index, message.commit_index, + message.view_number, oper_buf); + } + + // VRR 4.1: If the sender is behind, the receiver drops the message + if (message.view_number < state->view_number) + return HR_OK; + + // VRR 4.1: If the sender is ahead, the replica performs a state + // transfer: it requests information it is missing from + // the other replicas and uses this information to bring + // itself up to date before processing the message + if (message.view_number > state->view_number) { + state->view_number = message.view_number; + if (state->num_future < FUTURE_LIMIT) + state->future[state->num_future++] = message; + begin_state_transfer(state, message.sender_idx); + return HR_OK; + } + + if (message.log_index < state->log.count) + return HR_OK; // Message refers to an old entry. Ignore. + + if (message.log_index > state->log.count) { + if (state->num_future < FUTURE_LIMIT) + state->future[state->num_future++] = message; + begin_state_transfer(state, message.sender_idx); + return HR_OK; + } + + LogEntry log_entry = { + .oper = message.oper, + .votes = 0, + .view_number = state->view_number, + .client_id = message.client_id, + .request_id = message.request_id, + }; + if (log_append(&state->log, log_entry) < 0) + return HR_OUT_OF_MEMORY; + + PrepareOKMessage ok_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count-1, + .view_number = state->view_number, + }; + send_to_peer(state, message.sender_idx, &ok_message.base); + node_log(state, "SEND PREPARE_OK", "to=%d idx=%d %s/%s", + message.sender_idx, state->log.count-1, message.oper.bucket, message.oper.key); + + process_future_list(state); + advance_commit_index(state, message.commit_index, false); + + state->heartbeat = state->now; + return HR_OK; +} + +static HandlerResult +process_commit(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + CommitMessage message; + if (msg.len != sizeof(CommitMessage)) + return HR_INVALID_MESSAGE; + memcpy(&message, msg.ptr, sizeof(message)); + + node_log(state, "RECV COMMIT", "commit=%d", message.commit_index); + + if (message.view_number < state->view_number) + return HR_OK; // Stale peer + + if (message.view_number > state->view_number) { + begin_state_transfer(state, message.sender_idx); + return HR_OK; + } + + advance_commit_index(state, message.commit_index, false); + + state->heartbeat = state->now; + return HR_OK; +} + +static HandlerResult +process_new_state(ServerState *state, int conn_idx, ByteView msg) +{ + (void) conn_idx; + + NewStateMessage new_state_message; + if (msg.len < sizeof(NewStateMessage)) + return HR_INVALID_MESSAGE; + memcpy(&new_state_message, msg.ptr, sizeof(new_state_message)); + + node_log(state, "RECV NEW_STATE", "entries=%d commit=%d view=%lu", + new_state_message.op_number, new_state_message.commit_index, new_state_message.view_number); + + // Ignore if we're in a different view + if (new_state_message.view_number != state->view_number) + return HR_OK; + + int num_entries = (msg.len - sizeof(NewStateMessage)) / sizeof(LogEntry); + if (num_entries != new_state_message.op_number) + return HR_INVALID_MESSAGE; + + if (num_entries == 0) + return HR_OK; + + // Append received entries to our log. + // The entries array is a suffix of the sender's log starting at + // global position start_index. We skip entries we already have. + LogEntry *entries = (LogEntry *)((uint8_t *)msg.ptr + sizeof(NewStateMessage)); + int start_index = new_state_message.start_index; + for (int i = 0; i < num_entries; i++) { + + int global_idx = start_index + i; + if (global_idx < state->log.count) + continue; // Already have this entry + + LogEntry entry = { + .oper = entries[i].oper, + .votes = 0, + .view_number = state->view_number, + .client_id = entries[i].client_id, + .request_id = entries[i].request_id, + }; + if (log_append(&state->log, entry) < 0) + return HR_OUT_OF_MEMORY; + + // Send PREPARE_OK for each appended entry + PrepareOKMessage prepare_ok_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_PREPARE_OK, + .length = sizeof(PrepareOKMessage), + }, + .sender_idx = self_idx(state), + .log_index = state->log.count - 1, + .view_number = state->view_number, + }; + send_to_peer(state, leader_idx(state), &prepare_ok_message.base); + node_log(state, "SEND PREPARE_OK", "to=%d idx=%d %s/%s", leader_idx(state), state->log.count - 1, + state->log.entries[state->log.count - 1].oper.bucket, state->log.entries[state->log.count - 1].oper.key); + } + + process_future_list(state); + advance_commit_index(state, new_state_message.commit_index, false); + + state->state_transfer_pending = false; + state->heartbeat = state->now; // TODO: Should only do this if the sender is the primary + return HR_OK; +} + +static HandlerResult +send_redirect(ServerState *state, int conn_idx) +{ + RedirectMessage redirect_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_REDIRECT, + .length = sizeof(RedirectMessage), + }, + .view_number = state->view_number, + }; + + node_log(state, "SEND REDIRECT", "-> conn %d view=%lu leader=%d", + tcp_get_tag(&state->tcp, conn_idx), + (unsigned long)state->view_number, leader_idx(state)); + + ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx); + assert(output); + + byte_queue_write(output, &redirect_message, redirect_message.base.length); + return HR_OK; +} + +static HandlerResult +process_message_as_replica(ServerState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + switch (type) { + case MESSAGE_TYPE_REQUEST: + if (state->status == STATUS_NORMAL) + return send_redirect(state, conn_idx); + break; + + case MESSAGE_TYPE_COMMIT_PUT: + // CommitPut must go to the leader; redirect the client. + if (state->status == STATUS_NORMAL) + return send_redirect(state, conn_idx); + break; + + case MESSAGE_TYPE_PREPARE: + if (state->status == STATUS_NORMAL) + return process_prepare(state, msg); + break; + + case MESSAGE_TYPE_COMMIT: + if (state->status == STATUS_NORMAL) + return process_commit(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + if (state->status != STATUS_RECOVERY) + return process_begin_view_change(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_BEGIN_VIEW: + if (state->status != STATUS_RECOVERY) + return process_begin_view(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_RECOVERY: + if (state->status == STATUS_NORMAL) + return process_recovery(state, conn_idx, msg); + break; + + case MESSAGE_TYPE_RECOVERY_RESPONSE: + if (state->status == STATUS_RECOVERY) + return process_recovery_response(state, msg); + break; + + case MESSAGE_TYPE_NEW_STATE: + return process_new_state(state, conn_idx, msg); + + default: + break; + } + return HR_OK; +} + +static HandlerResult +process_message(ServerState *state, + int conn_idx, uint8_t type, ByteView msg) +{ + // Tag incoming connections with the sender's node index so that + // the connection can be used bidirectionally. Without this, when + // node A connects to node B and sends a message, node B can't + // send back to node A through the same connection (the tag is + // only set on the connector's side). + { + int sender_idx = -1; + switch (type) { + case MESSAGE_TYPE_PREPARE: + if (msg.len >= sizeof(PrepareMessage)) { + PrepareMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_PREPARE_OK: + if (msg.len >= sizeof(PrepareOKMessage)) { + PrepareOKMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_BEGIN_VIEW_CHANGE: + if (msg.len >= sizeof(BeginViewChangeMessage)) { + BeginViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_DO_VIEW_CHANGE: + if (msg.len >= sizeof(DoViewChangeMessage)) { + DoViewChangeMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_RECOVERY: + if (msg.len >= sizeof(RecoveryMessage)) { + RecoveryMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_RECOVERY_RESPONSE: + if (msg.len >= sizeof(RecoveryResponseMessage)) { + RecoveryResponseMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_GET_STATE: + if (msg.len >= sizeof(GetStateMessage)) { + GetStateMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + case MESSAGE_TYPE_FETCH_CHUNK: + if (msg.len >= sizeof(FetchChunkMessage)) { + FetchChunkMessage m; memcpy(&m, msg.ptr, sizeof(m)); + sender_idx = m.sender_idx; + } + break; + } + if (sender_idx >= 0 && sender_idx < state->num_nodes) { + int existing = tcp_index_from_tag(&state->tcp, sender_idx); + if (existing < 0) { + // No connection tagged with this peer yet, tag this one + tcp_set_tag(&state->tcp, conn_idx, sender_idx, false); + } + // If a different connection is already tagged for this peer, + // keep it. Closing it would also disconnect the peer end, + // which may be carrying data in the opposite direction (e.g. + // a DO_VIEW_CHANGE queued on the cross-connection). Stale + // connections are detected and cleaned up when sends fail. + } + } + + // Chunk and blob messages are handled by any server regardless of + // leader/replica role. They bypass the VSR log. + switch (type) { + case MESSAGE_TYPE_STORE_CHUNK: + return process_store_chunk(state, conn_idx, msg); + case MESSAGE_TYPE_FETCH_CHUNK: + return process_fetch_chunk(state, conn_idx, msg); + case MESSAGE_TYPE_GET_BLOB: + return process_get_blob(state, conn_idx, msg); + case MESSAGE_TYPE_STORE_CHUNK_ACK: + case MESSAGE_TYPE_FETCH_CHUNK_RESPONSE: + case MESSAGE_TYPE_GET_BLOB_RESPONSE: + // These are responses that servers send TO clients, not + // messages servers receive from clients. Ignore. + return HR_OK; + default: + break; + } + + if (is_leader(state)) { + return process_message_as_leader(state, conn_idx, type, msg); + } else { + return process_message_as_replica(state, conn_idx, type, msg); + } +} + +int server_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) +{ + ServerState *state = state_; + + Time now = get_current_time(); + if (now == INVALID_TIME) { + fprintf(stderr, "Node :: Couldn't get current time\n"); + return -1; + } + if (now > state->now) + state->now = now; + + state->num_nodes = 0; + + bool self_addr_set = false; + const char *chunks_path = "chunks"; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--addr")) { + if (self_addr_set) { + fprintf(stderr, "Option --addr specified twice\n"); + return -1; + } + self_addr_set = true; + i++; + if (i == argc) { + fprintf(stderr, "Option --addr missing value. Usage is --addr :\n"); + return -1; + } + // TODO: Check address is not duplicated + int ret = parse_addr_arg(argv[i], &state->self_addr); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --addr option\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + state->node_addrs[state->num_nodes++] = state->self_addr; + } else if (!strcmp(argv[i], "--peer")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --peer missing value. Usage is --peer :\n"); + return -1; + } + if (state->num_nodes == NODE_LIMIT) { + fprintf(stderr, "Node limit of %d reached\n", NODE_LIMIT); + return -1; + } + // TODO: Check address is not duplicated + int ret = parse_addr_arg(argv[i], &state->node_addrs[state->num_nodes]); + if (ret < 0) { + fprintf(stderr, "Malformed : pair for --peer option\n"); + return -1; + } + state->num_nodes++; + } else if (!strcmp(argv[i], "--chunks")) { + i++; + if (i == argc) { + fprintf(stderr, "Option --chunks missing value. Usage is --chunks \n"); + return -1; + } + chunks_path = argv[i]; + } else { + printf("Ignoring option '%s'\n", argv[i]); + } + } + + // Now sort the addresses + addr_sort(state->node_addrs, state->num_nodes); + + Time deadline = INVALID_TIME; + + state->view_number = 0; + state->last_normal_view = 0; + state->heartbeat = now; + state->commit_index = 0; + state->num_future = 0; + state->state_transfer_pending = false; + state->state_transfer_time = 0; + + // View change state + state->view_change_begin_votes = 0; + state->view_change_apply_votes = 0; + state->view_change_old_view = 0; + state->view_change_commit = 0; + log_init(&state->view_change_log); + + // Recovery state + state->recovery_votes = 0; + state->recovery_commit = 0; + state->recovery_view = 0; + state->recovery_log_view = 0; + log_init(&state->recovery_log); + + // Detect whether this is a restart after a crash by checking for a + // boot marker file on disk. The disk persists across crashes, so if + // the marker exists, this node previously ran and crashed. In that + // case, enter recovery mode to learn the current view from peers + // before participating in the protocol. + // + // We use open() directly (without O_CREAT) instead of file_exists() + // because access() is not available in the simulation environment. + int marker_fd = open("vsr_boot_marker", O_RDONLY, 0); + bool previously_crashed = (marker_fd >= 0); + if (previously_crashed) + close(marker_fd); + if (previously_crashed) { + state->status = STATUS_RECOVERY; + state->recovery_nonce = now; + state->recovery_time = now; + } else { + state->status = STATUS_NORMAL; + } + log_init(&state->log); // Initialize early so node_log can read log.count + node_log(state, "INIT", "nodes=%d%s", state->num_nodes, previously_crashed ? " (recovering)" : ""); + + client_table_init(&state->client_table); + state->next_client_tag = NODE_LIMIT; // Make sure they don't overlap with node indices + + meta_store_init(&state->metastore); + if (chunk_store_init(&state->chunk_store, chunks_path) < 0) { + fprintf(stderr, "Node :: Couldn't initialize chunk store at '%s'\n", chunks_path); + return -1; + } + + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP context\n"); + return -1; + } + + int ret = tcp_listen(&state->tcp, state->self_addr); + if (ret < 0) { + fprintf(stderr, "Node :: Couldn't setup TCP listener\n"); + tcp_context_free(&state->tcp); + return -1; + } + + // Write the boot marker to disk so that future restarts can detect + // a previous crash. This must happen after TCP init so that the + // marker is only written if the node successfully started. + if (!previously_crashed) { + int fd = open("vsr_boot_marker", O_WRONLY | O_CREAT, 0644); + if (fd >= 0) + close(fd); + } + + if (previously_crashed) { + node_log(state, "STATUS RECOVERY", "nonce=%lu (crash detected)", state->recovery_nonce); + + // Broadcast RECOVERY to all peers to learn the current view + RecoveryMessage recovery_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY, + .length = sizeof(RecoveryMessage), + }, + .sender_idx = self_idx(state), + .nonce = state->recovery_nonce, + }; + broadcast_to_peers(state, &recovery_message.base); + node_log(state, "SEND RECOVERY", "to=* nonce=%lu", state->recovery_nonce); + + nearest_deadline(&deadline, state->recovery_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL); + } + + *timeout = deadline_to_timeout(deadline, now); + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "Node :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int server_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) +{ + ServerState *state = state_; + + state->now = get_current_time(); + if (state->now == INVALID_TIME) + return -1; + + ///////////////////////////////////////////////////////////////// + // NETWORK EVENTS + ///////////////////////////////////////////////////////////////// + + Event events[TCP_EVENT_CAPACITY]; + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); + + for (int i = 0; i < num_events; i++) { + + if (events[i].type != EVENT_MESSAGE) + continue; + + int conn_idx = events[i].conn_idx; + for (;;) { + + ByteView msg; + uint16_t msg_type; + int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type); + if (ret == 0) + break; + if (ret < 0) { + tcp_close(&state->tcp, conn_idx); + break; + } + + HandlerResult hret = process_message(state, conn_idx, msg_type, msg); + if (hret == HR_INVALID_MESSAGE) { + tcp_close(&state->tcp, conn_idx); + break; + } + if (hret == HR_OUT_OF_MEMORY) + return -1; + assert(hret == HR_OK); + + tcp_consume_message(&state->tcp, conn_idx); + } + } + + ///////////////////////////////////////////////////////////////// + // TIME EVENTS + ///////////////////////////////////////////////////////////////// + + Time deadline = INVALID_TIME; + + if (state->status == STATUS_RECOVERY) { + // Recovery handling runs regardless of leader/replica position, + // since a recovering node must not act as leader until it learns + // the current view from its peers. + Time recovery_deadline = state->recovery_time + RECOVERY_TIMEOUT_SEC * 1000000000ULL; + if (recovery_deadline <= state->now) { + node_log_simple(state, "TIMEOUT RECOVERY"); + + RecoveryMessage recovery_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_RECOVERY, + .length = sizeof(RecoveryMessage), + }, + .sender_idx = self_idx(state), + .nonce = state->recovery_nonce, + }; + broadcast_to_peers(state, &recovery_message.base); + node_log(state, "SEND RECOVERY", "to=* nonce=%lu", state->recovery_nonce); + + // Don't reset recovery_votes or recovery_log_view here. + // The nonce is unchanged across retries, so responses from + // earlier rounds are still valid. Resetting votes would + // discard valid responses that took longer than one timeout + // interval to arrive (e.g. large log transfers from the + // primary over a slow link). + state->recovery_time = state->now; + + } else { + nearest_deadline(&deadline, recovery_deadline); + } + } else if (state->status == STATUS_CHANGE_VIEW) { + + Time view_change_deadline = state->heartbeat + VIEW_CHANGE_TIMEOUT_SEC * 1000000000ULL; + if (view_change_deadline <= state->now) { + + node_log_simple(state, "TIMEOUT CHANGE_VIEW"); + + clear_view_change_fields(state); + + add_vote(&state->view_change_begin_votes, self_idx(state)); + + state->view_number++; + state->heartbeat = state->now; + + BeginViewChangeMessage begin_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = state->view_number, + .sender_idx = self_idx(state), + }; + node_log(state, "SEND BEGIN_VIEW_CHG", "to=* view=%lu", state->view_number); + broadcast_to_peers(state, &begin_view_change_message.base); + + } else { + nearest_deadline(&deadline, view_change_deadline); + } + } else { + assert(state->status == STATUS_NORMAL); + + if (is_leader(state)) { + Time heartbeat_deadline = state->heartbeat + HEARTBEAT_INTERVAL_SEC * 1000000000ULL; + if (heartbeat_deadline <= state->now) { // TODO: check the time conversion here + + CommitMessage commit_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_COMMIT, + .length = sizeof(CommitMessage), + }, + .view_number = state->view_number, + .sender_idx = self_idx(state), + .commit_index = state->commit_index, + }; + broadcast_to_peers(state, &commit_message.base); + node_log(state, "SEND COMMIT", "to=* commit=%d", state->commit_index); + + state->heartbeat = state->now; + + } else { + nearest_deadline(&deadline, heartbeat_deadline); + } + } else { + Time death_deadline = state->heartbeat + PRIMARY_DEATH_TIMEOUT_SEC * 1000000000ULL; + if (death_deadline <= state->now) { + + node_log_simple(state, "TIMEOUT PRIMARY_DEATH"); + + clear_view_change_fields(state); + + add_vote(&state->view_change_begin_votes, self_idx(state)); + + state->view_number++; + state->status = STATUS_CHANGE_VIEW; + state->heartbeat = state->now; + + BeginViewChangeMessage begin_view_change_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + .length = sizeof(BeginViewChangeMessage), + }, + .view_number = state->view_number, + .sender_idx = self_idx(state), + }; + node_log(state, "SEND BEGIN_VIEW_CHG", "to=* view=%lu", state->view_number); + broadcast_to_peers(state, &begin_view_change_message.base); + + node_log(state, "STATUS CHANGE_VIEW", "view=%lu", state->view_number); + + } else { + nearest_deadline(&deadline, death_deadline); + } + } + } + + // State transfer retry: if we're waiting for missing log entries + // and the timeout has elapsed, re-send GET_STATE to the primary. + if (state->state_transfer_pending) { + + Time st_deadline = state->state_transfer_time + STATE_TRANSFER_TIMEOUT_SEC * 1000000000ULL; + if (st_deadline <= state->now) { + node_log(state, "TIMEOUT STATE_TRANSFER", "op=%d", state->log.count); + + GetStateMessage get_state_message = { + .base = { + .version = MESSAGE_VERSION, + .type = MESSAGE_TYPE_GET_STATE, + .length = sizeof(GetStateMessage), + }, + .view_number = state->view_number, + .op_number = state->log.count, + .sender_idx = self_idx(state), + }; + send_to_peer(state, leader_idx(state), &get_state_message.base); + node_log(state, "SEND GET_STATE", "to=%d op=%d", leader_idx(state), state->log.count); + + state->state_transfer_time = state->now; + + } else { + nearest_deadline(&deadline, st_deadline); + } + } + + *timeout = deadline_to_timeout(deadline, state->now); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int server_free(void *state_) +{ + ServerState *state = state_; + + node_log_simple(state, "CRASHED"); + + log_free(&state->log); + log_free(&state->recovery_log); + log_free(&state->view_change_log); + tcp_context_free(&state->tcp); + client_table_free(&state->client_table); + meta_store_free(&state->metastore); + chunk_store_free(&state->chunk_store); + return 0; +} + diff --git a/src/server.h b/src/server.h new file mode 100644 index 0000000..741ec79 --- /dev/null +++ b/src/server.h @@ -0,0 +1,301 @@ +#ifndef NODE_INCLUDED +#define NODE_INCLUDED + +#include +#include +#include + +#include "log.h" +#include "config.h" +#include "metadata.h" +#include "chunk_store.h" +#include "client_table.h" + +enum { + + // Normal Protocol + MESSAGE_TYPE_REQUEST, + MESSAGE_TYPE_REPLY, + MESSAGE_TYPE_PREPARE, + MESSAGE_TYPE_PREPARE_OK, + MESSAGE_TYPE_COMMIT, + + // View Change Protocol + MESSAGE_TYPE_BEGIN_VIEW_CHANGE, + MESSAGE_TYPE_DO_VIEW_CHANGE, + MESSAGE_TYPE_BEGIN_VIEW, + + // Recovery Protocol + MESSAGE_TYPE_RECOVERY, + MESSAGE_TYPE_RECOVERY_RESPONSE, + + // State Transfer Protocol + MESSAGE_TYPE_GET_STATE, + MESSAGE_TYPE_NEW_STATE, + + // Client Redirect + MESSAGE_TYPE_REDIRECT, + + // Chunk Storage Protocol (bypasses log) + MESSAGE_TYPE_STORE_CHUNK, + MESSAGE_TYPE_STORE_CHUNK_ACK, + MESSAGE_TYPE_FETCH_CHUNK, + MESSAGE_TYPE_FETCH_CHUNK_RESPONSE, + + // Blob Client Protocol + MESSAGE_TYPE_COMMIT_PUT, + MESSAGE_TYPE_GET_BLOB, + MESSAGE_TYPE_GET_BLOB_RESPONSE, +}; + +typedef struct { + MessageHeader base; + MetaOper oper; + uint64_t client_id; + uint64_t request_id; +} RequestMessage; + +typedef struct { + MessageHeader base; + MetaOper oper; + int sender_idx; + int log_index; + int commit_index; + uint64_t view_number; + uint64_t client_id; + uint64_t request_id; +} PrepareMessage; + +typedef struct { + MessageHeader base; + int sender_idx; + int log_index; + uint64_t view_number; +} PrepareOKMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int sender_idx; + int commit_index; +} CommitMessage; + +typedef struct { + MessageHeader base; + bool rejected; + MetaResult result; + uint64_t request_id; +} ReplyMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int sender_idx; +} BeginViewChangeMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; // The new view number + uint64_t old_view_number; // Last view number when replica was in normal status + int op_number; // Number of entries in the log + int commit_index; + int sender_idx; + // Followed by: LogEntry log[op_number] +} DoViewChangeMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int commit_index; + int op_number; // Number of log entries that follow + // Followed by: LogEntry log[op_number] +} BeginViewMessage; + +typedef struct { + MessageHeader base; + int sender_idx; + uint64_t nonce; +} RecoveryMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int op_number; + uint64_t nonce; + int commit_index; + int sender_idx; +} RecoveryResponseMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int op_number; // Requester's current log count + int sender_idx; +} GetStateMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; + int op_number; // Number of log entries that follow + int commit_index; + int start_index; // Global log index of the first entry in the suffix + // Followed by: LogEntry log[op_number] +} NewStateMessage; + +typedef struct { + MessageHeader base; + uint64_t view_number; +} RedirectMessage; + +// StoreChunk: client -> any server. Carries chunk data as trailing bytes. +typedef struct { + MessageHeader base; + SHA256 hash; + uint32_t size; + // Followed by: uint8_t data[size] +} StoreChunkMessage; + +// StoreChunkAck: server -> client. +typedef struct { + MessageHeader base; + SHA256 hash; + bool success; +} StoreChunkAckMessage; + +// FetchChunk: client/server -> server. Request a chunk by hash. +typedef struct { + MessageHeader base; + SHA256 hash; + int sender_idx; // -1 if from a client +} FetchChunkMessage; + +// FetchChunkResponse: server -> client/server. Chunk data as trailing bytes. +typedef struct { + MessageHeader base; + SHA256 hash; + uint32_t size; // 0 if chunk not found + // Followed by: uint8_t data[size] +} FetchChunkResponseMessage; + +// CommitPut: blob client -> leader. Commits metadata after chunks uploaded. +// Processed like a REQUEST (goes through VSR log). +typedef struct { + MessageHeader base; + MetaOper oper; + uint64_t client_id; + uint64_t request_id; +} CommitPutMessage; + +// GetBlob: client -> any server. Request metadata for a blob. +typedef struct { + MessageHeader base; + char bucket[META_BUCKET_MAX]; + char key[META_KEY_MAX]; +} GetBlobMessage; + +// GetBlobResponse: server -> client. Returns blob metadata. +typedef struct { + MessageHeader base; + bool found; + uint64_t size; + SHA256 content_hash; + uint32_t num_chunks; + ChunkRef chunks[META_CHUNKS_MAX]; +} GetBlobResponseMessage; + +typedef enum { + STATUS_NORMAL, + STATUS_CHANGE_VIEW, + STATUS_RECOVERY, +} Status; + +typedef struct { + + TCP tcp; + + Address self_addr; + Address node_addrs[NODE_LIMIT]; + int num_nodes; + + Status status; + + ClientTable client_table; + int next_client_tag; + + uint64_t view_number; + uint64_t last_normal_view; // Latest view where status was NORMAL + + // These fields are used in recovery mode + uint32_t recovery_votes; + uint64_t recovery_nonce; + uint64_t recovery_view; + Log recovery_log; + uint64_t recovery_log_view; + Time recovery_time; + int recovery_commit; + + /////////////////////////////////////////////////////////// + // VIEW CHANGE + + uint32_t view_change_begin_votes; + uint32_t view_change_apply_votes; + Log view_change_log; // Best log seen + uint64_t view_change_old_view; // Best old_view_number seen in DoViewChange + int view_change_commit; // Best commit_index seen + + /////////////////////////////////////////////////////////// + + // If this is the leader, commit_index is the index of the next uncommitted element + // of the log (if no uncommitted elements are present, it's equal to the total number + // of log elements) + int commit_index; + + PrepareMessage future[FUTURE_LIMIT]; + int num_future; + + bool state_transfer_pending; + Time state_transfer_time; + + Log log; + + Time heartbeat; + + MetaStore metastore; + ChunkStore chunk_store; + + // Set at each wakeup + Time now; + +} ServerState; + +struct pollfd; + +int server_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int server_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int server_free(void *state); + +typedef struct { + int last_min_commit; + int last_max_commit; + Status prev_status[NODE_LIMIT]; + + // External shadow log of committed operations (unbounded, dynamically allocated) + MetaOper *shadow_log; + int shadow_count; + int shadow_capacity; +} InvariantChecker; + +void invariant_checker_init(InvariantChecker *ic); +void invariant_checker_free(InvariantChecker *ic); + +// node_handles: opaque QuakeyNode handles for entering host context. +// Pass NULL when running outside the simulation (real mode). +void invariant_checker_run(InvariantChecker *ic, ServerState **nodes, int num_nodes, + unsigned long long *node_handles); + +#endif // NODE_INCLUDED diff --git a/src/sha256.c b/src/sha256.c deleted file mode 100644 index 70196ab..0000000 --- a/src/sha256.c +++ /dev/null @@ -1,249 +0,0 @@ -#include "sha256.h" - -//usr/bin/env clang -Ofast -Wall -Wextra -pedantic ${0} -o ${0%%.c*} $* ;exit $? -// -// SHA-256 implementation, Mark 2 -// -// Copyright (c) 2010,2014 Literatecode, http://www.literatecode.com -// Copyright (c) 2022 Ilia Levin (ilia@levin.sg) -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -// - -#define SHA256_SIZE_BYTES (32) - -typedef struct { - uint8_t buf[64]; - uint32_t hash[8]; - uint32_t bits[2]; - uint32_t len; - uint32_t rfu__; - uint32_t W[64]; -} sha256_context; - -#ifndef _cbmc_ -#define __CPROVER_assume(...) do {} while(0) -#endif - -#define FN_ static inline __attribute__((const)) - -static const uint32_t K[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -}; - -FN_ uint8_t _shb(uint32_t x, uint32_t n) -{ - return ((x >> (n & 31)) & 0xff); -} - -FN_ uint32_t _shw(uint32_t x, uint32_t n) -{ - return ((x << (n & 31)) & 0xffffffff); -} - -FN_ uint32_t _r(uint32_t x, uint8_t n) -{ - return ((x >> n) | _shw(x, 32 - n)); -} - -FN_ uint32_t _Ch(uint32_t x, uint32_t y, uint32_t z) -{ - return ((x & y) ^ ((~x) & z)); -} - -FN_ uint32_t _Ma(uint32_t x, uint32_t y, uint32_t z) -{ - return ((x & y) ^ (x & z) ^ (y & z)); -} - -FN_ uint32_t _S0(uint32_t x) -{ - return (_r(x, 2) ^ _r(x, 13) ^ _r(x, 22)); -} - -FN_ uint32_t _S1(uint32_t x) -{ - return (_r(x, 6) ^ _r(x, 11) ^ _r(x, 25)); -} - -FN_ uint32_t _G0(uint32_t x) -{ - return (_r(x, 7) ^ _r(x, 18) ^ (x >> 3)); -} - -FN_ uint32_t _G1(uint32_t x) -{ - return (_r(x, 17) ^ _r(x, 19) ^ (x >> 10)); -} - -FN_ uint32_t _word(uint8_t *c) -{ - return (_shw(c[0], 24) | _shw(c[1], 16) | _shw(c[2], 8) | (c[3])); -} - -static void _addbits(sha256_context *ctx, uint32_t n) -{ - __CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx)); - - if (ctx->bits[0] > (0xffffffff - n)) { - ctx->bits[1] = (ctx->bits[1] + 1) & 0xFFFFFFFF; - } - ctx->bits[0] = (ctx->bits[0] + n) & 0xFFFFFFFF; -} // _addbits - -static void _hash(sha256_context *ctx) -{ - __CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx)); - - register uint32_t a, b, c, d, e, f, g, h; - uint32_t t[2]; - - a = ctx->hash[0]; - b = ctx->hash[1]; - c = ctx->hash[2]; - d = ctx->hash[3]; - e = ctx->hash[4]; - f = ctx->hash[5]; - g = ctx->hash[6]; - h = ctx->hash[7]; - - for (uint32_t i = 0; i < 64; i++) { - if (i < 16) { - ctx->W[i] = _word(&ctx->buf[_shw(i, 2)]); - } else { - ctx->W[i] = _G1(ctx->W[i - 2]) + ctx->W[i - 7] + - _G0(ctx->W[i - 15]) + ctx->W[i - 16]; - } - - t[0] = h + _S1(e) + _Ch(e, f, g) + K[i] + ctx->W[i]; - t[1] = _S0(a) + _Ma(a, b, c); - h = g; - g = f; - f = e; - e = d + t[0]; - d = c; - c = b; - b = a; - a = t[0] + t[1]; - } - - ctx->hash[0] += a; - ctx->hash[1] += b; - ctx->hash[2] += c; - ctx->hash[3] += d; - ctx->hash[4] += e; - ctx->hash[5] += f; - ctx->hash[6] += g; - ctx->hash[7] += h; -} - -static void sha256_init(sha256_context *ctx) -{ - if (ctx != NULL) { - ctx->bits[0] = ctx->bits[1] = ctx->len = 0; - ctx->hash[0] = 0x6a09e667; - ctx->hash[1] = 0xbb67ae85; - ctx->hash[2] = 0x3c6ef372; - ctx->hash[3] = 0xa54ff53a; - ctx->hash[4] = 0x510e527f; - ctx->hash[5] = 0x9b05688c; - ctx->hash[6] = 0x1f83d9ab; - ctx->hash[7] = 0x5be0cd19; - } -} - -static void sha256_hash(sha256_context *ctx, const void *data, size_t len) -{ - const uint8_t *bytes = (const uint8_t *)data; - - if ((ctx != NULL) && (bytes != NULL) && (ctx->len < sizeof(ctx->buf))) { - __CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(bytes)); - __CPROVER_assume(__CPROVER_DYNAMIC_OBJECT(ctx)); - for (size_t i = 0; i < len; i++) { - ctx->buf[ctx->len++] = bytes[i]; - if (ctx->len == sizeof(ctx->buf)) { - _hash(ctx); - _addbits(ctx, sizeof(ctx->buf) * 8); - ctx->len = 0; - } - } - } -} - -static void sha256_done(sha256_context *ctx, uint8_t *hash) -{ - register uint32_t i, j; - - if (ctx != NULL) { - j = ctx->len % sizeof(ctx->buf); - ctx->buf[j] = 0x80; - for (i = j + 1; i < sizeof(ctx->buf); i++) { - ctx->buf[i] = 0x00; - } - - if (ctx->len > 55) { - _hash(ctx); - for (j = 0; j < sizeof(ctx->buf); j++) { - ctx->buf[j] = 0x00; - } - } - - _addbits(ctx, ctx->len * 8); - ctx->buf[63] = _shb(ctx->bits[0], 0); - ctx->buf[62] = _shb(ctx->bits[0], 8); - ctx->buf[61] = _shb(ctx->bits[0], 16); - ctx->buf[60] = _shb(ctx->bits[0], 24); - ctx->buf[59] = _shb(ctx->bits[1], 0); - ctx->buf[58] = _shb(ctx->bits[1], 8); - ctx->buf[57] = _shb(ctx->bits[1], 16); - ctx->buf[56] = _shb(ctx->bits[1], 24); - _hash(ctx); - - if (hash != NULL) { - for (i = 0, j = 24; i < 4; i++, j -= 8) { - hash[i + 0] = _shb(ctx->hash[0], j); - hash[i + 4] = _shb(ctx->hash[1], j); - hash[i + 8] = _shb(ctx->hash[2], j); - hash[i + 12] = _shb(ctx->hash[3], j); - hash[i + 16] = _shb(ctx->hash[4], j); - hash[i + 20] = _shb(ctx->hash[5], j); - hash[i + 24] = _shb(ctx->hash[6], j); - hash[i + 28] = _shb(ctx->hash[7], j); - } - } - } -} - -void sha256(const void *data, size_t len, uint8_t *hash) -{ - sha256_context ctx; - - sha256_init(&ctx); - sha256_hash(&ctx, data, len); - sha256_done(&ctx, hash); -} diff --git a/src/sha256.h b/src/sha256.h deleted file mode 100644 index ce317f2..0000000 --- a/src/sha256.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef SHA256_INCLUDED -#define SHA256_INCLUDED - -#include -#include - -void sha256(const void *data, size_t len, uint8_t *hash); - -#endif // SHA256_INCLUDED diff --git a/src/test_client.c b/src/test_client.c deleted file mode 100644 index 75e697b..0000000 --- a/src/test_client.c +++ /dev/null @@ -1,640 +0,0 @@ -#ifdef MAIN_TEST - -#define QUAKEY_ENABLE_MOCKS -#include -#include - -#include "tcp.h" -#include "test_client.h" - -// Helper function to parse address and port from command line -static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port) -{ - // Default to metadata server - *addr = "127.0.0.1"; - *port = 8080; - - for (int i = 0; i < argc - 1; i++) { - if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) { - *addr = argv[i + 1]; - if (i + 2 < argc) { - - errno = 0; - char *end; - long val = strtol(argv[i+2], &end, 10); - - if (end == argv[i+2] || *end != '\0' || errno == ERANGE) - break; - - if (val < 0 || val > UINT16_MAX) - break; - - *port = (uint16_t) val; - return true; - } - } - } - return true; -} - -int test_client_init(void *state_, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - TestClient *client = state_; - - char *addr; - uint16_t port; - parse_server_addr(argc, argv, &addr, &port); - - client->toasty = toasty_connect((ToastyString) { addr, strlen(addr) }, port); - if (client->toasty == NULL) - return -1; - - client->state = TEST_CLIENT_STATE_INIT; - client->tick = 0; - client->tests_passed = 0; - - printf("Client set up (remote=%s:%d)\n", addr, port); - fflush(stdout); - - *timeout = -1; - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); - return 0; -} - -// Static test data -static char lorem_ipsum[] = - "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " - "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut " - "enim ad minim veniam, quis nostrud exercitation ullamco laboris " - "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor " - "in reprehenderit in voluptate velit esse cillum dolore eu fugiat " - "nulla pariatur. Excepteur sint occaecat cupidatat non proident, " - "sunt in culpa qui officia deserunt mollit anim id est laborum."; - -static char short_msg[] = "Hello World!"; -static char offset_msg[] = "INSERTED_DATA"; -static char truncate_msg[] = "Truncated content"; - -// Test path constants -static ToastyString file_path = TOASTY_STR("some_file.txt"); -static ToastyString dir_path = TOASTY_STR("/testdir"); -static ToastyString subdir_path = TOASTY_STR("/testdir/subdir"); -static ToastyString file_in_dir_path = TOASTY_STR("/testdir/nested_file.txt"); -static ToastyString auto_create_path = TOASTY_STR("/auto_created_file.txt"); -static ToastyString truncate_file_path = TOASTY_STR("/truncate_test.txt"); -static ToastyString offset_file_path = TOASTY_STR("/offset_test.txt"); - -#define TEST_PASS(name) do { \ - client->tests_passed++; \ - printf("Test PASSED: %s\n", name); \ - fflush(stdout); \ -} while(0) - -#define TEST_FAIL(name) do { \ - printf("Test FAILED: %s\n", name); \ - fflush(stdout); \ - exit(1); \ -} while(0) - -int test_client_tick(void *state_, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, - int *timeout) -{ - TestClient *client = state_; - - // Process any pending events from the network and get new poll descriptors - *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); - - int ret; - ToastyResult result; - - switch (client->state) { - - //========================================================================== - // PHASE 1: Basic file create/write/read test - //========================================================================== - case TEST_CLIENT_STATE_INIT: - if (client->tick < 10) { - *timeout = 0; - client->tick++; - return 0; - } - printf("\n=== Phase 1: Basic file operations ===\n"); - client->handle = toasty_begin_create_file(client->toasty, file_path, 128); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create file"); - } - printf("Create file started\n"); - client->state = TEST_CLIENT_STATE_CREATE_FILE_WAIT; - break; - - case TEST_CLIENT_STATE_CREATE_FILE_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create file"); - if (ret == 1) break; // Still in progress - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create file result"); - } - TEST_PASS("Create file"); - toasty_free_result(&result); - - // Start write - client->handle = toasty_begin_write(client->toasty, file_path, 0, - lorem_ipsum, sizeof(lorem_ipsum)-1, TOASTY_VERSION_TAG_EMPTY, 0); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin write"); - } - printf("Write started\n"); - client->state = TEST_CLIENT_STATE_WRITE_WAIT; - break; - - case TEST_CLIENT_STATE_WRITE_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for write"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_WRITE_SUCCESS) { - TEST_FAIL("write result"); - } - TEST_PASS("Write data"); - toasty_free_result(&result); - - // Start read - client->handle = toasty_begin_read(client->toasty, file_path, 0, - client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin read"); - } - printf("Read started\n"); - client->state = TEST_CLIENT_STATE_READ_WAIT; - break; - - case TEST_CLIENT_STATE_READ_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for read"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_READ_SUCCESS) { - TEST_FAIL("read result type"); - } - if (result.bytes_read != sizeof(lorem_ipsum)-1) { - printf("Expected %zu bytes, got %d\n", sizeof(lorem_ipsum)-1, result.bytes_read); - TEST_FAIL("read bytes count"); - } - if (memcmp(client->buf, lorem_ipsum, sizeof(lorem_ipsum)-1)) { - TEST_FAIL("read data mismatch"); - } - TEST_PASS("Read data matches written data"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_CREATE_DIR; - *timeout = 0; // Trigger immediate next iteration - return 0; - - //========================================================================== - // PHASE 2: Directory operations - //========================================================================== - case TEST_CLIENT_STATE_CREATE_DIR: - printf("\n=== Phase 2: Directory operations ===\n"); - client->handle = toasty_begin_create_dir(client->toasty, dir_path); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create dir"); - } - printf("Create directory started\n"); - client->state = TEST_CLIENT_STATE_CREATE_DIR_WAIT; - break; - - case TEST_CLIENT_STATE_CREATE_DIR_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create dir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create dir result"); - } - TEST_PASS("Create directory"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_CREATE_SUBDIR; - *timeout = 0; return 0; - - case TEST_CLIENT_STATE_CREATE_SUBDIR: - client->handle = toasty_begin_create_dir(client->toasty, subdir_path); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create subdir"); - } - printf("Create subdirectory started\n"); - client->state = TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT; - break; - - case TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create subdir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create subdir result"); - } - TEST_PASS("Create subdirectory"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_CREATE_FILE_IN_DIR; - *timeout = 0; return 0; - - case TEST_CLIENT_STATE_CREATE_FILE_IN_DIR: - client->handle = toasty_begin_create_file(client->toasty, file_in_dir_path, 256); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create file in dir"); - } - printf("Create file in directory started\n"); - client->state = TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT; - break; - - case TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create file in dir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create file in dir result"); - } - TEST_PASS("Create file in directory"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_LIST_DIR; - *timeout = 0; return 0; - - case TEST_CLIENT_STATE_LIST_DIR: - client->handle = toasty_begin_list(client->toasty, dir_path, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin list dir"); - } - printf("List directory started\n"); - client->state = TEST_CLIENT_STATE_LIST_DIR_WAIT; - break; - - case TEST_CLIENT_STATE_LIST_DIR_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for list dir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_LIST_SUCCESS) { - TEST_FAIL("list dir result type"); - } - printf("Directory listing count: %d\n", result.listing.count); - // Directory should contain subdir and nested_file.txt - if (result.listing.count != 2) { - printf("Expected 2 entries, got %d\n", result.listing.count); - TEST_FAIL("list dir count"); - } - // Print the entries - for (int i = 0; i < result.listing.count; i++) { - printf(" Entry %d: %s (is_dir=%d)\n", i, - result.listing.items[i].name, - result.listing.items[i].is_dir); - } - TEST_PASS("List directory"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_LIST_ROOT; - *timeout = 0; return 0; - - case TEST_CLIENT_STATE_LIST_ROOT: - client->handle = toasty_begin_list(client->toasty, TOASTY_STR("/"), TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin list root"); - } - printf("List root directory started\n"); - client->state = TEST_CLIENT_STATE_LIST_ROOT_WAIT; - break; - - case TEST_CLIENT_STATE_LIST_ROOT_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for list root"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_LIST_SUCCESS) { - TEST_FAIL("list root result type"); - } - printf("Root listing count: %d\n", result.listing.count); - for (int i = 0; i < result.listing.count; i++) { - printf(" Entry %d: %s (is_dir=%d)\n", i, - result.listing.items[i].name, - result.listing.items[i].is_dir); - } - // Root should have some entries at this point - if (result.listing.count < 1) { - TEST_FAIL("list root count"); - } - TEST_PASS("List root directory"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING; - *timeout = 0; return 0; - - //========================================================================== - // PHASE 3: Write flags tests - //========================================================================== - case TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING: - printf("\n=== Phase 3: Additional write tests ===\n"); - // Create a file for testing additional writes - client->handle = toasty_begin_create_file(client->toasty, auto_create_path, 256); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create auto file"); - } - printf("Create auto file started\n"); - client->state = TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT; - break; - - case TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create auto file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create auto file result"); - } - TEST_PASS("Create auto file"); - toasty_free_result(&result); - - // Now write to the file - client->handle = toasty_begin_write(client->toasty, auto_create_path, 0, - short_msg, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY, 0); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin write to auto file"); - } - printf("Write to auto file started\n"); - client->state = TEST_CLIENT_STATE_READ_CREATED_FILE; - break; - - case TEST_CLIENT_STATE_READ_CREATED_FILE: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for write to auto file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_WRITE_SUCCESS) { - TEST_FAIL("write to auto file result"); - } - TEST_PASS("Write to auto file"); - toasty_free_result(&result); - - // Read back the file - memset(client->buf, 0, sizeof(client->buf)); - client->handle = toasty_begin_read(client->toasty, auto_create_path, 0, - client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin read auto file"); - } - printf("Read auto file started\n"); - client->state = TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT; - break; - - case TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for read auto file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_READ_SUCCESS) { - TEST_FAIL("read auto file result type"); - } - if (result.bytes_read != sizeof(short_msg)-1) { - printf("Expected %zu bytes, got %d\n", sizeof(short_msg)-1, result.bytes_read); - TEST_FAIL("read auto file bytes count"); - } - if (memcmp(client->buf, short_msg, sizeof(short_msg)-1)) { - TEST_FAIL("read auto file data mismatch"); - } - TEST_PASS("Read auto file"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_WRITE_TRUNCATE; - *timeout = 0; return 0; - - case TEST_CLIENT_STATE_WRITE_TRUNCATE: - // Create a file for testing multiple writes - client->handle = toasty_begin_create_file(client->toasty, truncate_file_path, 256); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create multi-write file"); - } - printf("Create multi-write test file started\n"); - client->state = TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT; - break; - - case TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create multi-write file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create multi-write file result"); - } - TEST_PASS("Create multi-write file"); - toasty_free_result(&result); - - // Write initial content - client->handle = toasty_begin_write(client->toasty, truncate_file_path, 0, - lorem_ipsum, sizeof(lorem_ipsum)-1, TOASTY_VERSION_TAG_EMPTY, 0); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin write multi-write content"); - } - printf("Write multi-write content started\n"); - client->state = TEST_CLIENT_STATE_READ_TRUNCATED; - break; - - case TEST_CLIENT_STATE_READ_TRUNCATED: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for write multi-write content"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_WRITE_SUCCESS) { - TEST_FAIL("write multi-write content result"); - } - TEST_PASS("Write multi-write content"); - toasty_free_result(&result); - - // Read back the file - memset(client->buf, 0, sizeof(client->buf)); - client->handle = toasty_begin_read(client->toasty, truncate_file_path, 0, - client->buf, sizeof(client->buf), TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin read multi-write file"); - } - printf("Read multi-write file started\n"); - client->state = TEST_CLIENT_STATE_READ_TRUNCATED_WAIT; - break; - - case TEST_CLIENT_STATE_READ_TRUNCATED_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for read multi-write file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_READ_SUCCESS) { - TEST_FAIL("read multi-write file result type"); - } - if (result.bytes_read != sizeof(lorem_ipsum)-1) { - printf("Expected %zu bytes, got %d\n", sizeof(lorem_ipsum)-1, result.bytes_read); - TEST_FAIL("read multi-write file bytes count"); - } - TEST_PASS("Read multi-write file"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_WRITE_AT_OFFSET; - *timeout = 0; return 0; - - //========================================================================== - // PHASE 4: Offset read/write tests - //========================================================================== - case TEST_CLIENT_STATE_WRITE_AT_OFFSET: - printf("\n=== Phase 4: Offset read/write tests ===\n"); - // Create a new file for offset testing with larger chunk size - client->handle = toasty_begin_create_file(client->toasty, offset_file_path, 256); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin create offset file"); - } - printf("Create offset test file started\n"); - client->state = TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT; - break; - - case TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for create offset file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - TEST_FAIL("create offset file result"); - } - TEST_PASS("Create offset file"); - toasty_free_result(&result); - - // Write initial data (small amount) - client->handle = toasty_begin_write(client->toasty, offset_file_path, 0, - short_msg, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY, 0); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin write initial offset data"); - } - printf("Write initial offset data started\n"); - client->state = TEST_CLIENT_STATE_READ_AT_OFFSET; - break; - - case TEST_CLIENT_STATE_READ_AT_OFFSET: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for write initial offset"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_WRITE_SUCCESS) { - TEST_FAIL("write initial offset result"); - } - TEST_PASS("Write initial offset data"); - toasty_free_result(&result); - - // Read from the beginning to verify - memset(client->buf, 0, sizeof(client->buf)); - client->handle = toasty_begin_read(client->toasty, offset_file_path, 0, - client->buf, sizeof(short_msg)-1, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin read offset file"); - } - printf("Read offset file started\n"); - client->state = TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT; - break; - - case TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for read offset file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_READ_SUCCESS) { - TEST_FAIL("read offset file result type"); - } - if (result.bytes_read != sizeof(short_msg)-1) { - printf("Expected %zu bytes, got %d\n", sizeof(short_msg)-1, result.bytes_read); - TEST_FAIL("read offset file bytes count"); - } - TEST_PASS("Read offset file"); - toasty_free_result(&result); - client->state = TEST_CLIENT_STATE_DELETE_FILE; - *timeout = 0; return 0; - - //========================================================================== - // PHASE 5: Delete operations - //========================================================================== - case TEST_CLIENT_STATE_DELETE_FILE: - printf("\n=== Phase 5: Delete operations ===\n"); - // Delete the auto-created file - client->handle = toasty_begin_delete(client->toasty, auto_create_path, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin delete file"); - } - printf("Delete auto file started\n"); - client->state = TEST_CLIENT_STATE_DELETE_FILE_WAIT; - break; - - case TEST_CLIENT_STATE_DELETE_FILE_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for delete file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_DELETE_SUCCESS) { - TEST_FAIL("delete file result"); - } - TEST_PASS("Delete file"); - toasty_free_result(&result); - - // Delete the nested file first (before deleting directory) - client->handle = toasty_begin_delete(client->toasty, file_in_dir_path, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin delete nested file"); - } - printf("Delete nested file started\n"); - client->state = TEST_CLIENT_STATE_DELETE_DIR; - break; - - case TEST_CLIENT_STATE_DELETE_DIR: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for delete nested file"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_DELETE_SUCCESS) { - TEST_FAIL("delete nested file result"); - } - TEST_PASS("Delete nested file"); - toasty_free_result(&result); - - // Delete the subdirectory - client->handle = toasty_begin_delete(client->toasty, subdir_path, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin delete subdir"); - } - printf("Delete subdirectory started\n"); - client->state = TEST_CLIENT_STATE_DELETE_DIR_WAIT; - break; - - case TEST_CLIENT_STATE_DELETE_DIR_WAIT: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for delete subdir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_DELETE_SUCCESS) { - TEST_FAIL("delete subdir result"); - } - TEST_PASS("Delete subdirectory"); - toasty_free_result(&result); - - // Delete the parent directory - client->handle = toasty_begin_delete(client->toasty, dir_path, TOASTY_VERSION_TAG_EMPTY); - if (client->handle == TOASTY_INVALID) { - TEST_FAIL("begin delete dir"); - } - printf("Delete directory started\n"); - client->state = TEST_CLIENT_STATE_DONE; - break; - - case TEST_CLIENT_STATE_DONE: - ret = toasty_get_result(client->toasty, client->handle, &result); - if (ret < 0) TEST_FAIL("get result for delete dir"); - if (ret == 1) break; - if (result.type != TOASTY_RESULT_DELETE_SUCCESS) { - TEST_FAIL("delete dir result"); - } - TEST_PASS("Delete directory"); - toasty_free_result(&result); - - printf("\n========================================\n"); - printf("All %d tests PASSED!\n", client->tests_passed); - printf("========================================\n"); - QUAKEY_SIGNAL("exit"); - } - - *timeout = -1; - if (pcap < TCP_POLL_CAPACITY) - return -1; - *pnum = toasty_process_events(client->toasty, ctxs, pdata, 0); - return 0; -} - -int test_client_free(void *state_) -{ - TestClient *client = state_; - - toasty_disconnect(client->toasty); - return 0; -} - -#endif // MAIN_TEST diff --git a/src/test_client.h b/src/test_client.h deleted file mode 100644 index aeac19b..0000000 --- a/src/test_client.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef TEST_CLIENT_INCLUDED -#define TEST_CLIENT_INCLUDED - -#include "ToastyFS.h" - -typedef enum { - // Basic file create/write/read test (existing) - TEST_CLIENT_STATE_INIT, - TEST_CLIENT_STATE_CREATE_FILE_WAIT, - TEST_CLIENT_STATE_WRITE_WAIT, - TEST_CLIENT_STATE_READ_WAIT, - - // Directory tests - TEST_CLIENT_STATE_CREATE_DIR, - TEST_CLIENT_STATE_CREATE_DIR_WAIT, - TEST_CLIENT_STATE_CREATE_SUBDIR, - TEST_CLIENT_STATE_CREATE_SUBDIR_WAIT, - TEST_CLIENT_STATE_CREATE_FILE_IN_DIR, - TEST_CLIENT_STATE_CREATE_FILE_IN_DIR_WAIT, - TEST_CLIENT_STATE_LIST_DIR, - TEST_CLIENT_STATE_LIST_DIR_WAIT, - TEST_CLIENT_STATE_LIST_ROOT, - TEST_CLIENT_STATE_LIST_ROOT_WAIT, - - // Delete tests - TEST_CLIENT_STATE_DELETE_FILE, - TEST_CLIENT_STATE_DELETE_FILE_WAIT, - TEST_CLIENT_STATE_DELETE_DIR, - TEST_CLIENT_STATE_DELETE_DIR_WAIT, - - // Write flags tests - TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING, - TEST_CLIENT_STATE_WRITE_CREATE_IF_MISSING_WAIT, - TEST_CLIENT_STATE_READ_CREATED_FILE, - TEST_CLIENT_STATE_READ_CREATED_FILE_WAIT, - TEST_CLIENT_STATE_WRITE_TRUNCATE, - TEST_CLIENT_STATE_WRITE_TRUNCATE_WAIT, - TEST_CLIENT_STATE_READ_TRUNCATED, - TEST_CLIENT_STATE_READ_TRUNCATED_WAIT, - - // Offset read/write tests - TEST_CLIENT_STATE_WRITE_AT_OFFSET, - TEST_CLIENT_STATE_WRITE_AT_OFFSET_WAIT, - TEST_CLIENT_STATE_READ_AT_OFFSET, - TEST_CLIENT_STATE_READ_AT_OFFSET_WAIT, - - // Test done - TEST_CLIENT_STATE_DONE, -} TestClientState; - -typedef struct { - ToastyFS* toasty; - ToastyHandle handle; - TestClientState state; - int tick; - int tests_passed; - char buf[1<<10]; - char buf2[1<<10]; // Secondary buffer for additional reads -} TestClient; - -struct pollfd; - -int test_client_init(void *state, int argc, char **argv, - void **ctxs, struct pollfd *pdata, int pcap, int *pnum, - int *timeout); - -int test_client_tick(void *state, void **ctxs, - struct pollfd *pdata, int pcap, int *pnum, int *timeout); - -int test_client_free(void *state); - -#endif // TEST_CLIENT_INCLUDED \ No newline at end of file diff --git a/src/wal.c b/src/wal.c deleted file mode 100644 index a4077da..0000000 --- a/src/wal.c +++ /dev/null @@ -1,630 +0,0 @@ - -#if defined(MAIN_SIMULATION) || defined(MAIN_TEST) -#define QUAKEY_ENABLE_MOCKS -#endif - -#include -#include - -#include "wal.h" -#include "file_tree.h" -#include "file_system.h" - -#define WAL_MAGIC 0xcafebebe -#define WAL_VERSION 1 - -typedef struct { - uint32_t magic; - uint32_t version; - uint64_t reserved; -} WALHeader; - -typedef enum { - WAL_ENTRY_CREATE, - WAL_ENTRY_DELETE, - WAL_ENTRY_WRITE, -} WALEntryType; - -typedef struct { - WALEntryType type; - - // create, delete, write - string path; - - // create, write - uint64_t chunk_size; - - // delete, write - uint64_t expect_gen; - - // create - bool is_dir; - - // write - uint64_t offset; - uint64_t length; - uint32_t num_chunks; - SHA256 *hashes; - -} WALEntry; - -static int write_exact(Handle handle, char *src, int len) -{ - int copied = 0; - while (copied < len) { - int ret = file_write(handle, src + copied, len - copied); - if (ret < 0) - return -1; - copied += ret; - } - return 0; -} - -typedef struct { - Handle handle; -} WriteSnapshotContext; - -static int -serialize_callback(char *src, int num, void *data) -{ - WriteSnapshotContext *wsc = data; - return write_exact(wsc->handle, src, num); -} - -static int write_snapshot(FileTree *file_tree, Handle handle) -{ - WriteSnapshotContext wsc; - wsc.handle = handle; - if (file_tree_serialize(file_tree, serialize_callback, &wsc) < 0) - return -1; - return 0; -} - -typedef struct { - Handle handle; -} ReadSnapshotContext; - -static int -deserialize_callback(char *dst, int num, void *data) -{ - ReadSnapshotContext *rsc = data; - int copied = 0; - while (copied < num) { - int ret = file_read(rsc->handle, dst + copied, num - copied); - if (ret < 0) - return -1; - if (ret == 0) - break; - copied += ret; - } - return copied; -} - -static int read_snapshot(FileTree *file_tree, Handle handle) -{ - ReadSnapshotContext rsc; - rsc.handle = handle; - int num = file_tree_deserialize(file_tree, deserialize_callback, &rsc); - if (num < 0) - return -1; - return num; -} - -static int swap_file(WAL *wal) -{ - // Create a temporary file path - char temp_path_buf[1<<10]; - if (wal->file_path.len + 5 > (int) sizeof(temp_path_buf)) - return -1; - - memcpy(temp_path_buf, wal->file_path.ptr, wal->file_path.len); - memcpy(temp_path_buf + wal->file_path.len, ".tmp", 4); - temp_path_buf[wal->file_path.len + 4] = '\0'; - - string temp_path = { temp_path_buf, wal->file_path.len + 4 }; - - // Create and open the temporary file - Handle temp_handle; - if (file_open(temp_path, &temp_handle) < 0) - return -1; - - // Write the WAL header - WALHeader header; - header.magic = WAL_MAGIC; - header.version = WAL_VERSION; - header.reserved = 0; - - if (write_exact(temp_handle, (char*) &header, sizeof(header)) < 0) { - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - - // Serialize the current file tree to the new file - if (write_snapshot(wal->file_tree, temp_handle) < 0) { - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - - // Sync the temporary file to ensure it's written to disk - if (file_sync(temp_handle) < 0) { - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - - // Lock the new file before switching - if (file_lock(temp_handle) < 0) { - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - - // Unlock and close the old file - file_unlock(wal->handle); - file_close(wal->handle); - - // Atomically rename the temporary file to replace the old file - // On Unix: rename() atomically replaces the destination - // On Windows: we use MoveFileEx with MOVEFILE_REPLACE_EXISTING for atomicity -#ifdef _WIN32 - // On Windows, use MoveFileEx for atomic replace - char old_path_zt[1<<10]; - if (wal->file_path.len >= (int) sizeof(old_path_zt)) { - file_unlock(temp_handle); - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - memcpy(old_path_zt, wal->file_path.ptr, wal->file_path.len); - old_path_zt[wal->file_path.len] = '\0'; - - WCHAR old_path_w[MAX_PATH]; - WCHAR temp_path_w[MAX_PATH]; - - if (!MultiByteToWideChar(CP_UTF8, 0, old_path_zt, -1, old_path_w, MAX_PATH) || - !MultiByteToWideChar(CP_UTF8, 0, temp_path_buf, -1, temp_path_w, MAX_PATH)) { - file_unlock(temp_handle); - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } - - // MOVEFILE_REPLACE_EXISTING allows atomic overwrite - if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) { - file_unlock(temp_handle); - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } -#else - // On Unix/Linux, rename() atomically replaces the destination - if (rename_file_or_dir(temp_path, wal->file_path) < 0) { - file_unlock(temp_handle); - file_close(temp_handle); - remove_file_or_dir(temp_path); - return -1; - } -#endif - - // Update the WAL to use the new file handle - wal->handle = temp_handle; - wal->entry_count = 0; - - return 0; -} - -static int read_exact(Handle handle, char *dst, int len) -{ - int copied = 0; - while (copied < len) { - int ret = file_read(handle, dst + copied, len - copied); - if (ret < 0) - return -1; - if (ret == 0) - return 0; // EOF - copied += ret; - } - return copied; -} - -static int read_u8(Handle handle, uint8_t *value) -{ - return read_exact(handle, (char*) value, sizeof(*value)); -} - -static int read_u16(Handle handle, uint16_t *value) -{ - return read_exact(handle, (char*) value, sizeof(*value)); -} - -static int read_u32(Handle handle, uint32_t *value) -{ - return read_exact(handle, (char*) value, sizeof(*value)); -} - -static int read_u64(Handle handle, uint64_t *value) -{ - return read_exact(handle, (char*) value, sizeof(*value)); -} - -static int next_entry(Handle handle, WALEntry *entry) -{ - // Initialize pointers to NULL for cleanup on error - entry->path.ptr = NULL; - entry->hashes = NULL; - - uint8_t type; - int ret = read_u8(handle, &type); - if (ret == 0) - return 0; // EOF - if (ret < 0) - return -1; - - entry->type = (WALEntryType) type; - - uint16_t path_len; - if (read_u16(handle, &path_len) <= 0) - return -1; - - // Dynamically allocate path buffer - char *path_buffer = malloc(path_len); - if (!path_buffer) - return -1; - - if (read_exact(handle, path_buffer, path_len) <= 0) { - free(path_buffer); - return -1; - } - - entry->path.ptr = path_buffer; - entry->path.len = path_len; - - switch (entry->type) { - case WAL_ENTRY_CREATE: - { - uint8_t is_dir; - if (read_u8(handle, &is_dir) <= 0) - goto cleanup_error; - entry->is_dir = is_dir; - - if (!is_dir) { - if (read_u64(handle, &entry->chunk_size) <= 0) - goto cleanup_error; - } else { - entry->chunk_size = 0; - } - } - break; - - case WAL_ENTRY_DELETE: - { - if (read_u64(handle, &entry->expect_gen) <= 0) - goto cleanup_error; - } - break; - - case WAL_ENTRY_WRITE: - { - if (read_u64(handle, &entry->expect_gen) <= 0) - goto cleanup_error; - if (read_u64(handle, &entry->offset) <= 0) - goto cleanup_error; - if (read_u64(handle, &entry->length) <= 0) - goto cleanup_error; - if (read_u32(handle, &entry->num_chunks) <= 0) - goto cleanup_error; - - // Dynamically allocate hash buffers - SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256)); - if (!hashes_buffer) { - goto cleanup_error; - } - - if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) { - free(hashes_buffer); - goto cleanup_error; - } - - entry->hashes = hashes_buffer; - } - break; - - default: - goto cleanup_error; - } - - return 1; - -cleanup_error: - if (entry->path.ptr) - free((char*) entry->path.ptr); - if (entry->hashes) - free(entry->hashes); - return -1; -} - -int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit) -{ - wal->entry_count = 0; - wal->entry_limit = entry_limit; - wal->file_tree = file_tree; - wal->file_path.ptr = NULL; - - // Copy file_path since the passed string may not have the same lifetime as WAL - char *path_copy = malloc(file_path.len); - if (!path_copy) - return -1; - memcpy(path_copy, file_path.ptr, file_path.len); - wal->file_path.ptr = path_copy; - wal->file_path.len = file_path.len; - - Handle handle; - if (file_open(file_path, &handle) < 0) - goto error_cleanup_path; - - if (file_lock(handle) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - // Check if the file is empty (newly created) and initialize it - size_t size; - if (file_size(handle, &size) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - if (size == 0) { - // Initialize a new WAL file - WALHeader header; - header.magic = WAL_MAGIC; - header.version = WAL_VERSION; - header.reserved = 0; - - if (write_exact(handle, (char*) &header, sizeof(header)) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - if (write_snapshot(file_tree, handle) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - if (file_sync(handle) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - // Reset to beginning after initialization - if (file_set_offset(handle, 0) < 0) { - file_close(handle); - goto error_cleanup_path; - } - } - - // Read file header - // NOTE: For now we don't worry about fixing endianess - WALHeader header; - for (int copied = 0; copied < (int) sizeof(header); ) { - int ret = file_read(handle, (char*) &header + copied, (int) sizeof(header) - copied); - if (ret <= 0) { - file_close(handle); // TODO: what happens if I close a file without unlocking it? - goto error_cleanup_path; - } - copied += ret; - } - - // Validate header fields - if (header.magic != WAL_MAGIC) { - file_close(handle); - goto error_cleanup_path; - } - if (header.version != WAL_VERSION) { - file_close(handle); - goto error_cleanup_path; - } - - // The read_snapshot function may read more - // bytes than necessary from the buffer, so - // we need to save our current position to - // later restore it to this offset plus what - // read_snapshot really consumed. - int saved_offset; - if (file_get_offset(handle, &saved_offset) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - int num = read_snapshot(file_tree, handle); - if (num < 0) { - file_close(handle); - goto error_cleanup_path; - } - - // Now restore the offset to the correct position. - if (num > INT_MAX - saved_offset) { - file_close(handle); - goto error_cleanup_path; - } - if (file_set_offset(handle, saved_offset + num) < 0) { - file_close(handle); - goto error_cleanup_path; - } - - WALEntry entry; - for (;;) { - - int ret = next_entry(handle, &entry); - if (ret == 0) - break; - if (ret < 0) { - file_close(handle); - goto error_cleanup_path; - } - assert(ret == 1); - - switch (entry.type) { - uint64_t gen; - case WAL_ENTRY_CREATE: - file_tree_create_entity(file_tree, entry.path, entry.is_dir, entry.chunk_size, &gen); - break; - case WAL_ENTRY_DELETE: - file_tree_delete_entity(file_tree, entry.path, entry.expect_gen); - break; - case WAL_ENTRY_WRITE: - // WAL replay: use false for truncate_after since truncation was already handled - file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.expect_gen, &gen, entry.hashes, NULL, NULL, false); - break; - default: - UNREACHABLE; - } - - // Free dynamically allocated fields from next_entry - if (entry.path.ptr) - free((char*) entry.path.ptr); - if (entry.hashes) - free(entry.hashes); - - wal->entry_count++; - } - - wal->handle = handle; - - if (wal->entry_count >= wal->entry_limit) { - if (swap_file(wal) < 0) { - goto error_cleanup_path; - } - } - return 0; - -error_cleanup_path: - free(path_copy); - return -1; -} - -void wal_close(WAL *wal) -{ - file_unlock(wal->handle); - file_close(wal->handle); - if (wal->file_path.ptr) - free((char*) wal->file_path.ptr); -} - -static int write_u8(Handle handle, uint8_t value) -{ - return write_exact(handle, (char*) &value, sizeof(value)); -} - -static int write_u16(Handle handle, uint16_t value) -{ - return write_exact(handle, (char*) &value, sizeof(value)); -} - -static int write_u32(Handle handle, uint32_t value) -{ - return write_exact(handle, (char*) &value, sizeof(value)); -} - -static int write_u64(Handle handle, uint64_t value) -{ - return write_exact(handle, (char*) &value, sizeof(value)); -} - -static int write_str(Handle handle, string value) -{ - return write_exact(handle, value.ptr, value.len); -} - -static int append_begin(WAL *wal) -{ - if (wal->entry_count >= wal->entry_limit) { - if (swap_file(wal) < 0) - return -1; - } - return 0; -} - -static int append_end(WAL *wal) -{ - if (file_sync(wal->handle) < 0) - return -1; - wal->entry_count++; - return 0; -} - -int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size) -{ - if (path.len > UINT16_MAX) - return -1; - - if (append_begin(wal) < 0) - return -1; - - write_u8(wal->handle, WAL_ENTRY_CREATE); - write_u16(wal->handle, path.len); - write_str(wal->handle, path); - write_u8(wal->handle, is_dir); - if (!is_dir) - write_u64(wal->handle, chunk_size); - - if (append_end(wal) < 0) - return -1; - - return 0; -} - -int wal_append_delete(WAL *wal, string path, uint64_t expect_gen) -{ - if (path.len > UINT16_MAX) - return -1; - - if (append_begin(wal) < 0) - return -1; - - // TODO: check for errors - write_u8(wal->handle, WAL_ENTRY_DELETE); - write_u16(wal->handle, path.len); - write_str(wal->handle, path); - write_u64(wal->handle, expect_gen); - - if (append_end(wal) < 0) - return -1; - - return 0; -} - -int wal_append_write(WAL *wal, string path, uint64_t off, - uint64_t len, uint32_t num_chunks, uint64_t expect_gen, - SHA256 *hashes) -{ - if (path.len > UINT16_MAX) - return -1; - - if (append_begin(wal) < 0) - return -1; - - if (write_u8(wal->handle, WAL_ENTRY_WRITE) < 0) - return -1; - if (write_u16(wal->handle, path.len) < 0) - return -1; - if (write_str(wal->handle, path) < 0) - return -1; - if (write_u64(wal->handle, expect_gen) < 0) - return -1; - if (write_u64(wal->handle, off) < 0) - return -1; - if (write_u64(wal->handle, len) < 0) - return -1; - if (write_u32(wal->handle, num_chunks) < 0) - return -1; - if (write_exact(wal->handle, (char*) hashes, num_chunks * sizeof(SHA256)) < 0) - return -1; - - if (append_end(wal) < 0) - return -1; - - return 0; -} diff --git a/src/wal.h b/src/wal.h deleted file mode 100644 index bf54054..0000000 --- a/src/wal.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef WAL_INCLUDED -#define WAL_INCLUDED - -#include "file_tree.h" -#include "file_system.h" - -typedef struct { - Handle handle; - FileTree *file_tree; - string file_path; - int entry_count; - int entry_limit; -} WAL; - -int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit); - -void wal_close(WAL *wal); - -int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size); - -int wal_append_delete(WAL *wal, string path, uint64_t expect_gen); - -int wal_append_write(WAL *wal, string path, uint64_t off, - uint64_t len, uint32_t num_chunks, uint64_t expect_gen, - SHA256 *hashes); - -#endif // WAL_INCLUDED