From 4bcee9a617b14c61498626e90edf4483180559df Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Dec 2025 19:49:48 +0000 Subject: [PATCH] Add TOASTY_WRITE_TRUNCATE_AFTER flag for HTTP PUT semantics Adds support for truncating files after write operations, enabling proper HTTP PUT semantics where the entire file content should be replaced. Implementation: - Added TOASTY_WRITE_TRUNCATE_AFTER flag (0x02) to ToastyFS API - Updated file_tree_write() to accept truncate_after parameter: * When true, sets file size to exactly offset+length * Removes chunks beyond the new file size * Marks removed chunks for garbage collection - Metadata server extracts and passes truncate flag to file_tree_write() - WAL replay passes false for truncate_after (already handled in original write) - Web proxy PUT now uses both CREATE_IF_MISSING and TRUNCATE_AFTER flags - Updated documentation in PROTOCOL.txt, DESIGN.txt, and web/DESIGN.txt Benefits: - Proper HTTP PUT semantics (replace entire file content) - Efficient: single write operation truncates and updates file - Works with both sync and async APIs - Garbage collection automatically removes orphaned chunks --- inc/ToastyFS.h | 17 +++++++++++------ misc/DESIGN.txt | 5 +++++ misc/PROTOCOL.txt | 4 ++++ src/file_tree.c | 21 +++++++++++++++++++-- src/file_tree.h | 3 ++- src/metadata_server.c | 10 +++++++--- src/wal.c | 3 ++- web/DESIGN.txt | 11 +++++++---- web/src/proxy_put.c | 7 ++++--- 9 files changed, 61 insertions(+), 20 deletions(-) diff --git a/inc/ToastyFS.h b/inc/ToastyFS.h index 6404f55..92a0c5f 100644 --- a/inc/ToastyFS.h +++ b/inc/ToastyFS.h @@ -58,6 +58,7 @@ typedef uint64_t ToastyVersionTag; // Write operation flags #define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) // Create file if it doesn't exist +#define TOASTY_WRITE_TRUNCATE_AFTER (1 << 1) // Truncate file after write offset+length // Creates a directory at the specified path. // Returns 0 on success, -1 on error. @@ -129,9 +130,11 @@ int toasty_read(ToastyFS *toasty, ToastyString path, int off, // // For how vtag works, see toasty_read. // -// The flags parameter can be set to TOASTY_WRITE_CREATE_IF_MISSING -// to automatically create the file if it doesn't exist. A default -// chunk size of 4096 bytes will be used for the created file. +// The flags parameter accepts the following values: +// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it +// doesn't exist. A default chunk size of 4096 bytes will be used. +// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len, +// discarding any data beyond the write range. Useful for HTTP PUT semantics. int toasty_write(ToastyFS *toasty, ToastyString path, int off, void *src, int len, ToastyVersionTag *vtag, uint32_t flags); @@ -185,9 +188,11 @@ ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, // If vtag is not 0, the operation only succedes if the // tag matches the remote entity's. // -// The flags parameter can be set to TOASTY_WRITE_CREATE_IF_MISSING -// to automatically create the file if it doesn't exist. A default -// chunk size of 4096 bytes will be used for the created file. +// The flags parameter accepts the following values: +// - TOASTY_WRITE_CREATE_IF_MISSING: Automatically create the file if it +// doesn't exist. A default chunk size of 4096 bytes will be used. +// - TOASTY_WRITE_TRUNCATE_AFTER: Truncate the file after offset+len, +// discarding any data beyond the write range. Useful for HTTP PUT semantics. ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, int off, void *src, int len, ToastyVersionTag vtag, uint32_t flags); diff --git a/misc/DESIGN.txt b/misc/DESIGN.txt index 44fc759..f30c5db 100644 --- a/misc/DESIGN.txt +++ b/misc/DESIGN.txt @@ -54,6 +54,11 @@ Architecture with a default chunk size (4096 bytes) and retries the write. This operation is logged to the WAL to ensure crash consistency. + If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is + truncated after the write, setting its size to exactly offset+length + and discarding any data beyond that point. This is useful for HTTP + PUT semantics where the entire file content should be replaced. + Note that write failures may cause chunks to be orphaned on chunk servers. This is solved by a garbage collection algorithm implemented by the synchronization messages diff --git a/misc/PROTOCOL.txt b/misc/PROTOCOL.txt index 4aedb6d..31c4c29 100644 --- a/misc/PROTOCOL.txt +++ b/misc/PROTOCOL.txt @@ -122,6 +122,10 @@ the metadata server: the metadata server will atomically create it with a default chunk size of 4096 bytes and retry the write operation. The creation is logged to the WAL for crash consistency. + - TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write + operation, setting the file size to exactly offset+length. Any data + beyond this point is discarded. Useful for HTTP PUT semantics where + the entire file content should be replaced. The offset and length mark the region that is being written to. diff --git a/src/file_tree.c b/src/file_tree.c index 1c65f24..82340d3 100644 --- a/src/file_tree.c +++ b/src/file_tree.c @@ -369,7 +369,8 @@ int file_tree_write( uint64_t* new_gen, SHA256* hashes, SHA256* removed_hashes, - int* num_removed) + int* num_removed, + bool truncate_after) { // WRITE operations cannot use expect_gen=0 if (expect_gen == NO_GENERATION) @@ -430,8 +431,24 @@ int file_tree_write( // Update file size (last byte written + 1) uint64_t new_size = off + len; - if (new_size > f->file_size) + if (truncate_after) { + // With truncation, set file size to exactly new_size and remove chunks beyond + uint64_t new_num_chunks = last_chunk_index + 1; + + // Add any chunks beyond the write to the overwritten list (they'll be removed) + for (uint64_t i = new_num_chunks; i < f->num_chunks; i++) { + if (num_overwritten_hashes < 100) { // Respect the limit + overwritten_hashes[num_overwritten_hashes++] = f->chunks[i]; + } + } + + f->num_chunks = new_num_chunks; f->file_size = new_size; + } else { + // Without truncation, only grow the file + if (new_size > f->file_size) + f->file_size = new_size; + } // Now check which old hashes are no longer used // anywhere in the tree diff --git a/src/file_tree.h b/src/file_tree.h index bec2d55..8f8e192 100644 --- a/src/file_tree.h +++ b/src/file_tree.h @@ -77,7 +77,8 @@ int file_tree_write(FileTree *ft, string path, uint64_t *new_gen, SHA256 *hashes, SHA256 *removed_hashes, - int *num_removed); + int *num_removed, + bool truncate_after); int file_tree_read(FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size, diff --git a/src/metadata_server.c b/src/metadata_server.c index 6bef45b..441b02e 100644 --- a/src/metadata_server.c +++ b/src/metadata_server.c @@ -634,13 +634,17 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg) assert(0); // TODO } + // Extract flag values + #define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) + #define TOASTY_WRITE_TRUNCATE_AFTER (1 << 1) + bool truncate_after = (flags & TOASTY_WRITE_TRUNCATE_AFTER) != 0; + uint64_t new_gen; int ret = file_tree_write(&state->file_tree, path, offset, length, - num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed); + num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed, truncate_after); // If write failed because file doesn't exist and CREATE_IF_MISSING flag is set, // create the file and retry the write - #define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) if (ret == FILETREE_NOENT && (flags & TOASTY_WRITE_CREATE_IF_MISSING)) { // Create the file with default chunk size of 4096 bytes uint64_t chunk_size = 4096; @@ -656,7 +660,7 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg) if (create_ret == 0) { // File created successfully, retry the write with the new generation ret = file_tree_write(&state->file_tree, path, offset, length, - num_chunks, create_gen, &new_gen, new_hashes, removed_hashes, &num_removed); + num_chunks, create_gen, &new_gen, new_hashes, removed_hashes, &num_removed, truncate_after); } } diff --git a/src/wal.c b/src/wal.c index dc5077c..bbaa5ba 100644 --- a/src/wal.c +++ b/src/wal.c @@ -473,7 +473,8 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit) file_tree_delete_entity(file_tree, entry.path, entry.expect_gen); break; case WAL_ENTRY_WRITE: - file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.expect_gen, &gen, entry.hashes, NULL, NULL); + // WAL replay: use false for truncate_after since truncation was already handled + file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.expect_gen, &gen, entry.hashes, NULL, NULL, false); break; default: UNREACHABLE; diff --git a/web/DESIGN.txt b/web/DESIGN.txt index 896b0b0..f0a52dc 100644 --- a/web/DESIGN.txt +++ b/web/DESIGN.txt @@ -35,10 +35,13 @@ PUT operation fails with code 400. If the path refers to a file, the file is automatically created if it - doesn't exist, then the specified contents are written to it. This is - implemented using the TOASTY_WRITE_CREATE_IF_MISSING flag which performs - file creation and write atomically on the metadata server. Files are - created with a default chunk size of 4096 bytes. + doesn't exist, then the specified contents are written to it, replacing + the entire file content (HTTP PUT semantics). This is implemented using + two flags: + - TOASTY_WRITE_CREATE_IF_MISSING: Creates file if it doesn't exist + - TOASTY_WRITE_TRUNCATE_AFTER: Truncates file after the write, discarding + any data beyond the uploaded content + Files are created with a default chunk size of 4096 bytes. If a file or directory already existed at that path, it is overwritten unless the header "X-ToastyFS-Overwrite: no" is sent. diff --git a/web/src/proxy_put.c b/web/src/proxy_put.c index 0a6a590..2e4b033 100644 --- a/web/src/proxy_put.c +++ b/web/src/proxy_put.c @@ -7,11 +7,12 @@ bool process_request_put(ProxyState *state, ProxyOperation *operation, request->url.path.ptr, request->url.path.len, }; - // Use TOASTY_WRITE_CREATE_IF_MISSING to automatically create files - // when they don't exist, enabling PUT operations to create new files + // Use both flags for proper HTTP PUT semantics: + // - TOASTY_WRITE_CREATE_IF_MISSING: Create file if it doesn't exist + // - TOASTY_WRITE_TRUNCATE_AFTER: Replace entire file content (truncate after write) ToastyHandle handle = toasty_begin_write(state->backend, path, 0, request->body.ptr, request->body.len, TOASTY_VERSION_TAG_EMPTY, - TOASTY_WRITE_CREATE_IF_MISSING); + TOASTY_WRITE_CREATE_IF_MISSING | TOASTY_WRITE_TRUNCATE_AFTER); if (handle == TOASTY_INVALID) { http_response_builder_status(builder, 500); // Internal Server Error http_response_builder_send(builder);