diff --git a/examples/blocking_api.c b/examples/blocking_api.c index d5fc237..0065c2b 100644 --- a/examples/blocking_api.c +++ b/examples/blocking_api.c @@ -22,7 +22,7 @@ int main(void) } char data[] = "Hello, world!"; - ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, 0); + ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, NULL, 0); if (ret < 0) { printf("Couldn't write to file\n"); toasty_disconnect(toasty); diff --git a/inc/ToastyFS.h b/inc/ToastyFS.h index e68c4ad..6404f55 100644 --- a/inc/ToastyFS.h +++ b/inc/ToastyFS.h @@ -56,6 +56,9 @@ typedef uint64_t ToastyVersionTag; // TODO: comment #define TOASTY_VERSION_TAG_EMPTY ((ToastyVersionTag) 0) +// Write operation flags +#define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) // Create file if it doesn't exist + // Creates a directory at the specified path. // Returns 0 on success, -1 on error. // @@ -125,8 +128,12 @@ int toasty_read(ToastyFS *toasty, ToastyString path, int off, // on success, or -1 on error. // // For how vtag works, see toasty_read. +// +// The flags parameter can be set to TOASTY_WRITE_CREATE_IF_MISSING +// to automatically create the file if it doesn't exist. A default +// chunk size of 4096 bytes will be used for the created file. int toasty_write(ToastyFS *toasty, ToastyString path, int off, - void *src, int len, ToastyVersionTag *vtag); + void *src, int len, ToastyVersionTag *vtag, uint32_t flags); ////////////////////////////////////////////////////////////////////////////////// // ASYNCHRONOUS API @@ -177,8 +184,12 @@ ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, // // If vtag is not 0, the operation only succedes if the // tag matches the remote entity's. +// +// The flags parameter can be set to TOASTY_WRITE_CREATE_IF_MISSING +// to automatically create the file if it doesn't exist. A default +// chunk size of 4096 bytes will be used for the created file. ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, - int off, void *src, int len, ToastyVersionTag vtag); + int off, void *src, int len, ToastyVersionTag vtag, uint32_t flags); // Associate the pointer "user" to the handle. The user // pointer will be returned in the ToastyResult when the diff --git a/misc/DESIGN.txt b/misc/DESIGN.txt index a29f508..44fc759 100644 --- a/misc/DESIGN.txt +++ b/misc/DESIGN.txt @@ -41,11 +41,19 @@ Architecture modifications are successful, the client holds the set of old hashes and new hashes for that file range. It completes the write by telling the metadata server to swap the old - hashes with the new ones. If the old hashes don't match, - another write succeded in the mean time and touched that - range, therefore the write fails. If the old hashes match, - the write succeded. If the client fails to modify any - chunks, it doesn't commit the write with the metadata server. + hashes with the new ones (optionally including write flags + such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes + don't match, another write succeded in the mean time and + touched that range, therefore the write fails. If the old + hashes match, the write succeded. If the client fails to + modify any chunks, it doesn't commit the write with the + metadata server. + + If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING + flag is set, the metadata server atomically creates the file + with a default chunk size (4096 bytes) and retries the write. + This operation is logged to the WAL to ensure crash consistency. + Note that write failures may cause chunks to be orphaned on chunk servers. This is solved by a garbage collection algorithm implemented by the synchronization messages diff --git a/misc/PROTOCOL.txt b/misc/PROTOCOL.txt index 33abafe..4aedb6d 100644 --- a/misc/PROTOCOL.txt +++ b/misc/PROTOCOL.txt @@ -106,6 +106,7 @@ the metadata server: struct WriteMessage { Header header; // type=WRITE uint64_t expect_gen; + uint32_t flags; uint16_t path_len; char path[path_len]; uint32_t offset; @@ -116,6 +117,12 @@ the metadata server: If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata. + The flags field contains write operation flags. Currently defined flags: + - TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist, + the metadata server will atomically create it with a default chunk + size of 4096 bytes and retry the write operation. The creation is + logged to the WAL for crash consistency. + The offset and length mark the region that is being written to. Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to diff --git a/src/client.c b/src/client.c index f595675..77bc4be 100644 --- a/src/client.c +++ b/src/client.c @@ -124,6 +124,7 @@ typedef struct { uint32_t num_chunks; uint32_t chunk_size; uint64_t expect_gen; + uint32_t flags; // Write operation flags (e.g., TOASTY_WRITE_CREATE_IF_MISSING) UploadSchedule *uploads; int num_uploads; int cap_uploads; @@ -744,7 +745,7 @@ unlock_and_exit: ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, int off, void *src, int len, - ToastyVersionTag vtag) + ToastyVersionTag vtag, uint32_t flags) { int opidx = -1; ToastyHandle handle = TOASTY_INVALID; @@ -759,6 +760,7 @@ ToastyHandle toasty_begin_write(ToastyFS *toasty, goto unlock_and_exit; toasty->operations[opidx].path = path; // TODO: must be a copy + toasty->operations[opidx].flags = flags; if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_WRITE, path, off, len, vtag) < 0) { free_operation(toasty, opidx); @@ -1974,6 +1976,7 @@ static void process_event_for_write(ToastyFS *toasty, ToastyString path = toasty->operations[opidx].path; uint32_t offset = toasty->operations[opidx].off; uint32_t length = toasty->operations[opidx].len; + uint32_t flags = toasty->operations[opidx].flags; if (path.len > UINT16_MAX) { assert(0); // TODO @@ -1984,6 +1987,7 @@ static void process_event_for_write(ToastyFS *toasty, uint32_t num_chunks = num_upload_results; message_write(&writer, &expect_gen, sizeof(expect_gen)); + message_write(&writer, &flags, sizeof(flags)); message_write(&writer, &path_len, sizeof(path_len)); message_write(&writer, path.ptr, path.len); message_write(&writer, &offset, sizeof(offset)); @@ -2449,9 +2453,9 @@ int toasty_read(ToastyFS *toasty, ToastyString path, } int toasty_write(ToastyFS *toasty, ToastyString path, - int off, void *src, int len, ToastyVersionTag *vtag) + int off, void *src, int len, ToastyVersionTag *vtag, uint32_t flags) { - ToastyHandle handle = toasty_begin_write(toasty, path, off, src, len, vtag ? *vtag : 0); + ToastyHandle handle = toasty_begin_write(toasty, path, off, src, len, vtag ? *vtag : 0, flags); if (handle == TOASTY_INVALID) return -1; diff --git a/src/metadata_server.c b/src/metadata_server.c index b512320..6bef45b 100644 --- a/src/metadata_server.c +++ b/src/metadata_server.c @@ -537,6 +537,10 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg) if (!binary_read(&reader, &expect_gen, sizeof(expect_gen))) return -1; + uint32_t flags; + if (!binary_read(&reader, &flags, sizeof(flags))) + return -1; + char path_mem[1<<10]; uint16_t path_len; @@ -634,6 +638,28 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg) int ret = file_tree_write(&state->file_tree, path, offset, length, num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed); + // If write failed because file doesn't exist and CREATE_IF_MISSING flag is set, + // create the file and retry the write + #define TOASTY_WRITE_CREATE_IF_MISSING (1 << 0) + if (ret == FILETREE_NOENT && (flags & TOASTY_WRITE_CREATE_IF_MISSING)) { + // Create the file with default chunk size of 4096 bytes + uint64_t chunk_size = 4096; + + // Log the creation in the WAL + if (wal_append_create(&state->wal, path, false, chunk_size) < 0) { + assert(0); // TODO + } + + uint64_t create_gen; + int create_ret = file_tree_create_entity(&state->file_tree, path, false, chunk_size, &create_gen); + + if (create_ret == 0) { + // File created successfully, retry the write with the new generation + ret = file_tree_write(&state->file_tree, path, offset, length, + num_chunks, create_gen, &new_gen, new_hashes, removed_hashes, &num_removed); + } + } + if (ret < 0) { string desc = file_tree_strerror(ret); diff --git a/src/simulation_client.c b/src/simulation_client.c index 6cbe595..e54c510 100644 --- a/src/simulation_client.c +++ b/src/simulation_client.c @@ -209,7 +209,7 @@ int simulation_client_step(SimulationClient *client, void **contexts, ptr = malloc(len); if (ptr == NULL) assert(0); memset(ptr, 'a', len); - handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY); + handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY, 0); //printf("[Client] submit write (path=%s, off=%d, len=%d)\n", entry.path, off, len); break; } diff --git a/web/DESIGN.txt b/web/DESIGN.txt index 460795f..896b0b0 100644 --- a/web/DESIGN.txt +++ b/web/DESIGN.txt @@ -34,9 +34,14 @@ PUT ??? (TODO) is returned. If the payload the request is not empty, the operation fails with code 400. - If the path refers to a file, it creates that file with the specified - contents. If a file or directory already existed at that path, it is - overwritten unless the header "X-ToastyFS-Overwrite: no" is sent. + If the path refers to a file, the file is automatically created if it + doesn't exist, then the specified contents are written to it. This is + implemented using the TOASTY_WRITE_CREATE_IF_MISSING flag which performs + file creation and write atomically on the metadata server. Files are + created with a default chunk size of 4096 bytes. + + If a file or directory already existed at that path, it is overwritten + unless the header "X-ToastyFS-Overwrite: no" is sent. (TODO error codes) DELETE diff --git a/web/src/proxy_put.c b/web/src/proxy_put.c index a479c9e..0a6a590 100644 --- a/web/src/proxy_put.c +++ b/web/src/proxy_put.c @@ -7,8 +7,11 @@ bool process_request_put(ProxyState *state, ProxyOperation *operation, request->url.path.ptr, request->url.path.len, }; + // Use TOASTY_WRITE_CREATE_IF_MISSING to automatically create files + // when they don't exist, enabling PUT operations to create new files ToastyHandle handle = toasty_begin_write(state->backend, path, 0, - request->body.ptr, request->body.len, TOASTY_VERSION_TAG_EMPTY); + request->body.ptr, request->body.len, TOASTY_VERSION_TAG_EMPTY, + TOASTY_WRITE_CREATE_IF_MISSING); if (handle == TOASTY_INVALID) { http_response_builder_status(builder, 500); // Internal Server Error http_response_builder_send(builder);