Add MISSING_FILE_GENERATION special value for "file does not exist"

Implements a new special generation counter value (UINT64_MAX) that
allows clients to assert that a file should NOT exist when performing
operations.

Primary use case: Atomic "create-if-not-exists" operations
When combined with TOASTY_WRITE_CREATE_IF_MISSING flag:
- File doesn't exist → server creates file and writes (SUCCESS)
- File exists → gen_match fails → BADGEN error (prevents overwrite)

This enables race-condition-free file creation where you want to
create a NEW file or fail if it already exists.

Key changes:
- Added MISSING_FILE_GENERATION constant to file_tree.h
- Updated gen_match() to return false when checking existing entities
  against MISSING_FILE_GENERATION
- Modified file_tree_delete_entity() to succeed (no-op) when file is
  missing and MISSING_FILE_GENERATION is specified (idempotent delete)
- Modified file_tree_write() to validate against the new value
- Added documentation in metadata_server.c explaining how
  MISSING_FILE_GENERATION works WITH CREATE_IF_MISSING
- Updated protocol documentation in PROTOCOL.txt
- Added comprehensive feature documentation in MISSING_FILE_GENERATION.md

Additional use cases:
- Idempotent delete operations (delete only if not already gone)
- Detecting unexpected file creation in validation/testing scenarios

The implementation is backward compatible and type-safe. Files are
never assigned MISSING_FILE_GENERATION as their actual generation value.

misc/PROTOCOL.txt

@@ -13,6 +13,21 @@ All messages start with a shared header, defined as:
         uint32_t length;
     };
 
+1.1 Generation Counter Special Values
+
+Generation counters (uint64_t) have two special values:
+
+    NO_GENERATION (0):
+        When used in expect_gen, means "skip generation check entirely".
+        Files/directories are never assigned this generation value.
+
+    MISSING_FILE_GENERATION (UINT64_MAX):
+        When used in expect_gen, means "expect file/directory to NOT exist".
+        - For DELETE: succeeds if file doesn't exist (no-op)
+        - For WRITE: fails with BADGEN if file exists, fails with NOENT if missing
+        - For existing operations: causes generation mismatch if file exists
+        Files/directories are never assigned this generation value.
+
 2. Client Messages
 
 2.1 Client to Metadata Server messages

src/client.c

@@ -19,6 +19,7 @@ typedef pthread_mutex_t Mutex;
 #include "system.h"
 #include "config.h"
 #include "message.h"
+#include "file_tree.h"
 
 #include <ToastyFS.h>
 
@@ -1494,7 +1495,7 @@ static void process_event_for_write(ToastyFS *toasty,
                 return;
             }
 
-            gen = 0; // TODO: is setting the generation to 0 right?
+            gen = MISSING_FILE_GENERATION; // TODO: is setting the generation to 0 right?
             chunk_size = 4096; // The creation flag defaults to a chunk size of 4K
             num_hashes = 0;
 
@@ -1505,6 +1506,7 @@ static void process_event_for_write(ToastyFS *toasty,
                 toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
                 return;
             }
+            assert(gen != NO_GENERATION); // TODO: should this be an assertion?
 
             if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) {
                 toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
@@ -2007,6 +2009,7 @@ static void process_event_for_write(ToastyFS *toasty,
             }
             uint16_t path_len = path.len;
             uint64_t expect_gen = toasty->operations[opidx].expect_gen;
+            assert(expect_gen != NO_GENERATION);
 
             uint32_t num_chunks = num_upload_results;
 

src/file_tree.c

@@ -108,10 +108,17 @@ static void dir_free(Dir *d)
 static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
 {
     assert(entity_gen != NO_GENERATION);
+    assert(entity_gen != MISSING_FILE_GENERATION);
 
+    // NO_GENERATION means "skip generation check"
     if (expected_gen == NO_GENERATION)
         return true;
 
+    // MISSING_FILE_GENERATION means "expect file to NOT exist"
+    // Since we're checking against an existing entity, this is a mismatch
+    if (expected_gen == MISSING_FILE_GENERATION)
+        return false;
+
     return expected_gen == entity_gen;
 }
 
@@ -350,11 +357,17 @@ int file_tree_delete_entity(FileTree *ft, string path,
         return FILETREE_NOTDIR;
 
     int i = dir_find(&e->d, comps[num_comps-1]);
-    if (i == -1)
+    if (i == -1) {
+        // File doesn't exist
+        // If caller expected it not to exist (MISSING_FILE_GENERATION), succeed
+        if (expected_gen == MISSING_FILE_GENERATION)
+            return 0;
         return FILETREE_NOENT;
+    }
 
+    // File exists - check generation
     if (dir_remove(&e->d, i, expected_gen) < 0)
-        return -1; // TODO: proper error code
+        return FILETREE_BADGEN;
 
     return 0;
 }
@@ -385,14 +398,21 @@ int file_tree_write(
 
     Entity *e = resolve_path(&ft->root, comps, num_comps);
 
-    if (e == NULL)
+    if (e == NULL) {
+        // File doesn't exist
+        // If caller expected it not to exist (MISSING_FILE_GENERATION), that's correct
+        // but we still can't write to a non-existent file (need CREATE_IF_MISSING flag in client layer)
+        if (expect_gen == MISSING_FILE_GENERATION)
+            return FILETREE_NOENT;  // Expected behavior: file missing as expected, but can't write
         return FILETREE_NOENT;
+    }
 
     if (e->is_dir)
         return FILETREE_ISDIR;
 
+    // Check generation - will fail if expect_gen is MISSING_FILE_GENERATION (expects missing but file exists)
     if (!gen_match(expect_gen, e->gen))
-        return -1; // TODO: proper error code
+        return FILETREE_BADGEN;
 
     File *f = &e->f;
 
@@ -519,18 +539,26 @@ int file_tree_read(FileTree *ft, string path,
         *actual_bytes = len;
     }
 
-    if (len == 0)
+    if (len == 0) {
+        assert(e->gen != NO_GENERATION);
+        *gen = e->gen;
         return 0;
+    }
 
     uint64_t first_chunk_index = off / f->chunk_size;
     uint64_t  last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size;
 
-    if (first_chunk_index >= f->num_chunks)
+    if (first_chunk_index >= f->num_chunks) {
+        *gen = e->gen;
         return 0;
+    }
 
     if (last_chunk_index >= f->num_chunks) {
-        if (f->num_chunks == 0)
+        if (f->num_chunks == 0) {
+            assert(e->gen != NO_GENERATION);
+            *gen = e->gen;
             return 0;
+        }
         last_chunk_index = f->num_chunks-1;
     }
 
@@ -560,7 +588,7 @@ string file_tree_strerror(int code)
         case FILETREE_EXISTS : return S("File or directory already exists");
         case FILETREE_BADPATH: return S("Invalid path");
         case FILETREE_BADOP  : return S("Invalid operation");
-        case FILETREE_BADGEN : return S("Generation counter cannot be zero for write operations");
+        case FILETREE_BADGEN : return S("Generation counter mismatch or invalid value");
         default:break;
     }
     return S("Unknown error");

src/file_tree.h

@@ -3,7 +3,11 @@
 
 #include "basic.h"
 
+// Special generation counter values:
+// NO_GENERATION (0): Skip generation check (accept any generation)
+// MISSING_FILE_GENERATION (UINT64_MAX): Expect file/directory to NOT exist
 #define NO_GENERATION ((uint64_t) 0)
+#define MISSING_FILE_GENERATION ((uint64_t) UINT64_MAX)
 
 enum {
     FILETREE_NOMEM   = -1,

src/metadata_server.c

@@ -430,6 +430,7 @@ process_client_read(MetadataServer *state, int conn_idx, ByteView msg)
     uint64_t actual_bytes;
     SHA256 hashes[MAX_READ_HASHES];
     int ret = file_tree_read(&state->file_tree, path, offset, length, &gen, &chunk_size, hashes, MAX_READ_HASHES, &actual_bytes);
+    assert(gen != NO_GENERATION);
 
     if (ret < 0) {
 
@@ -476,6 +477,7 @@ process_client_read(MetadataServer *state, int conn_idx, ByteView msg)
         MessageWriter writer;
         message_writer_init(&writer, output, MESSAGE_TYPE_READ_SUCCESS);
 
+        assert(gen != NO_GENERATION);
         message_write(&writer, &gen, sizeof(gen));
 
         if (chunk_size > UINT32_MAX) {
@@ -668,7 +670,10 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
         num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed, truncate_after);
 
     // If write failed because file doesn't exist and CREATE_IF_MISSING flag is set,
-    // create the file and retry the write
+    // create the file and retry the write.
+    // Note: MISSING_FILE_GENERATION works WITH CREATE_IF_MISSING to implement
+    // atomic "create-only-if-not-exists" semantics: creates if missing (NOENT),
+    // but fails if file already exists (BADGEN from gen_match).
     if (ret == FILETREE_NOENT && (flags & TOASTY_WRITE_CREATE_IF_MISSING)) {
         // Create the file with default chunk size of 4096 bytes
         uint64_t chunk_size = 4096;

src/simulation_client.c

@@ -168,6 +168,7 @@ int simulation_client_step(SimulationClient *client, void **contexts,
 
             TableEntry entry;
             uint32_t chunk_size;
+            uint32_t flags;
 
             case PENDING_OPERATION_CREATE:
             entry = table[random_in_range(0, table_len-1)];
@@ -209,7 +210,25 @@ int simulation_client_step(SimulationClient *client, void **contexts,
             ptr = malloc(len);
             if (ptr == NULL) assert(0);
             memset(ptr, 'a', len);
-            handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY, 0);
+            flags = 0;
+            switch (random_in_range(0, 3)) {
+            case 0:
+                flags = 0;
+                break;
+            case 1:
+                flags = TOASTY_WRITE_CREATE_IF_MISSING;
+                break;
+            case 2:
+                flags = TOASTY_WRITE_TRUNCATE_AFTER;
+                break;
+            case 3:
+                flags = TOASTY_WRITE_CREATE_IF_MISSING
+                      | TOASTY_WRITE_TRUNCATE_AFTER;
+                break;
+            default:
+                assert(0);
+            }
+            handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY, flags);
             //printf("[Client] submit write  (path=%s, off=%d, len=%d)\n", entry.path, off, len);
             break;
         }