Document communication between metadata server and clients in the misc/PROTOCOL.txt file and add generation counters for files

misc/PROTOCOL.txt

@@ -0,0 +1,254 @@
+1. Introduction & Notation
+
+The DESIGN.txt file gives an overview of the system
+and how nodes of a cluster interact with each other.
+This file documents the specific binary format used
+to exchange information between nodes.
+
+All messages start with a shared header, defined as:
+
+    struct Header {
+        uint16_t version;
+        uint16_t type;
+        uint32_t length;
+    };
+
+2. Client Messages
+
+2.1 Client to Metadata Server messages
+
+Let's start from the interactions between a client and
+the metadata server:
+
+[ CREATE | C -> MS ]
+    Upon file creation, the client sends a "CREATE" message with the following layout:
+
+        struct CreateMessage {
+            Header   header; // type=CREATE
+            uint16_t path_len;
+            char     path[path_len];
+            uint8_t  is_dir;
+            if (is_dir != 0) {
+                uint32_t chunk_size;
+            }
+        };
+
+    Note that in general only paths up to 65K bytes are
+    supported and that the layout of fields may depend
+    on the value of others.
+
+    The server then responds with a CREATE_SUCCESS or CREATE_ERROR message.
+
+[ DELETE | C -> MS ]
+    When a client deletes a file, it sends a DELETE
+    message with the following layout:
+
+        struct DeleteMessage {
+            Header   header; // type=DELETE
+            uint64_t expect_gen;
+            uint16_t path_len;
+            char     path[path_len];
+        };
+
+    The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless.
+
+    The server then responds with a DELETE_SUCCESS or DELETE_ERROR message.
+
+[ LIST | C -> MS ]
+    When a client requests a directory listing, it sends a LIST message whith the following format:
+
+        struct ListMessage {
+            Header   header; // type=LIST
+            uint64_t expect_gen;
+            uint16_t path_len;
+            char     path[path_len];
+        };
+
+    If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped.
+
+    The server then responds with a LIST_SUCCESS or LIST_ERROR message.
+
+[ READ | C -> MS ]
+    When a client requests to read a file, it sends a READ message with the following format:
+
+        struct ReadMessage {
+            Header   header; // type=READ
+            uint64_t expect_gen;
+            uint16_t path_len;
+            char     path[path_len];
+            uint32_t offset;
+            uint32_t length;
+        };
+
+    The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped.
+
+    The offset and length fields determine the region to be read from the file.
+
+[ WRITE | C -> MS ]
+    When a client wants to write to a file, it sends a WRITE message with the following layout:
+
+        struct Location {
+            uint8_t is_ipv4;
+            if (is_ipv4) {
+                uint32_t ipv4;
+            } else {
+                uint128_t ipv6;
+            }
+            uint16_t port;
+        };
+
+        struct WriteChunk {
+            SHA256 hash;
+            uint32_t num_locations;
+            Location locations[num_locations];
+        };
+
+        struct WriteMessage {
+            Header   header; // type=WRITE
+            uint64_t expect_gen;
+            uint16_t path_len;
+            char     path[path_len];
+            uint32_t offset;
+            uint32_t length;
+            uint32_t num_chunks;
+            WriteChunk chunks[num_chunks];
+        };
+
+    If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata.
+
+    The offset and length mark the region that is being written to.
+
+    Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to
+
+        num_chunks == length / chunk_size
+
+    Where chunk_size is the one for the target file at the specified generation.
+
+    Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it.
+
+2.2 Client to Chunk Server messages
+
+    TODO
+
+3. Metadata Server messages
+
+The following is the list of messages the Metadata Server may send to a Client:
+
+[ CREATE_ERROR | MS -> C ]
+    When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout:
+
+        struct CreateErrorMessage {
+            Header   header; // type=CREATE_ERROR
+            uint16_t message_len;
+            char     message[message_len];
+        };
+
+[ CREATE_SUCCESS | MS -> C ]
+    When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout:
+
+        struct CreateSuccessMessage {
+            Header   header; // type=CREATE_SUCCESS
+            uint64_t gen;
+        };
+
+    The gen field is the generation counter given to the file.
+
+[ DELETE_ERROR | MS -> C ]
+    See CREATE_ERROR
+
+[ DELETE_SUCCESS | MS -> C ]
+    When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout:
+
+        struct DeleteSuccessMessage {
+            Header header; // type=DELETE_SUCCESS
+        };
+
+    It does not store any fields other than the header.
+
+[ LIST_ERROR | MS -> C ]
+    When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout:
+
+        struct ListErrorMessage {
+            Header   header; // type=LIST_ERROR
+            uint16_t message_len;
+            char     message[message_len];
+        };
+
+[ LIST_SUCCESS | MS -> C ]
+    When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout:
+
+        struct ListItem {
+            uint8_t  is_dir;
+            uint16_t name_len;
+            char     name[name_len];
+        };
+
+        struct ListSuccessMessage {
+            Header   header; // type=LIST_SUCCESS
+            uint64_t gen;
+            uint8_t  truncated;
+            uint32_t item_count;
+            ListItem items[item_count];
+        };
+
+    The gen field contains the generation counter for the directory.
+
+    If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message.
+
+[ READ_ERROR | MS -> C ]
+    See CREATE_ERROR
+
+[ READ_SUCCESS | MS -> C ]
+
+        struct IPv4AndPort {
+            uint32_t ipv4;
+            uint16_t port;
+        };
+
+        struct IPv6AndPort {
+            uint128_t ipv6;
+            uint16_t  port;
+        };
+
+        struct ChunkServerAddrList {
+            uint32_t num_ipv4;
+            IPv4AndPort ipv4s[num_ipv4];
+            uint32_t num_ipv6;
+            IPv6AndPort ipv6s[num_ipv6];
+        };
+
+        struct ReadChunk {
+            SHA256 hash;
+            uint32_t num_holders;
+            ChunkServerAddrList holders[num_holders];
+        };
+
+        struct ReadSuccessMessage {
+            Header header; // type=READ_SUCCESS
+            uint64_t gen;
+            uint32_t chunk_size;
+            uint32_t file_length;
+            uint32_t num_hashes;
+            ReadChunk chunks[num_hashes];
+            uint32_t num_write_locations;
+            ChunkServerAddrList write_locations[num_write_locations];
+        };
+
+    The message returns general information about the file such as its generation counter, length in bytes, and chunk size.
+
+    The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them.
+
+    The write locations are a list of chunk servers that clients may write new chunks to.
+
+[ WRITE_ERROR | MS -> C ]
+    See CREATE_ERROR
+
+[ WRITE_SUCCESS | MS -> C ]
+    When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout:
+
+        struct WriteSuccessMessage {
+            Header   header; // type=WRITE_SUCCESS
+            uint64_t gen;
+        };
+
+    The gen field is the new generation counter for that file.

src/file_tree.c

@@ -3,6 +3,7 @@
 #include <string.h>
 #include <stdlib.h>
 
+#include "basic.h"
 #include "system.h"
 #include "file_tree.h"
 
@@ -104,10 +105,32 @@ static void dir_free(Dir *d)
     sys_free(d->children);
 }
 
-static void dir_remove(Dir *d, int idx)
+static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
 {
+    assert(entity_gen != NO_GENERATION);
+
+    if (expected_gen == NO_GENERATION)
+        return true;
+
+    return expected_gen == entity_gen;
+}
+
+static uint64_t create_generation(uint64_t *next_gen)
+{
+    (*next_gen)++;
+    if (*next_gen == 0 || *next_gen == UINT64_MAX)
+        *next_gen = 1;
+    return *next_gen;
+}
+
+static int dir_remove(Dir *d, int idx, uint64_t expected_gen)
+{
+    if (!gen_match(expected_gen, d->children[idx].gen))
+        return -1;
+
     // TODO: pretty sure this leaks memory
     d->children[idx] = d->children[--d->num_children];
+    return 0;
 }
 
 static bool dir_uses_hash(Dir *d, SHA256 hash)
@@ -142,10 +165,14 @@ static bool file_uses_hash(File *f, SHA256 hash)
 
 // Fails when the name is too long
 static int entity_init(Entity *e, char *name, int name_len,
-    bool is_dir, uint64_t chunk_size)
+    bool is_dir, uint64_t chunk_size, uint64_t *next_gen)
 {
     if (name_len >= (int) sizeof(e->name))
         return -1;
+
+    e->gen = create_generation(next_gen);
+    assert(e->gen != NO_GENERATION);
+
     memcpy(e->name, name, name_len);
     e->name[name_len] = '\0';
     e->name_len = (uint16_t) name_len;
@@ -177,7 +204,9 @@ static bool entity_uses_hash(Entity *e, SHA256 hash)
 
 int file_tree_init(FileTree *ft)
 {
-    int ret = entity_init(&ft->root, "", 0, true, 0);
+    ft->next_gen = 1;
+
+    int ret = entity_init(&ft->root, "", 0, true, 0, &ft->next_gen);
     if (ret < 0) return -1;
 
     return 0;
@@ -194,7 +223,7 @@ bool file_tree_uses_hash(FileTree *ft, SHA256 hash)
 }
 
 int file_tree_list(FileTree *ft, string path,
-    ListItem *items, int max_items)
+    ListItem *items, int max_items, uint64_t *gen)
 {
     int num_comps;
     string comps[MAX_COMPS];
@@ -230,11 +259,14 @@ int file_tree_list(FileTree *ft, string path,
         items[i].is_dir = c->is_dir;
     }
 
+    assert(e->gen != NO_GENERATION);
+    *gen = e->gen;
+
     return d->num_children;
 }
 
 int file_tree_create_entity(FileTree *ft, string path,
-    bool is_dir, uint64_t chunk_size)
+    bool is_dir, uint64_t chunk_size, uint64_t *gen)
 {
     int num_comps;
     string comps[MAX_COMPS];
@@ -285,17 +317,21 @@ int file_tree_create_entity(FileTree *ft, string path,
     }
     Entity *c = &d->children[d->num_children];
 
-    int ret = entity_init(c, (char*) name.ptr, name.len, is_dir, chunk_size);
+    int ret = entity_init(c, (char*) name.ptr, name.len, is_dir, chunk_size, &ft->next_gen);
     if (ret < 0)
         // Invalid name for the new file
         return FILETREE_BADPATH;
 
+    assert(e->gen != NO_GENERATION);
+    *gen = e->gen;
+
     d->num_children++;
     return 0;
 }
 
 // TODO: this should return the list of unreferenced hashes
-int file_tree_delete_entity(FileTree *ft, string path)
+int file_tree_delete_entity(FileTree *ft, string path,
+    uint64_t expected_gen)
 {
     int num_comps;
     string comps[MAX_COMPS];
@@ -316,14 +352,18 @@ int file_tree_delete_entity(FileTree *ft, string path)
     if (i == -1)
         return FILETREE_NOENT;
 
-    dir_remove(&e->d, i);
+    if (dir_remove(&e->d, i, expected_gen) < 0)
+        return -1; // TODO: proper error code
+
     return 0;
 }
 
 int file_tree_write(FileTree *ft, string path,
     uint64_t off, uint64_t len, uint32_t num_chunks,
-    uint32_t chunk_size, SHA256 *prev_hashes,
+    uint64_t expect_gen,
-    SHA256 *hashes, SHA256 *removed_hashes,
+    uint64_t *new_gen,
+    SHA256 *hashes,
+    SHA256 *removed_hashes,
     int *num_removed)
 {
     int num_comps;
@@ -341,10 +381,10 @@ int file_tree_write(FileTree *ft, string path,
     if (e->is_dir)
         return FILETREE_ISDIR;
 
-    File *f = &e->f;
+    if (!gen_match(expect_gen, e->gen))
+        return -1; // TODO: proper error code
 
-    if (f->chunk_size != chunk_size)
+    File *f = &e->f;
-        return -1; // TODO: error code
 
     uint64_t first_chunk_index = off / f->chunk_size;
     uint64_t  last_chunk_index = first_chunk_index + (len - 1) / f->chunk_size;
@@ -367,14 +407,17 @@ int file_tree_write(FileTree *ft, string path,
             memset(&f->chunks[i], 0, sizeof(SHA256));
     }
 
-    // Verify prev_hashes match
+    int num_overwritten_hashes = 0;
-    for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++)
+    SHA256 overwritten_hashes[100]; // TODO: fix this limit
-        if (memcmp(&f->chunks[i], &prev_hashes[i - first_chunk_index], sizeof(SHA256)))
+    if (num_chunks > 100) {
-            return -1; // TODO: error code
+        assert(0); // TODO
+    }
 
     // Update chunks
-    for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++)
+    for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++) {
+        overwritten_hashes[num_overwritten_hashes++] = f->chunks[i];
         f->chunks[i] = hashes[i - first_chunk_index];
+    }
 
     // Update file size (last byte written + 1)
     uint64_t new_size = off + len;
@@ -388,13 +431,14 @@ int file_tree_write(FileTree *ft, string path,
     //       interested in which hashes are no longer reachable.
     if (removed_hashes != NULL) {
         *num_removed = 0;
-        for (uint64_t i = first_chunk_index; i <= last_chunk_index; i++) {
+        for (int i = 0; i < num_overwritten_hashes; i++) {
-            SHA256 old_hash = prev_hashes[i - first_chunk_index];
+
+            SHA256 hash = overwritten_hashes[i];
 
             // Skip zero hashes
             bool is_zero = true;
             for (int j = 0; j < (int) sizeof(SHA256); j++) {
-                if (old_hash.data[j] != 0) {
+                if (hash.data[j] != 0) {
                     is_zero = false;
                     break;
                 }
@@ -403,20 +447,22 @@ int file_tree_write(FileTree *ft, string path,
                 continue;
 
             // Check if this hash is still used anywhere in the tree
-            if (!entity_uses_hash(&ft->root, old_hash)) {
+            if (!entity_uses_hash(&ft->root, hash)) {
-                // Not used - add to removed list
+                removed_hashes[*num_removed] = hash;
-                if (removed_hashes)
-                    removed_hashes[*num_removed] = old_hash;
                 (*num_removed)++;
             }
         }
     }
 
+    e->gen = create_generation(&ft->next_gen);
+    assert(e->gen != NO_GENERATION);
+
+    *new_gen = e->gen;
     return 0;
 }
 
 int file_tree_read(FileTree *ft, string path,
-    uint64_t off, uint64_t len, uint64_t *chunk_size,
+    uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size,
     SHA256 *hashes, int max_hashes, uint64_t *actual_bytes)
 {
     int num_comps;
@@ -472,6 +518,9 @@ int file_tree_read(FileTree *ft, string path,
         num_hashes++;
     }
 
+    assert(e->gen != NO_GENERATION);
+    *gen = e->gen;
+
     return num_hashes;
 }
 

src/file_tree.h

@@ -3,6 +3,8 @@
 
 #include "basic.h"
 
+#define NO_GENERATION ((uint64_t) 0)
+
 enum {
     FILETREE_NOMEM   = -1,
     FILETREE_NOENT   = -2,
@@ -29,6 +31,7 @@ typedef struct {
 } Dir;
 
 struct Entity {
+    uint64_t gen;
     char name[1<<8];
     uint16_t name_len;
     bool is_dir;
@@ -40,6 +43,7 @@ struct Entity {
 
 typedef struct {
     Entity root;
+    uint64_t next_gen;
 } FileTree;
 
 typedef struct {
@@ -50,16 +54,37 @@ typedef struct {
 
 #define MAX_COMPS 32
 
-int    file_tree_init          (FileTree *ft);
+int file_tree_init(FileTree *ft);
-void   file_tree_free          (FileTree *ft);
+
-bool   file_tree_uses_hash     (FileTree *ft, SHA256 hash);
+void file_tree_free(FileTree *ft);
-int    file_tree_list          (FileTree *ft, string path, ListItem *items, int max_items);
+
-int    file_tree_create_entity (FileTree *ft, string path, bool is_dir, uint64_t chunk_size);
+bool file_tree_uses_hash(FileTree *ft, SHA256 hash);
-int    file_tree_delete_entity (FileTree *ft, string path);
+
-int    file_tree_write         (FileTree *ft, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes, SHA256 *removed_hashes, int *num_removed);
+int file_tree_list(FileTree *ft, string path,
-int    file_tree_read          (FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *chunk_size, SHA256 *hashes, int max_hashes, uint64_t *actual_bytes);
+    ListItem *items, int max_items, uint64_t *gen);
-string file_tree_strerror      (int code);
+
-int    file_tree_serialize     (FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data);
+int file_tree_create_entity(FileTree *ft, string path,
-int    file_tree_deserialize   (FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data);
+    bool is_dir, uint64_t chunk_size, uint64_t *gen);
+
+int file_tree_delete_entity(FileTree *ft, string path,
+    uint64_t expected_gen);
+
+int file_tree_write(FileTree *ft, string path,
+    uint64_t off, uint64_t len, uint32_t num_chunks,
+    uint64_t expect_gen,
+    uint64_t *new_gen,
+    SHA256 *hashes,
+    SHA256 *removed_hashes,
+    int *num_removed);
+
+int file_tree_read(FileTree *ft, string path,
+    uint64_t off, uint64_t len, uint64_t *gen, uint64_t *chunk_size,
+    SHA256 *hashes, int max_hashes, uint64_t *actual_bytes);
+
+string file_tree_strerror(int code);
+
+int file_tree_serialize(FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data);
+
+int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data);
 
 #endif // FILE_TREE_INCLUDED

src/metadata_server.c

@@ -183,7 +183,8 @@ process_client_create(MetadataServer *state, int conn_idx, ByteView msg)
         assert(0); // TODO
     }
 
-    int ret = file_tree_create_entity(&state->file_tree, path, is_dir, chunk_size);
+    uint64_t gen;
+    int ret = file_tree_create_entity(&state->file_tree, path, is_dir, chunk_size, &gen);
 
     if (ret < 0) {
 
@@ -207,7 +208,7 @@ process_client_create(MetadataServer *state, int conn_idx, ByteView msg)
 
         ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
         message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_SUCCESS);
-
+        // TODO: write the generation
         if (!message_writer_free(&writer))
             return -1;
     }
@@ -224,6 +225,9 @@ process_client_delete(MetadataServer *state, int conn_idx, ByteView msg)
     if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
         return -1;
 
+    uint64_t expect_gen = NO_GENERATION;
+    // TODO: read expected generation
+
     char     path_mem[1<<10];
     uint16_t path_len;
 
@@ -242,12 +246,12 @@ process_client_delete(MetadataServer *state, int conn_idx, ByteView msg)
     if (binary_read(&reader, NULL, 1))
         return -1;
 
-    if (wal_append_delete(&state->wal, path) < 0) {
+    if (wal_append_delete(&state->wal, path, expect_gen) < 0) {
         assert(0); // TODO
     }
 
     // TODO: return unused hashes and add them to the ms_rem_list of holder chunk servers
-    int ret = file_tree_delete_entity(&state->file_tree, path);
+    int ret = file_tree_delete_entity(&state->file_tree, path, expect_gen);
 
     if (ret < 0) {
 
@@ -309,8 +313,9 @@ process_client_list(MetadataServer *state, int conn_idx, ByteView msg)
 
     #define MAX_LIST_SIZE 128
 
+    uint64_t gen;
     ListItem items[MAX_LIST_SIZE];
-    int ret = file_tree_list(&state->file_tree, path, items, MAX_LIST_SIZE);
+    int ret = file_tree_list(&state->file_tree, path, items, MAX_LIST_SIZE, &gen);
 
     if (ret < 0) {
 
@@ -337,6 +342,8 @@ process_client_list(MetadataServer *state, int conn_idx, ByteView msg)
         MessageWriter writer;
         message_writer_init(&writer, output, MESSAGE_TYPE_LIST_SUCCESS);
 
+        // TODO: write generation
+
         uint32_t item_count = ret;
         uint8_t truncated = 0;
 
@@ -405,10 +412,11 @@ process_client_read(MetadataServer *state, int conn_idx, ByteView msg)
 
     #define MAX_READ_HASHES 128
 
+    uint64_t gen;
     uint64_t chunk_size;
     uint64_t actual_bytes;
     SHA256 hashes[MAX_READ_HASHES];
-    int ret = file_tree_read(&state->file_tree, path, offset, length, &chunk_size, hashes, MAX_READ_HASHES, &actual_bytes);
+    int ret = file_tree_read(&state->file_tree, path, offset, length, &gen, &chunk_size, hashes, MAX_READ_HASHES, &actual_bytes);
 
     if (ret < 0) {
 
@@ -433,6 +441,8 @@ process_client_read(MetadataServer *state, int conn_idx, ByteView msg)
         MessageWriter writer;
         message_writer_init(&writer, output, MESSAGE_TYPE_READ_SUCCESS);
 
+        // TODO: write generation
+
         if (chunk_size > UINT32_MAX) {
             message_writer_free(&writer);
             return -1;
@@ -512,6 +522,9 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
     if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
         return -1;
 
+    // TODO: read expected generation
+    uint64_t expect_gen = NO_GENERATION;
+
     char     path_mem[1<<10];
     uint16_t path_len;
 
@@ -538,7 +551,7 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
     if (!binary_read(&reader, &num_chunks, sizeof(num_chunks)))
         return -1;
 
-    uint32_t chunk_size;
+    uint32_t chunk_size; // TODO: Not needed anymore
     if (!binary_read(&reader, &chunk_size, sizeof(chunk_size)))
         return -1;
 
@@ -558,7 +571,7 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
 
     for (uint32_t i = 0; i < num_chunks; i++) {
 
-        SHA256 old_hash;
+        SHA256 old_hash; // TODO: old hashes are not necessary anymore
         if (!binary_read(&reader, &old_hash, sizeof(old_hash)))
             return -1;
 
@@ -607,19 +620,17 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
     SHA256 removed_hashes[MAX_CHUNKS_PER_WRITE];
     int num_removed = 0;
 
-    SHA256 old_hashes[MAX_CHUNKS_PER_WRITE];
     SHA256 new_hashes[MAX_CHUNKS_PER_WRITE];
-    for (uint32_t i = 0; i < num_chunks; i++) {
+    for (uint32_t i = 0; i < num_chunks; i++)
-        old_hashes[i] = results[i].old_hash;
         new_hashes[i] = results[i].new_hash;
-    }
 
-    if (wal_append_write(&state->wal, path, offset, length, num_chunks, chunk_size, old_hashes, new_hashes) < 0) {
+    if (wal_append_write(&state->wal, path, offset, length, num_chunks, expect_gen, new_hashes) < 0) {
         assert(0); // TODO
     }
 
+    uint64_t new_gen;
     int ret = file_tree_write(&state->file_tree, path, offset, length,
-        num_chunks, chunk_size, old_hashes, new_hashes, removed_hashes, &num_removed);
+        num_chunks, expect_gen, &new_gen, new_hashes, removed_hashes, &num_removed);
 
     if (ret < 0) {
 
@@ -671,6 +682,8 @@ process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
         MessageWriter writer;
         message_writer_init(&writer, output, MESSAGE_TYPE_WRITE_SUCCESS);
 
+        // TODO: write new generation
+
         if (!message_writer_free(&writer))
             return -1;
     }

src/wal.c

@@ -32,6 +32,9 @@ typedef struct {
     // create, write
     uint64_t chunk_size;
 
+    // delete, write
+    uint64_t expect_gen;
+
     // create
     bool is_dir;
 
@@ -39,8 +42,7 @@ typedef struct {
     uint64_t offset;
     uint64_t length;
     uint32_t num_chunks;
-    SHA256 *prev_hashes;
+    SHA256 *hashes;
-    SHA256 *next_hashes;
 
 } WALEntry;
 
@@ -249,8 +251,7 @@ static int next_entry(Handle handle, WALEntry *entry)
 {
     // Initialize pointers to NULL for cleanup on error
     entry->path.ptr = NULL;
-    entry->prev_hashes = NULL;
+    entry->hashes = NULL;
-    entry->next_hashes = NULL;
 
     uint8_t type;
     int ret = read_u8(handle, &type);
@@ -296,45 +297,35 @@ static int next_entry(Handle handle, WALEntry *entry)
         break;
 
     case WAL_ENTRY_DELETE:
-        // No additional fields
+        {
+            if (read_u64(handle, &entry->expect_gen) <= 0)
+                goto cleanup_error;
+        }
         break;
 
     case WAL_ENTRY_WRITE:
         {
+            if (read_u64(handle, &entry->expect_gen) <= 0)
+                goto cleanup_error;
             if (read_u64(handle, &entry->offset) <= 0)
                 goto cleanup_error;
             if (read_u64(handle, &entry->length) <= 0)
                 goto cleanup_error;
             if (read_u32(handle, &entry->num_chunks) <= 0)
                 goto cleanup_error;
-            if (read_u32(handle, (uint32_t*) &entry->chunk_size) <= 0)
-                goto cleanup_error;
 
             // Dynamically allocate hash buffers
-            SHA256 *prev_hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256));
+            SHA256 *hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256));
-            SHA256 *next_hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256));
+            if (!hashes_buffer) {
-
-            if (!prev_hashes_buffer || !next_hashes_buffer) {
-                if (prev_hashes_buffer)
-                    sys_free(prev_hashes_buffer);
-                if (next_hashes_buffer)
-                    sys_free(next_hashes_buffer);
                 goto cleanup_error;
             }
 
-            if (read_exact(handle, (char*) prev_hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
+            if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
-                sys_free(prev_hashes_buffer);
+                sys_free(hashes_buffer);
-                sys_free(next_hashes_buffer);
-                goto cleanup_error;
-            }
-            if (read_exact(handle, (char*) next_hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
-                sys_free(prev_hashes_buffer);
-                sys_free(next_hashes_buffer);
                 goto cleanup_error;
             }
 
-            entry->prev_hashes = prev_hashes_buffer;
+            entry->hashes = hashes_buffer;
-            entry->next_hashes = next_hashes_buffer;
         }
         break;
 
@@ -347,10 +338,8 @@ static int next_entry(Handle handle, WALEntry *entry)
 cleanup_error:
     if (entry->path.ptr)
         sys_free((char*) entry->path.ptr);
-    if (entry->prev_hashes)
+    if (entry->hashes)
-        sys_free(entry->prev_hashes);
+        sys_free(entry->hashes);
-    if (entry->next_hashes)
-        sys_free(entry->next_hashes);
     return -1;
 }
 
@@ -476,14 +465,15 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
         assert(ret == 1);
 
         switch (entry.type) {
+            uint64_t gen;
         case WAL_ENTRY_CREATE:
-            file_tree_create_entity(file_tree, entry.path, entry.is_dir, entry.chunk_size);
+            file_tree_create_entity(file_tree, entry.path, entry.is_dir, entry.chunk_size, &gen);
             break;
         case WAL_ENTRY_DELETE:
-            file_tree_delete_entity(file_tree, entry.path);
+            file_tree_delete_entity(file_tree, entry.path, entry.expect_gen);
             break;
         case WAL_ENTRY_WRITE:
-            file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.chunk_size, entry.prev_hashes, entry.next_hashes, NULL, NULL);
+            file_tree_write(file_tree, entry.path, entry.offset, entry.length, entry.num_chunks, entry.expect_gen, &gen, entry.hashes, NULL, NULL);
             break;
         default:
             UNREACHABLE;
@@ -492,10 +482,8 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
         // Free dynamically allocated fields from next_entry
         if (entry.path.ptr)
             sys_free((char*) entry.path.ptr);
-        if (entry.prev_hashes)
+        if (entry.hashes)
-            sys_free(entry.prev_hashes);
+            sys_free(entry.hashes);
-        if (entry.next_hashes)
-            sys_free(entry.next_hashes);
 
         wal->entry_count++;
     }
@@ -585,7 +573,7 @@ int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size)
     return 0;
 }
 
-int wal_append_delete(WAL *wal, string path)
+int wal_append_delete(WAL *wal, string path, uint64_t expect_gen)
 {
     if (path.len > UINT16_MAX)
         return -1;
@@ -593,9 +581,11 @@ int wal_append_delete(WAL *wal, string path)
     if (append_begin(wal) < 0)
         return -1;
 
+    // TODO: check for errors
     write_u8(wal->handle, WAL_ENTRY_DELETE);
     write_u16(wal->handle, path.len);
     write_str(wal->handle, path);
+    write_u64(wal->handle, expect_gen);
 
     if (append_end(wal) < 0)
         return -1;
@@ -604,8 +594,8 @@ int wal_append_delete(WAL *wal, string path)
 }
 
 int wal_append_write(WAL *wal, string path, uint64_t off,
-    uint64_t len, uint32_t num_chunks, uint32_t chunk_size,
+    uint64_t len, uint32_t num_chunks, uint64_t expect_gen,
-    SHA256 *prev_hashes, SHA256 *hashes)
+    SHA256 *hashes)
 {
     if (path.len > UINT16_MAX)
         return -1;
@@ -619,16 +609,14 @@ int wal_append_write(WAL *wal, string path, uint64_t off,
         return -1;
     if (write_str(wal->handle, path) < 0)
         return -1;
+    if (write_u64(wal->handle, expect_gen) < 0)
+        return -1;
     if (write_u64(wal->handle, off) < 0)
         return -1;
     if (write_u64(wal->handle, len) < 0)
         return -1;
     if (write_u32(wal->handle, num_chunks) < 0)
         return -1;
-    if (write_u32(wal->handle, chunk_size) < 0)
-        return -1;
-    if (write_exact(wal->handle, (char*) prev_hashes, num_chunks * sizeof(SHA256)) < 0)
-        return -1;
     if (write_exact(wal->handle, (char*) hashes, num_chunks * sizeof(SHA256)) < 0)
         return -1;
 

src/wal.h

@@ -12,10 +12,16 @@ typedef struct {
     int entry_limit;
 } WAL;
 
-int  wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit);
+int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit);
+
 void wal_close(WAL *wal);
-int  wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size);
+
-int  wal_append_delete(WAL *wal, string path);
+int wal_append_create(WAL *wal, string path, bool is_dir, uint64_t chunk_size);
-int  wal_append_write(WAL *wal, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes);
+
+int wal_append_delete(WAL *wal, string path, uint64_t expect_gen);
+
+int wal_append_write(WAL *wal, string path, uint64_t off,
+    uint64_t len, uint32_t num_chunks, uint64_t expect_gen,
+    SHA256 *hashes);
 
 #endif // WAL_INCLUDED

web/DESIGN.txt

@@ -42,3 +42,15 @@ PUT
 DELETE
     Deletes the given file or directory. Returns 204 on success, 404 if
     no file or directory existed at that path, else returns 5xx.
+
+-----------
+cases:
+    PUT file / unused path -> OK
+    PUT file / file exists -> overwrite?
+    PUT file / directory exists
+    PUT directory / unused path -> OK
+    PUT directory / file exists
+    PUT directory / directory exists
+
+Idea: Use patch on the parent directory to rename
+-----------