Implement FileTree serialization and deserialization routines

misc/DESIGN.txt

@@ -209,3 +209,12 @@ Chunk Management:
     already cached that chunk list or not. If it didn't, the chunk server sends the entire list
     in chunks during state updates, with an increased
     update frequency.
+
+Metadata Persistence & Crash Recovery:
+    The metadata server uses a write-ahead log (WAL) file to store its state on disk.
+
+    Log files start with a full snapshot of the metadata and continues with operation
+    log entries.
+
+    When a file gets too big, the metadata server creates a new WAL file by writing
+    a snapshot to it and continuing logging there.

src/file_tree.c

@@ -105,6 +105,7 @@ static void dir_free(Dir *d)
 
 static void dir_remove(Dir *d, int idx)
 {
+    // TODO: pretty sure this leaks memory
     d->children[idx] = d->children[--d->num_children];
 }
 
@@ -466,3 +467,221 @@ string file_tree_strerror(int code)
     }
     return S("Unknown error");
 }
+
+typedef struct {
+    int (*write_fn)(char*,int,void*);
+    void *write_data;
+    char *buffer;
+    int   buffer_size;
+    int   buffer_used;
+    bool  error;
+} SerializeContext;
+
+static void sc_flush(SerializeContext *sc)
+{
+    if (sc->error)
+        return;
+
+    int ret = sc->write_fn(sc->buffer, sc->buffer_used, sc->write_data);
+    if (ret < 0) {
+        sc->error = true;
+        return;
+    }
+
+    sc->buffer_used = 0;
+}
+
+static void sc_write_mem(SerializeContext *sc, char *src, int len)
+{
+    if (sc->error)
+        return;
+
+    if (sc->buffer_size - sc->buffer_used < len) {
+
+        if (len > sc->buffer_size) {
+            sc->error = true;
+            return;
+        }
+
+        sc_flush(sc);
+        if (sc->error)
+            return;
+    }
+
+    memcpy(sc->buffer + sc->buffer_used, src, len);
+    sc->buffer_used += len;
+}
+static void sc_write_u8  (SerializeContext *sc, uint8_t  value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
+static void sc_write_u16 (SerializeContext *sc, uint16_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
+static void sc_write_u64 (SerializeContext *sc, uint64_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
+static void sc_write_hash(SerializeContext *sc, SHA256   value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); }
+
+static void file_serialize(SerializeContext *sc, File *f)
+{
+    sc_write_u64(sc, f->chunk_size);
+    sc_write_u64(sc, f->num_chunks);
+    for (uint64_t i = 0; i < f->num_chunks; i++)
+        sc_write_hash(sc, f->chunks[i]);
+}
+
+static void entity_serialize(SerializeContext *sc, Entity *e);
+
+static void dir_serialize(SerializeContext *sc, Dir *d)
+{
+    sc_write_u64(sc, d->num_children);
+    for (uint64_t i = 0; i < d->num_children; i++)
+        entity_serialize(sc, &d->children[i]);
+}
+
+static void entity_serialize(SerializeContext *sc, Entity *e)
+{
+    sc_write_u16(sc, e->name_len);
+    sc_write_mem(sc, e->name, e->name_len);
+    sc_write_u8(sc, e->is_dir);
+    if (e->is_dir)
+        dir_serialize(sc, &e->d);
+    else
+        file_serialize(sc, &e->f);
+}
+
+int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *write_data)
+{
+    SerializeContext sc;
+    sc.write_fn = write_fn;
+    sc.write_data = write_data;
+    sc.buffer_used = 0;
+    sc.buffer_size = 1<<10;
+    sc.buffer = sys_malloc(sc.buffer_size);
+    sc.error = false;
+    if (sc.buffer == NULL)
+        sc.error = true;
+    entity_serialize(&sc, &ft->root);
+    sc_flush(&sc);
+    sys_free(sc.buffer);
+    if (sc.error)
+        return -1;
+    return 0;
+}
+
+typedef struct {
+    int (*read_fn)(char*,int,void*);
+    void *read_data;
+    char *buffer;
+    int   buffer_size;
+    int   buffer_used;
+    int   buffer_head;
+    bool  error;
+    uint64_t total_read;
+} DeserializeContext;
+
+static void dc_read_mem(DeserializeContext *dc, void *dst, int len)
+{
+    if (dc->error)
+        return;
+
+    if (dc->buffer_used < len) {
+
+        if (dc->buffer_size < len) {
+            dc->error = true;
+            return;
+        }
+
+        memmove(dc->buffer, dc->buffer + dc->buffer_head, dc->buffer_used);
+        dc->buffer_head = 0;
+
+        int ret = dc->read_fn(
+            dc->buffer      + dc->buffer_used,
+            dc->buffer_size - dc->buffer_used,
+            dc->read_data);
+        if (ret < 0) {
+            dc->error = true;
+            return;
+        }
+        dc->buffer_used += ret;
+
+        if (dc->buffer_used < len) {
+            dc->error = true;
+            return;
+        }
+    }
+
+    memcpy(dst, dc->buffer + dc->buffer_head, len);
+    dc->buffer_head += len;
+    dc->buffer_used -= len;
+    dc->total_read  += len;
+}
+static void dc_read_u8 (DeserializeContext *dc,  uint8_t  *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
+static void dc_read_u16(DeserializeContext *dc,  uint16_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
+static void dc_read_u64(DeserializeContext *dc,  uint64_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
+static void dc_read_hash(DeserializeContext *dc, SHA256   *dst) { dc_read_mem(dc, dst, sizeof(*dst)); }
+
+static void file_deserialize(DeserializeContext *dc, File *f)
+{
+    dc_read_u64(dc, &f->chunk_size);
+    dc_read_u64(dc, &f->num_chunks);
+
+    f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256));
+    if (f->chunks == NULL) {
+        assert(0); // TODO
+    }
+
+    for (uint64_t i = 0; i < f->num_chunks; i++)
+        dc_read_hash(dc, &f->chunks[i]);
+}
+
+static void entity_deserialize(DeserializeContext *dc, Entity *e);
+
+static void dir_deserialize(DeserializeContext *dc, Dir *d)
+{
+    dc_read_u64(dc, &d->num_children);
+
+    d->max_children = d->num_children;
+    d->children = sys_malloc(d->num_children * sizeof(Entity));
+    if (d->children == NULL) {
+        assert(0); // TODO
+    }
+
+    // TODO: not checking for errors is not okay as
+    //       the code will branch based on garbage
+    //       values.
+    for (uint64_t i = 0; i < d->num_children; i++)
+        entity_deserialize(dc, &d->children[i]);
+}
+
+static void entity_deserialize(DeserializeContext *dc, Entity *e)
+{
+    dc_read_u16(dc, &e->name_len); // TODO: make sure this doesn't go over the static buffer
+    dc_read_mem(dc, e->name, e->name_len);
+
+    uint8_t is_dir;
+    dc_read_u8 (dc, &is_dir);
+    e->is_dir = (is_dir != 0);
+
+    if (e->is_dir)
+        dir_deserialize(dc, &e->d);
+    else
+        file_deserialize(dc, &e->f);
+}
+
+int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data)
+{
+    DeserializeContext dc;
+    dc.read_fn = read_fn;
+    dc.read_data = read_data;
+    dc.buffer_head = 0;
+    dc.buffer_used = 0;
+    dc.buffer_size = 1<<10;
+    dc.buffer = sys_malloc(dc.buffer_size);
+    dc.error = false;
+    if (dc.buffer == NULL)
+        dc.error = true;
+    dc.total_read = 0;
+    entity_deserialize(&dc, &ft->root);
+    sys_free(dc.buffer);
+    if (dc.error)
+        return -1;
+    if (dc.total_read > INT_MAX) {
+        assert(0); // TODO
+    }
+    return dc.total_read;
+}

src/file_tree.h

@@ -16,8 +16,8 @@ enum {
 typedef struct Entity Entity;
 
 typedef struct {
-    uint64_t chunk_size;
+    uint64_t chunk_size; // TODO: this should be an u32
-    uint64_t num_chunks;
+    uint64_t num_chunks; // TODO: and this too
     SHA256 *chunks;
 } File;
 
@@ -49,14 +49,16 @@ typedef struct {
 
 #define MAX_COMPS 32
 
-int    file_tree_init(FileTree *ft);
+int    file_tree_init          (FileTree *ft);
-void   file_tree_free(FileTree *ft);
+void   file_tree_free          (FileTree *ft);
-bool   file_tree_uses_hash(FileTree *ft, SHA256 hash);
+bool   file_tree_uses_hash     (FileTree *ft, SHA256 hash);
-int    file_tree_list(FileTree *ft, string path, ListItem *items, int max_items);
+int    file_tree_list          (FileTree *ft, string path, ListItem *items, int max_items);
-int    file_tree_create_entity(FileTree *ft, string path, bool is_dir, uint64_t chunk_size);
+int    file_tree_create_entity (FileTree *ft, string path, bool is_dir, uint64_t chunk_size);
-int    file_tree_delete_entity(FileTree *ft, string path);
+int    file_tree_delete_entity (FileTree *ft, string path);
-int    file_tree_write(FileTree *ft, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes, SHA256 *removed_hashes, int *num_removed);
+int    file_tree_write         (FileTree *ft, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes, SHA256 *removed_hashes, int *num_removed);
-int    file_tree_read(FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *chunk_size, SHA256 *hashes, int max_hashes);
+int    file_tree_read          (FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *chunk_size, SHA256 *hashes, int max_hashes);
-string file_tree_strerror(int code);
+string file_tree_strerror      (int code);
+int    file_tree_serialize     (FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data);
+int    file_tree_deserialize   (FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data);
 
 #endif // FILE_TREE_INCLUDED