diff --git a/misc/DESIGN.txt b/misc/DESIGN.txt index 68cffe0..a29f508 100644 --- a/misc/DESIGN.txt +++ b/misc/DESIGN.txt @@ -209,3 +209,12 @@ Chunk Management: already cached that chunk list or not. If it didn't, the chunk server sends the entire list in chunks during state updates, with an increased update frequency. + +Metadata Persistence & Crash Recovery: + The metadata server uses a write-ahead log (WAL) file to store its state on disk. + + Log files start with a full snapshot of the metadata and continues with operation + log entries. + + When a file gets too big, the metadata server creates a new WAL file by writing + a snapshot to it and continuing logging there. diff --git a/src/file_tree.c b/src/file_tree.c index 121d93b..383a365 100644 --- a/src/file_tree.c +++ b/src/file_tree.c @@ -105,6 +105,7 @@ static void dir_free(Dir *d) static void dir_remove(Dir *d, int idx) { + // TODO: pretty sure this leaks memory d->children[idx] = d->children[--d->num_children]; } @@ -466,3 +467,221 @@ string file_tree_strerror(int code) } return S("Unknown error"); } + +typedef struct { + int (*write_fn)(char*,int,void*); + void *write_data; + char *buffer; + int buffer_size; + int buffer_used; + bool error; +} SerializeContext; + +static void sc_flush(SerializeContext *sc) +{ + if (sc->error) + return; + + int ret = sc->write_fn(sc->buffer, sc->buffer_used, sc->write_data); + if (ret < 0) { + sc->error = true; + return; + } + + sc->buffer_used = 0; +} + +static void sc_write_mem(SerializeContext *sc, char *src, int len) +{ + if (sc->error) + return; + + if (sc->buffer_size - sc->buffer_used < len) { + + if (len > sc->buffer_size) { + sc->error = true; + return; + } + + sc_flush(sc); + if (sc->error) + return; + } + + memcpy(sc->buffer + sc->buffer_used, src, len); + sc->buffer_used += len; +} +static void sc_write_u8 (SerializeContext *sc, uint8_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } +static void sc_write_u16 (SerializeContext *sc, uint16_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } +static void sc_write_u64 (SerializeContext *sc, uint64_t value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } +static void sc_write_hash(SerializeContext *sc, SHA256 value) { sc_write_mem(sc, (char*) &value, (int) sizeof(value)); } + +static void file_serialize(SerializeContext *sc, File *f) +{ + sc_write_u64(sc, f->chunk_size); + sc_write_u64(sc, f->num_chunks); + for (uint64_t i = 0; i < f->num_chunks; i++) + sc_write_hash(sc, f->chunks[i]); +} + +static void entity_serialize(SerializeContext *sc, Entity *e); + +static void dir_serialize(SerializeContext *sc, Dir *d) +{ + sc_write_u64(sc, d->num_children); + for (uint64_t i = 0; i < d->num_children; i++) + entity_serialize(sc, &d->children[i]); +} + +static void entity_serialize(SerializeContext *sc, Entity *e) +{ + sc_write_u16(sc, e->name_len); + sc_write_mem(sc, e->name, e->name_len); + sc_write_u8(sc, e->is_dir); + if (e->is_dir) + dir_serialize(sc, &e->d); + else + file_serialize(sc, &e->f); +} + +int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *write_data) +{ + SerializeContext sc; + sc.write_fn = write_fn; + sc.write_data = write_data; + sc.buffer_used = 0; + sc.buffer_size = 1<<10; + sc.buffer = sys_malloc(sc.buffer_size); + sc.error = false; + if (sc.buffer == NULL) + sc.error = true; + entity_serialize(&sc, &ft->root); + sc_flush(&sc); + sys_free(sc.buffer); + if (sc.error) + return -1; + return 0; +} + +typedef struct { + int (*read_fn)(char*,int,void*); + void *read_data; + char *buffer; + int buffer_size; + int buffer_used; + int buffer_head; + bool error; + uint64_t total_read; +} DeserializeContext; + +static void dc_read_mem(DeserializeContext *dc, void *dst, int len) +{ + if (dc->error) + return; + + if (dc->buffer_used < len) { + + if (dc->buffer_size < len) { + dc->error = true; + return; + } + + memmove(dc->buffer, dc->buffer + dc->buffer_head, dc->buffer_used); + dc->buffer_head = 0; + + int ret = dc->read_fn( + dc->buffer + dc->buffer_used, + dc->buffer_size - dc->buffer_used, + dc->read_data); + if (ret < 0) { + dc->error = true; + return; + } + dc->buffer_used += ret; + + if (dc->buffer_used < len) { + dc->error = true; + return; + } + } + + memcpy(dst, dc->buffer + dc->buffer_head, len); + dc->buffer_head += len; + dc->buffer_used -= len; + dc->total_read += len; +} +static void dc_read_u8 (DeserializeContext *dc, uint8_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } +static void dc_read_u16(DeserializeContext *dc, uint16_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } +static void dc_read_u64(DeserializeContext *dc, uint64_t *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } +static void dc_read_hash(DeserializeContext *dc, SHA256 *dst) { dc_read_mem(dc, dst, sizeof(*dst)); } + +static void file_deserialize(DeserializeContext *dc, File *f) +{ + dc_read_u64(dc, &f->chunk_size); + dc_read_u64(dc, &f->num_chunks); + + f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256)); + if (f->chunks == NULL) { + assert(0); // TODO + } + + for (uint64_t i = 0; i < f->num_chunks; i++) + dc_read_hash(dc, &f->chunks[i]); +} + +static void entity_deserialize(DeserializeContext *dc, Entity *e); + +static void dir_deserialize(DeserializeContext *dc, Dir *d) +{ + dc_read_u64(dc, &d->num_children); + + d->max_children = d->num_children; + d->children = sys_malloc(d->num_children * sizeof(Entity)); + if (d->children == NULL) { + assert(0); // TODO + } + + // TODO: not checking for errors is not okay as + // the code will branch based on garbage + // values. + for (uint64_t i = 0; i < d->num_children; i++) + entity_deserialize(dc, &d->children[i]); +} + +static void entity_deserialize(DeserializeContext *dc, Entity *e) +{ + dc_read_u16(dc, &e->name_len); // TODO: make sure this doesn't go over the static buffer + dc_read_mem(dc, e->name, e->name_len); + + uint8_t is_dir; + dc_read_u8 (dc, &is_dir); + e->is_dir = (is_dir != 0); + + if (e->is_dir) + dir_deserialize(dc, &e->d); + else + file_deserialize(dc, &e->f); +} + +int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data) +{ + DeserializeContext dc; + dc.read_fn = read_fn; + dc.read_data = read_data; + dc.buffer_head = 0; + dc.buffer_used = 0; + dc.buffer_size = 1<<10; + dc.buffer = sys_malloc(dc.buffer_size); + dc.error = false; + if (dc.buffer == NULL) + dc.error = true; + dc.total_read = 0; + entity_deserialize(&dc, &ft->root); + sys_free(dc.buffer); + if (dc.error) + return -1; + if (dc.total_read > INT_MAX) { + assert(0); // TODO + } + return dc.total_read; +} diff --git a/src/file_tree.h b/src/file_tree.h index 1f1a719..8785875 100644 --- a/src/file_tree.h +++ b/src/file_tree.h @@ -16,8 +16,8 @@ enum { typedef struct Entity Entity; typedef struct { - uint64_t chunk_size; - uint64_t num_chunks; + uint64_t chunk_size; // TODO: this should be an u32 + uint64_t num_chunks; // TODO: and this too SHA256 *chunks; } File; @@ -49,14 +49,16 @@ typedef struct { #define MAX_COMPS 32 -int file_tree_init(FileTree *ft); -void file_tree_free(FileTree *ft); -bool file_tree_uses_hash(FileTree *ft, SHA256 hash); -int file_tree_list(FileTree *ft, string path, ListItem *items, int max_items); -int file_tree_create_entity(FileTree *ft, string path, bool is_dir, uint64_t chunk_size); -int file_tree_delete_entity(FileTree *ft, string path); -int file_tree_write(FileTree *ft, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes, SHA256 *removed_hashes, int *num_removed); -int file_tree_read(FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *chunk_size, SHA256 *hashes, int max_hashes); -string file_tree_strerror(int code); +int file_tree_init (FileTree *ft); +void file_tree_free (FileTree *ft); +bool file_tree_uses_hash (FileTree *ft, SHA256 hash); +int file_tree_list (FileTree *ft, string path, ListItem *items, int max_items); +int file_tree_create_entity (FileTree *ft, string path, bool is_dir, uint64_t chunk_size); +int file_tree_delete_entity (FileTree *ft, string path); +int file_tree_write (FileTree *ft, string path, uint64_t off, uint64_t len, uint32_t num_chunks, uint32_t chunk_size, SHA256 *prev_hashes, SHA256 *hashes, SHA256 *removed_hashes, int *num_removed); +int file_tree_read (FileTree *ft, string path, uint64_t off, uint64_t len, uint64_t *chunk_size, SHA256 *hashes, int max_hashes); +string file_tree_strerror (int code); +int file_tree_serialize (FileTree *ft, int (*flush_fn)(char*,int,void*), void *flush_data); +int file_tree_deserialize (FileTree *ft, int (*read_fn)(char*,int,void*), void *read_data); #endif // FILE_TREE_INCLUDED