cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e7455cc4ed93508997a3723a55ae07feac5767e3
ToastyFS/src/metadata_server.c
T
cozis e7455cc4ed Progress
2025-11-06 01:00:39 +01:00

872 lines
24 KiB
C
Raw Blame History

#define _GNU_SOURCE
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include "message.h"
#include "metadata_server.h"
static void hash_list_init(HashList *hash_list)
{
hash_list->count = 0;
hash_list->capacity = 0;
hash_list->items = NULL;
}
static void hash_list_free(HashList *hash_list)
{
sys_free(hash_list->items);
}
static int hash_list_insert(HashList *hash_list, SHA256 hash)
{
// Avoid duplicates
for (int i = 0; i < hash_list->count; i++)
if (!memcmp(&hash_list->items[i], &hash, sizeof(SHA256)))
return 0; // Already present
if (hash_list->count == hash_list->capacity) {
int new_capacity = hash_list->capacity ? hash_list->capacity * 2 : 16;
SHA256 *new_items = sys_realloc(hash_list->items, new_capacity * sizeof(SHA256));
if (new_items == NULL)
return -1;
hash_list->items = new_items;
hash_list->capacity = new_capacity;
}
hash_list->items[hash_list->count++] = hash;
return 0;
}
static bool hash_list_contains(HashList *hash_list, SHA256 hash)
{
for (int j = 0; j < hash_list->count; j++)
if (!memcmp(&hash, &hash_list->items[j], sizeof(SHA256)))
return true;
return false;
}
static void chunk_server_peer_init(ChunkServerPeer *chunk_server)
{
chunk_server->auth = false;
chunk_server->num_addrs = 0;
hash_list_init(&chunk_server->old_list);
hash_list_init(&chunk_server->add_list);
hash_list_init(&chunk_server->rem_list);
}
static void chunk_server_peer_free(ChunkServerPeer *chunk_server)
{
hash_list_free(&chunk_server->rem_list);
hash_list_free(&chunk_server->add_list);
hash_list_free(&chunk_server->old_list);
}
static bool chunk_server_peer_contains(ChunkServerPeer *chunk_server, SHA256 hash)
{
return hash_list_contains(&chunk_server->old_list, hash)
|| hash_list_contains(&chunk_server->add_list, hash);
}
static bool chunk_server_peer_load(ChunkServerPeer *chunk_server)
{
return chunk_server->old_list.count + chunk_server->add_list.count;
}
// Returns all chunk servers holding the given chunk
//
// The indices of the chunk servers is stored into "out", but at
// most "max" indices are written. The return value is the number
// of indices that would be written if "max" were large enough to
// hold all indices.
static int
all_chunk_servers_holding_chunk(MetadataServer *state, SHA256 hash, int *out, int max)
{
int num = 0;
for (int i = 0; i < state->num_chunk_servers; i++) {
if (num < max && chunk_server_peer_contains(&state->chunk_servers[i], hash))
out[num] = i;
num++;
}
return num;
}
#ifdef _WIN32
static int compare_chunk_servers(void *data, const void *p1, const void *p2)
#else
static int compare_chunk_servers(const void *p1, const void *p2, void *data)
#endif
{
int a = *(int*) p1;
int b = *(int*) p2;
MetadataServer *state = data;
int l1 = chunk_server_peer_load(&state->chunk_servers[a]);
int l2 = chunk_server_peer_load(&state->chunk_servers[b]);
return l1 - l2;
}
// Returns the indices of chunk servers with lowest load in
// the "out" array. The return value is the number of indices
// written, but no more than "max" are written.
static int choose_servers_for_write(MetadataServer *state, int *out, int max)
{
int num = state->num_chunk_servers;
int indices[MAX_CHUNK_SERVERS];
for (int i = 0; i < num; i++)
indices[i] = i;
#ifdef _WIN32
qsort_s(indices, num, sizeof(*indices), compare_chunk_servers, state);
#else
qsort_r(indices, num, sizeof(*indices), compare_chunk_servers, state);
#endif
if (max > num) max = num;
for (int i = 0; i < max; i++)
out[i] = indices[i]; // Or maybe the other way around? indices[max - i - 1]?
return num;
}
static int find_chunk_server_by_addr(MetadataServer *state, Address addr)
{
for (int i = 0; i < state->num_chunk_servers; i++)
for (int j = 0; j < state->chunk_servers[i].num_addrs; j++)
if (addr_eql(state->chunk_servers[i].addrs[j], addr))
return j;
return -1;
}
// Serialize the list of addresses for the specified
// chunk server.
static void
message_write_server_addr(MessageWriter *writer, ChunkServerPeer *server)
{
uint32_t num_ipv4 = 0;
for (int i = 0; i < server->num_addrs; i++)
if (server->addrs[i].is_ipv4)
num_ipv4++;
message_write(writer, &num_ipv4, sizeof(num_ipv4));
for (int i = 0; i < server->num_addrs; i++)
if (server->addrs[i].is_ipv4) {
message_write(writer, &server->addrs[i].ipv4, sizeof(server->addrs[i].ipv4));
message_write(writer, &server->addrs[i].port, sizeof(server->addrs[i].port));
}
uint32_t num_ipv6 = 0;
for (int i = 0; i < server->num_addrs; i++)
if (!server->addrs[i].is_ipv4)
num_ipv6++;
message_write(writer, &num_ipv6, sizeof(num_ipv6));
for (int i = 0; i < server->num_addrs; i++)
if (!server->addrs[i].is_ipv4) {
message_write(writer, &server->addrs[i].ipv6, sizeof(server->addrs[i].ipv6));
message_write(writer, &server->addrs[i].port, sizeof(server->addrs[i].port));
}
}
static int
process_client_create(MetadataServer *state, int conn_idx, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
char path_mem[1<<10];
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len)))
return -1;
if (path_len > sizeof(path_mem))
return -2;
if (!binary_read(&reader, &path_mem, path_len))
return -1;
string path = { path_mem, path_len };
uint8_t is_dir;
if (!binary_read(&reader, &is_dir, sizeof(is_dir)))
return -1;
uint32_t chunk_size;
if (is_dir)
chunk_size = 0;
else {
if (!binary_read(&reader, &chunk_size, sizeof(chunk_size)))
return -1;
}
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
int ret = file_tree_create_entity(&state->file_tree, path, is_dir, chunk_size);
if (ret < 0) {
string desc = file_tree_strerror(ret);
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_ERROR);
uint16_t len = desc.len;
message_write(&writer, &len, sizeof(len));
message_write(&writer, desc.ptr, desc.len);
if (!message_writer_free(&writer))
return -1;
} else {
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_CREATE_SUCCESS);
if (!message_writer_free(&writer))
return -1;
}
return 0;
}
static int
process_client_delete(MetadataServer *state, int conn_idx, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
char path_mem[1<<10];
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len)))
return -1;
if (path_len > sizeof(path_mem))
return -2;
if (!binary_read(&reader, &path_mem, path_len))
return -1;
string path = { path_mem, path_len };
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
int ret = file_tree_delete_entity(&state->file_tree, path);
if (ret < 0) {
string desc = file_tree_strerror(ret);
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_DELETE_ERROR);
uint16_t len = desc.len;
message_write(&writer, &len, sizeof(len));
message_write(&writer, desc.ptr, desc.len);
if (!message_writer_free(&writer))
return -1;
} else {
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_DELETE_SUCCESS);
if (!message_writer_free(&writer))
return -1;
}
return 0;
}
static int
process_client_list(MetadataServer *state, int conn_idx, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
char path_mem[1<<10];
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len)))
return -1;
if (path_len > sizeof(path_mem))
return -2;
if (!binary_read(&reader, &path_mem, path_len))
return -1;
string path = { path_mem, path_len };
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
#define MAX_LIST_SIZE 128
ListItem items[MAX_LIST_SIZE];
int ret = file_tree_list(&state->file_tree, path, items, MAX_LIST_SIZE);
if (ret < 0) {
string desc = file_tree_strerror(ret);
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_LIST_ERROR);
uint16_t len = desc.len;
message_write(&writer, &len, sizeof(len));
message_write(&writer, desc.ptr, desc.len);
if (!message_writer_free(&writer))
return -1;
} else {
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_LIST_SUCCESS);
uint32_t item_count = ret;
uint8_t truncated = 0;
if (ret > MAX_LIST_SIZE) {
truncated = 1;
item_count = MAX_LIST_SIZE;
}
message_write(&writer, &item_count, sizeof(item_count));
message_write(&writer, &truncated, sizeof(truncated));
for (int i = 0; i < ret && i < MAX_LIST_SIZE; i++) {
uint8_t is_dir = items[i].is_dir;
message_write(&writer, &is_dir, sizeof(is_dir));
if (items[i].name_len > UINT16_MAX)
return -1;
uint16_t name_len = items[i].name_len;
message_write(&writer, &name_len, sizeof(name_len));
message_write(&writer, items[i].name, name_len);
}
if (!message_writer_free(&writer))
return -1;
}
return 0;
}
static int
process_client_read(MetadataServer *state, int conn_idx, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
char path_mem[1<<10];
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len)))
return -1;
if (path_len > sizeof(path_mem))
return -2;
if (!binary_read(&reader, &path_mem, path_len))
return -1;
string path = { path_mem, path_len };
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset)))
return -1;
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length)))
return -1;
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
#define MAX_READ_HASHES 128
uint64_t chunk_size;
SHA256 hashes[MAX_READ_HASHES];
int ret = file_tree_read(&state->file_tree, path, offset, length, &chunk_size, hashes, MAX_READ_HASHES);
if (ret < 0) {
string desc = file_tree_strerror(ret);
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_READ_ERROR);
uint16_t len = desc.len;
message_write(&writer, &len, sizeof(len));
message_write(&writer, desc.ptr, desc.len);
if (!message_writer_free(&writer))
return -1;
} else {
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_READ_SUCCESS);
if (chunk_size > UINT32_MAX) {
message_writer_free(&writer);
return -1;
}
uint32_t tmp = chunk_size;
message_write(&writer, &tmp, sizeof(tmp));
uint32_t num_hashes = ret;
message_write(&writer, &num_hashes, sizeof(num_hashes));
for (uint32_t i = 0; i < num_hashes; i++) {
int holders[MAX_CHUNK_SERVERS];
int num_holders = all_chunk_servers_holding_chunk(state, hashes[i], holders, state->replication_factor);
message_write(&writer, &hashes[i], sizeof(hashes[i]));
uint32_t tmp = num_holders;
message_write(&writer, &tmp, sizeof(tmp));
for (int j = 0; j < num_holders; j++)
message_write_server_addr(&writer, &state->chunk_servers[holders[j]]);
}
int locations[MAX_CHUNK_SERVERS];
int num_locations = choose_servers_for_write(state, locations, state->replication_factor);
uint32_t tmp_u32 = num_locations;
message_write(&writer, &tmp_u32, sizeof(tmp_u32));
for (int j = 0; j < num_locations; j++)
message_write_server_addr(&writer, &state->chunk_servers[locations[j]]);
if (!message_writer_free(&writer))
return -1;
}
return 0;
}
static int
process_client_write(MetadataServer *state, int conn_idx, ByteView msg)
{
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
char path_mem[1<<10];
uint16_t path_len;
if (!binary_read(&reader, &path_len, sizeof(path_len)))
return -1;
if (path_len > sizeof(path_mem))
return -2;
if (!binary_read(&reader, &path_mem, path_len))
return -1;
string path = { path_mem, path_len };
uint32_t offset;
if (!binary_read(&reader, &offset, sizeof(offset)))
return -1;
uint32_t length;
if (!binary_read(&reader, &length, sizeof(length)))
return -1;
uint32_t num_chunks;
if (!binary_read(&reader, &num_chunks, sizeof(num_chunks)))
return -1;
#define MAX_CHUNKS_PER_WRITE 32
Address addrs[MAX_CHUNKS_PER_WRITE];
SHA256 new_hashes[MAX_CHUNKS_PER_WRITE];
SHA256 old_hashes[MAX_CHUNKS_PER_WRITE];
for (uint32_t i = 0; i < num_chunks; i++) {
SHA256 old_hash;
if (!binary_read(&reader, &old_hash, sizeof(old_hash)))
return -1;
SHA256 new_hash;
if (!binary_read(&reader, &new_hash, sizeof(new_hash)))
return -1;
uint8_t is_ipv4;
if (!binary_read(&reader, &is_ipv4, sizeof(is_ipv4)))
return -1;
Address addr;
addr.is_ipv4 = is_ipv4;
if (is_ipv4) {
if (!binary_read(&reader, &addr.ipv4, sizeof(addr.ipv4)))
return -1;
} else {
if (!binary_read(&reader, &addr.ipv6, sizeof(addr.ipv6)))
return -1;
}
if (!binary_read(&reader, &addr.port, sizeof(addr.port)))
return -1;
addrs[i] = addr;
new_hashes[i] = new_hash;
old_hashes[i] = old_hash;
}
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
// Array to collect hashes that are no longer used anywhere in the file tree
SHA256 removed_hashes[MAX_CHUNKS_PER_WRITE];
int num_removed = 0;
int ret = file_tree_write(&state->file_tree, path, offset, length,
old_hashes, new_hashes, removed_hashes, &num_removed);
if (ret < 0) {
string desc = file_tree_strerror(ret);
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_WRITE_ERROR);
uint16_t len = desc.len;
message_write(&writer, &len, sizeof(len));
message_write(&writer, desc.ptr, desc.len);
if (!message_writer_free(&writer))
return -1;
} else {
// Add new chunks to add_list
for (uint32_t i = 0; i < num_chunks; i++) {
int j = find_chunk_server_by_addr(state, addrs[i]);
if (j == -1)
return -1;
if (!hash_list_insert(&state->chunk_servers[j].add_list, new_hashes[i]))
return -1;
}
// Mark removed chunks for deletion on all chunk servers that have them
// These are chunks that were overwritten and are no longer referenced anywhere
for (int i = 0; i < num_removed; i++) {
SHA256 removed_hash = removed_hashes[i];
// Add to rem_list for all chunk servers that have this chunk
for (int j = 0; j < state->num_chunk_servers; j++) {
if (chunk_server_peer_contains(&state->chunk_servers[j], removed_hash)) {
if (!hash_list_insert(&state->chunk_servers[j].rem_list, removed_hash))
return -1;
}
}
}
MessageWriter writer;
ByteQueue *output = tcp_output_buffer(&state->tcp, conn_idx);
message_writer_init(&writer, output, MESSAGE_TYPE_WRITE_SUCCESS);
if (!message_writer_free(&writer))
return -1;
}
return 0;
}
static int
process_client_message(MetadataServer *state,
int conn_idx, uint8_t type, ByteView msg)
{
switch (type) {
case MESSAGE_TYPE_CREATE: return process_client_create(state, conn_idx, msg);
case MESSAGE_TYPE_DELETE: return process_client_delete(state, conn_idx, msg);
case MESSAGE_TYPE_LIST : return process_client_list (state, conn_idx, msg);
case MESSAGE_TYPE_READ : return process_client_read (state, conn_idx, msg);
case MESSAGE_TYPE_WRITE : return process_client_write (state, conn_idx, msg);
default:break;
}
return -1;
}
static ChunkServerPeer*
chunk_server_from_conn(MetadataServer *state, int conn_idx)
{
int tag = tcp_get_tag(&state->tcp, conn_idx);
assert(tag >= 0);
return &state->chunk_servers[tag];
}
static int process_chunk_server_auth(MetadataServer *state,
int conn_idx, ByteView msg)
{
ChunkServerPeer *chunk_server = chunk_server_from_conn(state, conn_idx);
chunk_server->num_addrs = 0;
BinaryReader reader = { msg.ptr, msg.len, 0 };
// Read header
if (!binary_read(&reader, NULL, sizeof(MessageHeader)))
return -1;
// Read IPv4s
{
uint32_t num_ipv4;
if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4)))
return -1;
for (uint32_t i = 0; i < num_ipv4; i++) {
IPv4 ipv4;
if (!binary_read(&reader, &ipv4, sizeof(ipv4)))
return -1;
uint16_t port;
if (!binary_read(&reader, &port, sizeof(port)))
return -1;
if (chunk_server->num_addrs < MAX_SERVER_ADDRS)
chunk_server->addrs[chunk_server->num_addrs++] =
(Address) { .ipv4=ipv4, .is_ipv4=true, .port=port };
}
}
// Read IPv6s
{
uint32_t num_ipv6;
if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6)))
return -1;
for (uint32_t i = 0; i < num_ipv6; i++) {
IPv6 ipv6;
if (!binary_read(&reader, &ipv6, sizeof(ipv6)))
return -1;
uint16_t port;
if (!binary_read(&reader, &port, sizeof(port)))
return -1;
if (chunk_server->num_addrs < MAX_SERVER_ADDRS)
chunk_server->addrs[chunk_server->num_addrs++] =
(Address) { .is_ipv4=true, .ipv6=ipv6, .port=port };
}
}
// No addresses were wpecified
if (chunk_server->num_addrs == 0)
return -1;
// Check that there are no more bytes to read
if (binary_read(&reader, NULL, 1))
return -1;
// NOTE: In a production system, this should verify the authentication
// using the shared secret key mentioned in the architecture. For now,
// we accept all connections that provide valid address information.
chunk_server->auth = true;
return 0;
}
static int
process_chunk_server_message(MetadataServer *state,
int conn_idx, uint8_t type, ByteView msg)
{
switch (type) {
case MESSAGE_TYPE_AUTH:
return process_chunk_server_auth(state, conn_idx, msg);
}
return -1;
}
static bool is_chunk_server_message_type(uint16_t type)
{
switch (type) {
case MESSAGE_TYPE_AUTH:
case MESSAGE_TYPE_STATE_UPDATE_ERROR:
case MESSAGE_TYPE_STATE_UPDATE_SUCCESS:
return true;
default:
break;
}
return false;
}
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout)
{
string addr = getargs(argc, argv, "--addr", "127.0.0.1");
int port = getargi(argc, argv, "--port", 8080);
if (port <= 0 || port >= 1<<16)
return -1;
state->replication_factor = 3;
if (state->replication_factor > MAX_CHUNK_SERVERS)
return -1;
state->num_chunk_servers = 0;
tcp_context_init(&state->tcp);
int ret = tcp_listen(&state->tcp, addr, port);
if (ret < 0) {
tcp_context_free(&state->tcp);
return -1;
}
ret = file_tree_init(&state->file_tree);
if (ret < 0) {
tcp_context_free(&state->tcp);
return -1;
}
printf("Metadata server set up (local=%.*s:%d)\n",
addr.len,
addr.ptr,
port
);
*timeout = -1; // No timeout needed for metadata server
return tcp_register_events(&state->tcp, contexts, polled);
}
int metadata_server_free(MetadataServer *state)
{
file_tree_free(&state->file_tree);
tcp_context_free(&state->tcp);
return 0;
}
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
{
Event events[MAX_CONNS+1];
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
for (int i = 0; i < num_events; i++) {
int conn_idx = events[i].conn_idx;
switch (events[i].type) {
case EVENT_CONNECT:
printf("New connection to metadata server\n");
tcp_set_tag(&state->tcp, conn_idx, CONNECTION_TAG_UNKNOWN);
break;
case EVENT_DISCONNECT:
{
printf("Dropped connection to metadata server\n");
int tag = tcp_get_tag(&state->tcp, conn_idx);
if (tag >= 0) {
chunk_server_peer_free(&state->chunk_servers[tag]);
state->num_chunk_servers--;
}
}
break;
case EVENT_MESSAGE:
{
for (;;) {
ByteView msg;
uint16_t msg_type;
int ret = tcp_next_message(&state->tcp, conn_idx, &msg, &msg_type);
if (ret == 0)
break;
if (ret < 0) {
tcp_close(&state->tcp, conn_idx);
break;
}
printf("Processing message to metadata server\n");
if (tcp_get_tag(&state->tcp, conn_idx) == CONNECTION_TAG_UNKNOWN) {
if (is_chunk_server_message_type(msg_type)) {
int chunk_server_idx = state->num_chunk_servers++;
chunk_server_peer_init(&state->chunk_servers[chunk_server_idx]);
tcp_set_tag(&state->tcp, conn_idx, chunk_server_idx);
} else {
tcp_set_tag(&state->tcp, conn_idx, CONNECTION_TAG_CLIENT);
}
}
if (tcp_get_tag(&state->tcp, conn_idx) == CONNECTION_TAG_CLIENT)
ret = process_client_message(state, conn_idx, msg_type, msg);
else
ret = process_chunk_server_message(state, conn_idx, msg_type, msg);
if (ret < 0) {
tcp_close(&state->tcp, conn_idx);
break;
}
tcp_consume_message(&state->tcp, conn_idx);
}
}
break;
}
}
*timeout = -1; // No timeout needed for metadata server
return tcp_register_events(&state->tcp, contexts, polled);
}
Reference in New Issue View Git Blame Copy Permalink