diff --git a/DESIGN.txt b/DESIGN.txt index 7f4447d..accc156 100644 --- a/DESIGN.txt +++ b/DESIGN.txt @@ -137,26 +137,51 @@ Chunk Management: servers need to forget some copies. If they are under-replicated, some chunk servers need to copy chunks from elsewhere. - Event 1: A chunk server connects - If the chunk server is holding some chunks, for each chunk - one of the following must be true: - - The chunk is not used by the file system - => Mark the chunk for removal - - The chunk is used and, with this copy, perfectly replicated - => Do nothing - - The chunk is used and under-replicated - => ??? - - The chunk is used and over-replicated - => Mark the chunk for removal + The metadata server holds an authoritative list of hashes that each + chunk server needs to hold. These lists are enforced periodically and + automatically during the state updates, therefore the metadata server + only needs to reorganize the chunks in its in-memory lists. The changes + will propagate over the chunk servers in time. The metadata server's + in-memory representation is the ideal state the system is converging + to, which means every chunk if always replicated correctly. Given this + setup, a chunk can only be under-replicated when there are not enough + chunk servers. - Event 2: A write operation on metadata (adding chunks) + Case 1: The first chunk server connects with no chunks + CS connects + MS accepts + CS sends auth message + MS validates auth + MS requests list of chunks + CS sends the chunk list + MS notices it's empty + + Case 2: The first chunk server connects with some chunks + CS connects + MS accepts + CS sends auth message + MS valudates auth + MS requests list of chunks + CS sends the chunk list + MS iterates over each chunk. For each chunk it determines + whether it's useless, under-replicated, over-replicated, + or properly replicated. + MS adds all useless or over-replicated chunks to the rem_list + of the newly connected chunk server. Note that it's only + possible for a chunk to be over-replicated by 1, so removing + it from the new chunk server will fix the issue. + MS If a chunk is under-replicated, it means that there are not + enough chunk servers to hold all the replicas, so there is + nothing to be done. + + Case 3: Chunk server with some chunks disconnects TODO Event 3: A chunk server disconnects A number of chunks are lost, but there is a chance the server will come back in a short while (10s or so). - - TODO + => Wait for about 10s. If the chunk server doesn't come back, distribute + its hashes to other chunk servers Event 4: A write operation on metadata occurs (overwriting old chunks) Assuming the write overwrites some chunks: diff --git a/src/message.h b/src/message.h index 5e943d3..ddaf786 100644 --- a/src/message.h +++ b/src/message.h @@ -38,6 +38,7 @@ enum { // Chunk server -> Metadata server MESSAGE_TYPE_AUTH, + MESSAGE_TYPE_CHUNK_LIST, MESSAGE_TYPE_STATE_UPDATE_ERROR, MESSAGE_TYPE_STATE_UPDATE_SUCCESS, diff --git a/src/metadata_server.c b/src/metadata_server.c index f230cf1..879a111 100644 --- a/src/metadata_server.c +++ b/src/metadata_server.c @@ -768,55 +768,49 @@ static int process_chunk_server_auth(MetadataServer *state, if (!binary_read(&reader, NULL, sizeof(MessageHeader))) return -1; - // Read IPv4s - { - uint32_t num_ipv4; - if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) + uint32_t num_ipv4; + if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) + return -1; + + for (uint32_t i = 0; i < num_ipv4; i++) { + + IPv4 ipv4; + if (!binary_read(&reader, &ipv4, sizeof(ipv4))) return -1; - for (uint32_t i = 0; i < num_ipv4; i++) { + uint16_t port; + if (!binary_read(&reader, &port, sizeof(port))) + return -1; - IPv4 ipv4; - if (!binary_read(&reader, &ipv4, sizeof(ipv4))) - return -1; - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) - return -1; - - if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { - Address addr = {0}; - addr.ipv4 = ipv4; - addr.is_ipv4 = true; - addr.port = port; - chunk_server->addrs[chunk_server->num_addrs++] = addr; - } + if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { + Address addr = {0}; + addr.ipv4 = ipv4; + addr.is_ipv4 = true; + addr.port = port; + chunk_server->addrs[chunk_server->num_addrs++] = addr; } } - // Read IPv6s - { - uint32_t num_ipv6; - if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) + uint32_t num_ipv6; + if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) + return -1; + + for (uint32_t i = 0; i < num_ipv6; i++) { + + IPv6 ipv6; + if (!binary_read(&reader, &ipv6, sizeof(ipv6))) return -1; - for (uint32_t i = 0; i < num_ipv6; i++) { + uint16_t port; + if (!binary_read(&reader, &port, sizeof(port))) + return -1; - IPv6 ipv6; - if (!binary_read(&reader, &ipv6, sizeof(ipv6))) - return -1; - - uint16_t port; - if (!binary_read(&reader, &port, sizeof(port))) - return -1; - - if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { - Address addr = {0}; - addr.ipv6 = ipv6; - addr.is_ipv4 = false; - addr.port = port; - chunk_server->addrs[chunk_server->num_addrs++] = addr; - } + if (chunk_server->num_addrs < MAX_SERVER_ADDRS) { + Address addr = {0}; + addr.ipv6 = ipv6; + addr.is_ipv4 = false; + addr.port = port; + chunk_server->addrs[chunk_server->num_addrs++] = addr; } } @@ -833,9 +827,32 @@ static int process_chunk_server_auth(MetadataServer *state, // we accept all connections that provide valid address information. chunk_server->auth = true; + // TODO: Request the chunk list held by this chunk server + return 0; } +static int process_chunk_server_chunk_list(MetadataServer *state, + int conn_idx, ByteView msg) +{ + // Iterate over each chunk held by this chunk + // server and determine whether it's useless, + // under-replicated, over-replicated, or + // properly replicated. + // Chunks that are useless or over-replicated + // are added to the remove list of this chunk + // server (note that any given chunk can only + // be over-replicated by 1, so removing it from + // this chunk server will fix the issue). If + // the chunk is under-replicated or properly + // replicated, do nothing. Note that it's only + // possible for a chunk to be under-replicated + // when the number of chunk servers is lower + // than the replication factor. + + // TODO +} + static int process_chunk_server_state_update_success(MetadataServer *state, int conn_idx, ByteView msg) { @@ -948,6 +965,9 @@ process_chunk_server_message(MetadataServer *state, case MESSAGE_TYPE_AUTH: return process_chunk_server_auth(state, conn_idx, msg); + case MESSAGE_TYPE_CHUNK_LIST: + return process_chunk_server_chunk_list(state, conn_idx, msg); + case MESSAGE_TYPE_STATE_UPDATE_SUCCESS: return process_chunk_server_state_update_success(state, conn_idx, msg);