Document communication between metadata server and clients in the misc/PROTOCOL.txt file and add generation counters for files
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
1. Introduction & Notation
|
||||
|
||||
The DESIGN.txt file gives an overview of the system
|
||||
and how nodes of a cluster interact with each other.
|
||||
This file documents the specific binary format used
|
||||
to exchange information between nodes.
|
||||
|
||||
All messages start with a shared header, defined as:
|
||||
|
||||
struct Header {
|
||||
uint16_t version;
|
||||
uint16_t type;
|
||||
uint32_t length;
|
||||
};
|
||||
|
||||
2. Client Messages
|
||||
|
||||
2.1 Client to Metadata Server messages
|
||||
|
||||
Let's start from the interactions between a client and
|
||||
the metadata server:
|
||||
|
||||
[ CREATE | C -> MS ]
|
||||
Upon file creation, the client sends a "CREATE" message with the following layout:
|
||||
|
||||
struct CreateMessage {
|
||||
Header header; // type=CREATE
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint8_t is_dir;
|
||||
if (is_dir != 0) {
|
||||
uint32_t chunk_size;
|
||||
}
|
||||
};
|
||||
|
||||
Note that in general only paths up to 65K bytes are
|
||||
supported and that the layout of fields may depend
|
||||
on the value of others.
|
||||
|
||||
The server then responds with a CREATE_SUCCESS or CREATE_ERROR message.
|
||||
|
||||
[ DELETE | C -> MS ]
|
||||
When a client deletes a file, it sends a DELETE
|
||||
message with the following layout:
|
||||
|
||||
struct DeleteMessage {
|
||||
Header header; // type=DELETE
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
};
|
||||
|
||||
The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless.
|
||||
|
||||
The server then responds with a DELETE_SUCCESS or DELETE_ERROR message.
|
||||
|
||||
[ LIST | C -> MS ]
|
||||
When a client requests a directory listing, it sends a LIST message whith the following format:
|
||||
|
||||
struct ListMessage {
|
||||
Header header; // type=LIST
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
};
|
||||
|
||||
If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped.
|
||||
|
||||
The server then responds with a LIST_SUCCESS or LIST_ERROR message.
|
||||
|
||||
[ READ | C -> MS ]
|
||||
When a client requests to read a file, it sends a READ message with the following format:
|
||||
|
||||
struct ReadMessage {
|
||||
Header header; // type=READ
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint32_t offset;
|
||||
uint32_t length;
|
||||
};
|
||||
|
||||
The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped.
|
||||
|
||||
The offset and length fields determine the region to be read from the file.
|
||||
|
||||
[ WRITE | C -> MS ]
|
||||
When a client wants to write to a file, it sends a WRITE message with the following layout:
|
||||
|
||||
struct Location {
|
||||
uint8_t is_ipv4;
|
||||
if (is_ipv4) {
|
||||
uint32_t ipv4;
|
||||
} else {
|
||||
uint128_t ipv6;
|
||||
}
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct WriteChunk {
|
||||
SHA256 hash;
|
||||
uint32_t num_locations;
|
||||
Location locations[num_locations];
|
||||
};
|
||||
|
||||
struct WriteMessage {
|
||||
Header header; // type=WRITE
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint32_t offset;
|
||||
uint32_t length;
|
||||
uint32_t num_chunks;
|
||||
WriteChunk chunks[num_chunks];
|
||||
};
|
||||
|
||||
If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata.
|
||||
|
||||
The offset and length mark the region that is being written to.
|
||||
|
||||
Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to
|
||||
|
||||
num_chunks == length / chunk_size
|
||||
|
||||
Where chunk_size is the one for the target file at the specified generation.
|
||||
|
||||
Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it.
|
||||
|
||||
2.2 Client to Chunk Server messages
|
||||
|
||||
TODO
|
||||
|
||||
3. Metadata Server messages
|
||||
|
||||
The following is the list of messages the Metadata Server may send to a Client:
|
||||
|
||||
[ CREATE_ERROR | MS -> C ]
|
||||
When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout:
|
||||
|
||||
struct CreateErrorMessage {
|
||||
Header header; // type=CREATE_ERROR
|
||||
uint16_t message_len;
|
||||
char message[message_len];
|
||||
};
|
||||
|
||||
[ CREATE_SUCCESS | MS -> C ]
|
||||
When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout:
|
||||
|
||||
struct CreateSuccessMessage {
|
||||
Header header; // type=CREATE_SUCCESS
|
||||
uint64_t gen;
|
||||
};
|
||||
|
||||
The gen field is the generation counter given to the file.
|
||||
|
||||
[ DELETE_ERROR | MS -> C ]
|
||||
See CREATE_ERROR
|
||||
|
||||
[ DELETE_SUCCESS | MS -> C ]
|
||||
When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout:
|
||||
|
||||
struct DeleteSuccessMessage {
|
||||
Header header; // type=DELETE_SUCCESS
|
||||
};
|
||||
|
||||
It does not store any fields other than the header.
|
||||
|
||||
[ LIST_ERROR | MS -> C ]
|
||||
When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout:
|
||||
|
||||
struct ListErrorMessage {
|
||||
Header header; // type=LIST_ERROR
|
||||
uint16_t message_len;
|
||||
char message[message_len];
|
||||
};
|
||||
|
||||
[ LIST_SUCCESS | MS -> C ]
|
||||
When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout:
|
||||
|
||||
struct ListItem {
|
||||
uint8_t is_dir;
|
||||
uint16_t name_len;
|
||||
char name[name_len];
|
||||
};
|
||||
|
||||
struct ListSuccessMessage {
|
||||
Header header; // type=LIST_SUCCESS
|
||||
uint64_t gen;
|
||||
uint8_t truncated;
|
||||
uint32_t item_count;
|
||||
ListItem items[item_count];
|
||||
};
|
||||
|
||||
The gen field contains the generation counter for the directory.
|
||||
|
||||
If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message.
|
||||
|
||||
[ READ_ERROR | MS -> C ]
|
||||
See CREATE_ERROR
|
||||
|
||||
[ READ_SUCCESS | MS -> C ]
|
||||
|
||||
struct IPv4AndPort {
|
||||
uint32_t ipv4;
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct IPv6AndPort {
|
||||
uint128_t ipv6;
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct ChunkServerAddrList {
|
||||
uint32_t num_ipv4;
|
||||
IPv4AndPort ipv4s[num_ipv4];
|
||||
uint32_t num_ipv6;
|
||||
IPv6AndPort ipv6s[num_ipv6];
|
||||
};
|
||||
|
||||
struct ReadChunk {
|
||||
SHA256 hash;
|
||||
uint32_t num_holders;
|
||||
ChunkServerAddrList holders[num_holders];
|
||||
};
|
||||
|
||||
struct ReadSuccessMessage {
|
||||
Header header; // type=READ_SUCCESS
|
||||
uint64_t gen;
|
||||
uint32_t chunk_size;
|
||||
uint32_t file_length;
|
||||
uint32_t num_hashes;
|
||||
ReadChunk chunks[num_hashes];
|
||||
uint32_t num_write_locations;
|
||||
ChunkServerAddrList write_locations[num_write_locations];
|
||||
};
|
||||
|
||||
The message returns general information about the file such as its generation counter, length in bytes, and chunk size.
|
||||
|
||||
The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them.
|
||||
|
||||
The write locations are a list of chunk servers that clients may write new chunks to.
|
||||
|
||||
[ WRITE_ERROR | MS -> C ]
|
||||
See CREATE_ERROR
|
||||
|
||||
[ WRITE_SUCCESS | MS -> C ]
|
||||
When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout:
|
||||
|
||||
struct WriteSuccessMessage {
|
||||
Header header; // type=WRITE_SUCCESS
|
||||
uint64_t gen;
|
||||
};
|
||||
|
||||
The gen field is the new generation counter for that file.
|
||||
Reference in New Issue
Block a user