cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow users of libtoastyfs to operate on files/directories with specific version tags

Browse Source
This commit is contained in:
cozis
2025-11-24 14:34:57 +01:00
parent 1281bf0822
commit afad13f44f
6 changed files with 128 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/blocking_api.c
+2 -2
View File
@@ -14,7 +14,7 @@ int main(void)
ToastyString path = TOASTY_STR("/first_file");
int ret = toasty_create_file(toasty, path, 1024);
int ret = toasty_create_file(toasty, path, 1024, NULL);
if (ret < 0) {
printf("Couldn't create file\n");
toasty_disconnect(toasty);
@@ -22,7 +22,7 @@ int main(void)
}
char data[] = "Hello, world!";
ret = toasty_write(toasty, path, 0, data, sizeof(data)-1);
ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, 0);
if (ret < 0) {
printf("Couldn't write to file\n");
toasty_disconnect(toasty);
inc/ToastyFS.h
+54 -10
View File
@@ -50,19 +50,37 @@ int toasty_wakeup(ToastyFS *toasty);
// BLOCKING API
//////////////////////////////////////////////////////////////////////////////////
// TODO: comment
typedef uint64_t ToastyVersionTag;
// TODO: comment
#define TOASTY_VERSION_TAG_EMPTY ((ToastyVersionTag) 0)
// Creates a directory at the specified path.
// Returns 0 on success, -1 on error.
int toasty_create_dir(ToastyFS *toasty, ToastyString path);
//
// If the version tag is not NULL, it's used to
// return the version tag associated to the newly
// created file.
int toasty_create_dir(ToastyFS *toasty, ToastyString path,
ToastyVersionTag *vtag);
// Creates a file with the given chunk size at
// the specified path. Returns 0 on success, -1
// on error. The chunk size can't be 0.
//
// If the version tag is not NULL, it's used to
// return the version tag associated to the newly
// created file.
int toasty_create_file(ToastyFS *toasty, ToastyString path,
unsigned int chunk_size);
unsigned int chunk_size, ToastyVersionTag *vtag);
// Deletes a file or directory at the specified path.
// Returns 0 on success, -1 on error.
int toasty_delete(ToastyFS *toasty, ToastyString path);
//
// If the version tag is not 0, the file/directory
// is only deleted if the tags match.
int toasty_delete(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag);
typedef struct {
char name[128]; // TODO: Implement a proper name length
@@ -79,8 +97,13 @@ typedef struct {
// on success, returns -1 on error. The listing is
// a dynamic array that needs to be freed using
// "toasy_free_listing".
//
// If the version tag is not 0, the listing only
// succedes if the tag matches the remote one.
// If the operation succedes, the vtag is set to
// the remote tag.
int toasty_list(ToastyFS *toasty, ToastyString path,
ToastyListing *listing);
ToastyListing *listing, ToastyVersionTag *vtag);
// Frees a listing created by "toasty_list".
void toasty_free_listing(ToastyListing *listing);
@@ -88,14 +111,21 @@ void toasty_free_listing(ToastyListing *listing);
// Reads "len" bytes at offset "off" from the file at
// the given path. Returns the number of bytes read on
// success, or -1 on error.
//
// If vtag is not NULL, the read only succedes if the
// target version tag matches vtag or vtag was 0. If
// the operation succedes, the version tag is set to
// the remote entity's.
int toasty_read(ToastyFS *toasty, ToastyString path, int off,
void *dst, int len);
void *dst, int len, ToastyVersionTag *vtag);
// Writes "len" bytes at offset "off" to the file at
// the given path. Returns the number of bytes written
// on success, or -1 on error.
//
// For how vtag works, see toasty_read.
int toasty_write(ToastyFS *toasty, ToastyString path, int off,
void *src, int len);
void *src, int len, ToastyVersionTag *vtag);
//////////////////////////////////////////////////////////////////////////////////
// ASYNCHRONOUS API
@@ -119,22 +149,35 @@ ToastyHandle toasty_begin_create_file(ToastyFS *toasty, ToastyString path,
// Begins a file or directory deletion operation and
// returns a handle to it. On error, TOASTY_INVALID is
// returned.
ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path);
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path,
ToastyVersionTag vtag);
// Begins a directory listing operation and returns
// a handle to it. On error, TOASTY_INVALID is returned.
ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path);
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path, ToastyVersionTag vtag);
// Begins a read operation and returns a handle to it.
// On error, TOASTY_INVALID is returned.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path,
int off, void *dst, int len);
int off, void *dst, int len, ToastyVersionTag vtag);
// Begins a write operation and returns a handle to it.
// On error, TOASTY_INVALID is returned. Note that the source
// buffer must be valid until the operation completes.
//
// If vtag is not 0, the operation only succedes if the
// tag matches the remote entity's.
ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path,
int off, void *src, int len);
int off, void *src, int len, ToastyVersionTag vtag);
// Associate the pointer "user" to the handle. The user
// pointer will be returned in the ToastyResult when the
@@ -158,6 +201,7 @@ typedef enum {
typedef struct {
ToastyResultType type;
ToastyListing listing;
ToastyVersionTag vtag;
void* user;
int bytes_read; // For read operations: actual number of bytes read
} ToastyResult;
misc/TODO.txt
+1
View File
@@ -23,3 +23,4 @@
[ ] When WAL entries are partially written, remove partial data and fail
[ ] Find a way to ensure listing operations of large directories are handled
[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication?
[ ] Rename generation counters to version numbers
src/client.c
+40 -21
View File
@@ -123,6 +123,7 @@ typedef struct {
int num_hashes;
uint32_t num_chunks;
uint32_t chunk_size;
uint64_t expect_gen;
UploadSchedule *uploads;
int num_uploads;
int cap_uploads;
@@ -598,7 +599,8 @@ ToastyHandle toasty_begin_create_file(ToastyFS *toasty, ToastyString path,
return begin_create(toasty, path, false, chunk_size);
}
ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path)
ToastyHandle toasty_begin_delete(ToastyFS *toasty,
ToastyString path, ToastyVersionTag vtag)
{
int opidx = -1;
ToastyHandle handle = TOASTY_INVALID;
@@ -618,7 +620,7 @@ ToastyHandle toasty_begin_delete(ToastyFS *toasty, ToastyString path)
goto unlock_and_exit;
}
uint16_t path_len = path.len;
uint64_t expect_gen = 0; // 0 means skip generation check
uint64_t expect_gen = vtag;
MessageWriter writer;
metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_DELETE);
@@ -642,7 +644,8 @@ unlock_and_exit:
return handle;
}
ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path)
ToastyHandle toasty_begin_list(ToastyFS *toasty,
ToastyString path, ToastyVersionTag vtag)
{
int opidx = -1;
ToastyHandle handle = TOASTY_INVALID;
@@ -662,7 +665,7 @@ ToastyHandle toasty_begin_list(ToastyFS *toasty, ToastyString path)
goto unlock_and_exit;
}
uint16_t path_len = path.len;
uint64_t expect_gen = 0; // 0 means skip generation check
uint64_t expect_gen = vtag;
MessageWriter writer;
metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_LIST);
@@ -686,12 +689,14 @@ unlock_and_exit:
return handle;
}
static int send_read_message(ToastyFS *toasty, int opidx, int tag, ToastyString path, uint32_t offset, uint32_t length)
static int send_read_message(ToastyFS *toasty, int opidx,
int tag, ToastyString path, uint32_t offset,
uint32_t length, ToastyVersionTag vtag)
{
if (path.len > UINT16_MAX)
return -1;
uint16_t path_len = path.len;
uint64_t expect_gen = 0; // 0 means skip generation check
uint64_t expect_gen = vtag;
MessageWriter writer;
metadata_server_request_start(toasty, &writer, MESSAGE_TYPE_READ);
@@ -705,7 +710,8 @@ static int send_read_message(ToastyFS *toasty, int opidx, int tag, ToastyString
return 0;
}
ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, int off, void *dst, int len)
ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path,
int off, void *dst, int len, ToastyVersionTag vtag)
{
int opidx = -1;
ToastyHandle handle = TOASTY_INVALID;
@@ -719,7 +725,7 @@ ToastyHandle toasty_begin_read(ToastyFS *toasty, ToastyString path, int off, voi
if (opidx < 0)
goto unlock_and_exit;
if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_READ, path, off, len) < 0) {
if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_READ, path, off, len, vtag) < 0) {
free_operation(toasty, opidx);
opidx = -1;
goto unlock_and_exit;
@@ -736,7 +742,9 @@ unlock_and_exit:
return handle;
}
ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, int off, void *src, int len)
ToastyHandle toasty_begin_write(ToastyFS *toasty,
ToastyString path, int off, void *src, int len,
ToastyVersionTag vtag)
{
int opidx = -1;
ToastyHandle handle = TOASTY_INVALID;
@@ -752,7 +760,7 @@ ToastyHandle toasty_begin_write(ToastyFS *toasty, ToastyString path, int off, vo
toasty->operations[opidx].path = path; // TODO: must be a copy
if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_WRITE, path, off, len) < 0) {
if (send_read_message(toasty, opidx, TAG_RETRIEVE_METADATA_FOR_WRITE, path, off, len, vtag) < 0) {
free_operation(toasty, opidx);
opidx = -1;
goto unlock_and_exit;
@@ -1462,6 +1470,7 @@ static void process_event_for_write(ToastyFS *toasty,
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_WRITE_ERROR, .user=toasty->operations[opidx].user };
return;
}
toasty->operations[opidx].expect_gen = gen;
uint32_t chunk_size;
if (!binary_read(&reader, &chunk_size, sizeof(chunk_size))) {
@@ -1961,7 +1970,7 @@ static void process_event_for_write(ToastyFS *toasty,
assert(0); // TODO
}
uint16_t path_len = path.len;
uint64_t expect_gen = 0; // 0 means skip generation check - but per protocol, WRITE can't use 0
uint64_t expect_gen = toasty->operations[opidx].expect_gen;
uint32_t num_chunks = num_upload_results;
@@ -2315,7 +2324,8 @@ int toasty_wait_result(ToastyFS *toasty, ToastyHandle handle, ToastyResult *resu
return 0;
}
int toasty_create_dir(ToastyFS *toasty, ToastyString path)
int toasty_create_dir(ToastyFS *toasty, ToastyString path,
ToastyVersionTag *vtag)
{
ToastyHandle handle = toasty_begin_create_dir(toasty, path);
if (handle == TOASTY_INVALID)
@@ -2330,12 +2340,13 @@ int toasty_create_dir(ToastyFS *toasty, ToastyString path)
return -1;
}
if (vtag) *vtag = result.vtag;
toasty_free_result(&result);
return 0;
}
int toasty_create_file(ToastyFS *toasty, ToastyString path,
unsigned int chunk_size)
unsigned int chunk_size, ToastyVersionTag *vtag)
{
ToastyHandle handle = toasty_begin_create_file(toasty, path, chunk_size);
if (handle == TOASTY_INVALID)
@@ -2350,13 +2361,15 @@ int toasty_create_file(ToastyFS *toasty, ToastyString path,
return -1;
}
if (vtag) *vtag = result.vtag;
toasty_free_result(&result);
return 0;
}
int toasty_delete(ToastyFS *toasty, ToastyString path)
int toasty_delete(ToastyFS *toasty, ToastyString path,
ToastyVersionTag vtag)
{
ToastyHandle handle = toasty_begin_delete(toasty, path);
ToastyHandle handle = toasty_begin_delete(toasty, path, vtag);
if (handle == TOASTY_INVALID)
return -1;
@@ -2373,9 +2386,10 @@ int toasty_delete(ToastyFS *toasty, ToastyString path)
return 0;
}
int toasty_list(ToastyFS *toasty, ToastyString path, ToastyListing *listing)
int toasty_list(ToastyFS *toasty, ToastyString path,
ToastyListing *listing, ToastyVersionTag *vtag)
{
ToastyHandle handle = toasty_begin_list(toasty, path);
ToastyHandle handle = toasty_begin_list(toasty, path, vtag ? *vtag : 0);
if (handle == TOASTY_INVALID)
return -1;
@@ -2388,6 +2402,9 @@ int toasty_list(ToastyFS *toasty, ToastyString path, ToastyListing *listing)
return -1;
}
if (vtag)
*vtag = result.vtag;
if (listing)
*listing = result.listing;
else
@@ -2401,9 +2418,9 @@ void toasty_free_listing(ToastyListing *listing)
}
int toasty_read(ToastyFS *toasty, ToastyString path,
int off, void *dst, int len)
int off, void *dst, int len, ToastyVersionTag *vtag)
{
ToastyHandle handle = toasty_begin_read(toasty, path, off, dst, len);
ToastyHandle handle = toasty_begin_read(toasty, path, off, dst, len, vtag ? *vtag : 0);
if (handle == TOASTY_INVALID)
return -1;
@@ -2416,15 +2433,16 @@ int toasty_read(ToastyFS *toasty, ToastyString path,
return -1;
}
if (vtag) *vtag = result.vtag;
int bytes_read = result.bytes_read;
toasty_free_result(&result);
return bytes_read;
}
int toasty_write(ToastyFS *toasty, ToastyString path,
int off, void *src, int len)
int off, void *src, int len, ToastyVersionTag *vtag)
{
ToastyHandle handle = toasty_begin_write(toasty, path, off, src, len);
ToastyHandle handle = toasty_begin_write(toasty, path, off, src, len, vtag ? *vtag : 0);
if (handle == TOASTY_INVALID)
return -1;
@@ -2437,6 +2455,7 @@ int toasty_write(ToastyFS *toasty, ToastyString path,
return -1;
}
if (vtag) *vtag = result.vtag;
toasty_free_result(&result);
return 0; // TODO: return the number of bytes written?
}
src/simulation_client.c
+4 -4
View File
@@ -182,13 +182,13 @@ int simulation_client_step(SimulationClient *client, void **contexts,
case PENDING_OPERATION_DELETE:
entry = table[random_in_range(0, table_len-1)];
handle = toasty_begin_delete(client->toasty, entry.path);
handle = toasty_begin_delete(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit delete (path=%s)\n", entry.path);
break;
case PENDING_OPERATION_LIST:
entry = table[random_in_range(0, table_len-1)];
handle = toasty_begin_list(client->toasty, entry.path);
handle = toasty_begin_list(client->toasty, entry.path, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit list (path=%s)\n", entry.path);
break;
@@ -198,7 +198,7 @@ int simulation_client_step(SimulationClient *client, void **contexts,
len = random_in_range(0, 5000);
ptr = malloc(len);
if (ptr == NULL) assert(0);
handle = toasty_begin_read(client->toasty, entry.path, off, ptr, len);
handle = toasty_begin_read(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit read (path=%s, off=%d, len=%d)\n", entry.path, off, len);
break;
@@ -209,7 +209,7 @@ int simulation_client_step(SimulationClient *client, void **contexts,
ptr = malloc(len);
if (ptr == NULL) assert(0);
memset(ptr, 'a', len);
handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len);
handle = toasty_begin_write(client->toasty, entry.path, off, ptr, len, TOASTY_VERSION_TAG_EMPTY);
//printf("[Client] submit write (path=%s, off=%d, len=%d)\n", entry.path, off, len);
break;
}
web/main.c
+25 -3
View File
@@ -292,7 +292,7 @@ int main(int argc, char **argv)
proxied[i].request->url.path.ptr,
proxied[i].request->url.path.len,
};
proxied[i].handle = toasty_begin_read(toasty, path, proxied[i].transferred, dst, cap);
proxied[i].handle = toasty_begin_read(toasty, path, proxied[i].transferred, dst, cap, TOASTY_VERSION_TAG_EMPTY);
if (proxied[i].handle == TOASTY_INVALID) {
assert(0); // TODO
}
@@ -355,11 +355,22 @@ int main(int argc, char **argv)
break;
}
ToastyVersionTag vtag;
{
int i = http_find_header(request->headers, request->num_headers, HTTP_STR("ETag"));
if (i < 0)
vtag = TOASTY_VERSION_TAG_EMPTY;
else {
// TODO: parse ETag and store its value into vtag
vtag = -1;
}
}
ToastyString path = {
request->url.path.ptr,
request->url.path.len,
};
ToastyHandle handle = toasty_begin_write(toasty, path, 0, request->body.ptr, request->body.len);
ToastyHandle handle = toasty_begin_write(toasty, path, 0, request->body.ptr, request->body.len, vtag);
if (handle == TOASTY_INVALID) {
http_response_builder_status(builder, 500); // Internal Server Error
http_response_builder_send(builder);
@@ -383,11 +394,22 @@ int main(int argc, char **argv)
break;
}
ToastyVersionTag vtag;
{
int i = http_find_header(request->headers, request->num_headers, HTTP_STR("ETag"));
if (i < 0)
vtag = TOASTY_VERSION_TAG_EMPTY;
else {
// TODO: parse ETag and store its value into vtag
vtag = -1;
}
}
ToastyString path = {
request->url.path.ptr,
request->url.path.len,
};
ToastyHandle handle = toasty_begin_delete(toasty, path);
ToastyHandle handle = toasty_begin_delete(toasty, path, vtag);
if (handle == TOASTY_INVALID) {
http_response_builder_status(builder, 500); // Internal Server Error
http_response_builder_send(builder);