cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Quakey

Browse Source
This commit is contained in:
cozis
2026-01-16 14:45:09 +01:00
parent 18e74a92c0
commit c10493881b
66 changed files with 12760 additions and 11817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic.c
+17 -9
View File
@@ -1,7 +1,9 @@
#include <string.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
#include "system.h"
bool streq(string s1, string s2)
{
@@ -24,10 +26,10 @@ Time get_current_time(void)
int64_t freq;
int ok;
ok = sys_QueryPerformanceCounter((LARGE_INTEGER*) &count);
ok = QueryPerformanceCounter((LARGE_INTEGER*) &count);
if (!ok) return INVALID_TIME;
ok = sys_QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
if (!ok) return INVALID_TIME;
uint64_t res = 1000000000 * (double) count / freq;
@@ -37,7 +39,7 @@ Time get_current_time(void)
{
struct timespec time;
if (sys_clock_gettime(CLOCK_REALTIME, &time))
if (clock_gettime(CLOCK_REALTIME, &time))
return INVALID_TIME;
uint64_t res;
@@ -99,11 +101,17 @@ int getargi(int argc, char **argv, char *name, int fallback)
if (i == argc)
break;
int tmp = atoi(argv[i]);
if (tmp == 0 && argv[i][0] != '0') // best effort
errno = 0;
char *end;
long val = strtol(argv[i], &end, 10);
if (end == argv[i] || *end != '\0' || errno == ERANGE)
break;
return tmp;
if (val < INT_MIN || val > INT_MAX)
break;
return (int) val;
}
return fallback;
}
src/byte_queue.c
+12 -9
View File
@@ -1,8 +1,11 @@
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdint.h>
#include <assert.h>
#include "system.h"
#include "byte_queue.h"
// This is the implementation of a byte queue useful
@@ -32,12 +35,12 @@ void byte_queue_free(ByteQueue *queue)
{
if (queue->read_target) {
if (queue->read_target != queue->data)
sys_free(queue->read_target);
free(queue->read_target);
queue->read_target = NULL;
queue->read_target_size = 0;
}
sys_free(queue->data);
free(queue->data);
queue->data = NULL;
}
@@ -99,7 +102,7 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
if (queue->read_target) {
if (queue->read_target != queue->data)
sys_free(queue->read_target);
free(queue->read_target);
queue->read_target = NULL;
queue->read_target_size = 0;
}
@@ -224,7 +227,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
if (size > queue->limit)
size = queue->limit;
uint8_t *data = sys_malloc(size);
uint8_t *data = malloc(size);
if (!data) {
queue->flags |= BYTE_QUEUE_ERROR;
return 0;
@@ -234,7 +237,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
memcpy(data, queue->data + queue->head, queue->used);
if (queue->read_target != queue->data)
sys_free(queue->data);
free(queue->data);
queue->data = data;
queue->head = 0;
src/byte_queue.h
+2
View File
@@ -1,6 +1,8 @@
#ifndef BYTE_QUEUE_INCLUDED
#define BYTE_QUEUE_INCLUDED
#include <stddef.h>
#include "basic.h"
typedef struct {
src/chunk_server.c
+71 -37
View File
@@ -1,14 +1,18 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include <stdint.h>
#include "config.h"
#include "metadata_server.h"
#include "sha256.h"
#include "message.h"
#include "file_system.h"
#include "hash_set.h"
#include "config.h"
#include "tcp.h"
#include "message.h"
#include "byte_queue.h"
#include "chunk_server.h"
static string hash2path(ChunkServer *state, SHA256 hash, char *out)
@@ -99,12 +103,12 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
return -1;
if (patch_off > SIZE_MAX - patch.len) {
sys_free(data.ptr);
free(data.ptr);
return -1;
}
if (patch_off + (size_t) patch.len > (size_t) data.len) {
sys_free(data.ptr);
free(data.ptr);
return -1;
}
@@ -112,11 +116,11 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
ret = store_chunk(state, data, new_hash);
if (ret < 0) {
sys_free(data.ptr);
free(data.ptr);
return -1;
}
sys_free(data.ptr);
free(data.ptr);
return 0;
}
@@ -129,7 +133,7 @@ static void download_targets_init(DownloadTargets *targets)
static void download_targets_free(DownloadTargets *targets)
{
sys_free(targets->items);
free(targets->items);
}
static void download_targets_remove(DownloadTargets *targets,
@@ -161,13 +165,13 @@ static int download_targets_push(DownloadTargets *targets,
else
new_capacity = 2 * targets->capacity;
DownloadTarget *new_items = sys_malloc(new_capacity * sizeof(DownloadTarget));
DownloadTarget *new_items = malloc(new_capacity * sizeof(DownloadTarget));
if (new_items == NULL)
return -1;
if (targets->capacity > 0) {
memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0]));
sys_free(targets->items);
free(targets->items);
}
targets->items = new_items;
@@ -563,7 +567,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
if (binary_read(&reader, NULL, 1))
return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message"));
char *mem = sys_malloc(chunk_size);
char *mem = malloc(chunk_size);
if (mem == NULL)
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory"));
@@ -579,7 +583,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
SHA256 dummy;
int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy);
sys_free(mem);
free(mem);
if (ret < 0)
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error"));
@@ -694,7 +698,7 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
string slice;
if (full == 0) {
if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) {
sys_free(data.ptr);
free(data.ptr);
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range"));
}
slice = (string) { data.ptr + target_off, target_len };
@@ -710,11 +714,11 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
message_write(&writer, &target_len, sizeof(target_len));
message_write(&writer, slice.ptr, slice.len);
if (!message_writer_free(&writer)) {
sys_free(data.ptr);
free(data.ptr);
return -1;
}
sys_free(data.ptr);
free(data.ptr);
return 0;
}
@@ -812,8 +816,12 @@ static int send_sync_message(ChunkServer *state)
return 0;
}
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout)
int chunk_server_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
ChunkServer *state = state_;
string addr = getargs(argc, argv, "--addr", "127.0.0.1");
int port = getargi(argc, argv, "--port", 8081);
string path = getargs(argc, argv, "--path", "chunk_server_data/");
@@ -821,34 +829,45 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1");
int remote_port = getargi(argc, argv, "--remote-port", 8080);
if (port <= 0 || port >= 1<<16)
if (port <= 0 || port >= 1<<16) {
fprintf(stderr, "chunk server :: Invalid port\n");
return -1;
}
if (remote_port <= 0 || remote_port >= 1<<16)
if (remote_port <= 0 || remote_port >= 1<<16) {
fprintf(stderr, "chunk server :: Invalid remote port\n");
return -1;
}
Time current_time = get_current_time();
if (current_time == INVALID_TIME)
if (current_time == INVALID_TIME) {
fprintf(stderr, "chunk server :: Couldn't read the time\n");
return -1;
}
state->trace = trace;
state->reconnect_delay = 1; // 1 second
if (tcp_context_init(&state->tcp) < 0)
if (tcp_context_init(&state->tcp) < 0) {
fprintf(stderr, "chunk server :: Couldn't setup the TCP context\n");
return -1;
}
int ret = tcp_listen(&state->tcp, addr, port);
if (ret < 0) {
fprintf(stderr, "chunk server :: Couldn't setup the TCP listener\n");
tcp_context_free(&state->tcp);
return -1;
}
if (create_dir(path) && errno != EEXIST) {
fprintf(stderr, "chunk server :: Couldn't create chunk folder\n");
tcp_context_free(&state->tcp);
return -1;
}
if (get_full_path(path, state->path) < 0) {
fprintf(stderr, "chunk server :: Couldn't convert path to absolute\n");
tcp_context_free(&state->tcp);
return -1;
}
@@ -861,6 +880,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
char tmp[1<<10];
if (addr.len >= (int) sizeof(tmp)) {
fprintf(stderr, "chunk server :: Address is too long\n");
tcp_context_free(&state->tcp);
return -1;
}
@@ -868,6 +888,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
tmp[addr.len] = '\0';
state->local_addr.is_ipv4 = true;
if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) {
fprintf(stderr, "chunk server :: Couldn't parse address\n");
tcp_context_free(&state->tcp);
return -1;
}
@@ -876,6 +897,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
// Initialize metadata server address
// // TODO: This should also support IPv6
if (remote_addr.len >= (int) sizeof(tmp)) {
fprintf(stderr, "chunk server :: Remote address is too long\n");
tcp_context_free(&state->tcp);
return -1;
}
@@ -883,6 +905,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
tmp[remote_addr.len] = '\0';
state->remote_addr.is_ipv4 = true;
if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) {
fprintf(stderr, "chunk server :: Couldn't parse remote address\n");
tcp_context_free(&state->tcp);
return -1;
}
@@ -904,23 +927,20 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
);
*timeout = 0;
return tcp_register_events(&state->tcp, contexts, polled);
}
int chunk_server_free(ChunkServer *state)
{
download_targets_free(&state->download_targets);
timed_hash_set_free(&state->cs_rem_list);
hash_set_free(&state->cs_lst_list);
hash_set_free(&state->cs_add_list);
tcp_context_free(&state->tcp);
if (pcap < TCP_POLL_CAPACITY) {
fprintf(stderr, "chunk server :: Capacity isn't large enough\n");
return -1;
}
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
int chunk_server_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{
ChunkServer *state = state_;
Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
Time current_time = get_current_time();
if (current_time == INVALID_TIME)
@@ -1031,5 +1051,19 @@ int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled
}
*timeout = deadline_to_timeout(deadline, current_time);
return tcp_register_events(&state->tcp, contexts, polled);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int chunk_server_free(void *state_)
{
ChunkServer *state = state_;
download_targets_free(&state->download_targets);
timed_hash_set_free(&state->cs_rem_list);
hash_set_free(&state->cs_lst_list);
hash_set_free(&state->cs_add_list);
tcp_context_free(&state->tcp);
return 0;
}
src/chunk_server.h
+12 -10
View File
@@ -1,12 +1,6 @@
#ifndef CHUNK_SERVER_INCLUDED
#define CHUNK_SERVER_INCLUDED
#include <limits.h>
#include "basic.h"
#include "metadata_server.h"
#include "tcp.h"
#define TAG_METADATA_SERVER 1
#define TAG_CHUNK_SERVER 2
@@ -57,8 +51,16 @@ typedef struct {
} ChunkServer;
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout);
int chunk_server_free(ChunkServer *state);
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
struct pollfd;
#endif // CHUNK_SERVER_INCLUDED
int chunk_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_free(void *state);
#endif // CHUNK_SERVER_INCLUDED
src/client.c
+42 -40
View File
@@ -1,22 +1,24 @@
#include <assert.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#ifdef _WIN32
# define POLL WSAPoll
#else
# define POLL poll
#endif
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#define POLL WSAPoll
typedef CRITICAL_SECTION Mutex;
#else
#include <pthread.h>
#include <arpa/inet.h>
#define POLL poll
typedef pthread_mutex_t Mutex;
#endif
#include "tcp.h"
#include "system.h"
#include "config.h"
#include "message.h"
#include "file_tree.h"
@@ -322,7 +324,7 @@ static int handle_to_operation(ToastyFS *toasty, ToastyHandle handle)
ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
{
ToastyFS *toasty = sys_malloc(sizeof(ToastyFS));
ToastyFS *toasty = malloc(sizeof(ToastyFS));
if (toasty == NULL)
return NULL;
@@ -330,7 +332,7 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
{
char tmp[128];
if (addr.len >= (int) sizeof(tmp)) {
sys_free(toasty);
free(toasty);
return NULL;
}
memcpy(tmp, addr.ptr, addr.len);
@@ -339,26 +341,26 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
addr2.is_ipv4 = true;
addr2.port = port;
if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) {
sys_free(toasty);
free(toasty);
return NULL;
}
}
if (mutex_init(&toasty->mutex) < 0) {
sys_free(toasty);
free(toasty);
return NULL;
}
if (tcp_context_init(&toasty->tcp) < 0) {
mutex_free(&toasty->mutex);
sys_free(toasty);
free(toasty);
return NULL;
}
if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) {
tcp_context_free(&toasty->tcp);
mutex_free(&toasty->mutex);
sys_free(toasty);
free(toasty);
return NULL;
}
@@ -435,7 +437,7 @@ void toasty_disconnect(ToastyFS *toasty)
{
tcp_context_free(&toasty->tcp);
mutex_free(&toasty->mutex);
sys_free(toasty);
free(toasty);
}
static bool
@@ -947,7 +949,7 @@ static void process_event_for_list(ToastyFS *toasty,
return;
}
ToastyListingEntry *entities = sys_malloc(item_count * sizeof(ToastyListingEntry));
ToastyListingEntry *entities = malloc(item_count * sizeof(ToastyListingEntry));
if (entities == NULL) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
return;
@@ -959,28 +961,28 @@ static void process_event_for_list(ToastyFS *toasty,
uint64_t gen;
if (!binary_read(&reader, &gen, sizeof(gen))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
uint8_t is_dir;
if (!binary_read(&reader, &is_dir, sizeof(is_dir))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
uint16_t name_len;
if (!binary_read(&reader, &name_len, sizeof(name_len))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
char *name = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, name_len)) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
@@ -989,7 +991,7 @@ static void process_event_for_list(ToastyFS *toasty,
if (name_len > sizeof(entities[i].name)-1) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
memcpy(entities[i].name, name, name_len);
@@ -999,7 +1001,7 @@ static void process_event_for_list(ToastyFS *toasty,
// Check there is nothing else to read
if (binary_read(&reader, NULL, 1)) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities);
free(entities);
return;
}
@@ -1089,7 +1091,7 @@ static void process_event_for_read(ToastyFS *toasty,
}
// Allocate ranges
Range *ranges = sys_malloc(num_chunks_needed * sizeof(Range));
Range *ranges = malloc(num_chunks_needed * sizeof(Range));
if (ranges == NULL) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
@@ -1104,7 +1106,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Read hash
SHA256 hash;
if (!binary_read(&reader, &hash, sizeof(hash))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1112,7 +1114,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Read number of servers
uint32_t num_servers;
if (!binary_read(&reader, &num_servers, sizeof(num_servers))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1120,7 +1122,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Parse IPv4 addresses
uint32_t num_ipv4;
if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1134,7 +1136,7 @@ static void process_event_for_read(ToastyFS *toasty,
uint16_t port;
if (!binary_read(&reader, &ipv4, sizeof(ipv4)) ||
!binary_read(&reader, &port, sizeof(port))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1149,21 +1151,21 @@ static void process_event_for_read(ToastyFS *toasty,
// Skip IPv6 addresses
uint32_t num_ipv6;
if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
for (uint32_t j = 0; j < num_ipv6; j++) {
if (!binary_read(&reader, NULL, sizeof(IPv6)) ||
!binary_read(&reader, NULL, sizeof(uint16_t))) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
}
if (!found) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1208,7 +1210,7 @@ static void process_event_for_read(ToastyFS *toasty,
Range *r = &ranges[0];
int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL);
if (cs_idx < 0) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1216,7 +1218,7 @@ static void process_event_for_read(ToastyFS *toasty,
if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk,
r->length_within_chunk, opidx, 0) < 0) {
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return;
}
@@ -1225,7 +1227,7 @@ static void process_event_for_read(ToastyFS *toasty,
toasty->operations[opidx].ranges_head = 1;
} else {
// No chunks to download
sys_free(ranges);
free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
}
@@ -1299,7 +1301,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Check if done
if (toasty->operations[opidx].num_pending == 0) {
sys_free(toasty->operations[opidx].ranges);
free(toasty->operations[opidx].ranges);
toasty->operations[opidx].ranges = NULL;
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
}
@@ -1424,7 +1426,7 @@ static int schedule_upload(ToastyFS *toasty, int opidx, UploadSchedule upload)
else
new_cap_uploads = 2 * o->cap_uploads;
UploadSchedule *uploads = sys_malloc(new_cap_uploads * sizeof(UploadSchedule));
UploadSchedule *uploads = malloc(new_cap_uploads * sizeof(UploadSchedule));
if (uploads == NULL)
return -1;
@@ -1534,7 +1536,7 @@ static void process_event_for_write(ToastyFS *toasty,
toasty->operations[opidx].num_chunks = num_all_hasehs;
toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow
toasty->operations[opidx].hashes = sys_malloc(num_hashes * sizeof(SHA256));
toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256));
if (toasty->operations[opidx].hashes == NULL) {
assert(0); // TODO
}
@@ -1957,7 +1959,7 @@ static void process_event_for_write(ToastyFS *toasty,
} ChunkUploadResult;
int num_upload_results = toasty->operations[opidx].num_chunks;
ChunkUploadResult *upload_results = sys_malloc(num_upload_results * sizeof(ChunkUploadResult));
ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult));
if (upload_results == NULL) {
assert(0); // TODO
}
@@ -2471,7 +2473,7 @@ int toasty_list(ToastyFS *toasty, ToastyString path,
void toasty_free_listing(ToastyListing *listing)
{
sys_free(listing->items);
free(listing->items);
}
int toasty_read(ToastyFS *toasty, ToastyString path,
src/crash_logger.c
-433
View File
@@ -1,433 +0,0 @@
#if defined(__linux__) && defined(__x86_64__)
#define _GNU_SOURCE
#include <elf.h>
#include <fcntl.h>
#include <stdio.h>
#include <signal.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <stdbool.h>
#include <sys/reg.h>
#include <sys/stat.h>
#include <ucontext.h>
typedef struct {
uint64_t base_addr;
int count;
void *symbols;
void *strings;
} SymbolTable;
static int is_hex(char c)
{
return (c >= '0' && c <= '9') ||
(c >= 'a' && c <= 'f') ||
(c >= 'A' && c <= 'F');
}
static uint64_t query_base_addr(void)
{
int fd = open("/proc/self/maps", O_RDONLY);
if (fd < 0)
return -1;
char buf[128];
int ret = read(fd, buf, sizeof(buf));
if (ret < 0) {
close(fd);
return -1;
}
close(fd);
if (ret == 0 || !is_hex(buf[0]))
return -1;
int i = 0;
uint64_t base_addr = 0;
for (;;) {
char c = buf[i++];
int d;
if (0) {}
else if (c >= 'a' && c <= 'f') d = c - 'a' + 10;
else if (c >= 'A' && c <= 'F') d = c - 'A' + 10;
else d = c - '0';
if (base_addr > (UINT64_MAX - d) / 16)
return -1;
base_addr = base_addr * 16 + d;
if (i == ret)
return -1;
if (buf[i] == '-')
break;
if (!is_hex(buf[i]))
return -1;
}
return base_addr;
}
static int current_executable_path(char *dst, int cap)
{
if (cap == 0)
return -1;
int ret = readlink("/proc/self/exe", dst, cap-1);
if (ret < 0)
return -1;
dst[ret] = '\0';
return ret;
}
static int load_symbols_from_elf(void *src, int len, SymbolTable *st)
{
// NOTE: It's assumed is properly aligned
assert(((uintptr_t) src & 15) == 0);
// Check that the file contains a full header
if (len < (int) sizeof(Elf64_Ehdr))
return -1;
Elf64_Ehdr *ehdr = (Elf64_Ehdr*) src;
// Check that the file contains the full list
// of section headers
if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > len)
return -1;
Elf64_Shdr *shdrs = (Elf64_Shdr*) (src + ehdr->e_shoff);
Elf64_Shdr *shstrtab_hdr = &shdrs[ehdr->e_shstrndx]; // TODO: bounds check
char *shstrtab = src + shstrtab_hdr->sh_offset;
// Iterate over the section headers to find the
// one reative to symbols and their strings
Elf64_Shdr *symtab_hdr = NULL;
Elf64_Shdr *strtab_hdr = NULL;
for (int i = 0; i < ehdr->e_shnum; i++) {
char *section_name = shstrtab + shdrs[i].sh_name;
if (0) {}
else if (!strcmp(section_name, ".symtab")) symtab_hdr = &shdrs[i];
else if (!strcmp(section_name, ".strtab")) strtab_hdr = &shdrs[i];
}
if (symtab_hdr == NULL || strtab_hdr == NULL) {
return -1;
}
void *mem = malloc(symtab_hdr->sh_size + strtab_hdr->sh_size);
if (mem == NULL) {
return -1;
}
st->count = symtab_hdr->sh_size / sizeof(Elf64_Sym);
st->symbols = mem;
st->strings = (char*) st->symbols + symtab_hdr->sh_size;
memcpy(st->symbols, src + symtab_hdr->sh_offset, symtab_hdr->sh_size);
memcpy(st->strings, src + strtab_hdr->sh_offset, strtab_hdr->sh_size);
return 0;
}
static char *read_file(char *path, int *len)
{
int fd = open(path, O_RDONLY);
if (fd < 0)
return NULL;
struct stat buf;
if (fstat(fd, &buf) < 0) {
close(fd);
return NULL;
}
*len = buf.st_size;
char *ptr = malloc(*len + 1);
if (ptr == NULL) {
close(fd);
return NULL;
}
for (int num = 0; num < *len; ) {
int ret = read(fd, ptr + num, *len - num);
if (ret <= 0) {
free(ptr);
close(fd);
return NULL;
}
num += ret;
}
ptr[*len] = '\0';
return ptr;
}
static int symbol_table_from_current_process(SymbolTable *st)
{
uint64_t base_addr = query_base_addr();
if (base_addr == (uint64_t) -1)
return -1;
st->base_addr = base_addr;
char path[1<<10];
if (current_executable_path(path, sizeof(path)) < 0)
return -1;
char *exe_ptr;
int exe_len;
exe_ptr = read_file(path, &exe_len);
if (exe_ptr == NULL)
return -1;
if (load_symbols_from_elf(exe_ptr, exe_len, st) < 0) {
free(exe_ptr);
return -1;
}
free(exe_ptr);
return 0;
}
static void symbol_table_free(SymbolTable *st)
{
free(st->symbols);
}
static char *symbol_table_find(SymbolTable *st, uint64_t addr)
{
for (int i = 0; i < st->count; i++) {
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
if (sym->st_value == 0)
continue;
uint64_t sym_beg = st->base_addr + sym->st_value;
uint64_t sym_end = st->base_addr + sym->st_value + sym->st_size;
if (addr >= sym_beg && addr < sym_end)
return (char*) st->strings + sym->st_name;
}
return NULL;
}
#if 0
static void symbol_table_dump(SymbolTable *st)
{
for (int i = 0; i < st->count; i++) {
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
if (sym->st_value == 0)
continue;
char *name = (char*) st->strings + sym->st_name;
printf("%s\n", name);
}
}
#endif
typedef struct {
char *name;
uint64_t addr;
} StackFrame;
static int walk_stack(uint64_t rip, uint64_t rbp, SymbolTable *st, StackFrame *frames, int max_frames)
{
int frame_count = 0;
if (frame_count < max_frames) {
frames[frame_count].addr = rip - st->base_addr;
frames[frame_count].name = symbol_table_find(st, rip);
frame_count++;
}
while (rbp != 0) {
if (rbp & 0xF)
break;
uint64_t *frame_ptr = (uint64_t*) rbp;
uint64_t next_rbp = frame_ptr[0];
uint64_t return_addr = frame_ptr[1];
if (next_rbp != 0 && next_rbp <= rbp)
break;
if (return_addr == 0)
break;
if (frame_count == max_frames)
break;
frames[frame_count].addr = return_addr - st->base_addr;
frames[frame_count].name = symbol_table_find(st, return_addr);
frame_count++;
rbp = next_rbp;
}
return frame_count;
}
static bool crash_logger_symbol_init = false;
static char* crash_logger_file_name = NULL;
static SymbolTable crash_logger_symbol_table;
static char* crash_logger_signal_stack;
static void crash_handler(int sig, siginfo_t *info, void *ucontext)
{
(void) info;
if (crash_logger_symbol_init) {
// Buffer for evaluating format strings
char tmp[1<<9];
int len;
ucontext_t *ctx = (ucontext_t*) ucontext;
uint64_t rip = ctx->uc_mcontext.gregs[REG_RIP];
uint64_t rbp = ctx->uc_mcontext.gregs[REG_RBP];
StackFrame frames[64];
int count = walk_stack(rip, rbp, &crash_logger_symbol_table, frames, 64);
int fd = open(crash_logger_file_name, O_WRONLY | O_CREAT, 0666);
if (fd < 0)
exit(1);
char *sig_name = "";
switch (sig) {
case SIGSEGV: sig_name = "Segmentation fault"; break;
case SIGBUS : sig_name = "Bus error"; break;
case SIGILL : sig_name = "Illegal instruction"; break;
case SIGFPE : sig_name = "Floating point exception"; break;
case SIGTRAP: sig_name = "Trace trap"; break;
case SIGSYS : sig_name = "Bad system call"; break;
case SIGABRT: sig_name = "Abort"; break;
}
if (sig_name[0] == '\0') {
len = snprintf(tmp, sizeof(tmp), "(unknown signal %d)\n", sig);
write(fd, tmp, len);
} else {
write(fd, sig_name, strlen(sig_name));
write(fd, "\n", 1);
}
for (int i = 0; i < count; i++) {
len = snprintf(tmp, sizeof(tmp), " [%d] 0x%lx %s\n", i, frames[i].addr,
frames[i].name ? frames[i].name : "?");
write(fd, tmp, len);
}
close(fd);
}
exit(1);
}
int crash_logger_init(char *file_name, int file_name_len)
{
{
char *file_name_copy = malloc(file_name_len + 1);
if (file_name_copy == NULL)
return -1;
memcpy(file_name_copy, file_name, file_name_len);
file_name_copy[file_name_len] = '\0';
crash_logger_file_name = file_name_copy;
}
if (symbol_table_from_current_process(&crash_logger_symbol_table) < 0) {
free(crash_logger_file_name);
return -1;
}
// Set up alternate signal stack
{
crash_logger_signal_stack = malloc(SIGSTKSZ);
if (crash_logger_signal_stack == NULL) {
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
return -1;
}
stack_t ss;
ss.ss_sp = crash_logger_signal_stack;
ss.ss_size = SIGSTKSZ;
ss.ss_flags = 0;
if (sigaltstack(&ss, NULL) < 0) {
free(crash_logger_signal_stack);
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
return -1;
}
}
{
// Register the crash handler
struct sigaction sa;
sa.sa_sigaction = crash_handler;
sa.sa_flags = SA_SIGINFO | SA_ONSTACK; // Add SA_ONSTACK flag
sigemptyset(&sa.sa_mask);
// Memory errors
sigaction(SIGSEGV, &sa, NULL); // Segmentation fault (invalid memory access)
sigaction(SIGBUS, &sa, NULL); // Bus error (misaligned access, hardware error)
// Execution errors
sigaction(SIGILL, &sa, NULL); // Illegal instruction
sigaction(SIGFPE, &sa, NULL); // Floating point exception
sigaction(SIGTRAP, &sa, NULL); // Trace trap
// System/resource errors
sigaction(SIGSYS, &sa, NULL); // Bad system call
sigaction(SIGABRT, &sa, NULL); // Abort (from assert, abort(), etc.)
// Optional: Resource limit violations
sigaction(SIGXCPU, &sa, NULL); // CPU time limit exceeded
sigaction(SIGXFSZ, &sa, NULL); // File size limit exceeded
}
crash_logger_symbol_init = true;
return 0;
}
void crash_logger_free(void)
{
if (!crash_logger_symbol_init)
return;
free(crash_logger_signal_stack);
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
crash_logger_symbol_init = false;
}
#else
static int crash_logger_init(char *file_name, int file_name_len)
{
(void) file_name;
(void) file_name_len;
return -1;
}
static void crash_logger_free(void)
{
}
#endif
src/crash_logger.h
-7
View File
@@ -1,7 +0,0 @@
#ifndef CRASH_LOGGER_INCLUDED
#define CRASH_LOGGER_INCLUDED
int crash_logger_init(char *file_name, int file_name_len);
void crash_logger_free(void);
#endif // CRASH_LOGGER_INCLUDED
src/file_system.c
+41 -40
View File
@@ -1,8 +1,9 @@
#include <limits.h>
#include <string.h>
#include <assert.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "system.h"
#include "file_system.h"
int rename_file_or_dir(string oldpath, string newpath);
@@ -16,7 +17,7 @@ int file_open(string path, Handle *fd)
memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0';
int ret = sys_open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
if (ret < 0)
return -1;
@@ -29,7 +30,7 @@ int file_open(string path, Handle *fd)
MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH);
wpath[path.len] = L'\0';
HANDLE h = sys_CreateFileW(
HANDLE h = CreateFileW(
wpath,
GENERIC_WRITE | GENERIC_READ,
0,
@@ -49,18 +50,18 @@ int file_open(string path, Handle *fd)
void file_close(Handle fd)
{
#ifdef __linux__
sys_close((int) fd.data);
close((int) fd.data);
#endif
#ifdef _WIN32
sys_CloseHandle((HANDLE) fd.data);
CloseHandle((HANDLE) fd.data);
#endif
}
int file_set_offset(Handle fd, int off)
{
#ifdef __linux__
off_t ret = sys_lseek((int) fd.data, off, SEEK_SET);
off_t ret = lseek((int) fd.data, off, SEEK_SET);
if (ret < 0)
return -1;
return 0;
@@ -69,8 +70,8 @@ int file_set_offset(Handle fd, int off)
#ifdef _WIN32
LARGE_INTEGER distance;
distance.QuadPart = off;
if (!sys_SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
if (GetLastError() != NO_ERROR)
if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
if (GetLastError() != 0)
return -1;
return 0;
#endif
@@ -79,7 +80,7 @@ int file_set_offset(Handle fd, int off)
int file_get_offset(Handle fd, int *off)
{
#ifdef __linux__
off_t ret = sys_lseek((int) fd.data, 0, SEEK_CUR);
off_t ret = lseek((int) fd.data, 0, SEEK_CUR);
if (ret < 0)
return -1;
*off = (int) ret;
@@ -87,8 +88,8 @@ int file_get_offset(Handle fd, int *off)
#endif
#ifdef _WIN32
DWORD pos = sys_SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
if (pos == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR)
DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0)
return -1;
*off = (int) pos;
return 0;
@@ -98,13 +99,13 @@ int file_get_offset(Handle fd, int *off)
int file_lock(Handle fd)
{
#ifdef __linux__
if (sys_flock((int) fd.data, LOCK_EX) < 0)
if (flock((int) fd.data, LOCK_EX) < 0)
return -1;
return 0;
#endif
#ifdef _WIN32
if (!sys_LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
return -1;
return 0;
#endif
@@ -113,13 +114,13 @@ int file_lock(Handle fd)
int file_unlock(Handle fd)
{
#ifdef __linux__
if (sys_flock((int) fd.data, LOCK_UN) < 0)
if (flock((int) fd.data, LOCK_UN) < 0)
return -1;
return 0;
#endif
#ifdef _WIN32
if (!sys_UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
return -1;
return 0;
#endif
@@ -128,13 +129,13 @@ int file_unlock(Handle fd)
int file_sync(Handle fd)
{
#ifdef __linux__
if (sys_fsync((int) fd.data) < 0)
if (fsync((int) fd.data) < 0)
return -1;
return 0;
#endif
#ifdef _WIN32
if (!sys_FlushFileBuffers((HANDLE) fd.data))
if (!FlushFileBuffers((HANDLE) fd.data))
return -1;
return 0;
#endif
@@ -143,12 +144,12 @@ int file_sync(Handle fd)
int file_read(Handle fd, char *dst, int max)
{
#ifdef __linux__
return sys_read((int) fd.data, dst, max);
return read((int) fd.data, dst, max);
#endif
#ifdef _WIN32
DWORD num;
if (!sys_ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
return -1;
if (num > INT_MAX)
return -1;
@@ -159,12 +160,12 @@ int file_read(Handle fd, char *dst, int max)
int file_write(Handle fd, char *src, int len)
{
#ifdef __linux__
return sys_write((int) fd.data, src, len);
return write((int) fd.data, src, len);
#endif
#ifdef _WIN32
DWORD num;
if (!sys_WriteFile((HANDLE) fd.data, src, len, &num, NULL))
if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL))
return -1;
if (num > INT_MAX)
return -1;
@@ -176,7 +177,7 @@ int file_size(Handle fd, size_t *len)
{
#ifdef __linux__
struct stat buf;
if (sys_fstat((int) fd.data, &buf) < 0)
if (fstat((int) fd.data, &buf) < 0)
return -1;
if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX)
return -1;
@@ -186,7 +187,7 @@ int file_size(Handle fd, size_t *len)
#ifdef _WIN32
LARGE_INTEGER buf;
if (!sys_GetFileSizeEx((HANDLE) fd.data, &buf))
if (!GetFileSizeEx((HANDLE) fd.data, &buf))
return -1;
if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX)
return -1;
@@ -204,10 +205,10 @@ int create_dir(string path)
zt[path.len] = '\0';
#ifdef _WIN32
if (sys__mkdir(zt) < 0)
if (_mkdir(zt) < 0)
return -1;
#else
if (sys_mkdir(zt, 0766))
if (mkdir(zt, 0766))
return -1;
#endif
@@ -228,7 +229,7 @@ int rename_file_or_dir(string oldpath, string newpath)
memcpy(newpath_zt, newpath.ptr, newpath.len);
newpath_zt[newpath.len] = '\0';
if (sys_rename(oldpath_zt, newpath_zt))
if (rename(oldpath_zt, newpath_zt))
return -1;
return 0;
}
@@ -241,7 +242,7 @@ int remove_file_or_dir(string path)
memcpy(path_zt, path.ptr, path.len);
path_zt[path.len] = '\0';
if (sys_remove(path_zt))
if (remove(path_zt))
return -1;
return 0;
}
@@ -255,12 +256,12 @@ int get_full_path(string path, char *dst)
path_zt[path.len] = '\0';
#ifdef __linux__
if (sys_realpath(path_zt, dst) == NULL)
if (realpath(path_zt, dst) == NULL)
return -1;
#endif
#ifdef _WIN32
if (sys__fullpath(path_zt, dst, PATH_MAX) == NULL)
if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
return -1;
#endif
@@ -285,7 +286,7 @@ int file_read_all(string path, string *data)
return -1;
}
char *dst = sys_malloc(len);
char *dst = malloc(len);
if (dst == NULL) {
file_close(fd);
return -1;
@@ -295,7 +296,7 @@ int file_read_all(string path, string *data)
while ((size_t) copied < len) {
ret = file_read(fd, dst + copied, len - copied);
if (ret < 0) {
sys_free(dst);
free(dst);
file_close(fd);
return -1;
}
@@ -316,7 +317,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
if (ret < 0 || ret >= (int) sizeof(pattern))
return -1;
scanner->handle = sys_FindFirstFileA(pattern, &scanner->find_data);
scanner->handle = FindFirstFileA(pattern, &scanner->find_data);
if (scanner->handle == INVALID_HANDLE_VALUE) {
if (GetLastError() == ERROR_FILE_NOT_FOUND) {
scanner->done = true;
@@ -336,7 +337,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
return 1;
if (!scanner->first) {
BOOL ok = sys_FindNextFileA(scanner->handle, &scanner->find_data);
BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data);
if (!ok) {
scanner->done = true;
if (GetLastError() == ERROR_NO_MORE_FILES)
@@ -354,7 +355,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
void directory_scanner_free(DirectoryScanner *scanner)
{
sys_FindClose(scanner->handle);
FindClose(scanner->handle);
}
#else
@@ -367,7 +368,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
memcpy(path_copy, path.ptr, path.len);
path_copy[path.len] = '\0';
scanner->d = sys_opendir(path_copy);
scanner->d = opendir(path_copy);
if (scanner->d == NULL) {
scanner->done = true;
return -1;
@@ -382,7 +383,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
if (scanner->done)
return 1;
scanner->e = sys_readdir(scanner->d);
scanner->e = readdir(scanner->d);
if (scanner->e == NULL) {
scanner->done = true;
return 1;
@@ -394,7 +395,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
void directory_scanner_free(DirectoryScanner *scanner)
{
sys_closedir(scanner->d);
closedir(scanner->d);
}
#endif
src/file_system.h
+6 -8
View File
@@ -1,14 +1,12 @@
#ifndef FILE_SYSTEM_INCLUDED
#define FILE_SYSTEM_INCLUDED
#include "basic.h"
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#else
#include <dirent.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
typedef struct {
uint64_t data;
@@ -50,4 +48,4 @@ int directory_scanner_init(DirectoryScanner *scanner, string path);
int directory_scanner_next(DirectoryScanner *scanner, string *name);
void directory_scanner_free(DirectoryScanner *scanner);
#endif // FILE_SYSTEM_INCLUDED
#endif // FILE_SYSTEM_INCLUDED
src/file_tree.c
+18 -16
View File
@@ -1,10 +1,12 @@
#include <limits.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <quakey.h>
#include "basic.h"
#include "system.h"
#include "file_tree.h"
static int parse_path(string path, string *comps, int max)
@@ -102,7 +104,7 @@ static void dir_free(Dir *d)
{
for (uint64_t i = 0; i < d->num_children; i++)
entity_free(&d->children[i]);
sys_free(d->children);
free(d->children);
}
static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
@@ -158,7 +160,7 @@ static void file_init(File *f, uint64_t chunk_size)
static void file_free(File *f)
{
sys_free(f->chunks);
free(f->chunks);
f->chunks = NULL;
}
@@ -312,14 +314,14 @@ int file_tree_create_entity(FileTree *ft, string path,
if (new_max == 0)
new_max = 8;
Entity *p = sys_malloc(sizeof(Entity) * new_max);
Entity *p = malloc(sizeof(Entity) * new_max);
if (p == NULL)
return FILETREE_NOMEM;
for (uint64_t i = 0; i < d->num_children; i++)
p[i] = d->children[i];
sys_free(d->children);
free(d->children);
d->children = p;
d->max_children = new_max;
}
@@ -423,13 +425,13 @@ int file_tree_write(
if (last_chunk_index >= f->num_chunks) {
uint64_t old_num_chunks = f->num_chunks;
SHA256 *new_chunks = sys_malloc((last_chunk_index+1) * sizeof(SHA256));
SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256));
if (new_chunks == NULL)
return FILETREE_NOMEM;
if (f->chunks) {
if (f->num_chunks > 0)
memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256));
sys_free(f->chunks);
free(f->chunks);
}
f->chunks = new_chunks;
f->num_chunks = last_chunk_index+1;
@@ -678,13 +680,13 @@ int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *wr
sc.write_data = write_data;
sc.buffer_used = 0;
sc.buffer_size = 1<<10;
sc.buffer = sys_malloc(sc.buffer_size);
sc.buffer = malloc(sc.buffer_size);
sc.error = false;
if (sc.buffer == NULL)
sc.error = true;
entity_serialize(&sc, &ft->root);
sc_flush(&sc);
sys_free(sc.buffer);
free(sc.buffer);
if (sc.error)
return -1;
return 0;
@@ -748,7 +750,7 @@ static void file_deserialize(DeserializeContext *dc, File *f)
dc_read_u64(dc, &f->num_chunks);
dc_read_u64(dc, &f->file_size);
f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256));
f->chunks = malloc(f->num_chunks * sizeof(SHA256));
if (f->chunks == NULL) {
assert(0); // TODO
}
@@ -764,7 +766,7 @@ static void dir_deserialize(DeserializeContext *dc, Dir *d)
dc_read_u64(dc, &d->num_children);
d->max_children = d->num_children;
d->children = sys_malloc(d->num_children * sizeof(Entity));
d->children = malloc(d->num_children * sizeof(Entity));
if (d->children == NULL) {
assert(0); // TODO
}
@@ -799,13 +801,13 @@ int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *r
dc.buffer_head = 0;
dc.buffer_used = 0;
dc.buffer_size = 1<<10;
dc.buffer = sys_malloc(dc.buffer_size);
dc.buffer = malloc(dc.buffer_size);
dc.error = false;
if (dc.buffer == NULL)
dc.error = true;
dc.total_read = 0;
entity_deserialize(&dc, &ft->root);
sys_free(dc.buffer);
free(dc.buffer);
if (dc.error)
return -1;
if (dc.total_read > INT_MAX) {
src/hash_set.c
+10 -9
View File
@@ -1,7 +1,8 @@
#include <assert.h>
#include <string.h>
#include "system.h"
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "hash_set.h"
void hash_set_init(HashSet *set)
@@ -13,7 +14,7 @@ void hash_set_init(HashSet *set)
void hash_set_free(HashSet *set)
{
sys_free(set->items);
free(set->items);
set->items = NULL;
}
@@ -40,7 +41,7 @@ int hash_set_insert(HashSet *set, SHA256 hash)
else
new_capacity = 2 * set->capacity;
SHA256 *new_items = sys_realloc(set->items, new_capacity * sizeof(SHA256));
SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256));
if (new_items == NULL)
return -1;
@@ -109,7 +110,7 @@ void timed_hash_set_init(TimedHashSet *set)
void timed_hash_set_free(TimedHashSet *set)
{
sys_free(set->items);
free(set->items);
set->items = NULL;
}
@@ -137,13 +138,13 @@ int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time)
else
new_capacity = 2 * set->capacity;
TimedHash *new_items = sys_malloc(new_capacity * sizeof(TimedHash));
TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash));
if (new_items == NULL)
return -1;
if (set->capacity > 0) {
memcpy(new_items, set->items, set->count * sizeof(set->items[0]));
sys_free(set->items);
free(set->items);
}
set->items = new_items;
src/main.c
+170
View File
@@ -0,0 +1,170 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#else
#define POLL_CAPACITY 1024
#endif
#include <stdint.h>
#include <quakey.h>
#include "metadata_server.h"
#include "chunk_server.h"
#include "random_client.h"
#ifdef MAIN_METADATA_SERVER
int main(int argc, char **argv)
{
int ret;
MetadataServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = metadata_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = metadata_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
metadata_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_CHUNK_SERVER
int main(int argc, char **argv)
{
int ret;
ChunkServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = chunk_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = chunk_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
chunk_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_SIMULATION
int main(void)
{
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
if (ret < 0)
return -1;
// Client
{
QuakeySpawn config = {
.state_size = sizeof(RandomClient),
.init_func = random_client_init,
.tick_func = random_client_tick,
.free_func = random_client_free,
.addrs = (char*[]) { "127.0.0.2" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.2");
}
// Metadata Server
{
QuakeySpawn config = {
.state_size = sizeof(MetadataServer),
.init_func = metadata_server_init,
.tick_func = metadata_server_tick,
.free_func = metadata_server_free,
.addrs = (char*[]) { "127.0.0.3" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "ms --addr 127.0.0.3");
}
// Chunk Server
{
QuakeySpawn config = {
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.addrs = (char*[]) { "127.0.0.4" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.4");
}
for (;;)
quakey_schedule_one(quakey);
quakey_free(quakey);
return 0;
}
#endif // MAIN_SIMULATION
src/main_server.c
-132
View File
@@ -1,132 +0,0 @@
#ifdef BUILD_SERVER
#include <time.h>
#include <string.h>
#ifdef _WIN32
#include <winsock2.h>
#define POLL WSAPoll
#else
#include <poll.h>
#define POLL poll
#endif
#include "crash_logger.h"
#include "chunk_server.h"
#include "metadata_server.h"
int metadata_server_main(int argc, char **argv)
{
{
time_t now = time(NULL);
struct tm unpacked_now;
#ifdef _WIN32
if (gmtime_s(&unpacked_now, &now) != 0)
return -1;
#else
if (gmtime_r(&now, &unpacked_now) == NULL)
return -1;
#endif
char time[sizeof("YYYYMMDDthhmmssz")];
int ret = strftime(time, sizeof(time),
"%Y%m%dT%H%M%SZ", &unpacked_now);
if (ret != sizeof(time)-1)
return -1;
char path[1<<10];
ret = snprintf(path, sizeof(path), "crash_%s_MS_%d.txt", time, getpid());
if (ret < 0 || ret >= (int) sizeof(path)) {
return -1;
}
int path_len = ret;
crash_logger_init(path, path_len);
}
void *contexts[TCP_POLL_CAPACITY];
struct pollfd polled[TCP_POLL_CAPACITY];
int num_polled;
int timeout = -1;
MetadataServer state;
num_polled = metadata_server_init(
&state, argc, argv, contexts, polled, &timeout);
if (num_polled < 0) return -1;
for (;;) {
POLL(polled, num_polled, timeout);
timeout = -1;
num_polled = metadata_server_step(
&state, contexts, polled, num_polled, &timeout);
if (num_polled < 0) return -1;
}
metadata_server_free(&state);
crash_logger_free();
return 0;
}
int chunk_server_main(int argc, char **argv)
{
{
time_t now = time(NULL);
struct tm unpacked_now;
#ifdef _WIN32
if (gmtime_s(&unpacked_now, &now) != 0)
return -1;
#else
if (gmtime_r(&now, &unpacked_now) == NULL)
return -1;
#endif
char time[sizeof("YYYYMMDDthhmmssz")];
int ret = strftime(time, sizeof(time),
"%Y%m%dT%H%M%SZ", &unpacked_now);
if (ret != sizeof(time)-1)
return -1;
char path[1<<10];
ret = snprintf(path, sizeof(path), "crash_%s_CS_%d.txt", time, getpid());
if (ret < 0 || ret >= (int) sizeof(path)) {
return -1;
}
int path_len = ret;
crash_logger_init(path, path_len);
}
void *contexts[TCP_POLL_CAPACITY];
struct pollfd polled[TCP_POLL_CAPACITY];
int num_polled;
int timeout = -1;
ChunkServer state;
num_polled = chunk_server_init(
&state, argc, argv, contexts, polled, &timeout);
if (num_polled < 0) return -1;
for (;;) {
POLL(polled, num_polled, timeout);
timeout = -1;
num_polled = chunk_server_step(
&state, contexts, polled, num_polled, &timeout);
if (num_polled < 0) return -1;
}
chunk_server_free(&state);
crash_logger_free();
return 0;
}
int main(int argc, char **argv)
{
if (getargb(argc, argv, "--leader"))
return metadata_server_main(argc, argv);
else
return chunk_server_main(argc, argv);
}
#endif
src/main_test.c
-57
View File
@@ -1,57 +0,0 @@
#ifdef BUILD_TEST
#include <stdio.h>
#include <signal.h>
#include <stdbool.h>
#include "system.h"
static sig_atomic_t simulation_should_stop = false;
static void signal_handler(int signum)
{
(void)signum;
simulation_should_stop = true;
}
int main(int argc, char **argv)
{
(void)argc;
(void)argv;
// Set up signal handlers for clean shutdown
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
startup_simulation(2);
// Spawn metadata server (leader)
spawn_simulated_process("--addr 127.0.0.1 --port 8080 --leader");
// Spawn chunk servers
spawn_simulated_process("--addr 127.0.0.1 --port 8081 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_0/");
spawn_simulated_process("--addr 127.0.0.1 --port 8082 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_1/");
spawn_simulated_process("--addr 127.0.0.1 --port 8083 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_2/");
spawn_simulated_process("--addr 127.0.0.1 --port 8084 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_3/");
spawn_simulated_process("--addr 127.0.0.1 --port 8085 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_4/");
spawn_simulated_process("--addr 127.0.0.1 --port 8086 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_5/");
spawn_simulated_process("--addr 127.0.0.1 --port 8087 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_6/");
spawn_simulated_process("--addr 127.0.0.1 --port 8088 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_7/");
spawn_simulated_process("--addr 127.0.0.1 --port 8089 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_8/");
spawn_simulated_process("--addr 127.0.0.1 --port 8090 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_9/");
// Spawn simulation client
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
while (!simulation_should_stop)
update_simulation();
cleanup_simulation();
return 0;
}
#endif
src/message.c
+4 -7
View File
@@ -1,11 +1,8 @@
#include <string.h>
#ifdef _WIN32
#include <ws2tcpip.h>
#else
#include <arpa/inet.h> // inet_ntop
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "message.h"
bool binary_read(BinaryReader *reader, void *dst, int len)
src/message.h
+5 -3
View File
@@ -1,9 +1,11 @@
#ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED
#include <stdio.h>
#include <stdbool.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
#include "byte_queue.h"
src/metadata_server.c
+46 -18
View File
@@ -1,8 +1,10 @@
#define _GNU_SOURCE
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <string.h>
#include <quakey.h>
#include <stdint.h>
#include <assert.h>
#include <stdlib.h>
#include "message.h"
#include "metadata_server.h"
@@ -1061,19 +1063,27 @@ static bool is_chunk_server_message_type(uint16_t type)
return false;
}
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout)
int metadata_server_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
MetadataServer *state = state_;
string addr = getargs(argc, argv, "--addr", "127.0.0.1");
int port = getargi(argc, argv, "--port", 8080);
bool trace = getargb(argc, argv, "--trace");
string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal");
int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit
if (port <= 0 || port >= 1<<16)
if (port <= 0 || port >= 1<<16) {
fprintf(stderr, "metadata server :: Invalid port\n");
return -1;
}
if (wal_limit < 0)
if (wal_limit < 0) {
fprintf(stderr, "metadata server :: Invalid WAL limit\n");
return -1;
}
state->trace = trace;
state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro?
@@ -1084,22 +1094,27 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
for (int i = 0; i < MAX_CHUNK_SERVERS; i++)
state->chunk_servers[i].used = false;
if (tcp_context_init(&state->tcp) < 0)
if (tcp_context_init(&state->tcp) < 0) {
fprintf(stderr, "metadata server :: Couldn't setup TCP context\n");
return -1;
}
int ret = tcp_listen(&state->tcp, addr, port);
if (ret < 0) {
fprintf(stderr, "metadata server :: Couldn't setup TCP listener\n");
tcp_context_free(&state->tcp);
return -1;
}
ret = file_tree_init(&state->file_tree);
if (ret < 0) {
fprintf(stderr, "metadata server :: Couldn't setup file tree\n");
tcp_context_free(&state->tcp);
return -1;
}
if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) {
fprintf(stderr, "metadata server :: Couldn't setup WAL\n");
assert(0); // TODO
}
@@ -1110,21 +1125,21 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
);
*timeout = -1; // No timeout until we have chunk servers
return tcp_register_events(&state->tcp, contexts, polled);
}
int metadata_server_free(MetadataServer *state)
{
wal_close(&state->wal);
file_tree_free(&state->file_tree);
tcp_context_free(&state->tcp);
if (pcap < TCP_POLL_CAPACITY) {
fprintf(stderr, "metadata server :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
return -1;
}
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
int metadata_server_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{
MetadataServer *state = state_;
Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
Time current_time = get_current_time();
if (current_time == INVALID_TIME)
@@ -1243,5 +1258,18 @@ int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *
}
*timeout = deadline_to_timeout(next_wakeup, current_time);
return tcp_register_events(&state->tcp, contexts, polled);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int metadata_server_free(void *state_)
{
MetadataServer *state = state_;
wal_close(&state->wal);
file_tree_free(&state->file_tree);
tcp_context_free(&state->tcp);
return 0;
}
src/metadata_server.h
+11 -4
View File
@@ -52,8 +52,15 @@ typedef struct {
} MetadataServer;
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout);
int metadata_server_free(MetadataServer *state);
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
struct pollfd;
#endif // METADATA_SERVER_INCLUDED
int metadata_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int metadata_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int metadata_server_free(void *state);
#endif // METADATA_SERVER_INCLUDED
src/simulation_client.c → src/random_client.c
+42 -16
View File
@@ -1,12 +1,11 @@
#ifdef BUILD_TEST
#ifdef MAIN_SIMULATION
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define QUAKEY_ENABLE_MOCKS
#include <quakey.h>
#include <assert.h>
#include "simulation_client.h"
#include "tcp.h"
#include "random_client.h"
// Helper function to parse address and port from command line
static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port)
@@ -19,7 +18,18 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) {
*addr = argv[i + 1];
if (i + 2 < argc) {
*port = (uint16_t)atoi(argv[i + 2]);
errno = 0;
char *end;
long val = strtol(argv[i+2], &end, 10);
if (end == argv[i+2] || *end != '\0' || errno == ERANGE)
break;
if (val < 0 || val > UINT16_MAX)
break;
*port = (uint16_t) val;
return true;
}
}
@@ -27,9 +37,12 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
return true;
}
int simulation_client_init(SimulationClient *client, int argc, char **argv,
void **contexts, struct pollfd *polled, int *timeout)
int random_client_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
RandomClient *client = state_;
char *addr;
uint16_t port;
parse_server_addr(argc, argv, &addr, &port);
@@ -43,20 +56,26 @@ int simulation_client_init(SimulationClient *client, int argc, char **argv,
printf("Client set up (remote=%s:%d)\n", addr, port);
*timeout = 0;
return toasty_process_events(client->toasty, contexts, polled, 0);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
return 0;
}
static int random_in_range(int min, int max)
{
uint64_t n = simulation_random_number();
uint64_t n = quakey_random();
return min + n % (max - min + 1);
}
int simulation_client_step(SimulationClient *client, void **contexts,
struct pollfd *polled, int num_polled, int *timeout)
int random_client_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{
RandomClient *client = state_;
// Process any pending events from the network and get new poll descriptors
num_polled = toasty_process_events(client->toasty, contexts, polled, num_polled);
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
for (int i = 0; i < client->num_pending; i++) {
@@ -241,12 +260,19 @@ int simulation_client_step(SimulationClient *client, void **contexts,
*timeout = 10;
else
*timeout = -1;
return toasty_process_events(client->toasty, contexts, polled, 0);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, 0);
return 0;
}
void simulation_client_free(SimulationClient *client)
int random_client_free(void *state_)
{
RandomClient *client = state_;
toasty_disconnect(client->toasty);
return 0;
}
#endif // BUILD_TEST
#endif // MAIN_SIMULATION
src/random_client.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef RANDOM_CLIENT_INCLUDED
#define RANDOM_CLIENT_INCLUDED
#include "ToastyFS.h"
#define MAX_PENDING_OPERATION 8
typedef enum {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
} PendingOperationType;
typedef struct {
PendingOperationType type;
ToastyHandle handle;
void *ptr;
} PendingOperation;
typedef struct {
ToastyFS *toasty;
int num_pending;
PendingOperation pending[MAX_PENDING_OPERATION];
} RandomClient;
struct pollfd;
int random_client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int random_client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int random_client_free(void *state);
#endif // RANDOM_CLIENT_INCLUDED
src/simulation_client.h
-43
View File
@@ -1,43 +0,0 @@
#ifndef SIMULATION_CLIENT_INCLUDED
#define SIMULATION_CLIENT_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#ifdef _WIN32
#include <winsock2.h>
#else
#include <poll.h>
#endif
#include "ToastyFS.h"
#define MAX_PENDING_OPERATION 8
typedef enum {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
} PendingOperationType;
typedef struct {
PendingOperationType type;
ToastyHandle handle;
void *ptr;
} PendingOperation;
typedef struct {
ToastyFS *toasty;
int num_pending;
PendingOperation pending[MAX_PENDING_OPERATION];
} SimulationClient;
int simulation_client_init(SimulationClient *client, int argc, char **argv,
void **contexts, struct pollfd *polled, int *timeout);
int simulation_client_step(SimulationClient *client, void **contexts,
struct pollfd *polled, int num_polled, int *timeout);
void simulation_client_free(SimulationClient *client);
#endif // SIMULATION_CLIENT_INCLUDED
src/system.c
-2060
View File
File diff suppressed because it is too large Load Diff
src/system.h
-209
View File
@@ -1,209 +0,0 @@
#ifndef SYSTEM_INCLUDED
#define SYSTEM_INCLUDED
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <direct.h> // _mkdir
#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <poll.h>
#include <time.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/socket.h>
#include <sys/random.h>
#include <arpa/inet.h>
#define SOCKET int
#define INVALID_SOCKET ((SOCKET) -1)
#endif
#ifdef BUILD_TEST
void startup_simulation(uint64_t seed_);
int spawn_simulated_process(char *args);
void update_simulation(void);
void cleanup_simulation(void);
uint64_t simulation_random_number(void);
void* mock_malloc(size_t len);
void* mock_realloc(void *ptr, size_t len);
void mock_free(void *ptr);
int mock_remove(char *path);
int mock_rename(char *oldpath, char *newpath);
SOCKET mock_socket(int domain, int type, int protocol);
int mock_bind(SOCKET fd, void *addr, size_t addr_len);
int mock_listen(SOCKET fd, int backlog);
SOCKET mock_accept(SOCKET fd, void *addr, socklen_t *addr_len);
int mock_getsockopt(SOCKET fd, int level, int optname, void *optval, socklen_t *optlen);
int mock_recv(SOCKET fd, void *dst, int len, int flags);
int mock_send(SOCKET fd, void *src, int len, int flags);
int mock_connect(SOCKET fd, void *addr, size_t addr_len);
#ifdef _WIN32
int mock_closesocket(SOCKET fd);
int mock_ioctlsocket(SOCKET fd, long cmd, u_long *argp);
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
BOOL mock_CloseHandle(HANDLE handle);
BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
BOOL mock_FlushFileBuffers(HANDLE handle);
BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
char* mock__fullpath(char *path, char *dst, int cap);
int mock__mkdir(char *path);
HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindClose(HANDLE hFindFile);
BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
#else
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_close(int fd);
int mock_flock(int fd, int op);
int mock_fsync(int fd);
int mock_read(int fd, char *dst, int len);
int mock_write(int fd, char *src, int len);
off_t mock_lseek(int fd, off_t offset, int whence);
int mock_fstat(int fd, struct stat *buf);
int mock_mkstemp(char *path);
char* mock_realpath(char *path, char *dst);
int mock_mkdir(char *path, mode_t mode);
int mock_fcntl(int fd, int cmd, ...);
DIR* mock_opendir(char *name);
struct dirent* mock_readdir(DIR *dirp);
int mock_closedir(DIR *dirp);
#endif
// Common
#define sys_malloc mock_malloc
#define sys_realloc mock_realloc
#define sys_free mock_free
#define sys_remove mock_remove
#define sys_rename mock_rename
#define sys_socket mock_socket
#define sys_bind mock_bind
#define sys_listen mock_listen
#define sys_accept mock_accept
#define sys_getsockopt mock_getsockopt
#define sys_recv mock_recv
#define sys_send mock_send
#define sys_connect mock_connect
// Windows
#define sys__mkdir mock__mkdir
#define sys_closesocket mock_closesocket
#define sys_ioctlsocket mock_ioctlsocket
#define sys_CreateFileW mock_CreateFileW
#define sys_CloseHandle mock_CloseHandle
#define sys_LockFile mock_LockFile
#define sys_UnlockFile mock_UnlockFile
#define sys_FlushFileBuffers mock_FlushFileBuffers
#define sys_ReadFile mock_ReadFile
#define sys_WriteFile mock_WriteFile
#define sys_SetFilePointer mock_SetFilePointer
#define sys_GetFileSizeEx mock_GetFileSizeEx
#define sys__fullpath mock__fullpath
#define sys_QueryPerformanceCounter mock_QueryPerformanceCounter
#define sys_QueryPerformanceFrequency mock_QueryPerformanceFrequency
#define sys_FindFirstFileA mock_FindFirstFileA
#define sys_FindNextFileA mock_FindNextFileA
#define sys_FindClose mock_FindClose
#define sys_MoveFileExW mock_MoveFileExW
// Linux
#define sys_mkdir mock_mkdir
#define sys_open mock_open
#define sys_close mock_close
#define sys_flock mock_flock
#define sys_fsync mock_fsync
#define sys_read mock_read
#define sys_write mock_write
#define sys_lseek mock_lseek
#define sys_fstat mock_fstat
#define sys_mkstemp mock_mkstemp
#define sys_realpath mock_realpath
#define sys_clock_gettime mock_clock_gettime
#define sys_fcntl mock_fcntl
#define sys_getrandom mock_getrandom
#define sys_opendir mock_opendir
#define sys_readdir mock_readdir
#define sys_closedir mock_closedir
#else
// Common
#define sys_malloc malloc
#define sys_realloc realloc
#define sys_free free
#define sys_remove remove
#define sys_rename rename
#define sys_socket socket
#define sys_bind bind
#define sys_listen listen
#define sys_accept accept
#define sys_getsockopt getsockopt
#define sys_recv recv
#define sys_send send
#define sys_connect connect
// Windows
#define sys__mkdir _mkdir
#define sys_closesocket closesocket
#define sys_ioctlsocket ioctlsocket
#define sys_CreateFileW CreateFileW
#define sys_CloseHandle CloseHandle
#define sys_LockFile LockFile
#define sys_UnlockFile UnlockFile
#define sys_FlushFileBuffers FlushFileBuffers
#define sys_ReadFile ReadFile
#define sys_WriteFile WriteFile
#define sys_SetFilePointer SetFilePointer
#define sys_GetFileSizeEx GetFileSizeEx
#define sys__fullpath _fullpath
#define sys_QueryPerformanceCounter QueryPerformanceCounter
#define sys_QueryPerformanceFrequency QueryPerformanceFrequency
#define sys_FindFirstFileA FindFirstFileA
#define sys_FindNextFileA FindNextFileA
#define sys_FindClose FindClose
#define sys_MoveFileExW MoveFileExW
// Linux
#define sys_mkdir mkdir
#define sys_open open
#define sys_close close
#define sys_flock flock
#define sys_fsync fsync
#define sys_read read
#define sys_write write
#define sys_lseek lseek
#define sys_fstat fstat
#define sys_mkstemp mkstemp
#define sys_realpath realpath
#define sys_clock_gettime clock_gettime
#define sys_fcntl fcntl
#define sys_opendir opendir
#define sys_readdir readdir
#define sys_closedir closedir
#endif
#endif // SYSTEM_INCLUDED
src/tcp.c
+19 -15
View File
@@ -1,8 +1,12 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include <string.h>
#include "tcp.h"
#include "system.h"
#include "message.h"
bool addr_eql(Address a, Address b)
@@ -28,15 +32,15 @@ static int set_socket_blocking(SOCKET sock, bool value)
{
#ifdef _WIN32
u_long mode = !value;
if (sys_ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
return -1;
#else
int flags = sys_fcntl(sock, F_GETFL, 0);
int flags = fcntl(sock, F_GETFL, 0);
if (flags < 0)
return -1;
if (value) flags &= ~O_NONBLOCK;
else flags |= O_NONBLOCK;
if (sys_fcntl(sock, F_SETFL, flags) < 0)
if (fcntl(sock, F_SETFL, flags) < 0)
return -1;
#endif
@@ -45,7 +49,7 @@ static int set_socket_blocking(SOCKET sock, bool value)
static SOCKET create_listen_socket(string addr, uint16_t port)
{
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0);
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET)
return INVALID_SOCKET;
@@ -72,13 +76,13 @@ static SOCKET create_listen_socket(string addr, uint16_t port)
return INVALID_SOCKET;
}
if (sys_bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
int backlog = 32;
if (sys_listen(fd, backlog) < 0) {
if (listen(fd, backlog) < 0) {
CLOSE_SOCKET(fd);
return INVALID_SOCKET;
}
@@ -319,7 +323,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
assert(contexts[i] == NULL);
if (polled[i].revents & POLLIN) {
SOCKET new_fd = sys_accept(tcp->listen_fd, NULL, NULL);
SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
if (new_fd != INVALID_SOCKET) {
if (set_socket_blocking(new_fd, false) < 0)
@@ -347,7 +351,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
int err = 0;
socklen_t len = sizeof(err);
if (sys_getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
defer_close = true;
else {
conn->connecting = false;
@@ -360,7 +364,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
if (polled[i].revents & POLLIN) {
byte_queue_write_setmincap(&conn->input, 1<<9);
ByteView buf = byte_queue_write_buf(&conn->input);
int num = sys_recv(conn->fd, (char*) buf.ptr, buf.len, 0);
int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num == 0)
defer_close = true;
else if (num < 0) {
@@ -388,7 +392,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
if (polled[i].revents & POLLOUT) {
ByteView buf = byte_queue_read_buf(&conn->output);
int num = sys_send(conn->fd, (char*) buf.ptr, buf.len, 0);
int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
defer_close = true;
@@ -432,7 +436,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
return -1;
int conn_idx = tcp->num_conns;
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0);
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET)
return -1;
@@ -447,13 +451,13 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
buf.sin_family = AF_INET;
buf.sin_port = htons(addr.port);
memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf));
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
} else {
struct sockaddr_in6 buf;
buf.sin6_family = AF_INET6;
buf.sin6_port = htons(addr.port);
memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf));
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
}
bool connecting;
src/tcp.h
+12 -4
View File
@@ -1,15 +1,23 @@
#ifndef TCP_INCLUDED
#define TCP_INCLUDED
#include <stdbool.h>
#ifdef MAIN_SIMULATION
# define QUAKEY_ENABLE_MOCKS
# include <quakey.h>
#else
# ifdef _WIN32
# include <winsock2.h>
# endif
#endif
#include "system.h"
#include "byte_queue.h"
#ifdef _WIN32
#define CLOSE_SOCKET sys_closesocket
#define CLOSE_SOCKET closesocket
#else
#define CLOSE_SOCKET sys_close
#define SOCKET int
#define INVALID_SOCKET -1
#define CLOSE_SOCKET close
#endif
#ifndef TCP_CONNECTION_LIMIT
src/wal.c
+18 -16
View File
@@ -1,12 +1,14 @@
#include <stddef.h>
#include <limits.h>
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h>
#include <string.h>
#include "wal.h"
#include "file_system.h"
#include "file_tree.h"
#include "system.h"
#define WAL_MAGIC 0xcafebebe
#define WAL_VERSION 1
@@ -190,7 +192,7 @@ static int swap_file(WAL *wal)
}
// MOVEFILE_REPLACE_EXISTING allows atomic overwrite
if (!sys_MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
file_unlock(temp_handle);
file_close(temp_handle);
remove_file_or_dir(temp_path);
@@ -267,12 +269,12 @@ static int next_entry(Handle handle, WALEntry *entry)
return -1;
// Dynamically allocate path buffer
char *path_buffer = sys_malloc(path_len);
char *path_buffer = malloc(path_len);
if (!path_buffer)
return -1;
if (read_exact(handle, path_buffer, path_len) <= 0) {
sys_free(path_buffer);
free(path_buffer);
return -1;
}
@@ -315,13 +317,13 @@ static int next_entry(Handle handle, WALEntry *entry)
goto cleanup_error;
// Dynamically allocate hash buffers
SHA256 *hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256));
SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256));
if (!hashes_buffer) {
goto cleanup_error;
}
if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
sys_free(hashes_buffer);
free(hashes_buffer);
goto cleanup_error;
}
@@ -337,9 +339,9 @@ static int next_entry(Handle handle, WALEntry *entry)
cleanup_error:
if (entry->path.ptr)
sys_free((char*) entry->path.ptr);
free((char*) entry->path.ptr);
if (entry->hashes)
sys_free(entry->hashes);
free(entry->hashes);
return -1;
}
@@ -351,7 +353,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
wal->file_path.ptr = NULL;
// Copy file_path since the passed string may not have the same lifetime as WAL
char *path_copy = sys_malloc(file_path.len);
char *path_copy = malloc(file_path.len);
if (!path_copy)
return -1;
memcpy(path_copy, file_path.ptr, file_path.len);
@@ -482,9 +484,9 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
// Free dynamically allocated fields from next_entry
if (entry.path.ptr)
sys_free((char*) entry.path.ptr);
free((char*) entry.path.ptr);
if (entry.hashes)
sys_free(entry.hashes);
free(entry.hashes);
wal->entry_count++;
}
@@ -499,7 +501,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
return 0;
error_cleanup_path:
sys_free(path_copy);
free(path_copy);
return -1;
}
@@ -508,7 +510,7 @@ void wal_close(WAL *wal)
file_unlock(wal->handle);
file_close(wal->handle);
if (wal->file_path.ptr)
sys_free((char*) wal->file_path.ptr);
free((char*) wal->file_path.ptr);
}
static int write_u8(Handle handle, uint8_t value)