cozis/ToastyFS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Quakey

Browse Source
This commit is contained in:
cozis
2026-01-16 14:45:09 +01:00
parent 18e74a92c0
commit c10493881b
66 changed files with 12760 additions and 11817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
-2
View File
@@ -1,2 +0,0 @@
# Ensure shell scripts always use LF line endings
*.sh text eol=lf
.gitignore
+3 -19
View File
@@ -1,19 +1,3 @@
*.exe toasty_metadata_server
*.out toasty_chunk_server
*.o toasty_simulation
*.a
chunk_server_data_*
crash_*.txt
# Coverage reports
coverage_report/
*.gcov
*.gcda
*.gcno
*.wal
*.tmp
# Cluster demo artifacts
.cluster_demo.pids
cluster_logs/
cluster_data/
Makefile
-67
View File
@@ -1,67 +0,0 @@
CFLAGS = -Wall -Wextra -ggdb
COVERAGE_CFLAGS = $(CFLAGS) --coverage
COVERAGE_LFLAGS = --coverage
ifeq ($(OS),Windows_NT)
LFLAGS = -lws2_32
EXT = .exe
else
LFLAGS =
EXT = .out
endif
CFILES = $(shell find src -name '*.c')
HFILES = $(shell find src -name '*.h')
OFILES = $(CFILES:.c=.o)
WEB_CFILES = $(shell find web/src web/3p -name '*.c')
WEB_HFILES = $(shell find web/src web/3p -name '*.h')
.PHONY: all clean coverage coverage-report coverage-html
all: toastyfs$(EXT) toastyfs_web$(EXT) toastyfs_random_test$(EXT) example_async_api$(EXT) example_blocking_api$(EXT) libtoastyfs.a
coverage: toastyfs_random_test_coverage$(EXT)
coverage-report:
@./scripts/measure_coverage.sh 60
coverage-html:
@./scripts/measure_coverage.sh 60 --html
toastyfs$(EXT): $(CFILES) $(HFILES)
gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_SERVER
toastyfs_random_test$(EXT): $(CFILES) $(HFILES)
gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_TEST
toastyfs_random_test_coverage$(EXT): $(CFILES) $(HFILES)
gcc -o $@ $(CFILES) $(COVERAGE_CFLAGS) $(LFLAGS) $(COVERAGE_LFLAGS) -Iinclude -DBUILD_TEST
example_async_api$(EXT): libtoastyfs.a examples/async_api.c
gcc -o $@ examples/async_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L.
example_blocking_api$(EXT): libtoastyfs.a examples/blocking_api.c
gcc -o $@ examples/blocking_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L.
toastyfs_web$(EXT): libtoastyfs.a $(WEB_CFILES) $(WEB_HFILES)
gcc -o $@ $(WEB_CFILES) $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -Iweb/3p -L.
%.o: %.c $(HFILES)
gcc -c -o $@ $< $(CFLAGS) -Iinclude
libtoastyfs.a: $(OFILES)
ar rcs $@ $^
clean:
rm -f \
*.exe \
*.out \
*.a \
*.gcov \
src/*.o \
src/*.gcda \
src/*.gcno \
*.gcda \
*.gcno
README.md
-24
View File
@@ -1,24 +0,0 @@
# ToastyFS
ToastyFS is a distributed file system designed for self-hosting, so it aims to be pragmatic, understandable, and robust. You can use ToastyFS to store your files reliably over multiple machines knowing they will be automatically replicated and healed in case of failures.
To try it out, run the script `./scripts/cluster_demo.sh`. It will spawn a local cluster with a web interface at `https://127.0.0.1:8090/` which will allow you to PUT, GET, DELETE files.
⚠️ Note that ToastyFS is still in early development ⚠️
🎵 Now let's get toasty 🎵
## Features
- Cross-platform (runs on Windows and Linux)
- Automatic Replication & Self-Healing
- Automatic content deduplication via internal content-addressing
- Configurable file chunk sizes
- Small and understandable
But ToastyFS is still in early development, so here are the missing features:
- No master replication
- No authentication or encryption
## Testing
ToastyFS is tested by running an in-memory simulation of a cluster with many clients running hundreds of random operations in parallel. The test is run for long periods of times under valgrind or compiled with sanitizers.
app/app.py
-54
View File
@@ -1,54 +0,0 @@
import asyncio
from pathlib import Path
import aiohttp
async def fetch(session, url):
async with session.get(url) as response:
if response.status != 200:
raise "Didn't work" # TODO
data = await response.text()
files = []
lines = data.splitlines()
for line in lines:
name, vtag = line.split(" ")
# Process name
is_dir = False
if len(name) > 0 and name[-1] == "/":
name = name[:-1]
is_dir = True
# Process vtag
vtag = int(vtag)
files.append({"name": name, "is_dir": is_dir, "vtag": vtag})
return files
async def main():
async with aiohttp.ClientSession() as session:
remote = "http://127.0.0.1:8090" # Must not end with a "/"
remote_dir = remote + "/"
local_dir = "local_root"
local_dir = Path(local_dir)
while True:
# List of files in the remote root directory
remote_files = await fetch(session, remote_dir)
# List of files in the local root directory
local_files = list(local_dir.iterdir())
await asyncio.sleep(1)
try:
asyncio.run(main())
except KeyboardInterrupt:
pass
build.sh
+3
View File
@@ -0,0 +1,3 @@
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c quakey/src/lfs.c quakey/src/lfs_util.c quakey/src/quakey.c -o toasty_simulation -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_metadata_server -DMAIN_METADATA_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_chunk_server -DMAIN_CHUNK_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
examples/async_api.c
-60
View File
@@ -1,60 +0,0 @@
#include <stdio.h>
#include <ToastyFS.h>
int main(void)
{
ToastyString remote_addr = TOASTY_STR("127.0.0.1");
uint16_t remote_port = 8080;
ToastyFS *toasty = toasty_connect(remote_addr, remote_port);
if (toasty == NULL) {
printf("Couldn't connect to metadata server");
return -1;
}
ToastyString path_1 = TOASTY_STR("/first_file");
ToastyString path_2 = TOASTY_STR("/second_file");
// Begin creation operation. This does not block.
ToastyHandle create_handle_1 = toasty_begin_create_file(toasty, path_1, 1024);
if (create_handle_1 == TOASTY_INVALID) {
printf("Couldn't create file 1");
return -1;
}
// This doesn't block either.
ToastyHandle create_handle_2 = toasty_begin_create_file(toasty, path_2, 1024);
if (create_handle_2 == TOASTY_INVALID) {
printf("Couldn't create file 2");
return -1;
}
// Now block execution by overlapping the waiting
// times for both file creations.
ToastyResult result;
int ret = toasty_wait_result(toasty, create_handle_1, &result, -1);
if (ret < 0) {
printf("Couldn't wait for completion\n");
return -1;
}
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
printf("Couldn't create file 1\n");
return -1;
}
ret = toasty_wait_result(toasty, create_handle_2, &result, -1);
if (ret < 0) {
printf("Couldn't wait for completion\n");
return -1;
}
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
printf("Couldn't create file 2\n");
return -1;
}
printf("All files were created!\n");
toasty_disconnect(toasty);
return 0;
}
examples/blocking_api.c
-34
View File
@@ -1,34 +0,0 @@
#include <stdio.h>
#include <ToastyFS.h>
int main(void)
{
ToastyString remote_addr = TOASTY_STR("127.0.0.1");
uint16_t remote_port = 8080;
ToastyFS *toasty = toasty_connect(remote_addr, remote_port);
if (toasty == NULL) {
printf("Couldn't connect to metadata server");
return -1;
}
ToastyString path = TOASTY_STR("/first_file");
int ret = toasty_create_file(toasty, path, 1024, NULL);
if (ret < 0) {
printf("Couldn't create file\n");
toasty_disconnect(toasty);
return -1;
}
char data[] = "Hello, world!";
ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, NULL, 0);
if (ret < 0) {
printf("Couldn't write to file\n");
toasty_disconnect(toasty);
return -1;
}
toasty_disconnect(toasty);
return 0;
}
misc/DESIGN.txt
-233
View File
@@ -1,233 +0,0 @@
Architecture
A ToastyFS instance is composed by a metadata server, a number
of chunk servers, and a number of clients.
The metadata server stores the full file system hieararchy,
except instead of storing the file contents, it stores an
array of hashes of the chunks of each file. A "chunk" is a
file range that is fixed for a single file but may vary
between files. Chunk servers hold an array of chunks that
are identified by their hash. The metadata server keeps
track of which chunks each chunk server is holding.
Clients are users of the file system that can read and
write metadata and files. They are assumed to behave
correctly.
Any read and write operation that doesn't involve file
contents can be performed by clients by talking to the
metadata server directly. Such operations include creating
an empty file or a directory, deleting a file or directory,
listing files.
If a client wants to read a range of bytes from a file,
it sends the metadata server the file name and range.
The metadata server responds with the chunk size of that
file, the list of hashes for the chunks involved in the
read, and the IP addresses of the chunk servers that hold
each chunk. The metadata server also adds the IP addresses
of three chunk servers any new chunks should be written
to. The client can then download the chunks from the chunk
servers and reassemble the result.
If a client wants to write at a range of bytes of a file,
it starts by reading that range from the metadata server,
getting the list of hashes it will modify, their locations,
and locations for any new chunks. The client then modifies
the chunk by sending to each chunk server the hash to modify
and the patch (a range of bytes within a chunk plus the new
data). The chunk server creates a new modified chunk and
keeps the old version, then returns the new hash. If all
modifications are successful, the client holds the set of
old hashes and new hashes for that file range. It completes
the write by telling the metadata server to swap the old
hashes with the new ones (optionally including write flags
such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes
don't match, another write succeded in the mean time and
touched that range, therefore the write fails. If the old
hashes match, the write succeded. If the client fails to
modify any chunks, it doesn't commit the write with the
metadata server.
If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING
flag is set, the metadata server atomically creates the file
with a default chunk size (4096 bytes) and retries the write.
This operation is logged to the WAL to ensure crash consistency.
If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is
truncated after the write, setting its size to exactly offset+length
and discarding any data beyond that point. This is useful for HTTP
PUT semantics where the entire file content should be replaced.
Note that write failures may cause chunks to be orphaned
on chunk servers. This is solved by a garbage collection
algorithm implemented by the synchronization messages
between metadata and chunk server.
Note that clients may cache chunks and index them by their
hash. When they read a file and receive its hashes, they may
avoid reaching for the chunk servers if they already cached
the chunks with those hashes. This allows reading files with
only one round trip at no cost of correctness. If getting
the up-to-date contents is not a concern, clients may also
cache file metadata.
Metadata and chunk server exchange:
The metadata server is only aware of each chunk server
as long as they have a TCP connection. When a chunk server
first connects to the metadata server, it authenticates
itself and sends its own IP addresses. If the server is
authentic, the metadata server requests the full list
of chunks the chunk server is holding. Upon receiving the
state of chunk server, the metadata server adds all useful
chunks to the "old_list" and all useless chunks to the
"rem_list", then sends the rem_list to the chunk server
which removes those chunks.
When writes are committed to the metadata server involving
new chunks to a chunk server, the metadata server adds those
hashes to an "add_list" and any hashes that are not useful
anymore to the rem_list.
Periodically, the metadata server sends the add_list and
rem_list to the chunk server. These list tell the chunk
server the ideal state it should have from the point of
view of the metadata server. Elements in the add_list should
already be in the chunk servers, and elements from the
rem_list are to be removed. A chunk server marks any chunk
in the rem_list as to be removed and checks that hashes
in the add list are present. If a chunk in the add list
is marked as to be removed, it is unmarked. When a chunk
is marked as to be removed for a certain amount of time,
it is permanently deleted. When the synchronization is
complete, the metadata server merges the add_list into
the old_list and clears the rem_list. If chunks in the
add_list are not present in the chunk server, it responds
with an error message containing the list of missing chunks.
The metadata server then responds with a list of chunk
server addresses where the chunk server with the missing
chunk can download it from. Each chunk server goes
through its download list one at the time downloading
the missing chunks.
Note that if the chunk server finds that its holding some
chunks that are not in the hash list of the metadata server,
that does not mean they are orphaned. It's possible that
some writes are being performed by clients that have uploaded
chunks to that chunk server but didn't yet acknowledge it
to the metadata server. If all goes well and the write
succeded, the metadata server will add those hashes to the
hash list. Chunk servers should only drop chunks if they
are not referenced by the metadata server for a period of
time (say, 30 minutes).
Security
All nodes of the system share a secret key and use it to
authenticate each other and encrypt messages. This allows
the server to accept new chunk servers and clients with
no prior setup
Reliability
The metadata server is a single point of failure. To reduce
the impact of crashes, the metadata server stores all write
operations into a write-ahead log that is replayed any time
the process goes online.
Chunk Management:
Chunks are added to the system when:
1. A chunk server connects
2. A write operation on metadata occurs (adding chunks)
They are removed when:
1. A chunk server disconnects
2. A write operation on metadata occurs (overwriting old chunks)
3. A delete operation on metadata occurs
4. A chunk is corrupted or removed forcefully from the chunk server
The system must make sure that chunks are not over-replicated
or under-replicated. If they are over-replicated, some chunk
servers need to forget some copies. If they are under-replicated,
some chunk servers need to copy chunks from elsewhere.
Metadata server variables for a chunk server:
ms_old_list: List of chunks that are known to be held by CS
ms_add_list: List of chunks that should be held by CS
ms_rem_list: List of chunks that may be held by CS but should removed from it
Chunk server variables:
cs_add_list: List of chunks added since the last update
cs_rem_list: List of chunks marked for removal after a timeout
cs_lst_list: List of chunks that were lost due to errors or forceful removals of chunk files
Metadata change for write:
When clients commit a write by adding new hashes to the metadata,
MS adds those hashes to the ms_add_list for the CS where the client
uploaded the chunks. If a hash was overwritten and became useless,
that hash is added to the ms_rem_list for all CS holding it.
Metadata change for delete:
All hashes that are no longer reachable by the file tree are added
to the ms_rem_list of their holders
Chunk upload:
When a chunk is uploaded to a chunk server, its hash is added to
the cs_add_list.
Periodically on the chunk server:
Elements in the cs_rem_list have a 30 minute timeout after which they are deleted
permanently.
Periodically:
CS sends cs_add_list to MS
MS may add a subset of cs_add_list to ms_add_list based on the chunk replication and distribution policy
MS sends ms_add_list and ms_rem_list to CS
CS (1) Adds all elements from ms_rem_list to cs_rem_list
(2) Elements in ms_add_list that are not held by the chunk server are added to
a temporary list tmp_list
(3) Removes elements in ms_add_list from cs_add_list and cs_rem_list, then merges cs_add_list into cs_rem_list and clears cs_add_list.
(4) Elements in cs_lst_list are added to tmp_list, then cs_lst_list is cleared.
(6) tmp_list is sent to MS
(7) cs_add_list is cleared
MS (1) Receives tmp_list and sends download locations to CS for those chunks
(2) Merges ms_add_list into ms_old_list, then removes all items in tmp_list from ms_old_list
(3) Sets ms_add_list equal to tmp_list
Chunk replication and distribution policy:
During an update, when CS reports a new chunk to MS, MS has to decide whether
to allow the CS to keep it or not.
There are 4 cases:
- The chunk is useless (not referenced by any file)
- The chunk is under-replicated even counting the new copy
- The chunk is properly replicated with the new copy
- The chunk is over-replicated with the new copy
If the chunk is not referenced by the file tree, do nothing.
If the chunk is properly replicated or under-replicated, add it to the ms_add_list.
If the chunk is over-replicated, either don't add it to the ms_add_list or add it to the ms_rem_list of some other holder.
TODO: The way chunk servers tell about the chunks they are holding
recently changed. Now instead of sending a full chunk list when
connecting, they send batches of chunks to the metadata server
during state updates. It also changed that now chunk servers
initiate updates.
When a chunk server connects (and authenticates)
it sends alongside the auth message the hash of
the hash list of all chunks. The metadata server
then replies with a message saying whether it
already cached that chunk list or not. If it didn't, the chunk server sends the entire list
in chunks during state updates, with an increased
update frequency.
Metadata Persistence & Crash Recovery:
The metadata server uses a write-ahead log (WAL) file to store its state on disk.
Log files start with a full snapshot of the metadata and continues with operation
log entries.
When a file gets too big, the metadata server creates a new WAL file by writing
a snapshot to it and continuing logging there.
misc/PROTOCOL.txt
-281
View File
@@ -1,281 +0,0 @@
1. Introduction & Notation
The DESIGN.txt file gives an overview of the system
and how nodes of a cluster interact with each other.
This file documents the specific binary format used
to exchange information between nodes.
All messages start with a shared header, defined as:
struct Header {
uint16_t version;
uint16_t type;
uint32_t length;
};
1.1 Generation Counter Special Values
Generation counters (uint64_t) have two special values:
NO_GENERATION (0):
When used in expect_gen, means "skip generation check entirely".
Files/directories are never assigned this generation value.
MISSING_FILE_GENERATION (UINT64_MAX):
When used in expect_gen, means "expect file/directory to NOT exist".
- For DELETE: succeeds if file doesn't exist (no-op)
- For WRITE: fails with BADGEN if file exists, fails with NOENT if missing
- For existing operations: causes generation mismatch if file exists
Files/directories are never assigned this generation value.
2. Client Messages
2.1 Client to Metadata Server messages
Let's start from the interactions between a client and
the metadata server:
[ CREATE | C -> MS ]
Upon file creation, the client sends a "CREATE" message with the following layout:
struct CreateMessage {
Header header; // type=CREATE
uint16_t path_len;
char path[path_len];
uint8_t is_dir;
if (is_dir != 0) {
uint32_t chunk_size;
}
};
Note that in general only paths up to 65K bytes are
supported and that the layout of fields may depend
on the value of others.
The server then responds with a CREATE_SUCCESS or CREATE_ERROR message.
[ DELETE | C -> MS ]
When a client deletes a file, it sends a DELETE
message with the following layout:
struct DeleteMessage {
Header header; // type=DELETE
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
};
The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless.
The server then responds with a DELETE_SUCCESS or DELETE_ERROR message.
[ LIST | C -> MS ]
When a client requests a directory listing, it sends a LIST message whith the following format:
struct ListMessage {
Header header; // type=LIST
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
};
If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped.
The server then responds with a LIST_SUCCESS or LIST_ERROR message.
[ READ | C -> MS ]
When a client requests to read a file, it sends a READ message with the following format:
struct ReadMessage {
Header header; // type=READ
uint64_t expect_gen;
uint16_t path_len;
char path[path_len];
uint32_t offset;
uint32_t length;
};
The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped.
The offset and length fields determine the region to be read from the file.
[ WRITE | C -> MS ]
When a client wants to write to a file, it sends a WRITE message with the following layout:
struct Location {
uint8_t is_ipv4;
if (is_ipv4) {
uint32_t ipv4;
} else {
uint128_t ipv6;
}
uint16_t port;
};
struct WriteChunk {
SHA256 hash;
uint32_t num_locations;
Location locations[num_locations];
};
struct WriteMessage {
Header header; // type=WRITE
uint64_t expect_gen;
uint32_t flags;
uint16_t path_len;
char path[path_len];
uint32_t offset;
uint32_t length;
uint32_t num_chunks;
WriteChunk chunks[num_chunks];
};
If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata.
The flags field contains write operation flags. Currently defined flags:
- TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist,
the metadata server will atomically create it with a default chunk
size of 4096 bytes and retry the write operation. The creation is
logged to the WAL for crash consistency.
- TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write
operation, setting the file size to exactly offset+length. Any data
beyond this point is discarded. Useful for HTTP PUT semantics where
the entire file content should be replaced.
The offset and length mark the region that is being written to.
Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to
num_chunks == length / chunk_size
Where chunk_size is the one for the target file at the specified generation.
Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it.
2.2 Client to Chunk Server messages
TODO
3. Metadata Server messages
The following is the list of messages the Metadata Server may send to a Client:
[ CREATE_ERROR | MS -> C ]
When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout:
struct CreateErrorMessage {
Header header; // type=CREATE_ERROR
uint16_t message_len;
char message[message_len];
};
[ CREATE_SUCCESS | MS -> C ]
When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout:
struct CreateSuccessMessage {
Header header; // type=CREATE_SUCCESS
uint64_t gen;
};
The gen field is the generation counter given to the file.
[ DELETE_ERROR | MS -> C ]
See CREATE_ERROR
[ DELETE_SUCCESS | MS -> C ]
When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout:
struct DeleteSuccessMessage {
Header header; // type=DELETE_SUCCESS
};
It does not store any fields other than the header.
[ LIST_ERROR | MS -> C ]
When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout:
struct ListErrorMessage {
Header header; // type=LIST_ERROR
uint16_t message_len;
char message[message_len];
};
[ LIST_SUCCESS | MS -> C ]
When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout:
struct ListItem {
uint64_t gen;
uint8_t is_dir;
uint16_t name_len;
char name[name_len];
};
struct ListSuccessMessage {
Header header; // type=LIST_SUCCESS
uint64_t gen;
uint8_t truncated;
uint32_t item_count;
ListItem items[item_count];
};
The ListSuccessMessage gen field contains the generation counter for the directory, while the gen field in ListItem is the counter for that specific child.
If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message.
[ READ_ERROR | MS -> C ]
TODO
[ READ_SUCCESS | MS -> C ]
struct IPv4AndPort {
uint32_t ipv4;
uint16_t port;
};
struct IPv6AndPort {
uint128_t ipv6;
uint16_t port;
};
struct ChunkServerAddrList {
uint32_t num_ipv4;
IPv4AndPort ipv4s[num_ipv4];
uint32_t num_ipv6;
IPv6AndPort ipv6s[num_ipv6];
};
struct ReadChunk {
SHA256 hash;
uint32_t num_holders;
ChunkServerAddrList holders[num_holders];
};
struct ReadSuccessMessage {
Header header; // type=READ_SUCCESS
uint64_t gen;
uint32_t chunk_size;
uint32_t file_length;
uint32_t num_hashes;
ReadChunk chunks[num_hashes];
uint32_t num_write_locations;
ChunkServerAddrList write_locations[num_write_locations];
};
The message returns general information about the file such as its generation counter, length in bytes, and chunk size.
The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them.
The write locations are a list of chunk servers that clients may write new chunks to.
[ WRITE_ERROR | MS -> C ]
See CREATE_ERROR
[ WRITE_SUCCESS | MS -> C ]
When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout:
struct WriteSuccessMessage {
Header header; // type=WRITE_SUCCESS
uint64_t gen;
};
The gen field is the new generation counter for that file.
misc/SIMTEST.txt
-3
View File
@@ -1,3 +0,0 @@
Notes on simulation testing:
- Make sure your program works without fault injections
- Trace the branch coverage of the test and find why some branches are never reached
misc/TODO.txt
-26
View File
@@ -1,26 +0,0 @@
[X] Implement proper operation handles that allow users to wait for specific events
[X] Implement metadata server WAL
[X] Check that the corner case where read and writes go over the length of the file work correctly
[X] Return the number of bytes read or written in the ToastyFS_Result struct
[X] Add a change counter to FileTree. The counter is initialized to 0 and updated any time the tree is changed. When a new file is created, the current change counter is assigned to it. This makes it possible to verify that the file didn't change in between read and write operations without that clumsy previous hash list and previous chunk size.
[ ] Clean up the read operation for clients
[ ] Make parallel uploads/downloads configurable
[ ] Recalculate next write locations whenever a write occurs, not at each read
[ ] add logging (when chunk servers connect and disconnect to the metadata server)
[ ] Should list scenarios that need testing, like those where chunks would be dropped
[ ] Update DESIGN.txt and the code to remove the chunk list message. The information of chunks held by chunk servers is now transmitted to the metadata server during state updates
[ ] clean up the replication factor business
[ ] Add failt injections to the simulation
[ ] Add authentication
[ ] Add release build and allow switching between debug, release, coverage, sanitizer builds while calling make
[ ] When an operation is committed to the WAL, it may then fail due to not deterministic reasons (out of memory). When the metadata server restarts and replays the log, the operation may succede and cause the system to diverge from the last instance of the process. Is this not a problem? How do we solve it?
[ ] Add notes on who inspired the testing approach
[ ] Add notes on benchmarks and that we don't do batching
[ ] Fix endianess when writing to network and the WAL
[ ] WAL entry checksum
[ ] Check write() errors when appending WAL entries
[ ] When WAL entries are partially written, remove partial data and fail
[ ] Find a way to ensure listing operations of large directories are handled
[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication?
[ ] Rename generation counters to version numbers
[ ] Add a flag for write operations to create the file if it doesn't exist alraedy
quakey/include/quakey.h
+190
View File
@@ -0,0 +1,190 @@
#ifndef QUAKEY_INCLUDED
#define QUAKEY_INCLUDED
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#ifdef _WIN32
#include <winsock2.h>
#include <ws2tcpip.h>
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <direct.h>
#else
#include <time.h>
#include <poll.h>
#include <errno.h>
#include <unistd.h>
#include <limits.h>
#include <string.h>
#include <dirent.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <pthread.h>
#endif
typedef struct {} Quakey;
// Function pointers to a simulated program's code
typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
typedef int (*QuakeyFreeFunc)(void *state);
typedef enum {
QUAKEY_LINUX,
QUAKEY_WINDOWS,
} QuakeyPlatform;
typedef struct {
// Size of the opaque state struct
int state_size;
// Pointers to program code
QuakeyInitFunc init_func;
QuakeyTickFunc tick_func;
QuakeyFreeFunc free_func;
// Network addresses enabled on the process
char **addrs;
int num_addrs;
// Disk size for the process
int disk_size;
// Platform used by this program
QuakeyPlatform platform;
} QuakeySpawn;
typedef unsigned long long QuakeyUInt64;
// Start a simulation
int quakey_init(Quakey **quakey, QuakeyUInt64 seed);
// Stop a simulation
void quakey_free(Quakey *quakey);
// Add a program to the simulation
void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg);
// Schedule and executes one program until it would block, then returns
int quakey_schedule_one(Quakey *quakey);
// Generate a random u64
QuakeyUInt64 quakey_random(void);
int *mock_errno_ptr(void);
#ifdef _WIN32
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
BOOL mock_CloseHandle(HANDLE handle);
BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
BOOL mock_FlushFileBuffers(HANDLE handle);
BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
char* mock__fullpath(char *path, char *dst, int cap);
HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindClose(HANDLE hFindFile);
int mock__mkdir(char *path);
#else
int mock_socket(int domain, int type, int protocol);
int mock_closesocket(int fd);
int mock_ioctlsocket(int fd, long cmd, unsigned long *argp);
int mock_bind(int fd, void *addr, unsigned long addr_len);
int mock_connect(int fd, void *addr, unsigned long addr_len);
int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
int mock_listen(int fd, int backlog);
int mock_accept(int fd, void *addr, socklen_t *addr_len);
int mock_recv(int fd, char *dst, int len, int flags);
int mock_send(int fd, char *src, int len, int flags);
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_fcntl(int fd, int cmd, int flags);
int mock_close(int fd);
int mock_fstat(int fd, struct stat *buf);
int mock_read(int fd, char *dst, int len);
int mock_write(int fd, char *src, int len);
off_t mock_lseek(int fd, off_t offset, int whence);
int mock_flock(int fd, int op);
int mock_fsync(int fd);
int mock_mkstemp(char *path);
int mock_mkdir(char *path, mode_t mode);
int mock_remove(char *path);
int mock_rename(char *oldpath, char *newpath);
char* mock_realpath(char *path, char *dst);
DIR* mock_opendir(char *name);
struct dirent* mock_readdir(DIR *dirp);
int mock_closedir(DIR *dirp);
#endif
void *mock_malloc(size_t size);
void *mock_realloc(void *ptr, size_t size);
void mock_free(void *ptr);
#ifdef QUAKEY_ENABLE_MOCKS
#undef errno
#define errno (*mock_errno_ptr())
#define malloc mock_malloc
#define realloc mock_realloc
#define free mock_free
#define socket mock_socket
#define closesocket mock_closesocket
#define ioctlsocket mock_ioctlsocket
#define bind mock_bind
#define connect mock_connect
#define getsockopt mock_getsockopt
#define listen mock_listen
#define accept mock_accept
#define recv mock_recv
#define send mock_send
#define clock_gettime mock_clock_gettime
#define QueryPerformanceCounter mock_QueryPerformanceCounter
#define QueryPerformanceFrequency mock_QueryPerformanceFrequency
#define open mock_open
#define fcntl mock_fcntl
#define close mock_close
#define CreateFileW mock_CreateFileW
#define CloseHandle mock_CloseHandle
#define ReadFile mock_ReadFile
#define WriteFile mock_WriteFile
#define read mock_read
#define write mock_write
#define fstat mock_fstat
#define GetFileSizeEx mock_GetFileSizeEx
#define lseek mock_lseek
#define SetFilePointer mock_SetFilePointer
#define flock mock_flock
#define LockFile mock_LockFile
#define UnlockFile mock_UnlockFile
#define fsync mock_fsync
#define FlushFileBuffers mock_FlushFileBuffers
#define mkstemp mock_mkstemp
#define mkdir mock_mkdir
#define _mkdir mock__mkdir
#define remove mock_remove
#define rename mock_rename
#define MoveFileExW mock_MoveFileExW
#define realpath mock_realpath
#define _fullpath mock__fullpath
#define opendir mock_opendir
#define readdir mock_readdir
#define closedir mock_closedir
#define FindFirstFileA mock_FindFirstFileA
#define FindNextFileA mock_FindNextFileA
#define FindClose mock_FindClose
#endif
#endif // QUAKEY_INCLUDED
quakey/src/lfs.c
+6548
View File
File diff suppressed because it is too large Load Diff
quakey/src/lfs.h
+801
View File
@@ -0,0 +1,801 @@
/*
* The little filesystem
*
* Copyright (c) 2022, The littlefs authors.
* Copyright (c) 2017, Arm Limited. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef LFS_H
#define LFS_H
#include "lfs_util.h"
#ifdef __cplusplus
extern "C"
{
#endif
/// Version info ///
// Software library version
// Major (top-nibble), incremented on backwards incompatible changes
// Minor (bottom-nibble), incremented on feature additions
#define LFS_VERSION 0x0002000b
#define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
#define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0))
// Version of On-disk data structures
// Major (top-nibble), incremented on backwards incompatible changes
// Minor (bottom-nibble), incremented on feature additions
#define LFS_DISK_VERSION 0x00020001
#define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16))
#define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0))
/// Definitions ///
// Type definitions
typedef uint32_t lfs_size_t;
typedef uint32_t lfs_off_t;
typedef int32_t lfs_ssize_t;
typedef int32_t lfs_soff_t;
typedef uint32_t lfs_block_t;
// Maximum name size in bytes, may be redefined to reduce the size of the
// info struct. Limited to <= 1022. Stored in superblock and must be
// respected by other littlefs drivers.
#ifndef LFS_NAME_MAX
#define LFS_NAME_MAX 255
#endif
// Maximum size of a file in bytes, may be redefined to limit to support other
// drivers. Limited on disk to <= 2147483647. Stored in superblock and must be
// respected by other littlefs drivers.
#ifndef LFS_FILE_MAX
#define LFS_FILE_MAX 2147483647
#endif
// Maximum size of custom attributes in bytes, may be redefined, but there is
// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored
// in superblock and must be respected by other littlefs drivers.
#ifndef LFS_ATTR_MAX
#define LFS_ATTR_MAX 1022
#endif
// Possible error codes, these are negative to allow
// valid positive return values
enum lfs_error {
LFS_ERR_OK = 0, // No error
LFS_ERR_IO = -5, // Error during device operation
LFS_ERR_CORRUPT = -84, // Corrupted
LFS_ERR_NOENT = -2, // No directory entry
LFS_ERR_EXIST = -17, // Entry already exists
LFS_ERR_NOTDIR = -20, // Entry is not a dir
LFS_ERR_ISDIR = -21, // Entry is a dir
LFS_ERR_NOTEMPTY = -39, // Dir is not empty
LFS_ERR_BADF = -9, // Bad file number
LFS_ERR_FBIG = -27, // File too large
LFS_ERR_INVAL = -22, // Invalid parameter
LFS_ERR_NOSPC = -28, // No space left on device
LFS_ERR_NOMEM = -12, // No more memory available
LFS_ERR_NOATTR = -61, // No data/attr available
LFS_ERR_NAMETOOLONG = -36, // File name too long
};
// File types
enum lfs_type {
// file types
LFS_TYPE_REG = 0x001,
LFS_TYPE_DIR = 0x002,
// internally used types
LFS_TYPE_SPLICE = 0x400,
LFS_TYPE_NAME = 0x000,
LFS_TYPE_STRUCT = 0x200,
LFS_TYPE_USERATTR = 0x300,
LFS_TYPE_FROM = 0x100,
LFS_TYPE_TAIL = 0x600,
LFS_TYPE_GLOBALS = 0x700,
LFS_TYPE_CRC = 0x500,
// internally used type specializations
LFS_TYPE_CREATE = 0x401,
LFS_TYPE_DELETE = 0x4ff,
LFS_TYPE_SUPERBLOCK = 0x0ff,
LFS_TYPE_DIRSTRUCT = 0x200,
LFS_TYPE_CTZSTRUCT = 0x202,
LFS_TYPE_INLINESTRUCT = 0x201,
LFS_TYPE_SOFTTAIL = 0x600,
LFS_TYPE_HARDTAIL = 0x601,
LFS_TYPE_MOVESTATE = 0x7ff,
LFS_TYPE_CCRC = 0x500,
LFS_TYPE_FCRC = 0x5ff,
// internal chip sources
LFS_FROM_NOOP = 0x000,
LFS_FROM_MOVE = 0x101,
LFS_FROM_USERATTRS = 0x102,
};
// File open flags
enum lfs_open_flags {
// open flags
LFS_O_RDONLY = 1, // Open a file as read only
#ifndef LFS_READONLY
LFS_O_WRONLY = 2, // Open a file as write only
LFS_O_RDWR = 3, // Open a file as read and write
LFS_O_CREAT = 0x0100, // Create a file if it does not exist
LFS_O_EXCL = 0x0200, // Fail if a file already exists
LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size
LFS_O_APPEND = 0x0800, // Move to end of file on every write
#endif
// internally used flags
#ifndef LFS_READONLY
LFS_F_DIRTY = 0x010000, // File does not match storage
LFS_F_WRITING = 0x020000, // File has been written since last flush
#endif
LFS_F_READING = 0x040000, // File has been read since last flush
#ifndef LFS_READONLY
LFS_F_ERRED = 0x080000, // An error occurred during write
#endif
LFS_F_INLINE = 0x100000, // Currently inlined in directory entry
};
// File seek flags
enum lfs_whence_flags {
LFS_SEEK_SET = 0, // Seek relative to an absolute position
LFS_SEEK_CUR = 1, // Seek relative to the current file position
LFS_SEEK_END = 2, // Seek relative to the end of the file
};
// Configuration provided during initialization of the littlefs
struct lfs_config {
// Opaque user provided context that can be used to pass
// information to the block device operations
void *context;
// Read a region in a block. Negative error codes are propagated
// to the user.
int (*read)(const struct lfs_config *c, lfs_block_t block,
lfs_off_t off, void *buffer, lfs_size_t size);
// Program a region in a block. The block must have previously
// been erased. Negative error codes are propagated to the user.
// May return LFS_ERR_CORRUPT if the block should be considered bad.
int (*prog)(const struct lfs_config *c, lfs_block_t block,
lfs_off_t off, const void *buffer, lfs_size_t size);
// Erase a block. A block must be erased before being programmed.
// The state of an erased block is undefined. Negative error codes
// are propagated to the user.
// May return LFS_ERR_CORRUPT if the block should be considered bad.
int (*erase)(const struct lfs_config *c, lfs_block_t block);
// Sync the state of the underlying block device. Negative error codes
// are propagated to the user.
int (*sync)(const struct lfs_config *c);
#ifdef LFS_THREADSAFE
// Lock the underlying block device. Negative error codes
// are propagated to the user.
int (*lock)(const struct lfs_config *c);
// Unlock the underlying block device. Negative error codes
// are propagated to the user.
int (*unlock)(const struct lfs_config *c);
#endif
// Minimum size of a block read in bytes. All read operations will be a
// multiple of this value.
lfs_size_t read_size;
// Minimum size of a block program in bytes. All program operations will be
// a multiple of this value.
lfs_size_t prog_size;
// Size of an erasable block in bytes. This does not impact ram consumption
// and may be larger than the physical erase size. However, non-inlined
// files take up at minimum one block. Must be a multiple of the read and
// program sizes.
lfs_size_t block_size;
// Number of erasable blocks on the device. Defaults to block_count stored
// on disk when zero.
lfs_size_t block_count;
// Number of erase cycles before littlefs evicts metadata logs and moves
// the metadata to another block. Suggested values are in the
// range 100-1000, with large values having better performance at the cost
// of less consistent wear distribution.
//
// Set to -1 to disable block-level wear-leveling.
int32_t block_cycles;
// Size of block caches in bytes. Each cache buffers a portion of a block in
// RAM. The littlefs needs a read cache, a program cache, and one additional
// cache per file. Larger caches can improve performance by storing more
// data and reducing the number of disk accesses. Must be a multiple of the
// read and program sizes, and a factor of the block size.
lfs_size_t cache_size;
// Size of the lookahead buffer in bytes. A larger lookahead buffer
// increases the number of blocks found during an allocation pass. The
// lookahead buffer is stored as a compact bitmap, so each byte of RAM
// can track 8 blocks.
lfs_size_t lookahead_size;
// Threshold for metadata compaction during lfs_fs_gc in bytes. Metadata
// pairs that exceed this threshold will be compacted during lfs_fs_gc.
// Defaults to ~88% block_size when zero, though the default may change
// in the future.
//
// Note this only affects lfs_fs_gc. Normal compactions still only occur
// when full.
//
// Set to -1 to disable metadata compaction during lfs_fs_gc.
lfs_size_t compact_thresh;
// Optional statically allocated read buffer. Must be cache_size.
// By default lfs_malloc is used to allocate this buffer.
void *read_buffer;
// Optional statically allocated program buffer. Must be cache_size.
// By default lfs_malloc is used to allocate this buffer.
void *prog_buffer;
// Optional statically allocated lookahead buffer. Must be lookahead_size.
// By default lfs_malloc is used to allocate this buffer.
void *lookahead_buffer;
// Optional upper limit on length of file names in bytes. No downside for
// larger names except the size of the info struct which is controlled by
// the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX or name_max stored on
// disk when zero.
lfs_size_t name_max;
// Optional upper limit on files in bytes. No downside for larger files
// but must be <= LFS_FILE_MAX. Defaults to LFS_FILE_MAX or file_max stored
// on disk when zero.
lfs_size_t file_max;
// Optional upper limit on custom attributes in bytes. No downside for
// larger attributes size but must be <= LFS_ATTR_MAX. Defaults to
// LFS_ATTR_MAX or attr_max stored on disk when zero.
lfs_size_t attr_max;
// Optional upper limit on total space given to metadata pairs in bytes. On
// devices with large blocks (e.g. 128kB) setting this to a low size (2-8kB)
// can help bound the metadata compaction time. Must be <= block_size.
// Defaults to block_size when zero.
lfs_size_t metadata_max;
// Optional upper limit on inlined files in bytes. Inlined files live in
// metadata and decrease storage requirements, but may be limited to
// improve metadata-related performance. Must be <= cache_size, <=
// attr_max, and <= block_size/8. Defaults to the largest possible
// inline_max when zero.
//
// Set to -1 to disable inlined files.
lfs_size_t inline_max;
#ifdef LFS_MULTIVERSION
// On-disk version to use when writing in the form of 16-bit major version
// + 16-bit minor version. This limiting metadata to what is supported by
// older minor versions. Note that some features will be lost. Defaults to
// to the most recent minor version when zero.
uint32_t disk_version;
#endif
};
// File info structure
struct lfs_info {
// Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR
uint8_t type;
// Size of the file, only valid for REG files. Limited to 32-bits.
lfs_size_t size;
// Name of the file stored as a null-terminated string. Limited to
// LFS_NAME_MAX+1, which can be changed by redefining LFS_NAME_MAX to
// reduce RAM. LFS_NAME_MAX is stored in superblock and must be
// respected by other littlefs drivers.
char name[LFS_NAME_MAX+1];
};
// Filesystem info structure
struct lfs_fsinfo {
// On-disk version.
uint32_t disk_version;
// Size of a logical block in bytes.
lfs_size_t block_size;
// Number of logical blocks in filesystem.
lfs_size_t block_count;
// Upper limit on the length of file names in bytes.
lfs_size_t name_max;
// Upper limit on the size of files in bytes.
lfs_size_t file_max;
// Upper limit on the size of custom attributes in bytes.
lfs_size_t attr_max;
};
// Custom attribute structure, used to describe custom attributes
// committed atomically during file writes.
struct lfs_attr {
// 8-bit type of attribute, provided by user and used to
// identify the attribute
uint8_t type;
// Pointer to buffer containing the attribute
void *buffer;
// Size of attribute in bytes, limited to LFS_ATTR_MAX
lfs_size_t size;
};
// Optional configuration provided during lfs_file_opencfg
struct lfs_file_config {
// Optional statically allocated file buffer. Must be cache_size.
// By default lfs_malloc is used to allocate this buffer.
void *buffer;
// Optional list of custom attributes related to the file. If the file
// is opened with read access, these attributes will be read from disk
// during the open call. If the file is opened with write access, the
// attributes will be written to disk every file sync or close. This
// write occurs atomically with update to the file's contents.
//
// Custom attributes are uniquely identified by an 8-bit type and limited
// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller
// than the buffer, it will be padded with zeros. If the stored attribute
// is larger, then it will be silently truncated. If the attribute is not
// found, it will be created implicitly.
struct lfs_attr *attrs;
// Number of custom attributes in the list
lfs_size_t attr_count;
};
/// internal littlefs data structures ///
typedef struct lfs_cache {
lfs_block_t block;
lfs_off_t off;
lfs_size_t size;
uint8_t *buffer;
} lfs_cache_t;
typedef struct lfs_mdir {
lfs_block_t pair[2];
uint32_t rev;
lfs_off_t off;
uint32_t etag;
uint16_t count;
bool erased;
bool split;
lfs_block_t tail[2];
} lfs_mdir_t;
// littlefs directory type
typedef struct lfs_dir {
struct lfs_dir *next;
uint16_t id;
uint8_t type;
lfs_mdir_t m;
lfs_off_t pos;
lfs_block_t head[2];
} lfs_dir_t;
// littlefs file type
typedef struct lfs_file {
struct lfs_file *next;
uint16_t id;
uint8_t type;
lfs_mdir_t m;
struct lfs_ctz {
lfs_block_t head;
lfs_size_t size;
} ctz;
uint32_t flags;
lfs_off_t pos;
lfs_block_t block;
lfs_off_t off;
lfs_cache_t cache;
const struct lfs_file_config *cfg;
} lfs_file_t;
typedef struct lfs_superblock {
uint32_t version;
lfs_size_t block_size;
lfs_size_t block_count;
lfs_size_t name_max;
lfs_size_t file_max;
lfs_size_t attr_max;
} lfs_superblock_t;
typedef struct lfs_gstate {
uint32_t tag;
lfs_block_t pair[2];
} lfs_gstate_t;
// The littlefs filesystem type
typedef struct lfs {
lfs_cache_t rcache;
lfs_cache_t pcache;
lfs_block_t root[2];
struct lfs_mlist {
struct lfs_mlist *next;
uint16_t id;
uint8_t type;
lfs_mdir_t m;
} *mlist;
uint32_t seed;
lfs_gstate_t gstate;
lfs_gstate_t gdisk;
lfs_gstate_t gdelta;
struct lfs_lookahead {
lfs_block_t start;
lfs_block_t size;
lfs_block_t next;
lfs_block_t ckpoint;
uint8_t *buffer;
} lookahead;
const struct lfs_config *cfg;
lfs_size_t block_count;
lfs_size_t name_max;
lfs_size_t file_max;
lfs_size_t attr_max;
lfs_size_t inline_max;
#ifdef LFS_MIGRATE
struct lfs1 *lfs1;
#endif
} lfs_t;
/// Filesystem functions ///
#ifndef LFS_READONLY
// Format a block device with the littlefs
//
// Requires a littlefs object and config struct. This clobbers the littlefs
// object, and does not leave the filesystem mounted. The config struct must
// be zeroed for defaults and backwards compatibility.
//
// Returns a negative error code on failure.
int lfs_format(lfs_t *lfs, const struct lfs_config *config);
#endif
// Mounts a littlefs
//
// Requires a littlefs object and config struct. Multiple filesystems
// may be mounted simultaneously with multiple littlefs objects. Both
// lfs and config must be allocated while mounted. The config struct must
// be zeroed for defaults and backwards compatibility.
//
// Returns a negative error code on failure.
int lfs_mount(lfs_t *lfs, const struct lfs_config *config);
// Unmounts a littlefs
//
// Does nothing besides releasing any allocated resources.
// Returns a negative error code on failure.
int lfs_unmount(lfs_t *lfs);
/// General operations ///
#ifndef LFS_READONLY
// Removes a file or directory
//
// If removing a directory, the directory must be empty.
// Returns a negative error code on failure.
int lfs_remove(lfs_t *lfs, const char *path);
#endif
#ifndef LFS_READONLY
// Rename or move a file or directory
//
// If the destination exists, it must match the source in type.
// If the destination is a directory, the directory must be empty.
//
// Returns a negative error code on failure.
int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
#endif
// Find info about a file or directory
//
// Fills out the info structure, based on the specified file or directory.
// Returns a negative error code on failure.
int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
// Get a custom attribute
//
// Custom attributes are uniquely identified by an 8-bit type and limited
// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than
// the buffer, it will be padded with zeros. If the stored attribute is larger,
// then it will be silently truncated. If no attribute is found, the error
// LFS_ERR_NOATTR is returned and the buffer is filled with zeros.
//
// Returns the size of the attribute, or a negative error code on failure.
// Note, the returned size is the size of the attribute on disk, irrespective
// of the size of the buffer. This can be used to dynamically allocate a buffer
// or check for existence.
lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
uint8_t type, void *buffer, lfs_size_t size);
#ifndef LFS_READONLY
// Set custom attributes
//
// Custom attributes are uniquely identified by an 8-bit type and limited
// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be
// implicitly created.
//
// Returns a negative error code on failure.
int lfs_setattr(lfs_t *lfs, const char *path,
uint8_t type, const void *buffer, lfs_size_t size);
#endif
#ifndef LFS_READONLY
// Removes a custom attribute
//
// If an attribute is not found, nothing happens.
//
// Returns a negative error code on failure.
int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
#endif
/// File operations ///
#ifndef LFS_NO_MALLOC
// Open a file
//
// The mode that the file is opened in is determined by the flags, which
// are values from the enum lfs_open_flags that are bitwise-ored together.
//
// Returns a negative error code on failure.
int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
const char *path, int flags);
// if LFS_NO_MALLOC is defined, lfs_file_open() will fail with LFS_ERR_NOMEM
// thus use lfs_file_opencfg() with config.buffer set.
#endif
// Open a file with extra configuration
//
// The mode that the file is opened in is determined by the flags, which
// are values from the enum lfs_open_flags that are bitwise-ored together.
//
// The config struct provides additional config options per file as described
// above. The config struct must remain allocated while the file is open, and
// the config struct must be zeroed for defaults and backwards compatibility.
//
// Returns a negative error code on failure.
int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
const char *path, int flags,
const struct lfs_file_config *config);
// Close a file
//
// Any pending writes are written out to storage as though
// sync had been called and releases any allocated resources.
//
// Returns a negative error code on failure.
int lfs_file_close(lfs_t *lfs, lfs_file_t *file);
// Synchronize a file on storage
//
// Any pending writes are written out to storage.
// Returns a negative error code on failure.
int lfs_file_sync(lfs_t *lfs, lfs_file_t *file);
// Read data from file
//
// Takes a buffer and size indicating where to store the read data.
// Returns the number of bytes read, or a negative error code on failure.
lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
void *buffer, lfs_size_t size);
#ifndef LFS_READONLY
// Write data to file
//
// Takes a buffer and size indicating the data to write. The file will not
// actually be updated on the storage until either sync or close is called.
//
// Returns the number of bytes written, or a negative error code on failure.
lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
const void *buffer, lfs_size_t size);
#endif
// Change the position of the file
//
// The change in position is determined by the offset and whence flag.
// Returns the new position of the file, or a negative error code on failure.
lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
lfs_soff_t off, int whence);
#ifndef LFS_READONLY
// Truncates the size of the file to the specified size
//
// Returns a negative error code on failure.
int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size);
#endif
// Return the position of the file
//
// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR)
// Returns the position of the file, or a negative error code on failure.
lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file);
// Change the position of the file to the beginning of the file
//
// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_SET)
// Returns a negative error code on failure.
int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file);
// Return the size of the file
//
// Similar to lfs_file_seek(lfs, file, 0, LFS_SEEK_END)
// Returns the size of the file, or a negative error code on failure.
lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file);
/// Directory operations ///
#ifndef LFS_READONLY
// Create a directory
//
// Returns a negative error code on failure.
int lfs_mkdir(lfs_t *lfs, const char *path);
#endif
// Open a directory
//
// Once open a directory can be used with read to iterate over files.
// Returns a negative error code on failure.
int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path);
// Close a directory
//
// Releases any allocated resources.
// Returns a negative error code on failure.
int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir);
// Read an entry in the directory
//
// Fills out the info structure, based on the specified file or directory.
// Returns a positive value on success, 0 at the end of directory,
// or a negative error code on failure.
int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info);
// Change the position of the directory
//
// The new off must be a value previous returned from tell and specifies
// an absolute offset in the directory seek.
//
// Returns a negative error code on failure.
int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off);
// Return the position of the directory
//
// The returned offset is only meant to be consumed by seek and may not make
// sense, but does indicate the current position in the directory iteration.
//
// Returns the position of the directory, or a negative error code on failure.
lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir);
// Change the position of the directory to the beginning of the directory
//
// Returns a negative error code on failure.
int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir);
/// Filesystem-level filesystem operations
// Find on-disk info about the filesystem
//
// Fills out the fsinfo structure based on the filesystem found on-disk.
// Returns a negative error code on failure.
int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo);
// Finds the current size of the filesystem
//
// Note: Result is best effort. If files share COW structures, the returned
// size may be larger than the filesystem actually is.
//
// Returns the number of allocated blocks, or a negative error code on failure.
lfs_ssize_t lfs_fs_size(lfs_t *lfs);
// Traverse through all blocks in use by the filesystem
//
// The provided callback will be called with each block address that is
// currently in use by the filesystem. This can be used to determine which
// blocks are in use or how much of the storage is available.
//
// Returns a negative error code on failure.
int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
#ifndef LFS_READONLY
// Attempt to make the filesystem consistent and ready for writing
//
// Calling this function is not required, consistency will be implicitly
// enforced on the first operation that writes to the filesystem, but this
// function allows the work to be performed earlier and without other
// filesystem changes.
//
// Returns a negative error code on failure.
int lfs_fs_mkconsistent(lfs_t *lfs);
#endif
#ifndef LFS_READONLY
// Attempt any janitorial work
//
// This currently:
// 1. Calls mkconsistent if not already consistent
// 2. Compacts metadata > compact_thresh
// 3. Populates the block allocator
//
// Though additional janitorial work may be added in the future.
//
// Calling this function is not required, but may allow the offloading of
// expensive janitorial work to a less time-critical code path.
//
// Returns a negative error code on failure. Accomplishing nothing is not
// an error.
int lfs_fs_gc(lfs_t *lfs);
#endif
#ifndef LFS_READONLY
// Grows the filesystem to a new size, updating the superblock with the new
// block count.
//
// If LFS_SHRINKNONRELOCATING is defined, this function will also accept
// block_counts smaller than the current configuration, after checking
// that none of the blocks that are being removed are in use.
// Note that littlefs's pseudorandom block allocation means that
// this is very unlikely to work in the general case.
//
// Returns a negative error code on failure.
int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count);
#endif
#ifndef LFS_READONLY
#ifdef LFS_MIGRATE
// Attempts to migrate a previous version of littlefs
//
// Behaves similarly to the lfs_format function. Attempts to mount
// the previous version of littlefs and update the filesystem so it can be
// mounted with the current version of littlefs.
//
// Requires a littlefs object and config struct. This clobbers the littlefs
// object, and does not leave the filesystem mounted. The config struct must
// be zeroed for defaults and backwards compatibility.
//
// Returns a negative error code on failure.
int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
#endif
#endif
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif
quakey/src/lfs_util.c
+37
View File
@@ -0,0 +1,37 @@
/*
* lfs util functions
*
* Copyright (c) 2022, The littlefs authors.
* Copyright (c) 2017, Arm Limited. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "lfs_util.h"
// Only compile if user does not provide custom config
#ifndef LFS_CONFIG
// If user provides their own CRC impl we don't need this
#ifndef LFS_CRC
// Software CRC implementation with small lookup table
uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) {
static const uint32_t rtable[16] = {
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac,
0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c,
0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c,
};
const uint8_t *data = buffer;
for (size_t i = 0; i < size; i++) {
crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 0)) & 0xf];
crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 4)) & 0xf];
}
return crc;
}
#endif
#endif
quakey/src/lfs_util.h
+277
View File
@@ -0,0 +1,277 @@
/*
* lfs utility functions
*
* Copyright (c) 2022, The littlefs authors.
* Copyright (c) 2017, Arm Limited. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef LFS_UTIL_H
#define LFS_UTIL_H
#define LFS_STRINGIZE(x) LFS_STRINGIZE2(x)
#define LFS_STRINGIZE2(x) #x
// Users can override lfs_util.h with their own configuration by defining
// LFS_CONFIG as a header file to include (-DLFS_CONFIG=lfs_config.h).
//
// If LFS_CONFIG is used, none of the default utils will be emitted and must be
// provided by the config file. To start, I would suggest copying lfs_util.h
// and modifying as needed.
#ifdef LFS_CONFIG
#include LFS_STRINGIZE(LFS_CONFIG)
#else
// Alternatively, users can provide a header file which defines
// macros and other things consumed by littlefs.
//
// For example, provide my_defines.h, which contains
// something like:
//
// #include <stddef.h>
// extern void *my_malloc(size_t sz);
// #define LFS_MALLOC(sz) my_malloc(sz)
//
// And build littlefs with the header by defining LFS_DEFINES.
// (-DLFS_DEFINES=my_defines.h)
#ifdef LFS_DEFINES
#include LFS_STRINGIZE(LFS_DEFINES)
#endif
// System includes
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include <inttypes.h>
#define LFS_NO_DEBUG
#define LFS_NO_WARN
#define LFS_NO_ERROR
#ifndef LFS_NO_MALLOC
#include <quakey.h>
#endif
#ifndef LFS_NO_ASSERT
#include <assert.h>
#endif
#if !defined(LFS_NO_DEBUG) || \
!defined(LFS_NO_WARN) || \
!defined(LFS_NO_ERROR) || \
defined(LFS_YES_TRACE)
#include <stdio.h>
#endif
#ifdef __cplusplus
extern "C"
{
#endif
// Macros, may be replaced by system specific wrappers. Arguments to these
// macros must not have side-effects as the macros can be removed for a smaller
// code footprint
// Logging functions
#ifndef LFS_TRACE
#ifdef LFS_YES_TRACE
#define LFS_TRACE_(fmt, ...) \
printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
#else
#define LFS_TRACE(...)
#endif
#endif
#ifndef LFS_DEBUG
#ifndef LFS_NO_DEBUG
#define LFS_DEBUG_(fmt, ...) \
printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
#define LFS_DEBUG(...) LFS_DEBUG_(__VA_ARGS__, "")
#else
#define LFS_DEBUG(...)
#endif
#endif
#ifndef LFS_WARN
#ifndef LFS_NO_WARN
#define LFS_WARN_(fmt, ...) \
printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
#define LFS_WARN(...) LFS_WARN_(__VA_ARGS__, "")
#else
#define LFS_WARN(...)
#endif
#endif
#ifndef LFS_ERROR
#ifndef LFS_NO_ERROR
#define LFS_ERROR_(fmt, ...) \
printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
#define LFS_ERROR(...) LFS_ERROR_(__VA_ARGS__, "")
#else
#define LFS_ERROR(...)
#endif
#endif
// Runtime assertions
#ifndef LFS_ASSERT
#ifndef LFS_NO_ASSERT
#define LFS_ASSERT(test) assert(test)
#else
#define LFS_ASSERT(test)
#endif
#endif
// Builtin functions, these may be replaced by more efficient
// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more
// expensive basic C implementation for debugging purposes
// Min/max functions for unsigned 32-bit numbers
static inline uint32_t lfs_max(uint32_t a, uint32_t b) {
return (a > b) ? a : b;
}
static inline uint32_t lfs_min(uint32_t a, uint32_t b) {
return (a < b) ? a : b;
}
// Align to nearest multiple of a size
static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) {
return a - (a % alignment);
}
static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) {
return lfs_aligndown(a + alignment-1, alignment);
}
// Find the smallest power of 2 greater than or equal to a
static inline uint32_t lfs_npw2(uint32_t a) {
#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
return 32 - __builtin_clz(a-1);
#else
uint32_t r = 0;
uint32_t s;
a -= 1;
s = (a > 0xffff) << 4; a >>= s; r |= s;
s = (a > 0xff ) << 3; a >>= s; r |= s;
s = (a > 0xf ) << 2; a >>= s; r |= s;
s = (a > 0x3 ) << 1; a >>= s; r |= s;
return (r | (a >> 1)) + 1;
#endif
}
// Count the number of trailing binary zeros in a
// lfs_ctz(0) may be undefined
static inline uint32_t lfs_ctz(uint32_t a) {
#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__)
return __builtin_ctz(a);
#else
return lfs_npw2((a & -a) + 1) - 1;
#endif
}
// Count the number of binary ones in a
static inline uint32_t lfs_popc(uint32_t a) {
#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
return __builtin_popcount(a);
#else
a = a - ((a >> 1) & 0x55555555);
a = (a & 0x33333333) + ((a >> 2) & 0x33333333);
return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
#endif
}
// Find the sequence comparison of a and b, this is the distance
// between a and b ignoring overflow
static inline int lfs_scmp(uint32_t a, uint32_t b) {
return (int)(unsigned)(a - b);
}
// Convert between 32-bit little-endian and native order
static inline uint32_t lfs_fromle32(uint32_t a) {
#if (defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \
(defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \
(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
return a;
#elif !defined(LFS_NO_INTRINSICS) && ( \
(defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \
(defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \
(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
return __builtin_bswap32(a);
#else
return ((uint32_t)((uint8_t*)&a)[0] << 0) |
((uint32_t)((uint8_t*)&a)[1] << 8) |
((uint32_t)((uint8_t*)&a)[2] << 16) |
((uint32_t)((uint8_t*)&a)[3] << 24);
#endif
}
static inline uint32_t lfs_tole32(uint32_t a) {
return lfs_fromle32(a);
}
// Convert between 32-bit big-endian and native order
static inline uint32_t lfs_frombe32(uint32_t a) {
#if !defined(LFS_NO_INTRINSICS) && ( \
(defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \
(defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \
(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
return __builtin_bswap32(a);
#elif (defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \
(defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \
(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
return a;
#else
return ((uint32_t)((uint8_t*)&a)[0] << 24) |
((uint32_t)((uint8_t*)&a)[1] << 16) |
((uint32_t)((uint8_t*)&a)[2] << 8) |
((uint32_t)((uint8_t*)&a)[3] << 0);
#endif
}
static inline uint32_t lfs_tobe32(uint32_t a) {
return lfs_frombe32(a);
}
// Calculate CRC-32 with polynomial = 0x04c11db7
#ifdef LFS_CRC
static inline uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) {
return LFS_CRC(crc, buffer, size);
}
#else
uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
#endif
// Allocate memory, only used if buffers are not provided to littlefs
//
// littlefs current has no alignment requirements, as it only allocates
// byte-level buffers.
static inline void *lfs_malloc(size_t size) {
#if defined(LFS_MALLOC)
return LFS_MALLOC(size);
#elif !defined(LFS_NO_MALLOC)
return malloc(size);
#else
(void)size;
return NULL;
#endif
}
// Deallocate memory, only used if buffers are not provided to littlefs
static inline void lfs_free(void *p) {
#if defined(LFS_FREE)
LFS_FREE(p);
#elif !defined(LFS_NO_MALLOC)
free(p);
#else
(void)p;
#endif
}
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif
#endif
quakey/src/quakey.c
+4304
View File
File diff suppressed because it is too large Load Diff
scripts/README.md
-85
View File
@@ -1,85 +0,0 @@
# Scripts
This directory contains utility scripts for ToastyFS development and testing.
## Cluster Demo Script
The `cluster_demo.sh` script allows you to easily spawn and manage a ToastyFS cluster for demo and testing purposes.
### Usage
```bash
# Start a cluster with 3 chunk servers (default)
./scripts/cluster_demo.sh start
# Start a cluster with a custom number of chunk servers
./scripts/cluster_demo.sh start 5
# Check cluster status
./scripts/cluster_demo.sh status
# Stop the cluster
./scripts/cluster_demo.sh stop
# Clean all cluster data and logs
./scripts/cluster_demo.sh clean
```
### What it does
The script:
- Builds ToastyFS if needed
- Starts one metadata server (leader) on port 8080
- Starts the specified number of chunk servers on ports 8081, 8082, etc.
- Stores all process IDs for easy management
- Captures logs to `cluster_logs/` directory
- Stores data in `cluster_data/` directory
All servers run in the background, making it easy to test the cluster with client applications.
## Branch Coverage Measurement
### Prerequisites
The HTML report generation requires `lcov`:
```bash
sudo apt-get install lcov
```
## Usage
### Quick text summary (5 second simulation)
```bash
make coverage-report
```
### HTML report with branch details (5 second simulation)
```bash
make coverage-html
```
### Custom duration
```bash
./scripts/measure_coverage.sh 10 # 10 second run, text output
./scripts/measure_coverage.sh 10 --html # 10 second run, HTML output
```
## What gets measured
The coverage tool:
- Builds the test binary with coverage instrumentation (`--coverage` flag)
- Runs the random simulation for the specified duration
- Generates reports showing which branches were executed
## Output
**Text report**: Shows per-file and total branch coverage percentages
**HTML report**: Interactive report generated by lcov/genhtml showing:
- Overall coverage summary
- Per-file coverage breakdown
- Source code with execution counts
- Branch coverage details
The HTML report is generated in `coverage_report/index.html`
scripts/cluster_demo.sh
-314
View File
@@ -1,314 +0,0 @@
#!/bin/bash
#
# ToastyFS Cluster Demo Script
# ===========================
# This script allows you to easily spawn and manage a ToastyFS cluster for demo purposes.
#
# Usage:
# ./scripts/cluster_demo.sh start [num_chunk_servers] - Start a cluster
# ./scripts/cluster_demo.sh stop - Stop the cluster
# ./scripts/cluster_demo.sh status - Show cluster status
# ./scripts/cluster_demo.sh clean - Clean data and log files
#
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
PID_FILE="$PROJECT_DIR/.cluster_demo.pids"
LOG_DIR="$PROJECT_DIR/cluster_logs"
DATA_DIR="$PROJECT_DIR/cluster_data"
METADATA_PORT=8080
CHUNK_SERVER_BASE_PORT=8081
WEB_SERVER_PORT=8090
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
print_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
print_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
build_if_needed() {
local need_build=false
if [ ! -f "$PROJECT_DIR/toastyfs.out" ] && [ ! -f "$PROJECT_DIR/toastyfs.exe" ]; then
need_build=true
fi
if [ ! -f "$PROJECT_DIR/toastyfs_web.out" ] && [ ! -f "$PROJECT_DIR/toastyfs_web.exe" ]; then
need_build=true
fi
if [ "$need_build" = true ]; then
print_info "Binaries not found. Building ToastyFS..."
cd "$PROJECT_DIR"
make
print_success "Build complete"
fi
}
get_binary() {
if [ -f "$PROJECT_DIR/toastyfs.out" ]; then
echo "$PROJECT_DIR/toastyfs.out"
elif [ -f "$PROJECT_DIR/toastyfs.exe" ]; then
echo "$PROJECT_DIR/toastyfs.exe"
else
print_error "ToastyFS binary not found"
exit 1
fi
}
get_web_binary() {
if [ -f "$PROJECT_DIR/toastyfs_web.out" ]; then
echo "$PROJECT_DIR/toastyfs_web.out"
elif [ -f "$PROJECT_DIR/toastyfs_web.exe" ]; then
echo "$PROJECT_DIR/toastyfs_web.exe"
else
print_error "ToastyFS web binary not found"
exit 1
fi
}
start_cluster() {
local num_chunk_servers=${1:-3}
if [ -f "$PID_FILE" ]; then
print_error "Cluster already running (PID file exists)"
print_info "Run '$0 stop' first or remove $PID_FILE if the cluster is not running"
exit 1
fi
build_if_needed
local binary=$(get_binary)
# Create directories
mkdir -p "$LOG_DIR"
mkdir -p "$DATA_DIR"
print_info "Starting ToastyFS cluster..."
print_info " Metadata server: 127.0.0.1:$METADATA_PORT"
print_info " Chunk servers: $num_chunk_servers"
print_info " Web server: 127.0.0.1:$WEB_SERVER_PORT"
echo
# Start metadata server
print_info "Starting metadata server on port $METADATA_PORT..."
local metadata_wal="$DATA_DIR/metadata.wal"
"$binary" --leader \
--addr 127.0.0.1 \
--port $METADATA_PORT \
--wal-file "$metadata_wal" \
> "$LOG_DIR/metadata.log" 2>&1 &
local metadata_pid=$!
echo "$metadata_pid" > "$PID_FILE"
print_success "Metadata server started (PID: $metadata_pid)"
# Wait a bit for metadata server to start
sleep 1
# Start chunk servers
for i in $(seq 1 $num_chunk_servers); do
local port=$((CHUNK_SERVER_BASE_PORT + i - 1))
local data_path="$DATA_DIR/chunk_server_$i/"
print_info "Starting chunk server $i on port $port..."
mkdir -p "$data_path"
"$binary" \
--addr 127.0.0.1 \
--port $port \
--path "$data_path" \
--remote-addr 127.0.0.1 \
--remote-port $METADATA_PORT \
> "$LOG_DIR/chunk_server_$i.log" 2>&1 &
local chunk_pid=$!
echo "$chunk_pid" >> "$PID_FILE"
print_success "Chunk server $i started (PID: $chunk_pid)"
# Small delay between starting servers
sleep 0.5
done
# Start web server
local web_binary=$(get_web_binary)
print_info "Starting web server on port $WEB_SERVER_PORT..."
"$web_binary" \
--upstream-addr 127.0.0.1 \
--upstream-port $METADATA_PORT \
--local-addr 127.0.0.1 \
--local-port $WEB_SERVER_PORT \
> "$LOG_DIR/web_server.log" 2>&1 &
local web_pid=$!
echo "$web_pid" >> "$PID_FILE"
print_success "Web server started (PID: $web_pid)"
echo
print_success "Cluster started successfully!"
echo
print_info "Connect to the cluster using:"
print_info " ToastyFS Protocol: 127.0.0.1:$METADATA_PORT"
print_info " HTTP Interface: http://127.0.0.1:$WEB_SERVER_PORT"
echo
print_info "View logs at: $LOG_DIR/"
print_info "Data stored at: $DATA_DIR/"
echo
print_info "To stop the cluster, run: $0 stop"
}
stop_cluster() {
if [ ! -f "$PID_FILE" ]; then
print_warning "No PID file found. Cluster may not be running."
exit 0
fi
print_info "Stopping ToastyFS cluster..."
local stopped=0
local failed=0
while IFS= read -r pid; do
if [ -n "$pid" ]; then
if kill -0 "$pid" 2>/dev/null; then
kill "$pid" 2>/dev/null && {
print_success "Stopped process $pid"
stopped=$((stopped + 1))
} || {
print_error "Failed to stop process $pid"
failed=$((failed + 1))
}
else
print_warning "Process $pid not running"
fi
fi
done < "$PID_FILE"
rm -f "$PID_FILE"
echo
print_success "Cluster stopped (Stopped: $stopped, Failed: $failed)"
}
show_status() {
if [ ! -f "$PID_FILE" ]; then
print_info "Cluster is NOT running"
exit 0
fi
print_info "Cluster status:"
echo
local running=0
local not_running=0
local line_num=0
local total_lines=$(wc -l < "$PID_FILE")
while IFS= read -r pid; do
if [ -n "$pid" ]; then
line_num=$((line_num + 1))
local server_type=""
if [ $line_num -eq 1 ]; then
server_type="Metadata server"
elif [ $line_num -eq $total_lines ]; then
server_type="Web server"
else
server_type="Chunk server $((line_num - 1))"
fi
if kill -0 "$pid" 2>/dev/null; then
echo -e " ${GREEN}●${NC} $server_type (PID: $pid) - ${GREEN}running${NC}"
running=$((running + 1))
else
echo -e " ${RED}●${NC} $server_type (PID: $pid) - ${RED}not running${NC}"
not_running=$((not_running + 1))
fi
fi
done < "$PID_FILE"
echo
if [ $not_running -eq 0 ]; then
print_success "All $running servers are running"
else
print_warning "$running running, $not_running not running"
fi
if [ -d "$LOG_DIR" ]; then
echo
print_info "Log directory: $LOG_DIR"
print_info "Recent log files:"
ls -lht "$LOG_DIR" | head -n 6 | tail -n 5 | awk '{print " " $9 " (" $5 ")"}'
fi
}
clean_data() {
if [ -f "$PID_FILE" ]; then
print_error "Cluster is running. Stop it first with: $0 stop"
exit 1
fi
print_warning "This will delete all cluster data and logs!"
read -p "Are you sure? (y/N) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
print_info "Cleaning data and logs..."
rm -rf "$DATA_DIR"
rm -rf "$LOG_DIR"
print_success "Cleaned successfully"
else
print_info "Cancelled"
fi
}
# Main command dispatcher
case "${1:-}" in
start)
start_cluster "${2:-3}"
;;
stop)
stop_cluster
;;
status)
show_status
;;
clean)
clean_data
;;
*)
echo "ToastyFS Cluster Demo Script"
echo
echo "Usage:"
echo " $0 start [num_chunk_servers] - Start a cluster (default: 3 chunk servers)"
echo " $0 stop - Stop the cluster"
echo " $0 status - Show cluster status"
echo " $0 clean - Clean data and log files"
echo
echo "Examples:"
echo " $0 start # Start with 3 chunk servers"
echo " $0 start 5 # Start with 5 chunk servers"
echo " $0 status # Check if cluster is running"
echo " $0 stop # Stop all servers"
exit 1
;;
esac
scripts/fix_line_endings.sh
-18
View File
@@ -1,18 +0,0 @@
#!/bin/bash
# Fix line endings for coverage scripts
echo "Fixing line endings for coverage scripts..."
# Convert CRLF to LF
dos2unix scripts/measure_coverage.sh 2>/dev/null || sed -i 's/\r$//' scripts/measure_coverage.sh
dos2unix scripts/generate_coverage_html.sh 2>/dev/null || sed -i 's/\r$//' scripts/generate_coverage_html.sh
# Ensure execute permissions
chmod +x scripts/measure_coverage.sh
chmod +x scripts/generate_coverage_html.sh
echo "Done! Line endings fixed and execute permissions set."
echo ""
echo "You can now run:"
echo " make coverage-report"
echo " make coverage-html"
scripts/generate_coverage_html.sh
-31
View File
@@ -1,31 +0,0 @@
#!/bin/bash
# Generate HTML coverage report using lcov
set -e
OUTPUT_DIR="coverage_report"
# Check if lcov is installed
if ! command -v lcov &> /dev/null; then
echo "Error: lcov is not installed"
echo "Please install it with: sudo apt-get install lcov"
exit 1
fi
# Capture coverage data with branch coverage enabled
echo "Capturing coverage data..."
lcov --capture --directory . --output-file coverage.info --rc lcov_branch_coverage=1
# Filter out system headers if any exist
echo "Filtering coverage data..."
lcov --remove coverage.info '/usr/*' --output-file coverage.info --ignore-errors unused --rc lcov_branch_coverage=1 || cp coverage.info coverage.info.bak
# Generate HTML report
echo "Generating HTML report..."
genhtml coverage.info --output-directory "$OUTPUT_DIR" --branch-coverage --rc lcov_branch_coverage=1
# Clean up
rm -f coverage.info
echo "HTML report generated in $OUTPUT_DIR/"
echo "Open $OUTPUT_DIR/index.html in your browser"
scripts/measure_coverage.sh
-113
View File
@@ -1,113 +0,0 @@
#!/bin/bash
# Script to measure branch coverage of the random simulation
set -e
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Branch Coverage Measurement Tool${NC}"
echo -e "${BLUE}========================================${NC}"
echo
# Clean previous coverage data
echo -e "${YELLOW}Cleaning previous coverage data...${NC}"
rm -f src/*.gcda src/*.gcno *.gcda *.gcno
make clean > /dev/null 2>&1
# Build with coverage
echo -e "${YELLOW}Building with coverage instrumentation...${NC}"
make coverage
# Run the simulation for a limited time
echo -e "${YELLOW}Running simulation...${NC}"
SIMULATION_TIME=${1:-5} # Default to 5 seconds if not specified
echo "Running for ${SIMULATION_TIME} seconds..."
# Run simulation in background and kill it after specified time
timeout ${SIMULATION_TIME}s ./toastyfs_random_test_coverage.out || true
# Generate coverage reports
echo
echo -e "${YELLOW}Generating coverage reports...${NC}"
# Find all .gcda files and generate coverage reports
GCDA_FILES=$(find . -name "*.gcda")
TOTAL_BRANCHES=0
TAKEN_BRANCHES=0
# Generate gcov reports from the coverage data files
for gcda_file in $GCDA_FILES; do
# Extract the source file name from the gcda filename
# Files are named like: toastyfs_random_test_coverage.out-basic.gcda
basename=$(basename "$gcda_file" .gcda)
source_name=${basename#*-} # Remove prefix up to and including '-'
# Run gcov to generate the .gcov file
gcov -b "$gcda_file" > /dev/null 2>&1 || true
done
# Parse gcov output to count branches
echo
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Branch Coverage Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo
# Process each .gcov file
for gcov_file in *.c.gcov; do
if [ -f "$gcov_file" ]; then
# Extract branch statistics from the gcov file
branches=$(grep -c "^branch" "$gcov_file" 2>/dev/null)
if [ -z "$branches" ]; then branches=0; fi
if [ "$branches" -gt 0 ]; then
taken_count=$(grep "^branch" "$gcov_file" 2>/dev/null | grep -c "taken [1-9]" 2>/dev/null || echo "0")
# Clean up the value - remove any whitespace/newlines
taken_count=$(echo "$taken_count" | tr -d '[:space:]')
if [ -z "$taken_count" ] || [ "$taken_count" = "" ]; then taken_count=0; fi
TOTAL_BRANCHES=$((TOTAL_BRANCHES + branches))
TAKEN_BRANCHES=$((TAKEN_BRANCHES + taken_count))
percentage=$((taken_count * 100 / branches))
filename=$(echo "$gcov_file" | sed 's/.gcov$//')
printf "%-40s %5d / %5d branches (%3d%%)\n" "$filename" "$taken_count" "$branches" "$percentage"
fi
fi
done
echo
echo -e "${BLUE}========================================${NC}"
if [ "$TOTAL_BRANCHES" -gt 0 ]; then
COVERAGE_PERCENT=$((TAKEN_BRANCHES * 100 / TOTAL_BRANCHES))
echo -e "${GREEN}Total: $TAKEN_BRANCHES / $TOTAL_BRANCHES branches reached (${COVERAGE_PERCENT}%)${NC}"
else
echo -e "${YELLOW}No branch coverage data found${NC}"
fi
echo -e "${BLUE}========================================${NC}"
echo
# Generate HTML report if requested
if [ "$2" == "--html" ]; then
echo -e "${YELLOW}Generating HTML coverage report...${NC}"
# Get the directory where this script is located
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
"$SCRIPT_DIR/generate_coverage_html.sh"
echo -e "${GREEN}HTML report generated in coverage_report/index.html${NC}"
echo "Open with: firefox coverage_report/index.html (or your preferred browser)"
echo
fi
# Clean up gcov files unless HTML was requested
if [ "$2" != "--html" ] && [ "$2" != "--detailed" ]; then
echo -e "${YELLOW}Cleaning up coverage files...${NC}"
rm -f *.gcov
fi
echo
echo -e "${GREEN}Coverage measurement complete!${NC}"
src/basic.c
+17 -9
View File
@@ -1,7 +1,9 @@
#include <string.h> #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h" #include "basic.h"
#include "system.h"
bool streq(string s1, string s2) bool streq(string s1, string s2)
{ {
@@ -24,10 +26,10 @@ Time get_current_time(void)
int64_t freq; int64_t freq;
int ok; int ok;
ok = sys_QueryPerformanceCounter((LARGE_INTEGER*) &count); ok = QueryPerformanceCounter((LARGE_INTEGER*) &count);
if (!ok) return INVALID_TIME; if (!ok) return INVALID_TIME;
ok = sys_QueryPerformanceFrequency((LARGE_INTEGER*) &freq); ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
if (!ok) return INVALID_TIME; if (!ok) return INVALID_TIME;
uint64_t res = 1000000000 * (double) count / freq; uint64_t res = 1000000000 * (double) count / freq;
@@ -37,7 +39,7 @@ Time get_current_time(void)
{ {
struct timespec time; struct timespec time;
if (sys_clock_gettime(CLOCK_REALTIME, &time)) if (clock_gettime(CLOCK_REALTIME, &time))
return INVALID_TIME; return INVALID_TIME;
uint64_t res; uint64_t res;
@@ -99,11 +101,17 @@ int getargi(int argc, char **argv, char *name, int fallback)
if (i == argc) if (i == argc)
break; break;
int tmp = atoi(argv[i]); errno = 0;
if (tmp == 0 && argv[i][0] != '0') // best effort char *end;
long val = strtol(argv[i], &end, 10);
if (end == argv[i] || *end != '\0' || errno == ERANGE)
break; break;
return tmp; if (val < INT_MIN || val > INT_MAX)
break;
return (int) val;
} }
return fallback; return fallback;
} }
src/byte_queue.c
+12 -9
View File
@@ -1,8 +1,11 @@
#include <string.h> #ifdef MAIN_SIMULATION
#include <assert.h> #define QUAKEY_ENABLE_MOCKS
#include <stdlib.h> #endif
#include <quakey.h>
#include <stdint.h>
#include <assert.h>
#include "system.h"
#include "byte_queue.h" #include "byte_queue.h"
// This is the implementation of a byte queue useful // This is the implementation of a byte queue useful
@@ -32,12 +35,12 @@ void byte_queue_free(ByteQueue *queue)
{ {
if (queue->read_target) { if (queue->read_target) {
if (queue->read_target != queue->data) if (queue->read_target != queue->data)
sys_free(queue->read_target); free(queue->read_target);
queue->read_target = NULL; queue->read_target = NULL;
queue->read_target_size = 0; queue->read_target_size = 0;
} }
sys_free(queue->data); free(queue->data);
queue->data = NULL; queue->data = NULL;
} }
@@ -99,7 +102,7 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
if (queue->read_target) { if (queue->read_target) {
if (queue->read_target != queue->data) if (queue->read_target != queue->data)
sys_free(queue->read_target); free(queue->read_target);
queue->read_target = NULL; queue->read_target = NULL;
queue->read_target_size = 0; queue->read_target_size = 0;
} }
@@ -224,7 +227,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
if (size > queue->limit) if (size > queue->limit)
size = queue->limit; size = queue->limit;
uint8_t *data = sys_malloc(size); uint8_t *data = malloc(size);
if (!data) { if (!data) {
queue->flags |= BYTE_QUEUE_ERROR; queue->flags |= BYTE_QUEUE_ERROR;
return 0; return 0;
@@ -234,7 +237,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
memcpy(data, queue->data + queue->head, queue->used); memcpy(data, queue->data + queue->head, queue->used);
if (queue->read_target != queue->data) if (queue->read_target != queue->data)
sys_free(queue->data); free(queue->data);
queue->data = data; queue->data = data;
queue->head = 0; queue->head = 0;
src/byte_queue.h
+2
View File
@@ -1,6 +1,8 @@
#ifndef BYTE_QUEUE_INCLUDED #ifndef BYTE_QUEUE_INCLUDED
#define BYTE_QUEUE_INCLUDED #define BYTE_QUEUE_INCLUDED
#include <stddef.h>
#include "basic.h" #include "basic.h"
typedef struct { typedef struct {
src/chunk_server.c
+71 -37
View File
@@ -1,14 +1,18 @@
#include <stdio.h> #ifdef MAIN_SIMULATION
#include <string.h> #define QUAKEY_ENABLE_MOCKS
#include <stdlib.h> #endif
#include <assert.h>
#include <quakey.h>
#include <assert.h>
#include <stdint.h>
#include "config.h"
#include "metadata_server.h"
#include "sha256.h" #include "sha256.h"
#include "message.h"
#include "file_system.h" #include "file_system.h"
#include "hash_set.h"
#include "config.h"
#include "tcp.h" #include "tcp.h"
#include "message.h"
#include "byte_queue.h"
#include "chunk_server.h" #include "chunk_server.h"
static string hash2path(ChunkServer *state, SHA256 hash, char *out) static string hash2path(ChunkServer *state, SHA256 hash, char *out)
@@ -99,12 +103,12 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
return -1; return -1;
if (patch_off > SIZE_MAX - patch.len) { if (patch_off > SIZE_MAX - patch.len) {
sys_free(data.ptr); free(data.ptr);
return -1; return -1;
} }
if (patch_off + (size_t) patch.len > (size_t) data.len) { if (patch_off + (size_t) patch.len > (size_t) data.len) {
sys_free(data.ptr); free(data.ptr);
return -1; return -1;
} }
@@ -112,11 +116,11 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
ret = store_chunk(state, data, new_hash); ret = store_chunk(state, data, new_hash);
if (ret < 0) { if (ret < 0) {
sys_free(data.ptr); free(data.ptr);
return -1; return -1;
} }
sys_free(data.ptr); free(data.ptr);
return 0; return 0;
} }
@@ -129,7 +133,7 @@ static void download_targets_init(DownloadTargets *targets)
static void download_targets_free(DownloadTargets *targets) static void download_targets_free(DownloadTargets *targets)
{ {
sys_free(targets->items); free(targets->items);
} }
static void download_targets_remove(DownloadTargets *targets, static void download_targets_remove(DownloadTargets *targets,
@@ -161,13 +165,13 @@ static int download_targets_push(DownloadTargets *targets,
else else
new_capacity = 2 * targets->capacity; new_capacity = 2 * targets->capacity;
DownloadTarget *new_items = sys_malloc(new_capacity * sizeof(DownloadTarget)); DownloadTarget *new_items = malloc(new_capacity * sizeof(DownloadTarget));
if (new_items == NULL) if (new_items == NULL)
return -1; return -1;
if (targets->capacity > 0) { if (targets->capacity > 0) {
memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0])); memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0]));
sys_free(targets->items); free(targets->items);
} }
targets->items = new_items; targets->items = new_items;
@@ -563,7 +567,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
if (binary_read(&reader, NULL, 1)) if (binary_read(&reader, NULL, 1))
return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message"));
char *mem = sys_malloc(chunk_size); char *mem = malloc(chunk_size);
if (mem == NULL) if (mem == NULL)
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory")); return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory"));
@@ -579,7 +583,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
SHA256 dummy; SHA256 dummy;
int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy); int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy);
sys_free(mem); free(mem);
if (ret < 0) if (ret < 0)
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error")); return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error"));
@@ -694,7 +698,7 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
string slice; string slice;
if (full == 0) { if (full == 0) {
if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) { if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) {
sys_free(data.ptr); free(data.ptr);
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range")); return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range"));
} }
slice = (string) { data.ptr + target_off, target_len }; slice = (string) { data.ptr + target_off, target_len };
@@ -710,11 +714,11 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
message_write(&writer, &target_len, sizeof(target_len)); message_write(&writer, &target_len, sizeof(target_len));
message_write(&writer, slice.ptr, slice.len); message_write(&writer, slice.ptr, slice.len);
if (!message_writer_free(&writer)) { if (!message_writer_free(&writer)) {
sys_free(data.ptr); free(data.ptr);
return -1; return -1;
} }
sys_free(data.ptr); free(data.ptr);
return 0; return 0;
} }
@@ -812,8 +816,12 @@ static int send_sync_message(ChunkServer *state)
return 0; return 0;
} }
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout) int chunk_server_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{ {
ChunkServer *state = state_;
string addr = getargs(argc, argv, "--addr", "127.0.0.1"); string addr = getargs(argc, argv, "--addr", "127.0.0.1");
int port = getargi(argc, argv, "--port", 8081); int port = getargi(argc, argv, "--port", 8081);
string path = getargs(argc, argv, "--path", "chunk_server_data/"); string path = getargs(argc, argv, "--path", "chunk_server_data/");
@@ -821,34 +829,45 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1"); string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1");
int remote_port = getargi(argc, argv, "--remote-port", 8080); int remote_port = getargi(argc, argv, "--remote-port", 8080);
if (port <= 0 || port >= 1<<16) if (port <= 0 || port >= 1<<16) {
fprintf(stderr, "chunk server :: Invalid port\n");
return -1; return -1;
}
if (remote_port <= 0 || remote_port >= 1<<16) if (remote_port <= 0 || remote_port >= 1<<16) {
fprintf(stderr, "chunk server :: Invalid remote port\n");
return -1; return -1;
}
Time current_time = get_current_time(); Time current_time = get_current_time();
if (current_time == INVALID_TIME) if (current_time == INVALID_TIME) {
fprintf(stderr, "chunk server :: Couldn't read the time\n");
return -1; return -1;
}
state->trace = trace; state->trace = trace;
state->reconnect_delay = 1; // 1 second state->reconnect_delay = 1; // 1 second
if (tcp_context_init(&state->tcp) < 0) if (tcp_context_init(&state->tcp) < 0) {
fprintf(stderr, "chunk server :: Couldn't setup the TCP context\n");
return -1; return -1;
}
int ret = tcp_listen(&state->tcp, addr, port); int ret = tcp_listen(&state->tcp, addr, port);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "chunk server :: Couldn't setup the TCP listener\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
if (create_dir(path) && errno != EEXIST) { if (create_dir(path) && errno != EEXIST) {
fprintf(stderr, "chunk server :: Couldn't create chunk folder\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
if (get_full_path(path, state->path) < 0) { if (get_full_path(path, state->path) < 0) {
fprintf(stderr, "chunk server :: Couldn't convert path to absolute\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
@@ -861,6 +880,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
char tmp[1<<10]; char tmp[1<<10];
if (addr.len >= (int) sizeof(tmp)) { if (addr.len >= (int) sizeof(tmp)) {
fprintf(stderr, "chunk server :: Address is too long\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
@@ -868,6 +888,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
tmp[addr.len] = '\0'; tmp[addr.len] = '\0';
state->local_addr.is_ipv4 = true; state->local_addr.is_ipv4 = true;
if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) { if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) {
fprintf(stderr, "chunk server :: Couldn't parse address\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
@@ -876,6 +897,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
// Initialize metadata server address // Initialize metadata server address
// // TODO: This should also support IPv6 // // TODO: This should also support IPv6
if (remote_addr.len >= (int) sizeof(tmp)) { if (remote_addr.len >= (int) sizeof(tmp)) {
fprintf(stderr, "chunk server :: Remote address is too long\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
@@ -883,6 +905,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
tmp[remote_addr.len] = '\0'; tmp[remote_addr.len] = '\0';
state->remote_addr.is_ipv4 = true; state->remote_addr.is_ipv4 = true;
if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) { if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) {
fprintf(stderr, "chunk server :: Couldn't parse remote address\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
@@ -904,23 +927,20 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
); );
*timeout = 0; *timeout = 0;
return tcp_register_events(&state->tcp, contexts, polled); if (pcap < TCP_POLL_CAPACITY) {
} fprintf(stderr, "chunk server :: Capacity isn't large enough\n");
return -1;
int chunk_server_free(ChunkServer *state) }
{ *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
download_targets_free(&state->download_targets);
timed_hash_set_free(&state->cs_rem_list);
hash_set_free(&state->cs_lst_list);
hash_set_free(&state->cs_add_list);
tcp_context_free(&state->tcp);
return 0; return 0;
} }
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout) int chunk_server_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{ {
ChunkServer *state = state_;
Event events[TCP_EVENT_CAPACITY]; Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled); int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
Time current_time = get_current_time(); Time current_time = get_current_time();
if (current_time == INVALID_TIME) if (current_time == INVALID_TIME)
@@ -1031,5 +1051,19 @@ int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled
} }
*timeout = deadline_to_timeout(deadline, current_time); *timeout = deadline_to_timeout(deadline, current_time);
return tcp_register_events(&state->tcp, contexts, polled); if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int chunk_server_free(void *state_)
{
ChunkServer *state = state_;
download_targets_free(&state->download_targets);
timed_hash_set_free(&state->cs_rem_list);
hash_set_free(&state->cs_lst_list);
hash_set_free(&state->cs_add_list);
tcp_context_free(&state->tcp);
return 0;
} }
src/chunk_server.h
+12 -10
View File
@@ -1,12 +1,6 @@
#ifndef CHUNK_SERVER_INCLUDED #ifndef CHUNK_SERVER_INCLUDED
#define CHUNK_SERVER_INCLUDED #define CHUNK_SERVER_INCLUDED
#include <limits.h>
#include "basic.h"
#include "metadata_server.h"
#include "tcp.h"
#define TAG_METADATA_SERVER 1 #define TAG_METADATA_SERVER 1
#define TAG_CHUNK_SERVER 2 #define TAG_CHUNK_SERVER 2
@@ -57,8 +51,16 @@ typedef struct {
} ChunkServer; } ChunkServer;
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout); struct pollfd;
int chunk_server_free(ChunkServer *state);
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
#endif // CHUNK_SERVER_INCLUDED int chunk_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int chunk_server_free(void *state);
#endif // CHUNK_SERVER_INCLUDED
src/client.c
+42 -40
View File
@@ -1,22 +1,24 @@
#include <assert.h> #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <assert.h>
#include <stdlib.h>
#ifdef _WIN32
# define POLL WSAPoll
#else
# define POLL poll
#endif
#ifdef _WIN32 #ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#define POLL WSAPoll
typedef CRITICAL_SECTION Mutex; typedef CRITICAL_SECTION Mutex;
#else #else
#include <pthread.h>
#include <arpa/inet.h>
#define POLL poll
typedef pthread_mutex_t Mutex; typedef pthread_mutex_t Mutex;
#endif #endif
#include "tcp.h" #include "tcp.h"
#include "system.h"
#include "config.h" #include "config.h"
#include "message.h" #include "message.h"
#include "file_tree.h" #include "file_tree.h"
@@ -322,7 +324,7 @@ static int handle_to_operation(ToastyFS *toasty, ToastyHandle handle)
ToastyFS *toasty_connect(ToastyString addr, uint16_t port) ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
{ {
ToastyFS *toasty = sys_malloc(sizeof(ToastyFS)); ToastyFS *toasty = malloc(sizeof(ToastyFS));
if (toasty == NULL) if (toasty == NULL)
return NULL; return NULL;
@@ -330,7 +332,7 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
{ {
char tmp[128]; char tmp[128];
if (addr.len >= (int) sizeof(tmp)) { if (addr.len >= (int) sizeof(tmp)) {
sys_free(toasty); free(toasty);
return NULL; return NULL;
} }
memcpy(tmp, addr.ptr, addr.len); memcpy(tmp, addr.ptr, addr.len);
@@ -339,26 +341,26 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
addr2.is_ipv4 = true; addr2.is_ipv4 = true;
addr2.port = port; addr2.port = port;
if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) { if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) {
sys_free(toasty); free(toasty);
return NULL; return NULL;
} }
} }
if (mutex_init(&toasty->mutex) < 0) { if (mutex_init(&toasty->mutex) < 0) {
sys_free(toasty); free(toasty);
return NULL; return NULL;
} }
if (tcp_context_init(&toasty->tcp) < 0) { if (tcp_context_init(&toasty->tcp) < 0) {
mutex_free(&toasty->mutex); mutex_free(&toasty->mutex);
sys_free(toasty); free(toasty);
return NULL; return NULL;
} }
if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) { if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) {
tcp_context_free(&toasty->tcp); tcp_context_free(&toasty->tcp);
mutex_free(&toasty->mutex); mutex_free(&toasty->mutex);
sys_free(toasty); free(toasty);
return NULL; return NULL;
} }
@@ -435,7 +437,7 @@ void toasty_disconnect(ToastyFS *toasty)
{ {
tcp_context_free(&toasty->tcp); tcp_context_free(&toasty->tcp);
mutex_free(&toasty->mutex); mutex_free(&toasty->mutex);
sys_free(toasty); free(toasty);
} }
static bool static bool
@@ -947,7 +949,7 @@ static void process_event_for_list(ToastyFS *toasty,
return; return;
} }
ToastyListingEntry *entities = sys_malloc(item_count * sizeof(ToastyListingEntry)); ToastyListingEntry *entities = malloc(item_count * sizeof(ToastyListingEntry));
if (entities == NULL) { if (entities == NULL) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
return; return;
@@ -959,28 +961,28 @@ static void process_event_for_list(ToastyFS *toasty,
uint64_t gen; uint64_t gen;
if (!binary_read(&reader, &gen, sizeof(gen))) { if (!binary_read(&reader, &gen, sizeof(gen))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
uint8_t is_dir; uint8_t is_dir;
if (!binary_read(&reader, &is_dir, sizeof(is_dir))) { if (!binary_read(&reader, &is_dir, sizeof(is_dir))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
uint16_t name_len; uint16_t name_len;
if (!binary_read(&reader, &name_len, sizeof(name_len))) { if (!binary_read(&reader, &name_len, sizeof(name_len))) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
char *name = (char*) reader.src + reader.cur; char *name = (char*) reader.src + reader.cur;
if (!binary_read(&reader, NULL, name_len)) { if (!binary_read(&reader, NULL, name_len)) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
@@ -989,7 +991,7 @@ static void process_event_for_list(ToastyFS *toasty,
if (name_len > sizeof(entities[i].name)-1) { if (name_len > sizeof(entities[i].name)-1) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
memcpy(entities[i].name, name, name_len); memcpy(entities[i].name, name, name_len);
@@ -999,7 +1001,7 @@ static void process_event_for_list(ToastyFS *toasty,
// Check there is nothing else to read // Check there is nothing else to read
if (binary_read(&reader, NULL, 1)) { if (binary_read(&reader, NULL, 1)) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
sys_free(entities); free(entities);
return; return;
} }
@@ -1089,7 +1091,7 @@ static void process_event_for_read(ToastyFS *toasty,
} }
// Allocate ranges // Allocate ranges
Range *ranges = sys_malloc(num_chunks_needed * sizeof(Range)); Range *ranges = malloc(num_chunks_needed * sizeof(Range));
if (ranges == NULL) { if (ranges == NULL) {
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
@@ -1104,7 +1106,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Read hash // Read hash
SHA256 hash; SHA256 hash;
if (!binary_read(&reader, &hash, sizeof(hash))) { if (!binary_read(&reader, &hash, sizeof(hash))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1112,7 +1114,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Read number of servers // Read number of servers
uint32_t num_servers; uint32_t num_servers;
if (!binary_read(&reader, &num_servers, sizeof(num_servers))) { if (!binary_read(&reader, &num_servers, sizeof(num_servers))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1120,7 +1122,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Parse IPv4 addresses // Parse IPv4 addresses
uint32_t num_ipv4; uint32_t num_ipv4;
if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) { if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1134,7 +1136,7 @@ static void process_event_for_read(ToastyFS *toasty,
uint16_t port; uint16_t port;
if (!binary_read(&reader, &ipv4, sizeof(ipv4)) || if (!binary_read(&reader, &ipv4, sizeof(ipv4)) ||
!binary_read(&reader, &port, sizeof(port))) { !binary_read(&reader, &port, sizeof(port))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1149,21 +1151,21 @@ static void process_event_for_read(ToastyFS *toasty,
// Skip IPv6 addresses // Skip IPv6 addresses
uint32_t num_ipv6; uint32_t num_ipv6;
if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) { if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
for (uint32_t j = 0; j < num_ipv6; j++) { for (uint32_t j = 0; j < num_ipv6; j++) {
if (!binary_read(&reader, NULL, sizeof(IPv6)) || if (!binary_read(&reader, NULL, sizeof(IPv6)) ||
!binary_read(&reader, NULL, sizeof(uint16_t))) { !binary_read(&reader, NULL, sizeof(uint16_t))) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
} }
if (!found) { if (!found) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1208,7 +1210,7 @@ static void process_event_for_read(ToastyFS *toasty,
Range *r = &ranges[0]; Range *r = &ranges[0];
int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL); int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL);
if (cs_idx < 0) { if (cs_idx < 0) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1216,7 +1218,7 @@ static void process_event_for_read(ToastyFS *toasty,
if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk, if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk,
r->length_within_chunk, opidx, 0) < 0) { r->length_within_chunk, opidx, 0) < 0) {
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
return; return;
} }
@@ -1225,7 +1227,7 @@ static void process_event_for_read(ToastyFS *toasty,
toasty->operations[opidx].ranges_head = 1; toasty->operations[opidx].ranges_head = 1;
} else { } else {
// No chunks to download // No chunks to download
sys_free(ranges); free(ranges);
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
} }
@@ -1299,7 +1301,7 @@ static void process_event_for_read(ToastyFS *toasty,
// Check if done // Check if done
if (toasty->operations[opidx].num_pending == 0) { if (toasty->operations[opidx].num_pending == 0) {
sys_free(toasty->operations[opidx].ranges); free(toasty->operations[opidx].ranges);
toasty->operations[opidx].ranges = NULL; toasty->operations[opidx].ranges = NULL;
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
} }
@@ -1424,7 +1426,7 @@ static int schedule_upload(ToastyFS *toasty, int opidx, UploadSchedule upload)
else else
new_cap_uploads = 2 * o->cap_uploads; new_cap_uploads = 2 * o->cap_uploads;
UploadSchedule *uploads = sys_malloc(new_cap_uploads * sizeof(UploadSchedule)); UploadSchedule *uploads = malloc(new_cap_uploads * sizeof(UploadSchedule));
if (uploads == NULL) if (uploads == NULL)
return -1; return -1;
@@ -1534,7 +1536,7 @@ static void process_event_for_write(ToastyFS *toasty,
toasty->operations[opidx].num_chunks = num_all_hasehs; toasty->operations[opidx].num_chunks = num_all_hasehs;
toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow
toasty->operations[opidx].hashes = sys_malloc(num_hashes * sizeof(SHA256)); toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256));
if (toasty->operations[opidx].hashes == NULL) { if (toasty->operations[opidx].hashes == NULL) {
assert(0); // TODO assert(0); // TODO
} }
@@ -1957,7 +1959,7 @@ static void process_event_for_write(ToastyFS *toasty,
} ChunkUploadResult; } ChunkUploadResult;
int num_upload_results = toasty->operations[opidx].num_chunks; int num_upload_results = toasty->operations[opidx].num_chunks;
ChunkUploadResult *upload_results = sys_malloc(num_upload_results * sizeof(ChunkUploadResult)); ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult));
if (upload_results == NULL) { if (upload_results == NULL) {
assert(0); // TODO assert(0); // TODO
} }
@@ -2471,7 +2473,7 @@ int toasty_list(ToastyFS *toasty, ToastyString path,
void toasty_free_listing(ToastyListing *listing) void toasty_free_listing(ToastyListing *listing)
{ {
sys_free(listing->items); free(listing->items);
} }
int toasty_read(ToastyFS *toasty, ToastyString path, int toasty_read(ToastyFS *toasty, ToastyString path,
src/crash_logger.c
-433
View File
@@ -1,433 +0,0 @@
#if defined(__linux__) && defined(__x86_64__)
#define _GNU_SOURCE
#include <elf.h>
#include <fcntl.h>
#include <stdio.h>
#include <signal.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <stdbool.h>
#include <sys/reg.h>
#include <sys/stat.h>
#include <ucontext.h>
typedef struct {
uint64_t base_addr;
int count;
void *symbols;
void *strings;
} SymbolTable;
static int is_hex(char c)
{
return (c >= '0' && c <= '9') ||
(c >= 'a' && c <= 'f') ||
(c >= 'A' && c <= 'F');
}
static uint64_t query_base_addr(void)
{
int fd = open("/proc/self/maps", O_RDONLY);
if (fd < 0)
return -1;
char buf[128];
int ret = read(fd, buf, sizeof(buf));
if (ret < 0) {
close(fd);
return -1;
}
close(fd);
if (ret == 0 || !is_hex(buf[0]))
return -1;
int i = 0;
uint64_t base_addr = 0;
for (;;) {
char c = buf[i++];
int d;
if (0) {}
else if (c >= 'a' && c <= 'f') d = c - 'a' + 10;
else if (c >= 'A' && c <= 'F') d = c - 'A' + 10;
else d = c - '0';
if (base_addr > (UINT64_MAX - d) / 16)
return -1;
base_addr = base_addr * 16 + d;
if (i == ret)
return -1;
if (buf[i] == '-')
break;
if (!is_hex(buf[i]))
return -1;
}
return base_addr;
}
static int current_executable_path(char *dst, int cap)
{
if (cap == 0)
return -1;
int ret = readlink("/proc/self/exe", dst, cap-1);
if (ret < 0)
return -1;
dst[ret] = '\0';
return ret;
}
static int load_symbols_from_elf(void *src, int len, SymbolTable *st)
{
// NOTE: It's assumed is properly aligned
assert(((uintptr_t) src & 15) == 0);
// Check that the file contains a full header
if (len < (int) sizeof(Elf64_Ehdr))
return -1;
Elf64_Ehdr *ehdr = (Elf64_Ehdr*) src;
// Check that the file contains the full list
// of section headers
if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > len)
return -1;
Elf64_Shdr *shdrs = (Elf64_Shdr*) (src + ehdr->e_shoff);
Elf64_Shdr *shstrtab_hdr = &shdrs[ehdr->e_shstrndx]; // TODO: bounds check
char *shstrtab = src + shstrtab_hdr->sh_offset;
// Iterate over the section headers to find the
// one reative to symbols and their strings
Elf64_Shdr *symtab_hdr = NULL;
Elf64_Shdr *strtab_hdr = NULL;
for (int i = 0; i < ehdr->e_shnum; i++) {
char *section_name = shstrtab + shdrs[i].sh_name;
if (0) {}
else if (!strcmp(section_name, ".symtab")) symtab_hdr = &shdrs[i];
else if (!strcmp(section_name, ".strtab")) strtab_hdr = &shdrs[i];
}
if (symtab_hdr == NULL || strtab_hdr == NULL) {
return -1;
}
void *mem = malloc(symtab_hdr->sh_size + strtab_hdr->sh_size);
if (mem == NULL) {
return -1;
}
st->count = symtab_hdr->sh_size / sizeof(Elf64_Sym);
st->symbols = mem;
st->strings = (char*) st->symbols + symtab_hdr->sh_size;
memcpy(st->symbols, src + symtab_hdr->sh_offset, symtab_hdr->sh_size);
memcpy(st->strings, src + strtab_hdr->sh_offset, strtab_hdr->sh_size);
return 0;
}
static char *read_file(char *path, int *len)
{
int fd = open(path, O_RDONLY);
if (fd < 0)
return NULL;
struct stat buf;
if (fstat(fd, &buf) < 0) {
close(fd);
return NULL;
}
*len = buf.st_size;
char *ptr = malloc(*len + 1);
if (ptr == NULL) {
close(fd);
return NULL;
}
for (int num = 0; num < *len; ) {
int ret = read(fd, ptr + num, *len - num);
if (ret <= 0) {
free(ptr);
close(fd);
return NULL;
}
num += ret;
}
ptr[*len] = '\0';
return ptr;
}
static int symbol_table_from_current_process(SymbolTable *st)
{
uint64_t base_addr = query_base_addr();
if (base_addr == (uint64_t) -1)
return -1;
st->base_addr = base_addr;
char path[1<<10];
if (current_executable_path(path, sizeof(path)) < 0)
return -1;
char *exe_ptr;
int exe_len;
exe_ptr = read_file(path, &exe_len);
if (exe_ptr == NULL)
return -1;
if (load_symbols_from_elf(exe_ptr, exe_len, st) < 0) {
free(exe_ptr);
return -1;
}
free(exe_ptr);
return 0;
}
static void symbol_table_free(SymbolTable *st)
{
free(st->symbols);
}
static char *symbol_table_find(SymbolTable *st, uint64_t addr)
{
for (int i = 0; i < st->count; i++) {
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
if (sym->st_value == 0)
continue;
uint64_t sym_beg = st->base_addr + sym->st_value;
uint64_t sym_end = st->base_addr + sym->st_value + sym->st_size;
if (addr >= sym_beg && addr < sym_end)
return (char*) st->strings + sym->st_name;
}
return NULL;
}
#if 0
static void symbol_table_dump(SymbolTable *st)
{
for (int i = 0; i < st->count; i++) {
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
if (sym->st_value == 0)
continue;
char *name = (char*) st->strings + sym->st_name;
printf("%s\n", name);
}
}
#endif
typedef struct {
char *name;
uint64_t addr;
} StackFrame;
static int walk_stack(uint64_t rip, uint64_t rbp, SymbolTable *st, StackFrame *frames, int max_frames)
{
int frame_count = 0;
if (frame_count < max_frames) {
frames[frame_count].addr = rip - st->base_addr;
frames[frame_count].name = symbol_table_find(st, rip);
frame_count++;
}
while (rbp != 0) {
if (rbp & 0xF)
break;
uint64_t *frame_ptr = (uint64_t*) rbp;
uint64_t next_rbp = frame_ptr[0];
uint64_t return_addr = frame_ptr[1];
if (next_rbp != 0 && next_rbp <= rbp)
break;
if (return_addr == 0)
break;
if (frame_count == max_frames)
break;
frames[frame_count].addr = return_addr - st->base_addr;
frames[frame_count].name = symbol_table_find(st, return_addr);
frame_count++;
rbp = next_rbp;
}
return frame_count;
}
static bool crash_logger_symbol_init = false;
static char* crash_logger_file_name = NULL;
static SymbolTable crash_logger_symbol_table;
static char* crash_logger_signal_stack;
static void crash_handler(int sig, siginfo_t *info, void *ucontext)
{
(void) info;
if (crash_logger_symbol_init) {
// Buffer for evaluating format strings
char tmp[1<<9];
int len;
ucontext_t *ctx = (ucontext_t*) ucontext;
uint64_t rip = ctx->uc_mcontext.gregs[REG_RIP];
uint64_t rbp = ctx->uc_mcontext.gregs[REG_RBP];
StackFrame frames[64];
int count = walk_stack(rip, rbp, &crash_logger_symbol_table, frames, 64);
int fd = open(crash_logger_file_name, O_WRONLY | O_CREAT, 0666);
if (fd < 0)
exit(1);
char *sig_name = "";
switch (sig) {
case SIGSEGV: sig_name = "Segmentation fault"; break;
case SIGBUS : sig_name = "Bus error"; break;
case SIGILL : sig_name = "Illegal instruction"; break;
case SIGFPE : sig_name = "Floating point exception"; break;
case SIGTRAP: sig_name = "Trace trap"; break;
case SIGSYS : sig_name = "Bad system call"; break;
case SIGABRT: sig_name = "Abort"; break;
}
if (sig_name[0] == '\0') {
len = snprintf(tmp, sizeof(tmp), "(unknown signal %d)\n", sig);
write(fd, tmp, len);
} else {
write(fd, sig_name, strlen(sig_name));
write(fd, "\n", 1);
}
for (int i = 0; i < count; i++) {
len = snprintf(tmp, sizeof(tmp), " [%d] 0x%lx %s\n", i, frames[i].addr,
frames[i].name ? frames[i].name : "?");
write(fd, tmp, len);
}
close(fd);
}
exit(1);
}
int crash_logger_init(char *file_name, int file_name_len)
{
{
char *file_name_copy = malloc(file_name_len + 1);
if (file_name_copy == NULL)
return -1;
memcpy(file_name_copy, file_name, file_name_len);
file_name_copy[file_name_len] = '\0';
crash_logger_file_name = file_name_copy;
}
if (symbol_table_from_current_process(&crash_logger_symbol_table) < 0) {
free(crash_logger_file_name);
return -1;
}
// Set up alternate signal stack
{
crash_logger_signal_stack = malloc(SIGSTKSZ);
if (crash_logger_signal_stack == NULL) {
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
return -1;
}
stack_t ss;
ss.ss_sp = crash_logger_signal_stack;
ss.ss_size = SIGSTKSZ;
ss.ss_flags = 0;
if (sigaltstack(&ss, NULL) < 0) {
free(crash_logger_signal_stack);
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
return -1;
}
}
{
// Register the crash handler
struct sigaction sa;
sa.sa_sigaction = crash_handler;
sa.sa_flags = SA_SIGINFO | SA_ONSTACK; // Add SA_ONSTACK flag
sigemptyset(&sa.sa_mask);
// Memory errors
sigaction(SIGSEGV, &sa, NULL); // Segmentation fault (invalid memory access)
sigaction(SIGBUS, &sa, NULL); // Bus error (misaligned access, hardware error)
// Execution errors
sigaction(SIGILL, &sa, NULL); // Illegal instruction
sigaction(SIGFPE, &sa, NULL); // Floating point exception
sigaction(SIGTRAP, &sa, NULL); // Trace trap
// System/resource errors
sigaction(SIGSYS, &sa, NULL); // Bad system call
sigaction(SIGABRT, &sa, NULL); // Abort (from assert, abort(), etc.)
// Optional: Resource limit violations
sigaction(SIGXCPU, &sa, NULL); // CPU time limit exceeded
sigaction(SIGXFSZ, &sa, NULL); // File size limit exceeded
}
crash_logger_symbol_init = true;
return 0;
}
void crash_logger_free(void)
{
if (!crash_logger_symbol_init)
return;
free(crash_logger_signal_stack);
symbol_table_free(&crash_logger_symbol_table);
free(crash_logger_file_name);
crash_logger_symbol_init = false;
}
#else
static int crash_logger_init(char *file_name, int file_name_len)
{
(void) file_name;
(void) file_name_len;
return -1;
}
static void crash_logger_free(void)
{
}
#endif
src/crash_logger.h
-7
View File
@@ -1,7 +0,0 @@
#ifndef CRASH_LOGGER_INCLUDED
#define CRASH_LOGGER_INCLUDED
int crash_logger_init(char *file_name, int file_name_len);
void crash_logger_free(void);
#endif // CRASH_LOGGER_INCLUDED
src/file_system.c
+41 -40
View File
@@ -1,8 +1,9 @@
#include <limits.h> #ifdef MAIN_SIMULATION
#include <string.h> #define QUAKEY_ENABLE_MOCKS
#include <assert.h> #endif
#include <stdint.h>
#include <quakey.h>
#include "system.h"
#include "file_system.h" #include "file_system.h"
int rename_file_or_dir(string oldpath, string newpath); int rename_file_or_dir(string oldpath, string newpath);
@@ -16,7 +17,7 @@ int file_open(string path, Handle *fd)
memcpy(zt, path.ptr, path.len); memcpy(zt, path.ptr, path.len);
zt[path.len] = '\0'; zt[path.len] = '\0';
int ret = sys_open(zt, O_RDWR | O_CREAT | O_APPEND, 0644); int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
if (ret < 0) if (ret < 0)
return -1; return -1;
@@ -29,7 +30,7 @@ int file_open(string path, Handle *fd)
MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH); MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH);
wpath[path.len] = L'\0'; wpath[path.len] = L'\0';
HANDLE h = sys_CreateFileW( HANDLE h = CreateFileW(
wpath, wpath,
GENERIC_WRITE | GENERIC_READ, GENERIC_WRITE | GENERIC_READ,
0, 0,
@@ -49,18 +50,18 @@ int file_open(string path, Handle *fd)
void file_close(Handle fd) void file_close(Handle fd)
{ {
#ifdef __linux__ #ifdef __linux__
sys_close((int) fd.data); close((int) fd.data);
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
sys_CloseHandle((HANDLE) fd.data); CloseHandle((HANDLE) fd.data);
#endif #endif
} }
int file_set_offset(Handle fd, int off) int file_set_offset(Handle fd, int off)
{ {
#ifdef __linux__ #ifdef __linux__
off_t ret = sys_lseek((int) fd.data, off, SEEK_SET); off_t ret = lseek((int) fd.data, off, SEEK_SET);
if (ret < 0) if (ret < 0)
return -1; return -1;
return 0; return 0;
@@ -69,8 +70,8 @@ int file_set_offset(Handle fd, int off)
#ifdef _WIN32 #ifdef _WIN32
LARGE_INTEGER distance; LARGE_INTEGER distance;
distance.QuadPart = off; distance.QuadPart = off;
if (!sys_SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN)) if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
if (GetLastError() != NO_ERROR) if (GetLastError() != 0)
return -1; return -1;
return 0; return 0;
#endif #endif
@@ -79,7 +80,7 @@ int file_set_offset(Handle fd, int off)
int file_get_offset(Handle fd, int *off) int file_get_offset(Handle fd, int *off)
{ {
#ifdef __linux__ #ifdef __linux__
off_t ret = sys_lseek((int) fd.data, 0, SEEK_CUR); off_t ret = lseek((int) fd.data, 0, SEEK_CUR);
if (ret < 0) if (ret < 0)
return -1; return -1;
*off = (int) ret; *off = (int) ret;
@@ -87,8 +88,8 @@ int file_get_offset(Handle fd, int *off)
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
DWORD pos = sys_SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT); DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
if (pos == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0)
return -1; return -1;
*off = (int) pos; *off = (int) pos;
return 0; return 0;
@@ -98,13 +99,13 @@ int file_get_offset(Handle fd, int *off)
int file_lock(Handle fd) int file_lock(Handle fd)
{ {
#ifdef __linux__ #ifdef __linux__
if (sys_flock((int) fd.data, LOCK_EX) < 0) if (flock((int) fd.data, LOCK_EX) < 0)
return -1; return -1;
return 0; return 0;
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
if (!sys_LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
return -1; return -1;
return 0; return 0;
#endif #endif
@@ -113,13 +114,13 @@ int file_lock(Handle fd)
int file_unlock(Handle fd) int file_unlock(Handle fd)
{ {
#ifdef __linux__ #ifdef __linux__
if (sys_flock((int) fd.data, LOCK_UN) < 0) if (flock((int) fd.data, LOCK_UN) < 0)
return -1; return -1;
return 0; return 0;
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
if (!sys_UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
return -1; return -1;
return 0; return 0;
#endif #endif
@@ -128,13 +129,13 @@ int file_unlock(Handle fd)
int file_sync(Handle fd) int file_sync(Handle fd)
{ {
#ifdef __linux__ #ifdef __linux__
if (sys_fsync((int) fd.data) < 0) if (fsync((int) fd.data) < 0)
return -1; return -1;
return 0; return 0;
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
if (!sys_FlushFileBuffers((HANDLE) fd.data)) if (!FlushFileBuffers((HANDLE) fd.data))
return -1; return -1;
return 0; return 0;
#endif #endif
@@ -143,12 +144,12 @@ int file_sync(Handle fd)
int file_read(Handle fd, char *dst, int max) int file_read(Handle fd, char *dst, int max)
{ {
#ifdef __linux__ #ifdef __linux__
return sys_read((int) fd.data, dst, max); return read((int) fd.data, dst, max);
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
DWORD num; DWORD num;
if (!sys_ReadFile((HANDLE) fd.data, dst, max, &num, NULL)) if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
return -1; return -1;
if (num > INT_MAX) if (num > INT_MAX)
return -1; return -1;
@@ -159,12 +160,12 @@ int file_read(Handle fd, char *dst, int max)
int file_write(Handle fd, char *src, int len) int file_write(Handle fd, char *src, int len)
{ {
#ifdef __linux__ #ifdef __linux__
return sys_write((int) fd.data, src, len); return write((int) fd.data, src, len);
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
DWORD num; DWORD num;
if (!sys_WriteFile((HANDLE) fd.data, src, len, &num, NULL)) if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL))
return -1; return -1;
if (num > INT_MAX) if (num > INT_MAX)
return -1; return -1;
@@ -176,7 +177,7 @@ int file_size(Handle fd, size_t *len)
{ {
#ifdef __linux__ #ifdef __linux__
struct stat buf; struct stat buf;
if (sys_fstat((int) fd.data, &buf) < 0) if (fstat((int) fd.data, &buf) < 0)
return -1; return -1;
if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX) if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX)
return -1; return -1;
@@ -186,7 +187,7 @@ int file_size(Handle fd, size_t *len)
#ifdef _WIN32 #ifdef _WIN32
LARGE_INTEGER buf; LARGE_INTEGER buf;
if (!sys_GetFileSizeEx((HANDLE) fd.data, &buf)) if (!GetFileSizeEx((HANDLE) fd.data, &buf))
return -1; return -1;
if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX) if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX)
return -1; return -1;
@@ -204,10 +205,10 @@ int create_dir(string path)
zt[path.len] = '\0'; zt[path.len] = '\0';
#ifdef _WIN32 #ifdef _WIN32
if (sys__mkdir(zt) < 0) if (_mkdir(zt) < 0)
return -1; return -1;
#else #else
if (sys_mkdir(zt, 0766)) if (mkdir(zt, 0766))
return -1; return -1;
#endif #endif
@@ -228,7 +229,7 @@ int rename_file_or_dir(string oldpath, string newpath)
memcpy(newpath_zt, newpath.ptr, newpath.len); memcpy(newpath_zt, newpath.ptr, newpath.len);
newpath_zt[newpath.len] = '\0'; newpath_zt[newpath.len] = '\0';
if (sys_rename(oldpath_zt, newpath_zt)) if (rename(oldpath_zt, newpath_zt))
return -1; return -1;
return 0; return 0;
} }
@@ -241,7 +242,7 @@ int remove_file_or_dir(string path)
memcpy(path_zt, path.ptr, path.len); memcpy(path_zt, path.ptr, path.len);
path_zt[path.len] = '\0'; path_zt[path.len] = '\0';
if (sys_remove(path_zt)) if (remove(path_zt))
return -1; return -1;
return 0; return 0;
} }
@@ -255,12 +256,12 @@ int get_full_path(string path, char *dst)
path_zt[path.len] = '\0'; path_zt[path.len] = '\0';
#ifdef __linux__ #ifdef __linux__
if (sys_realpath(path_zt, dst) == NULL) if (realpath(path_zt, dst) == NULL)
return -1; return -1;
#endif #endif
#ifdef _WIN32 #ifdef _WIN32
if (sys__fullpath(path_zt, dst, PATH_MAX) == NULL) if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
return -1; return -1;
#endif #endif
@@ -285,7 +286,7 @@ int file_read_all(string path, string *data)
return -1; return -1;
} }
char *dst = sys_malloc(len); char *dst = malloc(len);
if (dst == NULL) { if (dst == NULL) {
file_close(fd); file_close(fd);
return -1; return -1;
@@ -295,7 +296,7 @@ int file_read_all(string path, string *data)
while ((size_t) copied < len) { while ((size_t) copied < len) {
ret = file_read(fd, dst + copied, len - copied); ret = file_read(fd, dst + copied, len - copied);
if (ret < 0) { if (ret < 0) {
sys_free(dst); free(dst);
file_close(fd); file_close(fd);
return -1; return -1;
} }
@@ -316,7 +317,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
if (ret < 0 || ret >= (int) sizeof(pattern)) if (ret < 0 || ret >= (int) sizeof(pattern))
return -1; return -1;
scanner->handle = sys_FindFirstFileA(pattern, &scanner->find_data); scanner->handle = FindFirstFileA(pattern, &scanner->find_data);
if (scanner->handle == INVALID_HANDLE_VALUE) { if (scanner->handle == INVALID_HANDLE_VALUE) {
if (GetLastError() == ERROR_FILE_NOT_FOUND) { if (GetLastError() == ERROR_FILE_NOT_FOUND) {
scanner->done = true; scanner->done = true;
@@ -336,7 +337,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
return 1; return 1;
if (!scanner->first) { if (!scanner->first) {
BOOL ok = sys_FindNextFileA(scanner->handle, &scanner->find_data); BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data);
if (!ok) { if (!ok) {
scanner->done = true; scanner->done = true;
if (GetLastError() == ERROR_NO_MORE_FILES) if (GetLastError() == ERROR_NO_MORE_FILES)
@@ -354,7 +355,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
void directory_scanner_free(DirectoryScanner *scanner) void directory_scanner_free(DirectoryScanner *scanner)
{ {
sys_FindClose(scanner->handle); FindClose(scanner->handle);
} }
#else #else
@@ -367,7 +368,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
memcpy(path_copy, path.ptr, path.len); memcpy(path_copy, path.ptr, path.len);
path_copy[path.len] = '\0'; path_copy[path.len] = '\0';
scanner->d = sys_opendir(path_copy); scanner->d = opendir(path_copy);
if (scanner->d == NULL) { if (scanner->d == NULL) {
scanner->done = true; scanner->done = true;
return -1; return -1;
@@ -382,7 +383,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
if (scanner->done) if (scanner->done)
return 1; return 1;
scanner->e = sys_readdir(scanner->d); scanner->e = readdir(scanner->d);
if (scanner->e == NULL) { if (scanner->e == NULL) {
scanner->done = true; scanner->done = true;
return 1; return 1;
@@ -394,7 +395,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
void directory_scanner_free(DirectoryScanner *scanner) void directory_scanner_free(DirectoryScanner *scanner)
{ {
sys_closedir(scanner->d); closedir(scanner->d);
} }
#endif #endif
src/file_system.h
+6 -8
View File
@@ -1,14 +1,12 @@
#ifndef FILE_SYSTEM_INCLUDED #ifndef FILE_SYSTEM_INCLUDED
#define FILE_SYSTEM_INCLUDED #define FILE_SYSTEM_INCLUDED
#include "basic.h" #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#else
#include <dirent.h>
#endif #endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h"
typedef struct { typedef struct {
uint64_t data; uint64_t data;
@@ -50,4 +48,4 @@ int directory_scanner_init(DirectoryScanner *scanner, string path);
int directory_scanner_next(DirectoryScanner *scanner, string *name); int directory_scanner_next(DirectoryScanner *scanner, string *name);
void directory_scanner_free(DirectoryScanner *scanner); void directory_scanner_free(DirectoryScanner *scanner);
#endif // FILE_SYSTEM_INCLUDED #endif // FILE_SYSTEM_INCLUDED
src/file_tree.c
+18 -16
View File
@@ -1,10 +1,12 @@
#include <limits.h> #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <assert.h> #include <assert.h>
#include <string.h> #include <quakey.h>
#include <stdlib.h>
#include "basic.h" #include "basic.h"
#include "system.h"
#include "file_tree.h" #include "file_tree.h"
static int parse_path(string path, string *comps, int max) static int parse_path(string path, string *comps, int max)
@@ -102,7 +104,7 @@ static void dir_free(Dir *d)
{ {
for (uint64_t i = 0; i < d->num_children; i++) for (uint64_t i = 0; i < d->num_children; i++)
entity_free(&d->children[i]); entity_free(&d->children[i]);
sys_free(d->children); free(d->children);
} }
static bool gen_match(uint64_t expected_gen, uint64_t entity_gen) static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
@@ -158,7 +160,7 @@ static void file_init(File *f, uint64_t chunk_size)
static void file_free(File *f) static void file_free(File *f)
{ {
sys_free(f->chunks); free(f->chunks);
f->chunks = NULL; f->chunks = NULL;
} }
@@ -312,14 +314,14 @@ int file_tree_create_entity(FileTree *ft, string path,
if (new_max == 0) if (new_max == 0)
new_max = 8; new_max = 8;
Entity *p = sys_malloc(sizeof(Entity) * new_max); Entity *p = malloc(sizeof(Entity) * new_max);
if (p == NULL) if (p == NULL)
return FILETREE_NOMEM; return FILETREE_NOMEM;
for (uint64_t i = 0; i < d->num_children; i++) for (uint64_t i = 0; i < d->num_children; i++)
p[i] = d->children[i]; p[i] = d->children[i];
sys_free(d->children); free(d->children);
d->children = p; d->children = p;
d->max_children = new_max; d->max_children = new_max;
} }
@@ -423,13 +425,13 @@ int file_tree_write(
if (last_chunk_index >= f->num_chunks) { if (last_chunk_index >= f->num_chunks) {
uint64_t old_num_chunks = f->num_chunks; uint64_t old_num_chunks = f->num_chunks;
SHA256 *new_chunks = sys_malloc((last_chunk_index+1) * sizeof(SHA256)); SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256));
if (new_chunks == NULL) if (new_chunks == NULL)
return FILETREE_NOMEM; return FILETREE_NOMEM;
if (f->chunks) { if (f->chunks) {
if (f->num_chunks > 0) if (f->num_chunks > 0)
memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256)); memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256));
sys_free(f->chunks); free(f->chunks);
} }
f->chunks = new_chunks; f->chunks = new_chunks;
f->num_chunks = last_chunk_index+1; f->num_chunks = last_chunk_index+1;
@@ -678,13 +680,13 @@ int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *wr
sc.write_data = write_data; sc.write_data = write_data;
sc.buffer_used = 0; sc.buffer_used = 0;
sc.buffer_size = 1<<10; sc.buffer_size = 1<<10;
sc.buffer = sys_malloc(sc.buffer_size); sc.buffer = malloc(sc.buffer_size);
sc.error = false; sc.error = false;
if (sc.buffer == NULL) if (sc.buffer == NULL)
sc.error = true; sc.error = true;
entity_serialize(&sc, &ft->root); entity_serialize(&sc, &ft->root);
sc_flush(&sc); sc_flush(&sc);
sys_free(sc.buffer); free(sc.buffer);
if (sc.error) if (sc.error)
return -1; return -1;
return 0; return 0;
@@ -748,7 +750,7 @@ static void file_deserialize(DeserializeContext *dc, File *f)
dc_read_u64(dc, &f->num_chunks); dc_read_u64(dc, &f->num_chunks);
dc_read_u64(dc, &f->file_size); dc_read_u64(dc, &f->file_size);
f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256)); f->chunks = malloc(f->num_chunks * sizeof(SHA256));
if (f->chunks == NULL) { if (f->chunks == NULL) {
assert(0); // TODO assert(0); // TODO
} }
@@ -764,7 +766,7 @@ static void dir_deserialize(DeserializeContext *dc, Dir *d)
dc_read_u64(dc, &d->num_children); dc_read_u64(dc, &d->num_children);
d->max_children = d->num_children; d->max_children = d->num_children;
d->children = sys_malloc(d->num_children * sizeof(Entity)); d->children = malloc(d->num_children * sizeof(Entity));
if (d->children == NULL) { if (d->children == NULL) {
assert(0); // TODO assert(0); // TODO
} }
@@ -799,13 +801,13 @@ int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *r
dc.buffer_head = 0; dc.buffer_head = 0;
dc.buffer_used = 0; dc.buffer_used = 0;
dc.buffer_size = 1<<10; dc.buffer_size = 1<<10;
dc.buffer = sys_malloc(dc.buffer_size); dc.buffer = malloc(dc.buffer_size);
dc.error = false; dc.error = false;
if (dc.buffer == NULL) if (dc.buffer == NULL)
dc.error = true; dc.error = true;
dc.total_read = 0; dc.total_read = 0;
entity_deserialize(&dc, &ft->root); entity_deserialize(&dc, &ft->root);
sys_free(dc.buffer); free(dc.buffer);
if (dc.error) if (dc.error)
return -1; return -1;
if (dc.total_read > INT_MAX) { if (dc.total_read > INT_MAX) {
src/hash_set.c
+10 -9
View File
@@ -1,7 +1,8 @@
#include <assert.h> #ifdef MAIN_SIMULATION
#include <string.h> #define QUAKEY_ENABLE_MOCKS
#endif
#include "system.h" #include <stdint.h>
#include <quakey.h>
#include "hash_set.h" #include "hash_set.h"
void hash_set_init(HashSet *set) void hash_set_init(HashSet *set)
@@ -13,7 +14,7 @@ void hash_set_init(HashSet *set)
void hash_set_free(HashSet *set) void hash_set_free(HashSet *set)
{ {
sys_free(set->items); free(set->items);
set->items = NULL; set->items = NULL;
} }
@@ -40,7 +41,7 @@ int hash_set_insert(HashSet *set, SHA256 hash)
else else
new_capacity = 2 * set->capacity; new_capacity = 2 * set->capacity;
SHA256 *new_items = sys_realloc(set->items, new_capacity * sizeof(SHA256)); SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256));
if (new_items == NULL) if (new_items == NULL)
return -1; return -1;
@@ -109,7 +110,7 @@ void timed_hash_set_init(TimedHashSet *set)
void timed_hash_set_free(TimedHashSet *set) void timed_hash_set_free(TimedHashSet *set)
{ {
sys_free(set->items); free(set->items);
set->items = NULL; set->items = NULL;
} }
@@ -137,13 +138,13 @@ int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time)
else else
new_capacity = 2 * set->capacity; new_capacity = 2 * set->capacity;
TimedHash *new_items = sys_malloc(new_capacity * sizeof(TimedHash)); TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash));
if (new_items == NULL) if (new_items == NULL)
return -1; return -1;
if (set->capacity > 0) { if (set->capacity > 0) {
memcpy(new_items, set->items, set->count * sizeof(set->items[0])); memcpy(new_items, set->items, set->count * sizeof(set->items[0]));
sys_free(set->items); free(set->items);
} }
set->items = new_items; set->items = new_items;
src/main.c
+170
View File
@@ -0,0 +1,170 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#else
#define POLL_CAPACITY 1024
#endif
#include <stdint.h>
#include <quakey.h>
#include "metadata_server.h"
#include "chunk_server.h"
#include "random_client.h"
#ifdef MAIN_METADATA_SERVER
int main(int argc, char **argv)
{
int ret;
MetadataServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = metadata_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = metadata_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
metadata_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_CHUNK_SERVER
int main(int argc, char **argv)
{
int ret;
ChunkServer state;
void* poll_ctxs[POLL_CAPACITY];
struct pollfd poll_array[POLL_CAPACITY];
int poll_count;
int poll_timeout;
ret = chunk_server_init(
&state,
argc,
argv,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
for (;;) {
#ifdef _WIN32
WSAPoll(poll_array, poll_count, poll_timeout);
#else
poll(poll_array, poll_count, poll_timeout);
#endif
ret = chunk_server_tick(
&state,
poll_ctxs,
poll_array,
POLL_CAPACITY,
&poll_count,
&poll_timeout
);
if (ret < 0)
return -1;
}
chunk_server_free(&state);
return 0;
}
#endif
#ifdef MAIN_SIMULATION
int main(void)
{
Quakey *quakey;
int ret = quakey_init(&quakey, 1);
if (ret < 0)
return -1;
// Client
{
QuakeySpawn config = {
.state_size = sizeof(RandomClient),
.init_func = random_client_init,
.tick_func = random_client_tick,
.free_func = random_client_free,
.addrs = (char*[]) { "127.0.0.2" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.2");
}
// Metadata Server
{
QuakeySpawn config = {
.state_size = sizeof(MetadataServer),
.init_func = metadata_server_init,
.tick_func = metadata_server_tick,
.free_func = metadata_server_free,
.addrs = (char*[]) { "127.0.0.3" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "ms --addr 127.0.0.3");
}
// Chunk Server
{
QuakeySpawn config = {
.state_size = sizeof(ChunkServer),
.init_func = chunk_server_init,
.tick_func = chunk_server_tick,
.free_func = chunk_server_free,
.addrs = (char*[]) { "127.0.0.4" },
.num_addrs = 1,
.disk_size = 1<<20,
.platform = QUAKEY_LINUX,
};
quakey_spawn(quakey, config, "cs --addr 127.0.0.4");
}
for (;;)
quakey_schedule_one(quakey);
quakey_free(quakey);
return 0;
}
#endif // MAIN_SIMULATION
src/main_server.c
-132
View File
@@ -1,132 +0,0 @@
#ifdef BUILD_SERVER
#include <time.h>
#include <string.h>
#ifdef _WIN32
#include <winsock2.h>
#define POLL WSAPoll
#else
#include <poll.h>
#define POLL poll
#endif
#include "crash_logger.h"
#include "chunk_server.h"
#include "metadata_server.h"
int metadata_server_main(int argc, char **argv)
{
{
time_t now = time(NULL);
struct tm unpacked_now;
#ifdef _WIN32
if (gmtime_s(&unpacked_now, &now) != 0)
return -1;
#else
if (gmtime_r(&now, &unpacked_now) == NULL)
return -1;
#endif
char time[sizeof("YYYYMMDDthhmmssz")];
int ret = strftime(time, sizeof(time),
"%Y%m%dT%H%M%SZ", &unpacked_now);
if (ret != sizeof(time)-1)
return -1;
char path[1<<10];
ret = snprintf(path, sizeof(path), "crash_%s_MS_%d.txt", time, getpid());
if (ret < 0 || ret >= (int) sizeof(path)) {
return -1;
}
int path_len = ret;
crash_logger_init(path, path_len);
}
void *contexts[TCP_POLL_CAPACITY];
struct pollfd polled[TCP_POLL_CAPACITY];
int num_polled;
int timeout = -1;
MetadataServer state;
num_polled = metadata_server_init(
&state, argc, argv, contexts, polled, &timeout);
if (num_polled < 0) return -1;
for (;;) {
POLL(polled, num_polled, timeout);
timeout = -1;
num_polled = metadata_server_step(
&state, contexts, polled, num_polled, &timeout);
if (num_polled < 0) return -1;
}
metadata_server_free(&state);
crash_logger_free();
return 0;
}
int chunk_server_main(int argc, char **argv)
{
{
time_t now = time(NULL);
struct tm unpacked_now;
#ifdef _WIN32
if (gmtime_s(&unpacked_now, &now) != 0)
return -1;
#else
if (gmtime_r(&now, &unpacked_now) == NULL)
return -1;
#endif
char time[sizeof("YYYYMMDDthhmmssz")];
int ret = strftime(time, sizeof(time),
"%Y%m%dT%H%M%SZ", &unpacked_now);
if (ret != sizeof(time)-1)
return -1;
char path[1<<10];
ret = snprintf(path, sizeof(path), "crash_%s_CS_%d.txt", time, getpid());
if (ret < 0 || ret >= (int) sizeof(path)) {
return -1;
}
int path_len = ret;
crash_logger_init(path, path_len);
}
void *contexts[TCP_POLL_CAPACITY];
struct pollfd polled[TCP_POLL_CAPACITY];
int num_polled;
int timeout = -1;
ChunkServer state;
num_polled = chunk_server_init(
&state, argc, argv, contexts, polled, &timeout);
if (num_polled < 0) return -1;
for (;;) {
POLL(polled, num_polled, timeout);
timeout = -1;
num_polled = chunk_server_step(
&state, contexts, polled, num_polled, &timeout);
if (num_polled < 0) return -1;
}
chunk_server_free(&state);
crash_logger_free();
return 0;
}
int main(int argc, char **argv)
{
if (getargb(argc, argv, "--leader"))
return metadata_server_main(argc, argv);
else
return chunk_server_main(argc, argv);
}
#endif
src/main_test.c
-57
View File
@@ -1,57 +0,0 @@
#ifdef BUILD_TEST
#include <stdio.h>
#include <signal.h>
#include <stdbool.h>
#include "system.h"
static sig_atomic_t simulation_should_stop = false;
static void signal_handler(int signum)
{
(void)signum;
simulation_should_stop = true;
}
int main(int argc, char **argv)
{
(void)argc;
(void)argv;
// Set up signal handlers for clean shutdown
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
startup_simulation(2);
// Spawn metadata server (leader)
spawn_simulated_process("--addr 127.0.0.1 --port 8080 --leader");
// Spawn chunk servers
spawn_simulated_process("--addr 127.0.0.1 --port 8081 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_0/");
spawn_simulated_process("--addr 127.0.0.1 --port 8082 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_1/");
spawn_simulated_process("--addr 127.0.0.1 --port 8083 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_2/");
spawn_simulated_process("--addr 127.0.0.1 --port 8084 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_3/");
spawn_simulated_process("--addr 127.0.0.1 --port 8085 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_4/");
spawn_simulated_process("--addr 127.0.0.1 --port 8086 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_5/");
spawn_simulated_process("--addr 127.0.0.1 --port 8087 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_6/");
spawn_simulated_process("--addr 127.0.0.1 --port 8088 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_7/");
spawn_simulated_process("--addr 127.0.0.1 --port 8089 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_8/");
spawn_simulated_process("--addr 127.0.0.1 --port 8090 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_9/");
// Spawn simulation client
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
while (!simulation_should_stop)
update_simulation();
cleanup_simulation();
return 0;
}
#endif
src/message.c
+4 -7
View File
@@ -1,11 +1,8 @@
#include <string.h> #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#ifdef _WIN32
#include <ws2tcpip.h>
#else
#include <arpa/inet.h> // inet_ntop
#endif #endif
#include <stdint.h>
#include <quakey.h>
#include "message.h" #include "message.h"
bool binary_read(BinaryReader *reader, void *dst, int len) bool binary_read(BinaryReader *reader, void *dst, int len)
src/message.h
+5 -3
View File
@@ -1,9 +1,11 @@
#ifndef MESSAGE_INCLUDED #ifndef MESSAGE_INCLUDED
#define MESSAGE_INCLUDED #define MESSAGE_INCLUDED
#include <stdio.h> #ifdef MAIN_SIMULATION
#include <stdbool.h> #define QUAKEY_ENABLE_MOCKS
#endif
#include <stdint.h>
#include <quakey.h>
#include "basic.h" #include "basic.h"
#include "byte_queue.h" #include "byte_queue.h"
src/metadata_server.c
+46 -18
View File
@@ -1,8 +1,10 @@
#define _GNU_SOURCE #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <string.h> #include <quakey.h>
#include <stdint.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h>
#include "message.h" #include "message.h"
#include "metadata_server.h" #include "metadata_server.h"
@@ -1061,19 +1063,27 @@ static bool is_chunk_server_message_type(uint16_t type)
return false; return false;
} }
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout) int metadata_server_init(void *state_, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{ {
MetadataServer *state = state_;
string addr = getargs(argc, argv, "--addr", "127.0.0.1"); string addr = getargs(argc, argv, "--addr", "127.0.0.1");
int port = getargi(argc, argv, "--port", 8080); int port = getargi(argc, argv, "--port", 8080);
bool trace = getargb(argc, argv, "--trace"); bool trace = getargb(argc, argv, "--trace");
string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal"); string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal");
int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit
if (port <= 0 || port >= 1<<16) if (port <= 0 || port >= 1<<16) {
fprintf(stderr, "metadata server :: Invalid port\n");
return -1; return -1;
}
if (wal_limit < 0) if (wal_limit < 0) {
fprintf(stderr, "metadata server :: Invalid WAL limit\n");
return -1; return -1;
}
state->trace = trace; state->trace = trace;
state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro? state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro?
@@ -1084,22 +1094,27 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
for (int i = 0; i < MAX_CHUNK_SERVERS; i++) for (int i = 0; i < MAX_CHUNK_SERVERS; i++)
state->chunk_servers[i].used = false; state->chunk_servers[i].used = false;
if (tcp_context_init(&state->tcp) < 0) if (tcp_context_init(&state->tcp) < 0) {
fprintf(stderr, "metadata server :: Couldn't setup TCP context\n");
return -1; return -1;
}
int ret = tcp_listen(&state->tcp, addr, port); int ret = tcp_listen(&state->tcp, addr, port);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "metadata server :: Couldn't setup TCP listener\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
ret = file_tree_init(&state->file_tree); ret = file_tree_init(&state->file_tree);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "metadata server :: Couldn't setup file tree\n");
tcp_context_free(&state->tcp); tcp_context_free(&state->tcp);
return -1; return -1;
} }
if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) { if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) {
fprintf(stderr, "metadata server :: Couldn't setup WAL\n");
assert(0); // TODO assert(0); // TODO
} }
@@ -1110,21 +1125,21 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
); );
*timeout = -1; // No timeout until we have chunk servers *timeout = -1; // No timeout until we have chunk servers
return tcp_register_events(&state->tcp, contexts, polled); if (pcap < TCP_POLL_CAPACITY) {
} fprintf(stderr, "metadata server :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
return -1;
int metadata_server_free(MetadataServer *state) }
{ *pnum = tcp_register_events(&state->tcp, ctxs, pdata);
wal_close(&state->wal);
file_tree_free(&state->file_tree);
tcp_context_free(&state->tcp);
return 0; return 0;
} }
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout) int metadata_server_tick(void *state_, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
{ {
MetadataServer *state = state_;
Event events[TCP_EVENT_CAPACITY]; Event events[TCP_EVENT_CAPACITY];
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled); int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
Time current_time = get_current_time(); Time current_time = get_current_time();
if (current_time == INVALID_TIME) if (current_time == INVALID_TIME)
@@ -1243,5 +1258,18 @@ int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *
} }
*timeout = deadline_to_timeout(next_wakeup, current_time); *timeout = deadline_to_timeout(next_wakeup, current_time);
return tcp_register_events(&state->tcp, contexts, polled); if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
return 0;
}
int metadata_server_free(void *state_)
{
MetadataServer *state = state_;
wal_close(&state->wal);
file_tree_free(&state->file_tree);
tcp_context_free(&state->tcp);
return 0;
} }
src/metadata_server.h
+11 -4
View File
@@ -52,8 +52,15 @@ typedef struct {
} MetadataServer; } MetadataServer;
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout); struct pollfd;
int metadata_server_free(MetadataServer *state);
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
#endif // METADATA_SERVER_INCLUDED int metadata_server_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int metadata_server_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int metadata_server_free(void *state);
#endif // METADATA_SERVER_INCLUDED
src/simulation_client.c → src/random_client.c
+42 -16
View File
@@ -1,12 +1,11 @@
#ifdef BUILD_TEST #ifdef MAIN_SIMULATION
#include <stdio.h> #define QUAKEY_ENABLE_MOCKS
#include <stdlib.h> #include <quakey.h>
#include <string.h>
#include <assert.h> #include <assert.h>
#include "simulation_client.h"
#include "tcp.h" #include "tcp.h"
#include "random_client.h"
// Helper function to parse address and port from command line // Helper function to parse address and port from command line
static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port) static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port)
@@ -19,7 +18,18 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) { if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) {
*addr = argv[i + 1]; *addr = argv[i + 1];
if (i + 2 < argc) { if (i + 2 < argc) {
*port = (uint16_t)atoi(argv[i + 2]);
errno = 0;
char *end;
long val = strtol(argv[i+2], &end, 10);
if (end == argv[i+2] || *end != '\0' || errno == ERANGE)
break;
if (val < 0 || val > UINT16_MAX)
break;
*port = (uint16_t) val;
return true; return true;
} }
} }
@@ -27,9 +37,12 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
return true; return true;
} }
int simulation_client_init(SimulationClient *client, int argc, char **argv, int random_client_init(void *state_, int argc, char **argv,
void **contexts, struct pollfd *polled, int *timeout) void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{ {
RandomClient *client = state_;
char *addr; char *addr;
uint16_t port; uint16_t port;
parse_server_addr(argc, argv, &addr, &port); parse_server_addr(argc, argv, &addr, &port);
@@ -43,20 +56,26 @@ int simulation_client_init(SimulationClient *client, int argc, char **argv,
printf("Client set up (remote=%s:%d)\n", addr, port); printf("Client set up (remote=%s:%d)\n", addr, port);
*timeout = 0; *timeout = 0;
return toasty_process_events(client->toasty, contexts, polled, 0); if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
return 0;
} }
static int random_in_range(int min, int max) static int random_in_range(int min, int max)
{ {
uint64_t n = simulation_random_number(); uint64_t n = quakey_random();
return min + n % (max - min + 1); return min + n % (max - min + 1);
} }
int simulation_client_step(SimulationClient *client, void **contexts, int random_client_tick(void *state_, void **ctxs,
struct pollfd *polled, int num_polled, int *timeout) struct pollfd *pdata, int pcap, int *pnum,
int *timeout)
{ {
RandomClient *client = state_;
// Process any pending events from the network and get new poll descriptors // Process any pending events from the network and get new poll descriptors
num_polled = toasty_process_events(client->toasty, contexts, polled, num_polled); *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
for (int i = 0; i < client->num_pending; i++) { for (int i = 0; i < client->num_pending; i++) {
@@ -241,12 +260,19 @@ int simulation_client_step(SimulationClient *client, void **contexts,
*timeout = 10; *timeout = 10;
else else
*timeout = -1; *timeout = -1;
return toasty_process_events(client->toasty, contexts, polled, 0);
if (pcap < TCP_POLL_CAPACITY)
return -1;
*pnum = toasty_process_events(client->toasty, ctxs, pdata, 0);
return 0;
} }
void simulation_client_free(SimulationClient *client) int random_client_free(void *state_)
{ {
RandomClient *client = state_;
toasty_disconnect(client->toasty); toasty_disconnect(client->toasty);
return 0;
} }
#endif // BUILD_TEST #endif // MAIN_SIMULATION
src/random_client.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef RANDOM_CLIENT_INCLUDED
#define RANDOM_CLIENT_INCLUDED
#include "ToastyFS.h"
#define MAX_PENDING_OPERATION 8
typedef enum {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
} PendingOperationType;
typedef struct {
PendingOperationType type;
ToastyHandle handle;
void *ptr;
} PendingOperation;
typedef struct {
ToastyFS *toasty;
int num_pending;
PendingOperation pending[MAX_PENDING_OPERATION];
} RandomClient;
struct pollfd;
int random_client_init(void *state, int argc, char **argv,
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
int *timeout);
int random_client_tick(void *state, void **ctxs,
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
int random_client_free(void *state);
#endif // RANDOM_CLIENT_INCLUDED
src/simulation_client.h
-43
View File
@@ -1,43 +0,0 @@
#ifndef SIMULATION_CLIENT_INCLUDED
#define SIMULATION_CLIENT_INCLUDED
#include <stdint.h>
#include <stdbool.h>
#ifdef _WIN32
#include <winsock2.h>
#else
#include <poll.h>
#endif
#include "ToastyFS.h"
#define MAX_PENDING_OPERATION 8
typedef enum {
PENDING_OPERATION_CREATE,
PENDING_OPERATION_DELETE,
PENDING_OPERATION_LIST,
PENDING_OPERATION_READ,
PENDING_OPERATION_WRITE,
} PendingOperationType;
typedef struct {
PendingOperationType type;
ToastyHandle handle;
void *ptr;
} PendingOperation;
typedef struct {
ToastyFS *toasty;
int num_pending;
PendingOperation pending[MAX_PENDING_OPERATION];
} SimulationClient;
int simulation_client_init(SimulationClient *client, int argc, char **argv,
void **contexts, struct pollfd *polled, int *timeout);
int simulation_client_step(SimulationClient *client, void **contexts,
struct pollfd *polled, int num_polled, int *timeout);
void simulation_client_free(SimulationClient *client);
#endif // SIMULATION_CLIENT_INCLUDED
src/system.c
-2060
View File
File diff suppressed because it is too large Load Diff
src/system.h
-209
View File
@@ -1,209 +0,0 @@
#ifndef SYSTEM_INCLUDED
#define SYSTEM_INCLUDED
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <direct.h> // _mkdir
#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <poll.h>
#include <time.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/socket.h>
#include <sys/random.h>
#include <arpa/inet.h>
#define SOCKET int
#define INVALID_SOCKET ((SOCKET) -1)
#endif
#ifdef BUILD_TEST
void startup_simulation(uint64_t seed_);
int spawn_simulated_process(char *args);
void update_simulation(void);
void cleanup_simulation(void);
uint64_t simulation_random_number(void);
void* mock_malloc(size_t len);
void* mock_realloc(void *ptr, size_t len);
void mock_free(void *ptr);
int mock_remove(char *path);
int mock_rename(char *oldpath, char *newpath);
SOCKET mock_socket(int domain, int type, int protocol);
int mock_bind(SOCKET fd, void *addr, size_t addr_len);
int mock_listen(SOCKET fd, int backlog);
SOCKET mock_accept(SOCKET fd, void *addr, socklen_t *addr_len);
int mock_getsockopt(SOCKET fd, int level, int optname, void *optval, socklen_t *optlen);
int mock_recv(SOCKET fd, void *dst, int len, int flags);
int mock_send(SOCKET fd, void *src, int len, int flags);
int mock_connect(SOCKET fd, void *addr, size_t addr_len);
#ifdef _WIN32
int mock_closesocket(SOCKET fd);
int mock_ioctlsocket(SOCKET fd, long cmd, u_long *argp);
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
BOOL mock_CloseHandle(HANDLE handle);
BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
BOOL mock_FlushFileBuffers(HANDLE handle);
BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
char* mock__fullpath(char *path, char *dst, int cap);
int mock__mkdir(char *path);
HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
BOOL mock_FindClose(HANDLE hFindFile);
BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
#else
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
int mock_open(char *path, int flags, int mode);
int mock_close(int fd);
int mock_flock(int fd, int op);
int mock_fsync(int fd);
int mock_read(int fd, char *dst, int len);
int mock_write(int fd, char *src, int len);
off_t mock_lseek(int fd, off_t offset, int whence);
int mock_fstat(int fd, struct stat *buf);
int mock_mkstemp(char *path);
char* mock_realpath(char *path, char *dst);
int mock_mkdir(char *path, mode_t mode);
int mock_fcntl(int fd, int cmd, ...);
DIR* mock_opendir(char *name);
struct dirent* mock_readdir(DIR *dirp);
int mock_closedir(DIR *dirp);
#endif
// Common
#define sys_malloc mock_malloc
#define sys_realloc mock_realloc
#define sys_free mock_free
#define sys_remove mock_remove
#define sys_rename mock_rename
#define sys_socket mock_socket
#define sys_bind mock_bind
#define sys_listen mock_listen
#define sys_accept mock_accept
#define sys_getsockopt mock_getsockopt
#define sys_recv mock_recv
#define sys_send mock_send
#define sys_connect mock_connect
// Windows
#define sys__mkdir mock__mkdir
#define sys_closesocket mock_closesocket
#define sys_ioctlsocket mock_ioctlsocket
#define sys_CreateFileW mock_CreateFileW
#define sys_CloseHandle mock_CloseHandle
#define sys_LockFile mock_LockFile
#define sys_UnlockFile mock_UnlockFile
#define sys_FlushFileBuffers mock_FlushFileBuffers
#define sys_ReadFile mock_ReadFile
#define sys_WriteFile mock_WriteFile
#define sys_SetFilePointer mock_SetFilePointer
#define sys_GetFileSizeEx mock_GetFileSizeEx
#define sys__fullpath mock__fullpath
#define sys_QueryPerformanceCounter mock_QueryPerformanceCounter
#define sys_QueryPerformanceFrequency mock_QueryPerformanceFrequency
#define sys_FindFirstFileA mock_FindFirstFileA
#define sys_FindNextFileA mock_FindNextFileA
#define sys_FindClose mock_FindClose
#define sys_MoveFileExW mock_MoveFileExW
// Linux
#define sys_mkdir mock_mkdir
#define sys_open mock_open
#define sys_close mock_close
#define sys_flock mock_flock
#define sys_fsync mock_fsync
#define sys_read mock_read
#define sys_write mock_write
#define sys_lseek mock_lseek
#define sys_fstat mock_fstat
#define sys_mkstemp mock_mkstemp
#define sys_realpath mock_realpath
#define sys_clock_gettime mock_clock_gettime
#define sys_fcntl mock_fcntl
#define sys_getrandom mock_getrandom
#define sys_opendir mock_opendir
#define sys_readdir mock_readdir
#define sys_closedir mock_closedir
#else
// Common
#define sys_malloc malloc
#define sys_realloc realloc
#define sys_free free
#define sys_remove remove
#define sys_rename rename
#define sys_socket socket
#define sys_bind bind
#define sys_listen listen
#define sys_accept accept
#define sys_getsockopt getsockopt
#define sys_recv recv
#define sys_send send
#define sys_connect connect
// Windows
#define sys__mkdir _mkdir
#define sys_closesocket closesocket
#define sys_ioctlsocket ioctlsocket
#define sys_CreateFileW CreateFileW
#define sys_CloseHandle CloseHandle
#define sys_LockFile LockFile
#define sys_UnlockFile UnlockFile
#define sys_FlushFileBuffers FlushFileBuffers
#define sys_ReadFile ReadFile
#define sys_WriteFile WriteFile
#define sys_SetFilePointer SetFilePointer
#define sys_GetFileSizeEx GetFileSizeEx
#define sys__fullpath _fullpath
#define sys_QueryPerformanceCounter QueryPerformanceCounter
#define sys_QueryPerformanceFrequency QueryPerformanceFrequency
#define sys_FindFirstFileA FindFirstFileA
#define sys_FindNextFileA FindNextFileA
#define sys_FindClose FindClose
#define sys_MoveFileExW MoveFileExW
// Linux
#define sys_mkdir mkdir
#define sys_open open
#define sys_close close
#define sys_flock flock
#define sys_fsync fsync
#define sys_read read
#define sys_write write
#define sys_lseek lseek
#define sys_fstat fstat
#define sys_mkstemp mkstemp
#define sys_realpath realpath
#define sys_clock_gettime clock_gettime
#define sys_fcntl fcntl
#define sys_opendir opendir
#define sys_readdir readdir
#define sys_closedir closedir
#endif
#endif // SYSTEM_INCLUDED
src/tcp.c
+19 -15
View File
@@ -1,8 +1,12 @@
#ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h> #include <assert.h>
#include <string.h>
#include "tcp.h" #include "tcp.h"
#include "system.h"
#include "message.h" #include "message.h"
bool addr_eql(Address a, Address b) bool addr_eql(Address a, Address b)
@@ -28,15 +32,15 @@ static int set_socket_blocking(SOCKET sock, bool value)
{ {
#ifdef _WIN32 #ifdef _WIN32
u_long mode = !value; u_long mode = !value;
if (sys_ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR) if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
return -1; return -1;
#else #else
int flags = sys_fcntl(sock, F_GETFL, 0); int flags = fcntl(sock, F_GETFL, 0);
if (flags < 0) if (flags < 0)
return -1; return -1;
if (value) flags &= ~O_NONBLOCK; if (value) flags &= ~O_NONBLOCK;
else flags |= O_NONBLOCK; else flags |= O_NONBLOCK;
if (sys_fcntl(sock, F_SETFL, flags) < 0) if (fcntl(sock, F_SETFL, flags) < 0)
return -1; return -1;
#endif #endif
@@ -45,7 +49,7 @@ static int set_socket_blocking(SOCKET sock, bool value)
static SOCKET create_listen_socket(string addr, uint16_t port) static SOCKET create_listen_socket(string addr, uint16_t port)
{ {
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0); SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET) if (fd == INVALID_SOCKET)
return INVALID_SOCKET; return INVALID_SOCKET;
@@ -72,13 +76,13 @@ static SOCKET create_listen_socket(string addr, uint16_t port)
return INVALID_SOCKET; return INVALID_SOCKET;
} }
if (sys_bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) { if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
CLOSE_SOCKET(fd); CLOSE_SOCKET(fd);
return INVALID_SOCKET; return INVALID_SOCKET;
} }
int backlog = 32; int backlog = 32;
if (sys_listen(fd, backlog) < 0) { if (listen(fd, backlog) < 0) {
CLOSE_SOCKET(fd); CLOSE_SOCKET(fd);
return INVALID_SOCKET; return INVALID_SOCKET;
} }
@@ -319,7 +323,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
assert(contexts[i] == NULL); assert(contexts[i] == NULL);
if (polled[i].revents & POLLIN) { if (polled[i].revents & POLLIN) {
SOCKET new_fd = sys_accept(tcp->listen_fd, NULL, NULL); SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
if (new_fd != INVALID_SOCKET) { if (new_fd != INVALID_SOCKET) {
if (set_socket_blocking(new_fd, false) < 0) if (set_socket_blocking(new_fd, false) < 0)
@@ -347,7 +351,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
int err = 0; int err = 0;
socklen_t len = sizeof(err); socklen_t len = sizeof(err);
if (sys_getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0) if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
defer_close = true; defer_close = true;
else { else {
conn->connecting = false; conn->connecting = false;
@@ -360,7 +364,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
if (polled[i].revents & POLLIN) { if (polled[i].revents & POLLIN) {
byte_queue_write_setmincap(&conn->input, 1<<9); byte_queue_write_setmincap(&conn->input, 1<<9);
ByteView buf = byte_queue_write_buf(&conn->input); ByteView buf = byte_queue_write_buf(&conn->input);
int num = sys_recv(conn->fd, (char*) buf.ptr, buf.len, 0); int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num == 0) if (num == 0)
defer_close = true; defer_close = true;
else if (num < 0) { else if (num < 0) {
@@ -388,7 +392,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
if (polled[i].revents & POLLOUT) { if (polled[i].revents & POLLOUT) {
ByteView buf = byte_queue_read_buf(&conn->output); ByteView buf = byte_queue_read_buf(&conn->output);
int num = sys_send(conn->fd, (char*) buf.ptr, buf.len, 0); int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
if (num < 0) { if (num < 0) {
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
defer_close = true; defer_close = true;
@@ -432,7 +436,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
return -1; return -1;
int conn_idx = tcp->num_conns; int conn_idx = tcp->num_conns;
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0); SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd == INVALID_SOCKET) if (fd == INVALID_SOCKET)
return -1; return -1;
@@ -447,13 +451,13 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
buf.sin_family = AF_INET; buf.sin_family = AF_INET;
buf.sin_port = htons(addr.port); buf.sin_port = htons(addr.port);
memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4)); memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf)); ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
} else { } else {
struct sockaddr_in6 buf; struct sockaddr_in6 buf;
buf.sin6_family = AF_INET6; buf.sin6_family = AF_INET6;
buf.sin6_port = htons(addr.port); buf.sin6_port = htons(addr.port);
memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6)); memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf)); ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
} }
bool connecting; bool connecting;
src/tcp.h
+12 -4
View File
@@ -1,15 +1,23 @@
#ifndef TCP_INCLUDED #ifndef TCP_INCLUDED
#define TCP_INCLUDED #define TCP_INCLUDED
#include <stdbool.h> #ifdef MAIN_SIMULATION
# define QUAKEY_ENABLE_MOCKS
# include <quakey.h>
#else
# ifdef _WIN32
# include <winsock2.h>
# endif
#endif
#include "system.h"
#include "byte_queue.h" #include "byte_queue.h"
#ifdef _WIN32 #ifdef _WIN32
#define CLOSE_SOCKET sys_closesocket #define CLOSE_SOCKET closesocket
#else #else
#define CLOSE_SOCKET sys_close #define SOCKET int
#define INVALID_SOCKET -1
#define CLOSE_SOCKET close
#endif #endif
#ifndef TCP_CONNECTION_LIMIT #ifndef TCP_CONNECTION_LIMIT
src/wal.c
+18 -16
View File
@@ -1,12 +1,14 @@
#include <stddef.h>
#include <limits.h> #ifdef MAIN_SIMULATION
#define QUAKEY_ENABLE_MOCKS
#endif
#include <quakey.h>
#include <assert.h> #include <assert.h>
#include <string.h>
#include "wal.h" #include "wal.h"
#include "file_system.h" #include "file_system.h"
#include "file_tree.h" #include "file_tree.h"
#include "system.h"
#define WAL_MAGIC 0xcafebebe #define WAL_MAGIC 0xcafebebe
#define WAL_VERSION 1 #define WAL_VERSION 1
@@ -190,7 +192,7 @@ static int swap_file(WAL *wal)
} }
// MOVEFILE_REPLACE_EXISTING allows atomic overwrite // MOVEFILE_REPLACE_EXISTING allows atomic overwrite
if (!sys_MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) { if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
file_unlock(temp_handle); file_unlock(temp_handle);
file_close(temp_handle); file_close(temp_handle);
remove_file_or_dir(temp_path); remove_file_or_dir(temp_path);
@@ -267,12 +269,12 @@ static int next_entry(Handle handle, WALEntry *entry)
return -1; return -1;
// Dynamically allocate path buffer // Dynamically allocate path buffer
char *path_buffer = sys_malloc(path_len); char *path_buffer = malloc(path_len);
if (!path_buffer) if (!path_buffer)
return -1; return -1;
if (read_exact(handle, path_buffer, path_len) <= 0) { if (read_exact(handle, path_buffer, path_len) <= 0) {
sys_free(path_buffer); free(path_buffer);
return -1; return -1;
} }
@@ -315,13 +317,13 @@ static int next_entry(Handle handle, WALEntry *entry)
goto cleanup_error; goto cleanup_error;
// Dynamically allocate hash buffers // Dynamically allocate hash buffers
SHA256 *hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256)); SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256));
if (!hashes_buffer) { if (!hashes_buffer) {
goto cleanup_error; goto cleanup_error;
} }
if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) { if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
sys_free(hashes_buffer); free(hashes_buffer);
goto cleanup_error; goto cleanup_error;
} }
@@ -337,9 +339,9 @@ static int next_entry(Handle handle, WALEntry *entry)
cleanup_error: cleanup_error:
if (entry->path.ptr) if (entry->path.ptr)
sys_free((char*) entry->path.ptr); free((char*) entry->path.ptr);
if (entry->hashes) if (entry->hashes)
sys_free(entry->hashes); free(entry->hashes);
return -1; return -1;
} }
@@ -351,7 +353,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
wal->file_path.ptr = NULL; wal->file_path.ptr = NULL;
// Copy file_path since the passed string may not have the same lifetime as WAL // Copy file_path since the passed string may not have the same lifetime as WAL
char *path_copy = sys_malloc(file_path.len); char *path_copy = malloc(file_path.len);
if (!path_copy) if (!path_copy)
return -1; return -1;
memcpy(path_copy, file_path.ptr, file_path.len); memcpy(path_copy, file_path.ptr, file_path.len);
@@ -482,9 +484,9 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
// Free dynamically allocated fields from next_entry // Free dynamically allocated fields from next_entry
if (entry.path.ptr) if (entry.path.ptr)
sys_free((char*) entry.path.ptr); free((char*) entry.path.ptr);
if (entry.hashes) if (entry.hashes)
sys_free(entry.hashes); free(entry.hashes);
wal->entry_count++; wal->entry_count++;
} }
@@ -499,7 +501,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
return 0; return 0;
error_cleanup_path: error_cleanup_path:
sys_free(path_copy); free(path_copy);
return -1; return -1;
} }
@@ -508,7 +510,7 @@ void wal_close(WAL *wal)
file_unlock(wal->handle); file_unlock(wal->handle);
file_close(wal->handle); file_close(wal->handle);
if (wal->file_path.ptr) if (wal->file_path.ptr)
sys_free((char*) wal->file_path.ptr); free((char*) wal->file_path.ptr);
} }
static int write_u8(Handle handle, uint8_t value) static int write_u8(Handle handle, uint8_t value)
web/3p/chttp.c
-5077
View File
File diff suppressed because it is too large Load Diff
web/3p/chttp.h
-1367
View File
File diff suppressed because it is too large Load Diff
web/DESIGN.txt
-64
View File
@@ -1,64 +0,0 @@
The web interface allows working with the file system cluster using
basic HTTP requests. There are many ways to interface with a file system
using HTTP (like WebDAV), but for the firsts versions of ToastyFS I
opted for some custom minimal semantics. Here are they.
GET
If the path refers to a directory, the listing of its direct children
is returned as a newline-separated list of relative paths. Children
that are directories have ending slashes, while files don't:
image.jpg
Documents/
notes.txt
If the path refers to a file, its contents are returned. Note that
Range requests are not supported.
Returns 200 on success, 404 if the file/directory doesn't exist, else 5xx.
HEAD
Works like GET but doesn't return the content (The Content-Length is
set as if the payload was being send).
PUT
The server considers the resource being uploaded as a directory if the
path ends with a slash, and as a file otherwise:
photos/ directory
image.jpg file
If the path refers to a directory and no file/directory exists at that
path, a new one is created and status 201 is returned. If a directory
existd already, statis 204 is returned. If a file existed already status
??? (TODO) is returned. If the payload the request is not empty, the
operation fails with code 400.
If the path refers to a file, the file is automatically created if it
doesn't exist, then the specified contents are written to it, replacing
the entire file content (HTTP PUT semantics). This is implemented using
two flags:
- TOASTY_WRITE_CREATE_IF_MISSING: Creates file if it doesn't exist
- TOASTY_WRITE_TRUNCATE_AFTER: Truncates file after the write, discarding
any data beyond the uploaded content
Files are created with a default chunk size of 4096 bytes.
If a file or directory already existed at that path, it is overwritten
unless the header "X-ToastyFS-Overwrite: no" is sent.
(TODO error codes)
DELETE
Deletes the given file or directory. Returns 204 on success, 404 if
no file or directory existed at that path, else returns 5xx.
-----------
cases:
PUT file / unused path -> OK
PUT file / file exists -> overwrite?
PUT file / directory exists
PUT directory / unused path -> OK
PUT directory / file exists
PUT directory / directory exists
Idea: Use patch on the parent directory to rename
-----------
web/src/config.c
-76
View File
@@ -1,76 +0,0 @@
#include "config.h"
void parse_config_or_exit(ProxyConfig *config,
int argc, char **argv)
{
config->upstream_addr = TOASTY_STR("127.0.0.1");
config->upstream_port = 9000;
config->local_addr = HTTP_STR("127.0.0.1");
config->local_port = 8080;
config->reuse_addr = false;
config->trace_bytes = false;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h")) {
printf("TODO: print help\n");
exit(0);
}
if (!strcmp(argv[i], "--upstream-addr")) {
i++;
if (i == argc) {
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
exit(-1);
}
config->upstream_addr = (ToastyString) { argv[i], strlen(argv[i]) };
} else if (!strcmp(argv[i], "--upstream-port")) {
i++;
if (i == argc) {
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
exit(-1);
}
int tmp = atoi(argv[i]);
if (tmp < 1 || tmp > UINT16_MAX) {
fprintf(stderr, "Error: Invalid port %s\n", argv[i]);
exit(-1);
}
config->upstream_port = (uint16_t) tmp;
} else if (!strcmp(argv[i], "--local-addr")) {
i++;
if (i == argc) {
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
exit(-1);
}
config->local_addr = (HTTP_String) { argv[i], strlen(argv[i]) };
} else if (!strcmp(argv[i], "--local-port")) {
i++;
if (i == argc) {
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
exit(-1);
}
int tmp = atoi(argv[i]);
if (tmp < 1 || tmp > UINT16_MAX) {
fprintf(stderr, "Error: Invalid port %s\n", argv[i]);
exit(-1);
}
config->local_port = (uint16_t) tmp;
} else if(!strcmp(argv[i], "--reuse-addr")) {
config->reuse_addr = true;
} else if(!strcmp(argv[i], "--trace-bytes")) {
config->trace_bytes = true;
} else {
fprintf(stderr, "Error: Invalid option %s\n", argv[i]);
exit(-1);
}
}
}
web/src/config.h
-21
View File
@@ -1,21 +0,0 @@
#ifndef CONFIG_INCLUDED
#define CONFIG_INCLUDED
#include <chttp.h>
#include <ToastyFS.h>
typedef struct {
ToastyString upstream_addr;
uint16_t upstream_port;
HTTP_String local_addr;
uint16_t local_port;
bool reuse_addr;
bool trace_bytes;
} ProxyConfig;
void parse_config_or_exit(ProxyConfig *config,
int argc, char **argv);
#endif // CONFIG_INCLUDED
web/src/event_loop.c
-84
View File
@@ -1,84 +0,0 @@
#include "event_loop.h"
#ifdef _WIN32
#define POLL WSAPoll
#else
#define POLL poll
#endif
#define POLL_CAPACITY (HTTP_SERVER_POLL_CAPACITY + TOASTY_POLL_CAPACITY)
void event_loop_init(EventLoop *loop, HTTP_Server *server,
ToastyFS *backend)
{
loop->state = EVENT_LOOP_PROCESSING_COMPLETIONS;
loop->server = server;
loop->backend = backend;
}
void event_loop_free(EventLoop *loop)
{
(void) loop;
}
int event_loop_wait(EventLoop *loop, Event *event)
{
for (;;) {
switch (loop->state) {
int ret;
case EVENT_LOOP_PROCESSING_COMPLETIONS:
ret = toasty_get_result(loop->backend, TOASTY_INVALID, &event->completion);
if (ret < 0)
return -1; // Error
if (ret == 0) {
// Completed
event->type = EVENT_TYPE_COMPLETION;
return 0;
}
// fallthrough
case EVENT_LOOP_PROCESSING_REQUESTS:
if (http_server_next_request(loop->server, &event->request, &event->builder)) {
// Completed
event->type = EVENT_TYPE_REQUEST;
return 0;
}
// fallthrough
}
EventRegister reg;
void *ptrs[POLL_CAPACITY];
struct pollfd polled[POLL_CAPACITY];
void **http_ptrs = ptrs;
struct pollfd *http_polled = polled;
reg = (EventRegister) { ptrs, polled, 0 };
http_server_register_events(loop->server, &reg);
int num_http_polled = reg.num_polled;
void **toasty_ptrs = ptrs + num_http_polled;
struct pollfd *toasty_polled = polled + num_http_polled;
int num_toasty_polled = toasty_process_events(loop->backend, toasty_ptrs, toasty_polled, 0);
if (num_toasty_polled < 0)
return -1;
int num_polled = num_http_polled + num_toasty_polled;
if (num_http_polled != 0 && num_toasty_polled != 0)
POLL(polled, num_polled, -1);
if (toasty_process_events(loop->backend, toasty_ptrs, toasty_polled, num_toasty_polled) < 0)
return -1;
reg = (EventRegister) { http_ptrs, http_polled, num_http_polled };
http_server_process_events(loop->server, reg);
}
// Unreachable
return -1;
}
web/src/event_loop.h
-42
View File
@@ -1,42 +0,0 @@
#ifndef EVENT_LOOP_INCLUDED
#define EVENT_LOOP_INCLUDED
#include <ToastyFS.h>
#include "chttp.h"
typedef enum {
EVENT_TYPE_REQUEST,
EVENT_TYPE_COMPLETION,
} EventType;
typedef struct {
EventType type;
// Completion
ToastyResult completion;
// Request
HTTP_Request *request;
HTTP_ResponseBuilder builder;
} Event;
typedef enum {
EVENT_LOOP_PROCESSING_REQUESTS,
EVENT_LOOP_PROCESSING_COMPLETIONS,
} EventLoopState;
typedef struct {
EventLoopState state;
HTTP_Server *server;
ToastyFS *backend;
} EventLoop;
void event_loop_init(EventLoop *loop, HTTP_Server *server,
ToastyFS *backend);
void event_loop_free(EventLoop *loop);
int event_loop_wait(EventLoop *loop, Event *event);
#endif // EVENT_LOOP_INCLUDED
web/src/main.c
-52
View File
@@ -1,52 +0,0 @@
#include "proxy.h"
#include "config.h"
#include "event_loop.h"
int main(int argc, char **argv)
{
ProxyConfig config;
parse_config_or_exit(&config, argc, argv);
ToastyFS *backend = toasty_connect(
config.upstream_addr,
config.upstream_port);
if (backend == NULL) {
printf("toasty_connect error\n");
return -1;
}
HTTP_Server server;
if (http_server_init(&server) < 0) {
printf("http_server_init error\n");
return -1;
}
http_server_set_reuse_addr(&server, config.reuse_addr);
http_server_set_trace_bytes(&server, config.trace_bytes);
if (http_server_listen_tcp(&server, config.local_addr, config.local_port) < 0) {
printf("http_server_listen_tcp error\n");
return -1;
}
EventLoop loop;
event_loop_init(&loop, &server, backend);
ProxyState proxy;
proxy_init(&proxy, backend);
for (Event event; !event_loop_wait(&loop, &event); ) {
if (event.type == EVENT_TYPE_REQUEST) {
proxy_process_request(&proxy, event.request, event.builder);
} else {
assert(event.type == EVENT_TYPE_COMPLETION);
proxy_process_completion(&proxy, event.completion);
}
}
proxy_free(&proxy);
event_loop_free(&loop);
http_server_free(&server);
toasty_disconnect(backend);
return 0;
}
web/src/proxy.c
-126
View File
@@ -1,126 +0,0 @@
#include "proxy.h"
#include "proxy_get.h"
#include "proxy_put.h"
#include "proxy_delete.h"
void proxy_init(ProxyState *state, ToastyFS *backend)
{
state->backend = backend;
state->num_operations = 0;
for (int i = 0; i < PROXY_CAPACITY; i++)
state->operations[i].type = PO_FREE;
}
void proxy_free(ProxyState *state)
{
for (int i = 0; i < PROXY_CAPACITY; i++) {
ProxyOperation *operation = &state->operations[i];
if (operation->type != PO_FREE) {
http_response_builder_status(operation->builder, 500);
http_response_builder_send(operation->builder);
}
}
}
static ProxyOperation *find_unused_operation(ProxyState *state)
{
if (state->num_operations == PROXY_CAPACITY)
return NULL;
ProxyOperation *operation = state->operations;
while (operation->type != PO_FREE)
operation++;
return operation;
}
void proxy_process_request(ProxyState *state,
HTTP_Request *request, HTTP_ResponseBuilder builder)
{
ProxyOperation *operation = find_unused_operation(state);
bool created = false;
switch (request->method) {
case HTTP_METHOD_GET:
if (operation == NULL) {
http_response_builder_status(builder, 503); // Service Unavailable
http_response_builder_send(builder);
} else {
created = process_request_get(state, operation, request, builder);
}
break;
case HTTP_METHOD_HEAD:
if (operation == NULL) {
http_response_builder_status(builder, 503); // Service Unavailable
http_response_builder_send(builder);
} else {
created = process_request_head(state, operation, request, builder);
}
break;
case HTTP_METHOD_PUT:
if (operation == NULL) {
http_response_builder_status(builder, 503); // Service Unavailable
http_response_builder_send(builder);
} else {
created = process_request_put(state, operation, request, builder);
}
break;
case HTTP_METHOD_DELETE:
if (operation == NULL) {
http_response_builder_status(builder, 503); // Service Unavailable
http_response_builder_send(builder);
} else {
created = process_request_delete(state, operation, request, builder);
}
break;
case HTTP_METHOD_OPTIONS:
http_response_builder_status(builder, 200); // OK
http_response_builder_header(builder,
HTTP_STR("Allow: GET, HEAD, PUT, DELETE, OPTIONS"));
http_response_builder_send(builder);
break;
default:
http_response_builder_status(builder, 405); // Method not allowed
http_response_builder_header(builder,
HTTP_STR("Allow: GET, HEAD, PUT, DELETE, OPTIONS"));
http_response_builder_send(builder);
break;
}
if (created) {
state->num_operations++;
}
}
void proxy_process_completion(ProxyState *state,
ToastyResult completion)
{
ProxyOperation *operation = completion.user;
assert(operation);
bool completed;
switch (operation->type) {
case PO_CREATE_DIR:
completed = process_completion_create_dir(state, operation, completion);
break;
case PO_CREATE_FILE:
completed = process_completion_create_file(state, operation, completion);
break;
case PO_DELETE:
completed = process_completion_delete(state, operation, completion);
break;
case PO_READ_DIR:
completed = process_completion_read_dir(state, operation, completion);
break;
case PO_READ_FILE:
completed = process_completion_read_file(state, operation, completion);
break;
case PO_WRITE:
completed = process_completion_write(state, operation, completion);
break;
default:
}
if (completed) {
operation->type = PO_FREE;
state->num_operations--;
}
}
web/src/proxy.h
-44
View File
@@ -1,44 +0,0 @@
#ifndef PROXY_INCLUDED
#define PROXY_INCLUDED
#include <chttp.h>
#include <ToastyFS.h>
#define PROXY_CAPACITY (1<<10)
typedef enum {
PO_FREE,
PO_CREATE_DIR,
PO_CREATE_FILE,
PO_DELETE,
PO_READ_DIR,
PO_READ_FILE,
PO_WRITE,
} ProxyOperationType;
typedef struct {
ProxyOperationType type;
HTTP_Request* request;
HTTP_ResponseBuilder builder;
ToastyHandle handle;
bool head_only;
int transferred;
} ProxyOperation;
typedef struct {
ToastyFS *backend;
int num_operations;
ProxyOperation operations[PROXY_CAPACITY];
} ProxyState;
void proxy_init(ProxyState *state, ToastyFS *backend);
void proxy_free(ProxyState *state);
void proxy_process_request(ProxyState *state,
HTTP_Request *request, HTTP_ResponseBuilder builder);
void proxy_process_completion(ProxyState *state,
ToastyResult completion);
#endif // PROXY_INCLUDED
web/src/proxy_delete.c
-37
View File
@@ -1,37 +0,0 @@
#include "proxy_delete.h"
bool process_request_delete(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder)
{
// TODO: Implement If-Match and If-None-Match headers
HTTP_String path = request->url.path;
ToastyHandle handle = toasty_begin_delete(state->backend,
(ToastyString) { path.ptr, path.len }, TOASTY_VERSION_TAG_EMPTY);
if (handle == TOASTY_INVALID) {
http_response_builder_status(builder, 500); // Internal Server Error
http_response_builder_send(builder);
return false;
}
toasty_set_user(state->backend, handle, operation);
operation->type = PO_DELETE;
operation->request = request;
operation->builder = builder;
operation->handle = handle;
return true;
}
bool process_completion_delete(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
if (completion.type == TOASTY_RESULT_DELETE_SUCCESS) {
http_response_builder_status(operation->builder, 204); // No Content
http_response_builder_send(operation->builder);
} else {
// TODO: Should differentiate between error conditions
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
}
return true;
}
web/src/proxy_delete.h
-12
View File
@@ -1,12 +0,0 @@
#ifndef PROXY_DELETE_INCLUDED
#define PROXY_DELETE_INCLUDED
#include "proxy.h"
bool process_request_delete(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder);
bool process_completion_delete(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
#endif // PROXY_DELETE_INCLUDED
web/src/proxy_get.c
-143
View File
@@ -1,143 +0,0 @@
#include "proxy_get.h"
static bool path_refers_to_dir(ToastyString path)
{
return path.len > 0 && path.ptr[path.len-1] == '/';
}
bool process_request_get(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder)
{
assert(request->method == HTTP_METHOD_GET
|| request->method == HTTP_METHOD_HEAD);
http_response_builder_status(builder, 200); // OK
ToastyString path = {
request->url.path.ptr,
request->url.path.len
};
ToastyHandle handle;
if (path_refers_to_dir(path)) {
handle = toasty_begin_list(state->backend, path, TOASTY_VERSION_TAG_EMPTY);
} else {
http_response_builder_body_cap(builder, 1<<10); // TODO: pick the prealloc better
int cap;
char *dst = http_response_builder_body_buf(builder, &cap);
// TODO: check for NULL dst
handle = toasty_begin_read(state->backend, path, 0, dst, cap, TOASTY_VERSION_TAG_EMPTY);
}
if (handle == TOASTY_INVALID) {
if (!path_refers_to_dir(path))
http_response_builder_body_ack(builder, 0);
http_response_builder_status(builder, 500); // Internal Server Error
http_response_builder_send(builder);
return false;
}
toasty_set_user(state->backend, handle, operation);
operation->type = path_refers_to_dir(path) ? PO_READ_DIR : PO_READ_FILE;
operation->request = request;
operation->builder = builder;
operation->handle = handle;
operation->head_only = (request->method == HTTP_METHOD_HEAD);
operation->transferred = 0;
return true;
}
bool process_request_head(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder)
{
return process_request_get(state, operation, request, builder);
}
bool process_completion_read_dir(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
if (completion.type == TOASTY_RESULT_LIST_SUCCESS) {
http_response_builder_status(operation->builder, 200);
for (int i = 0; i < completion.listing.count; i++) {
ToastyListingEntry *entry = &completion.listing.items[i];
char vtag[32];
int ret = snprintf(vtag, sizeof(vtag), "%lu", entry->vtag);
if (ret < 0 || ret >= (int) sizeof(vtag)) {
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
return true;
}
http_response_builder_body(operation->builder, (HTTP_String) { entry->name, strlen(entry->name) });
if (entry->is_dir) {
http_response_builder_body(operation->builder, HTTP_STR("/ "));
} else {
http_response_builder_body(operation->builder, HTTP_STR(" "));
}
http_response_builder_body(operation->builder, (HTTP_String) { vtag, ret });
http_response_builder_body(operation->builder, HTTP_STR("\n"));
}
http_response_builder_send(operation->builder);
toasty_free_listing(&completion.listing);
} else {
// TODO: Should differentiate between error conditions
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
}
return true;
}
bool process_completion_read_file(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
bool again = false;
if (completion.type != TOASTY_RESULT_READ_SUCCESS) {
http_response_builder_body_ack(operation->builder, 0);
http_response_builder_status(operation->builder, 500);
http_response_builder_send(operation->builder);
} else {
// First, ACK the byte we just read, even if it's
// just 0 bytes (every bodybuf must by paired with
// a bodyack).
operation->transferred += completion.bytes_read;
int ack = operation->head_only ? 0 : completion.bytes_read;
http_response_builder_body_ack(operation->builder, ack);
// If we didn't reach the end of the file, start
// a new read.
if (completion.bytes_read > 0) {
// Make sure there is some free space in the buffer
int mincap = 1<<10; // TODO: Choose based on overall file size
http_response_builder_body_cap(operation->builder, mincap);
// Get the location of that buffer
int cap;
char *dst = http_response_builder_body_buf(operation->builder, &cap);
if (dst == NULL) {
assert(0); // TODO
}
ToastyString path = {
operation->request->url.path.ptr,
operation->request->url.path.len,
};
operation->handle = toasty_begin_read(state->backend, path,
operation->transferred, dst, cap, TOASTY_VERSION_TAG_EMPTY);
if (operation->handle == TOASTY_INVALID) {
assert(0); // TODO
}
toasty_set_user(state->backend, operation->handle, operation);
again = true;
} else {
http_response_builder_send(operation->builder);
}
}
return !again;
}
web/src/proxy_get.h
-18
View File
@@ -1,18 +0,0 @@
#ifndef PROXY_GET_INCLUDED
#define PROXY_GET_INCLUDED
#include "proxy.h"
bool process_request_get(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder);
bool process_request_head(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder);
bool process_completion_read_dir(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
bool process_completion_read_file(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
#endif // PROXY_GET_INCLUDED
web/src/proxy_put.c
-70
View File
@@ -1,70 +0,0 @@
#include "proxy_put.h"
bool process_request_put(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder)
{
ToastyString path = {
request->url.path.ptr,
request->url.path.len,
};
// Use both flags for proper HTTP PUT semantics:
// - TOASTY_WRITE_CREATE_IF_MISSING: Create file if it doesn't exist
// - TOASTY_WRITE_TRUNCATE_AFTER: Replace entire file content (truncate after write)
ToastyHandle handle = toasty_begin_write(state->backend, path, 0,
request->body.ptr, request->body.len, TOASTY_VERSION_TAG_EMPTY,
TOASTY_WRITE_CREATE_IF_MISSING | TOASTY_WRITE_TRUNCATE_AFTER);
if (handle == TOASTY_INVALID) {
http_response_builder_status(builder, 500); // Internal Server Error
http_response_builder_send(builder);
return false;
}
toasty_set_user(state->backend, handle, operation);
operation->type = PO_WRITE;
operation->request = request;
operation->builder = builder;
operation->handle = handle;
return true;
}
bool process_completion_create_dir(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
if (completion.type == TOASTY_RESULT_CREATE_SUCCESS) {
http_response_builder_status(operation->builder, 201); // Created
http_response_builder_send(operation->builder);
} else {
// TODO: Should differentiate between error conditions
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
}
return true;
}
bool process_completion_create_file(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
if (completion.type == TOASTY_RESULT_CREATE_SUCCESS) {
http_response_builder_status(operation->builder, 201); // Created
http_response_builder_send(operation->builder);
} else {
// TODO: Should differentiate between error conditions
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
}
return true;
}
bool process_completion_write(ProxyState *state,
ProxyOperation *operation, ToastyResult completion)
{
if (completion.type == TOASTY_RESULT_WRITE_SUCCESS) {
http_response_builder_status(operation->builder, 201); // Created
http_response_builder_send(operation->builder);
} else {
// TODO: Should differentiate between error conditions
http_response_builder_status(operation->builder, 500); // Internal Server Error
http_response_builder_send(operation->builder);
}
return true;
}
web/src/proxy_put.h
-18
View File
@@ -1,18 +0,0 @@
#ifndef PROXY_PUT_INCLUDED
#define PROXY_PUT_INCLUDED
#include "proxy.h"
bool process_request_put(ProxyState *state, ProxyOperation *operation,
HTTP_Request *request, HTTP_ResponseBuilder builder);
bool process_completion_create_dir(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
bool process_completion_create_file(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
bool process_completion_write(ProxyState *state,
ProxyOperation *operation, ToastyResult completion);
#endif // PROXY_PUT_INCLUDED