Add Quakey
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
# Ensure shell scripts always use LF line endings
|
||||
*.sh text eol=lf
|
||||
+3
-19
@@ -1,19 +1,3 @@
|
||||
*.exe
|
||||
*.out
|
||||
*.o
|
||||
*.a
|
||||
chunk_server_data_*
|
||||
crash_*.txt
|
||||
|
||||
# Coverage reports
|
||||
coverage_report/
|
||||
*.gcov
|
||||
*.gcda
|
||||
*.gcno
|
||||
*.wal
|
||||
*.tmp
|
||||
|
||||
# Cluster demo artifacts
|
||||
.cluster_demo.pids
|
||||
cluster_logs/
|
||||
cluster_data/
|
||||
toasty_metadata_server
|
||||
toasty_chunk_server
|
||||
toasty_simulation
|
||||
@@ -1,67 +0,0 @@
|
||||
|
||||
CFLAGS = -Wall -Wextra -ggdb
|
||||
COVERAGE_CFLAGS = $(CFLAGS) --coverage
|
||||
COVERAGE_LFLAGS = --coverage
|
||||
|
||||
ifeq ($(OS),Windows_NT)
|
||||
LFLAGS = -lws2_32
|
||||
EXT = .exe
|
||||
else
|
||||
LFLAGS =
|
||||
EXT = .out
|
||||
endif
|
||||
|
||||
CFILES = $(shell find src -name '*.c')
|
||||
HFILES = $(shell find src -name '*.h')
|
||||
OFILES = $(CFILES:.c=.o)
|
||||
|
||||
WEB_CFILES = $(shell find web/src web/3p -name '*.c')
|
||||
WEB_HFILES = $(shell find web/src web/3p -name '*.h')
|
||||
|
||||
.PHONY: all clean coverage coverage-report coverage-html
|
||||
|
||||
all: toastyfs$(EXT) toastyfs_web$(EXT) toastyfs_random_test$(EXT) example_async_api$(EXT) example_blocking_api$(EXT) libtoastyfs.a
|
||||
|
||||
coverage: toastyfs_random_test_coverage$(EXT)
|
||||
|
||||
coverage-report:
|
||||
@./scripts/measure_coverage.sh 60
|
||||
|
||||
coverage-html:
|
||||
@./scripts/measure_coverage.sh 60 --html
|
||||
|
||||
toastyfs$(EXT): $(CFILES) $(HFILES)
|
||||
gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_SERVER
|
||||
|
||||
toastyfs_random_test$(EXT): $(CFILES) $(HFILES)
|
||||
gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_TEST
|
||||
|
||||
toastyfs_random_test_coverage$(EXT): $(CFILES) $(HFILES)
|
||||
gcc -o $@ $(CFILES) $(COVERAGE_CFLAGS) $(LFLAGS) $(COVERAGE_LFLAGS) -Iinclude -DBUILD_TEST
|
||||
|
||||
example_async_api$(EXT): libtoastyfs.a examples/async_api.c
|
||||
gcc -o $@ examples/async_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L.
|
||||
|
||||
example_blocking_api$(EXT): libtoastyfs.a examples/blocking_api.c
|
||||
gcc -o $@ examples/blocking_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L.
|
||||
|
||||
toastyfs_web$(EXT): libtoastyfs.a $(WEB_CFILES) $(WEB_HFILES)
|
||||
gcc -o $@ $(WEB_CFILES) $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -Iweb/3p -L.
|
||||
|
||||
%.o: %.c $(HFILES)
|
||||
gcc -c -o $@ $< $(CFLAGS) -Iinclude
|
||||
|
||||
libtoastyfs.a: $(OFILES)
|
||||
ar rcs $@ $^
|
||||
|
||||
clean:
|
||||
rm -f \
|
||||
*.exe \
|
||||
*.out \
|
||||
*.a \
|
||||
*.gcov \
|
||||
src/*.o \
|
||||
src/*.gcda \
|
||||
src/*.gcno \
|
||||
*.gcda \
|
||||
*.gcno
|
||||
@@ -1,24 +0,0 @@
|
||||
# ToastyFS
|
||||
ToastyFS is a distributed file system designed for self-hosting, so it aims to be pragmatic, understandable, and robust. You can use ToastyFS to store your files reliably over multiple machines knowing they will be automatically replicated and healed in case of failures.
|
||||
|
||||
To try it out, run the script `./scripts/cluster_demo.sh`. It will spawn a local cluster with a web interface at `https://127.0.0.1:8090/` which will allow you to PUT, GET, DELETE files.
|
||||
|
||||
⚠️ Note that ToastyFS is still in early development ⚠️
|
||||
|
||||
🎵 Now let's get toasty 🎵
|
||||
|
||||
## Features
|
||||
|
||||
- Cross-platform (runs on Windows and Linux)
|
||||
- Automatic Replication & Self-Healing
|
||||
- Automatic content deduplication via internal content-addressing
|
||||
- Configurable file chunk sizes
|
||||
- Small and understandable
|
||||
|
||||
But ToastyFS is still in early development, so here are the missing features:
|
||||
|
||||
- No master replication
|
||||
- No authentication or encryption
|
||||
|
||||
## Testing
|
||||
ToastyFS is tested by running an in-memory simulation of a cluster with many clients running hundreds of random operations in parallel. The test is run for long periods of times under valgrind or compiled with sanitizers.
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
|
||||
|
||||
async def fetch(session, url):
|
||||
async with session.get(url) as response:
|
||||
if response.status != 200:
|
||||
raise "Didn't work" # TODO
|
||||
|
||||
data = await response.text()
|
||||
|
||||
files = []
|
||||
lines = data.splitlines()
|
||||
for line in lines:
|
||||
name, vtag = line.split(" ")
|
||||
|
||||
# Process name
|
||||
is_dir = False
|
||||
if len(name) > 0 and name[-1] == "/":
|
||||
name = name[:-1]
|
||||
is_dir = True
|
||||
|
||||
# Process vtag
|
||||
vtag = int(vtag)
|
||||
|
||||
files.append({"name": name, "is_dir": is_dir, "vtag": vtag})
|
||||
|
||||
return files
|
||||
|
||||
|
||||
async def main():
|
||||
async with aiohttp.ClientSession() as session:
|
||||
remote = "http://127.0.0.1:8090" # Must not end with a "/"
|
||||
remote_dir = remote + "/"
|
||||
local_dir = "local_root"
|
||||
|
||||
local_dir = Path(local_dir)
|
||||
|
||||
while True:
|
||||
# List of files in the remote root directory
|
||||
remote_files = await fetch(session, remote_dir)
|
||||
|
||||
# List of files in the local root directory
|
||||
local_files = list(local_dir.iterdir())
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@@ -0,0 +1,3 @@
|
||||
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c quakey/src/lfs.c quakey/src/lfs_util.c quakey/src/quakey.c -o toasty_simulation -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION
|
||||
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_metadata_server -DMAIN_METADATA_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
|
||||
gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_chunk_server -DMAIN_CHUNK_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include
|
||||
@@ -1,60 +0,0 @@
|
||||
#include <stdio.h>
|
||||
#include <ToastyFS.h>
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ToastyString remote_addr = TOASTY_STR("127.0.0.1");
|
||||
uint16_t remote_port = 8080;
|
||||
|
||||
ToastyFS *toasty = toasty_connect(remote_addr, remote_port);
|
||||
if (toasty == NULL) {
|
||||
printf("Couldn't connect to metadata server");
|
||||
return -1;
|
||||
}
|
||||
|
||||
ToastyString path_1 = TOASTY_STR("/first_file");
|
||||
ToastyString path_2 = TOASTY_STR("/second_file");
|
||||
|
||||
// Begin creation operation. This does not block.
|
||||
ToastyHandle create_handle_1 = toasty_begin_create_file(toasty, path_1, 1024);
|
||||
if (create_handle_1 == TOASTY_INVALID) {
|
||||
printf("Couldn't create file 1");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// This doesn't block either.
|
||||
ToastyHandle create_handle_2 = toasty_begin_create_file(toasty, path_2, 1024);
|
||||
if (create_handle_2 == TOASTY_INVALID) {
|
||||
printf("Couldn't create file 2");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Now block execution by overlapping the waiting
|
||||
// times for both file creations.
|
||||
|
||||
ToastyResult result;
|
||||
int ret = toasty_wait_result(toasty, create_handle_1, &result, -1);
|
||||
if (ret < 0) {
|
||||
printf("Couldn't wait for completion\n");
|
||||
return -1;
|
||||
}
|
||||
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
|
||||
printf("Couldn't create file 1\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
ret = toasty_wait_result(toasty, create_handle_2, &result, -1);
|
||||
if (ret < 0) {
|
||||
printf("Couldn't wait for completion\n");
|
||||
return -1;
|
||||
}
|
||||
if (result.type != TOASTY_RESULT_CREATE_SUCCESS) {
|
||||
printf("Couldn't create file 2\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
printf("All files were created!\n");
|
||||
|
||||
toasty_disconnect(toasty);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
#include <stdio.h>
|
||||
#include <ToastyFS.h>
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ToastyString remote_addr = TOASTY_STR("127.0.0.1");
|
||||
uint16_t remote_port = 8080;
|
||||
|
||||
ToastyFS *toasty = toasty_connect(remote_addr, remote_port);
|
||||
if (toasty == NULL) {
|
||||
printf("Couldn't connect to metadata server");
|
||||
return -1;
|
||||
}
|
||||
|
||||
ToastyString path = TOASTY_STR("/first_file");
|
||||
|
||||
int ret = toasty_create_file(toasty, path, 1024, NULL);
|
||||
if (ret < 0) {
|
||||
printf("Couldn't create file\n");
|
||||
toasty_disconnect(toasty);
|
||||
return -1;
|
||||
}
|
||||
|
||||
char data[] = "Hello, world!";
|
||||
ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, NULL, 0);
|
||||
if (ret < 0) {
|
||||
printf("Couldn't write to file\n");
|
||||
toasty_disconnect(toasty);
|
||||
return -1;
|
||||
}
|
||||
|
||||
toasty_disconnect(toasty);
|
||||
return 0;
|
||||
}
|
||||
-233
@@ -1,233 +0,0 @@
|
||||
Architecture
|
||||
A ToastyFS instance is composed by a metadata server, a number
|
||||
of chunk servers, and a number of clients.
|
||||
|
||||
The metadata server stores the full file system hieararchy,
|
||||
except instead of storing the file contents, it stores an
|
||||
array of hashes of the chunks of each file. A "chunk" is a
|
||||
file range that is fixed for a single file but may vary
|
||||
between files. Chunk servers hold an array of chunks that
|
||||
are identified by their hash. The metadata server keeps
|
||||
track of which chunks each chunk server is holding.
|
||||
|
||||
Clients are users of the file system that can read and
|
||||
write metadata and files. They are assumed to behave
|
||||
correctly.
|
||||
|
||||
Any read and write operation that doesn't involve file
|
||||
contents can be performed by clients by talking to the
|
||||
metadata server directly. Such operations include creating
|
||||
an empty file or a directory, deleting a file or directory,
|
||||
listing files.
|
||||
|
||||
If a client wants to read a range of bytes from a file,
|
||||
it sends the metadata server the file name and range.
|
||||
The metadata server responds with the chunk size of that
|
||||
file, the list of hashes for the chunks involved in the
|
||||
read, and the IP addresses of the chunk servers that hold
|
||||
each chunk. The metadata server also adds the IP addresses
|
||||
of three chunk servers any new chunks should be written
|
||||
to. The client can then download the chunks from the chunk
|
||||
servers and reassemble the result.
|
||||
|
||||
If a client wants to write at a range of bytes of a file,
|
||||
it starts by reading that range from the metadata server,
|
||||
getting the list of hashes it will modify, their locations,
|
||||
and locations for any new chunks. The client then modifies
|
||||
the chunk by sending to each chunk server the hash to modify
|
||||
and the patch (a range of bytes within a chunk plus the new
|
||||
data). The chunk server creates a new modified chunk and
|
||||
keeps the old version, then returns the new hash. If all
|
||||
modifications are successful, the client holds the set of
|
||||
old hashes and new hashes for that file range. It completes
|
||||
the write by telling the metadata server to swap the old
|
||||
hashes with the new ones (optionally including write flags
|
||||
such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes
|
||||
don't match, another write succeded in the mean time and
|
||||
touched that range, therefore the write fails. If the old
|
||||
hashes match, the write succeded. If the client fails to
|
||||
modify any chunks, it doesn't commit the write with the
|
||||
metadata server.
|
||||
|
||||
If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING
|
||||
flag is set, the metadata server atomically creates the file
|
||||
with a default chunk size (4096 bytes) and retries the write.
|
||||
This operation is logged to the WAL to ensure crash consistency.
|
||||
|
||||
If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is
|
||||
truncated after the write, setting its size to exactly offset+length
|
||||
and discarding any data beyond that point. This is useful for HTTP
|
||||
PUT semantics where the entire file content should be replaced.
|
||||
|
||||
Note that write failures may cause chunks to be orphaned
|
||||
on chunk servers. This is solved by a garbage collection
|
||||
algorithm implemented by the synchronization messages
|
||||
between metadata and chunk server.
|
||||
|
||||
Note that clients may cache chunks and index them by their
|
||||
hash. When they read a file and receive its hashes, they may
|
||||
avoid reaching for the chunk servers if they already cached
|
||||
the chunks with those hashes. This allows reading files with
|
||||
only one round trip at no cost of correctness. If getting
|
||||
the up-to-date contents is not a concern, clients may also
|
||||
cache file metadata.
|
||||
|
||||
Metadata and chunk server exchange:
|
||||
|
||||
The metadata server is only aware of each chunk server
|
||||
as long as they have a TCP connection. When a chunk server
|
||||
first connects to the metadata server, it authenticates
|
||||
itself and sends its own IP addresses. If the server is
|
||||
authentic, the metadata server requests the full list
|
||||
of chunks the chunk server is holding. Upon receiving the
|
||||
state of chunk server, the metadata server adds all useful
|
||||
chunks to the "old_list" and all useless chunks to the
|
||||
"rem_list", then sends the rem_list to the chunk server
|
||||
which removes those chunks.
|
||||
|
||||
When writes are committed to the metadata server involving
|
||||
new chunks to a chunk server, the metadata server adds those
|
||||
hashes to an "add_list" and any hashes that are not useful
|
||||
anymore to the rem_list.
|
||||
|
||||
Periodically, the metadata server sends the add_list and
|
||||
rem_list to the chunk server. These list tell the chunk
|
||||
server the ideal state it should have from the point of
|
||||
view of the metadata server. Elements in the add_list should
|
||||
already be in the chunk servers, and elements from the
|
||||
rem_list are to be removed. A chunk server marks any chunk
|
||||
in the rem_list as to be removed and checks that hashes
|
||||
in the add list are present. If a chunk in the add list
|
||||
is marked as to be removed, it is unmarked. When a chunk
|
||||
is marked as to be removed for a certain amount of time,
|
||||
it is permanently deleted. When the synchronization is
|
||||
complete, the metadata server merges the add_list into
|
||||
the old_list and clears the rem_list. If chunks in the
|
||||
add_list are not present in the chunk server, it responds
|
||||
with an error message containing the list of missing chunks.
|
||||
The metadata server then responds with a list of chunk
|
||||
server addresses where the chunk server with the missing
|
||||
chunk can download it from. Each chunk server goes
|
||||
through its download list one at the time downloading
|
||||
the missing chunks.
|
||||
|
||||
Note that if the chunk server finds that its holding some
|
||||
chunks that are not in the hash list of the metadata server,
|
||||
that does not mean they are orphaned. It's possible that
|
||||
some writes are being performed by clients that have uploaded
|
||||
chunks to that chunk server but didn't yet acknowledge it
|
||||
to the metadata server. If all goes well and the write
|
||||
succeded, the metadata server will add those hashes to the
|
||||
hash list. Chunk servers should only drop chunks if they
|
||||
are not referenced by the metadata server for a period of
|
||||
time (say, 30 minutes).
|
||||
|
||||
Security
|
||||
All nodes of the system share a secret key and use it to
|
||||
authenticate each other and encrypt messages. This allows
|
||||
the server to accept new chunk servers and clients with
|
||||
no prior setup
|
||||
|
||||
Reliability
|
||||
The metadata server is a single point of failure. To reduce
|
||||
the impact of crashes, the metadata server stores all write
|
||||
operations into a write-ahead log that is replayed any time
|
||||
the process goes online.
|
||||
|
||||
Chunk Management:
|
||||
Chunks are added to the system when:
|
||||
1. A chunk server connects
|
||||
2. A write operation on metadata occurs (adding chunks)
|
||||
|
||||
They are removed when:
|
||||
1. A chunk server disconnects
|
||||
2. A write operation on metadata occurs (overwriting old chunks)
|
||||
3. A delete operation on metadata occurs
|
||||
4. A chunk is corrupted or removed forcefully from the chunk server
|
||||
|
||||
The system must make sure that chunks are not over-replicated
|
||||
or under-replicated. If they are over-replicated, some chunk
|
||||
servers need to forget some copies. If they are under-replicated,
|
||||
some chunk servers need to copy chunks from elsewhere.
|
||||
|
||||
Metadata server variables for a chunk server:
|
||||
ms_old_list: List of chunks that are known to be held by CS
|
||||
ms_add_list: List of chunks that should be held by CS
|
||||
ms_rem_list: List of chunks that may be held by CS but should removed from it
|
||||
|
||||
Chunk server variables:
|
||||
cs_add_list: List of chunks added since the last update
|
||||
cs_rem_list: List of chunks marked for removal after a timeout
|
||||
cs_lst_list: List of chunks that were lost due to errors or forceful removals of chunk files
|
||||
|
||||
Metadata change for write:
|
||||
When clients commit a write by adding new hashes to the metadata,
|
||||
MS adds those hashes to the ms_add_list for the CS where the client
|
||||
uploaded the chunks. If a hash was overwritten and became useless,
|
||||
that hash is added to the ms_rem_list for all CS holding it.
|
||||
|
||||
Metadata change for delete:
|
||||
All hashes that are no longer reachable by the file tree are added
|
||||
to the ms_rem_list of their holders
|
||||
|
||||
Chunk upload:
|
||||
When a chunk is uploaded to a chunk server, its hash is added to
|
||||
the cs_add_list.
|
||||
|
||||
Periodically on the chunk server:
|
||||
Elements in the cs_rem_list have a 30 minute timeout after which they are deleted
|
||||
permanently.
|
||||
|
||||
Periodically:
|
||||
CS sends cs_add_list to MS
|
||||
MS may add a subset of cs_add_list to ms_add_list based on the chunk replication and distribution policy
|
||||
MS sends ms_add_list and ms_rem_list to CS
|
||||
CS (1) Adds all elements from ms_rem_list to cs_rem_list
|
||||
(2) Elements in ms_add_list that are not held by the chunk server are added to
|
||||
a temporary list tmp_list
|
||||
(3) Removes elements in ms_add_list from cs_add_list and cs_rem_list, then merges cs_add_list into cs_rem_list and clears cs_add_list.
|
||||
(4) Elements in cs_lst_list are added to tmp_list, then cs_lst_list is cleared.
|
||||
(6) tmp_list is sent to MS
|
||||
(7) cs_add_list is cleared
|
||||
MS (1) Receives tmp_list and sends download locations to CS for those chunks
|
||||
(2) Merges ms_add_list into ms_old_list, then removes all items in tmp_list from ms_old_list
|
||||
(3) Sets ms_add_list equal to tmp_list
|
||||
|
||||
Chunk replication and distribution policy:
|
||||
During an update, when CS reports a new chunk to MS, MS has to decide whether
|
||||
to allow the CS to keep it or not.
|
||||
|
||||
There are 4 cases:
|
||||
- The chunk is useless (not referenced by any file)
|
||||
- The chunk is under-replicated even counting the new copy
|
||||
- The chunk is properly replicated with the new copy
|
||||
- The chunk is over-replicated with the new copy
|
||||
|
||||
If the chunk is not referenced by the file tree, do nothing.
|
||||
|
||||
If the chunk is properly replicated or under-replicated, add it to the ms_add_list.
|
||||
|
||||
If the chunk is over-replicated, either don't add it to the ms_add_list or add it to the ms_rem_list of some other holder.
|
||||
|
||||
TODO: The way chunk servers tell about the chunks they are holding
|
||||
recently changed. Now instead of sending a full chunk list when
|
||||
connecting, they send batches of chunks to the metadata server
|
||||
during state updates. It also changed that now chunk servers
|
||||
initiate updates.
|
||||
|
||||
When a chunk server connects (and authenticates)
|
||||
it sends alongside the auth message the hash of
|
||||
the hash list of all chunks. The metadata server
|
||||
then replies with a message saying whether it
|
||||
already cached that chunk list or not. If it didn't, the chunk server sends the entire list
|
||||
in chunks during state updates, with an increased
|
||||
update frequency.
|
||||
|
||||
Metadata Persistence & Crash Recovery:
|
||||
The metadata server uses a write-ahead log (WAL) file to store its state on disk.
|
||||
|
||||
Log files start with a full snapshot of the metadata and continues with operation
|
||||
log entries.
|
||||
|
||||
When a file gets too big, the metadata server creates a new WAL file by writing
|
||||
a snapshot to it and continuing logging there.
|
||||
@@ -1,281 +0,0 @@
|
||||
1. Introduction & Notation
|
||||
|
||||
The DESIGN.txt file gives an overview of the system
|
||||
and how nodes of a cluster interact with each other.
|
||||
This file documents the specific binary format used
|
||||
to exchange information between nodes.
|
||||
|
||||
All messages start with a shared header, defined as:
|
||||
|
||||
struct Header {
|
||||
uint16_t version;
|
||||
uint16_t type;
|
||||
uint32_t length;
|
||||
};
|
||||
|
||||
1.1 Generation Counter Special Values
|
||||
|
||||
Generation counters (uint64_t) have two special values:
|
||||
|
||||
NO_GENERATION (0):
|
||||
When used in expect_gen, means "skip generation check entirely".
|
||||
Files/directories are never assigned this generation value.
|
||||
|
||||
MISSING_FILE_GENERATION (UINT64_MAX):
|
||||
When used in expect_gen, means "expect file/directory to NOT exist".
|
||||
- For DELETE: succeeds if file doesn't exist (no-op)
|
||||
- For WRITE: fails with BADGEN if file exists, fails with NOENT if missing
|
||||
- For existing operations: causes generation mismatch if file exists
|
||||
Files/directories are never assigned this generation value.
|
||||
|
||||
2. Client Messages
|
||||
|
||||
2.1 Client to Metadata Server messages
|
||||
|
||||
Let's start from the interactions between a client and
|
||||
the metadata server:
|
||||
|
||||
[ CREATE | C -> MS ]
|
||||
Upon file creation, the client sends a "CREATE" message with the following layout:
|
||||
|
||||
struct CreateMessage {
|
||||
Header header; // type=CREATE
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint8_t is_dir;
|
||||
if (is_dir != 0) {
|
||||
uint32_t chunk_size;
|
||||
}
|
||||
};
|
||||
|
||||
Note that in general only paths up to 65K bytes are
|
||||
supported and that the layout of fields may depend
|
||||
on the value of others.
|
||||
|
||||
The server then responds with a CREATE_SUCCESS or CREATE_ERROR message.
|
||||
|
||||
[ DELETE | C -> MS ]
|
||||
When a client deletes a file, it sends a DELETE
|
||||
message with the following layout:
|
||||
|
||||
struct DeleteMessage {
|
||||
Header header; // type=DELETE
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
};
|
||||
|
||||
The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless.
|
||||
|
||||
The server then responds with a DELETE_SUCCESS or DELETE_ERROR message.
|
||||
|
||||
[ LIST | C -> MS ]
|
||||
When a client requests a directory listing, it sends a LIST message whith the following format:
|
||||
|
||||
struct ListMessage {
|
||||
Header header; // type=LIST
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
};
|
||||
|
||||
If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped.
|
||||
|
||||
The server then responds with a LIST_SUCCESS or LIST_ERROR message.
|
||||
|
||||
[ READ | C -> MS ]
|
||||
When a client requests to read a file, it sends a READ message with the following format:
|
||||
|
||||
struct ReadMessage {
|
||||
Header header; // type=READ
|
||||
uint64_t expect_gen;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint32_t offset;
|
||||
uint32_t length;
|
||||
};
|
||||
|
||||
The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped.
|
||||
|
||||
The offset and length fields determine the region to be read from the file.
|
||||
|
||||
[ WRITE | C -> MS ]
|
||||
When a client wants to write to a file, it sends a WRITE message with the following layout:
|
||||
|
||||
struct Location {
|
||||
uint8_t is_ipv4;
|
||||
if (is_ipv4) {
|
||||
uint32_t ipv4;
|
||||
} else {
|
||||
uint128_t ipv6;
|
||||
}
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct WriteChunk {
|
||||
SHA256 hash;
|
||||
uint32_t num_locations;
|
||||
Location locations[num_locations];
|
||||
};
|
||||
|
||||
struct WriteMessage {
|
||||
Header header; // type=WRITE
|
||||
uint64_t expect_gen;
|
||||
uint32_t flags;
|
||||
uint16_t path_len;
|
||||
char path[path_len];
|
||||
uint32_t offset;
|
||||
uint32_t length;
|
||||
uint32_t num_chunks;
|
||||
WriteChunk chunks[num_chunks];
|
||||
};
|
||||
|
||||
If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata.
|
||||
|
||||
The flags field contains write operation flags. Currently defined flags:
|
||||
- TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist,
|
||||
the metadata server will atomically create it with a default chunk
|
||||
size of 4096 bytes and retry the write operation. The creation is
|
||||
logged to the WAL for crash consistency.
|
||||
- TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write
|
||||
operation, setting the file size to exactly offset+length. Any data
|
||||
beyond this point is discarded. Useful for HTTP PUT semantics where
|
||||
the entire file content should be replaced.
|
||||
|
||||
The offset and length mark the region that is being written to.
|
||||
|
||||
Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to
|
||||
|
||||
num_chunks == length / chunk_size
|
||||
|
||||
Where chunk_size is the one for the target file at the specified generation.
|
||||
|
||||
Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it.
|
||||
|
||||
2.2 Client to Chunk Server messages
|
||||
|
||||
TODO
|
||||
|
||||
3. Metadata Server messages
|
||||
|
||||
The following is the list of messages the Metadata Server may send to a Client:
|
||||
|
||||
[ CREATE_ERROR | MS -> C ]
|
||||
When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout:
|
||||
|
||||
struct CreateErrorMessage {
|
||||
Header header; // type=CREATE_ERROR
|
||||
uint16_t message_len;
|
||||
char message[message_len];
|
||||
};
|
||||
|
||||
[ CREATE_SUCCESS | MS -> C ]
|
||||
When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout:
|
||||
|
||||
struct CreateSuccessMessage {
|
||||
Header header; // type=CREATE_SUCCESS
|
||||
uint64_t gen;
|
||||
};
|
||||
|
||||
The gen field is the generation counter given to the file.
|
||||
|
||||
[ DELETE_ERROR | MS -> C ]
|
||||
See CREATE_ERROR
|
||||
|
||||
[ DELETE_SUCCESS | MS -> C ]
|
||||
When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout:
|
||||
|
||||
struct DeleteSuccessMessage {
|
||||
Header header; // type=DELETE_SUCCESS
|
||||
};
|
||||
|
||||
It does not store any fields other than the header.
|
||||
|
||||
[ LIST_ERROR | MS -> C ]
|
||||
When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout:
|
||||
|
||||
struct ListErrorMessage {
|
||||
Header header; // type=LIST_ERROR
|
||||
uint16_t message_len;
|
||||
char message[message_len];
|
||||
};
|
||||
|
||||
[ LIST_SUCCESS | MS -> C ]
|
||||
When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout:
|
||||
|
||||
struct ListItem {
|
||||
uint64_t gen;
|
||||
uint8_t is_dir;
|
||||
uint16_t name_len;
|
||||
char name[name_len];
|
||||
};
|
||||
|
||||
struct ListSuccessMessage {
|
||||
Header header; // type=LIST_SUCCESS
|
||||
uint64_t gen;
|
||||
uint8_t truncated;
|
||||
uint32_t item_count;
|
||||
ListItem items[item_count];
|
||||
};
|
||||
|
||||
The ListSuccessMessage gen field contains the generation counter for the directory, while the gen field in ListItem is the counter for that specific child.
|
||||
|
||||
If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message.
|
||||
|
||||
[ READ_ERROR | MS -> C ]
|
||||
TODO
|
||||
|
||||
[ READ_SUCCESS | MS -> C ]
|
||||
|
||||
struct IPv4AndPort {
|
||||
uint32_t ipv4;
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct IPv6AndPort {
|
||||
uint128_t ipv6;
|
||||
uint16_t port;
|
||||
};
|
||||
|
||||
struct ChunkServerAddrList {
|
||||
uint32_t num_ipv4;
|
||||
IPv4AndPort ipv4s[num_ipv4];
|
||||
uint32_t num_ipv6;
|
||||
IPv6AndPort ipv6s[num_ipv6];
|
||||
};
|
||||
|
||||
struct ReadChunk {
|
||||
SHA256 hash;
|
||||
uint32_t num_holders;
|
||||
ChunkServerAddrList holders[num_holders];
|
||||
};
|
||||
|
||||
struct ReadSuccessMessage {
|
||||
Header header; // type=READ_SUCCESS
|
||||
uint64_t gen;
|
||||
uint32_t chunk_size;
|
||||
uint32_t file_length;
|
||||
uint32_t num_hashes;
|
||||
ReadChunk chunks[num_hashes];
|
||||
uint32_t num_write_locations;
|
||||
ChunkServerAddrList write_locations[num_write_locations];
|
||||
};
|
||||
|
||||
The message returns general information about the file such as its generation counter, length in bytes, and chunk size.
|
||||
|
||||
The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them.
|
||||
|
||||
The write locations are a list of chunk servers that clients may write new chunks to.
|
||||
|
||||
[ WRITE_ERROR | MS -> C ]
|
||||
See CREATE_ERROR
|
||||
|
||||
[ WRITE_SUCCESS | MS -> C ]
|
||||
When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout:
|
||||
|
||||
struct WriteSuccessMessage {
|
||||
Header header; // type=WRITE_SUCCESS
|
||||
uint64_t gen;
|
||||
};
|
||||
|
||||
The gen field is the new generation counter for that file.
|
||||
@@ -1,3 +0,0 @@
|
||||
Notes on simulation testing:
|
||||
- Make sure your program works without fault injections
|
||||
- Trace the branch coverage of the test and find why some branches are never reached
|
||||
@@ -1,26 +0,0 @@
|
||||
[X] Implement proper operation handles that allow users to wait for specific events
|
||||
[X] Implement metadata server WAL
|
||||
[X] Check that the corner case where read and writes go over the length of the file work correctly
|
||||
[X] Return the number of bytes read or written in the ToastyFS_Result struct
|
||||
[X] Add a change counter to FileTree. The counter is initialized to 0 and updated any time the tree is changed. When a new file is created, the current change counter is assigned to it. This makes it possible to verify that the file didn't change in between read and write operations without that clumsy previous hash list and previous chunk size.
|
||||
[ ] Clean up the read operation for clients
|
||||
[ ] Make parallel uploads/downloads configurable
|
||||
[ ] Recalculate next write locations whenever a write occurs, not at each read
|
||||
[ ] add logging (when chunk servers connect and disconnect to the metadata server)
|
||||
[ ] Should list scenarios that need testing, like those where chunks would be dropped
|
||||
[ ] Update DESIGN.txt and the code to remove the chunk list message. The information of chunks held by chunk servers is now transmitted to the metadata server during state updates
|
||||
[ ] clean up the replication factor business
|
||||
[ ] Add failt injections to the simulation
|
||||
[ ] Add authentication
|
||||
[ ] Add release build and allow switching between debug, release, coverage, sanitizer builds while calling make
|
||||
[ ] When an operation is committed to the WAL, it may then fail due to not deterministic reasons (out of memory). When the metadata server restarts and replays the log, the operation may succede and cause the system to diverge from the last instance of the process. Is this not a problem? How do we solve it?
|
||||
[ ] Add notes on who inspired the testing approach
|
||||
[ ] Add notes on benchmarks and that we don't do batching
|
||||
[ ] Fix endianess when writing to network and the WAL
|
||||
[ ] WAL entry checksum
|
||||
[ ] Check write() errors when appending WAL entries
|
||||
[ ] When WAL entries are partially written, remove partial data and fail
|
||||
[ ] Find a way to ensure listing operations of large directories are handled
|
||||
[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication?
|
||||
[ ] Rename generation counters to version numbers
|
||||
[ ] Add a flag for write operations to create the file if it doesn't exist alraedy
|
||||
@@ -0,0 +1,190 @@
|
||||
#ifndef QUAKEY_INCLUDED
|
||||
#define QUAKEY_INCLUDED
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef _WIN32
|
||||
#include <winsock2.h>
|
||||
#include <ws2tcpip.h>
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#include <direct.h>
|
||||
#else
|
||||
#include <time.h>
|
||||
#include <poll.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <limits.h>
|
||||
#include <string.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/fcntl.h>
|
||||
#include <sys/socket.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <pthread.h>
|
||||
#endif
|
||||
|
||||
typedef struct {} Quakey;
|
||||
|
||||
// Function pointers to a simulated program's code
|
||||
typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
typedef int (*QuakeyFreeFunc)(void *state);
|
||||
|
||||
typedef enum {
|
||||
QUAKEY_LINUX,
|
||||
QUAKEY_WINDOWS,
|
||||
} QuakeyPlatform;
|
||||
|
||||
typedef struct {
|
||||
|
||||
// Size of the opaque state struct
|
||||
int state_size;
|
||||
|
||||
// Pointers to program code
|
||||
QuakeyInitFunc init_func;
|
||||
QuakeyTickFunc tick_func;
|
||||
QuakeyFreeFunc free_func;
|
||||
|
||||
// Network addresses enabled on the process
|
||||
char **addrs;
|
||||
int num_addrs;
|
||||
|
||||
// Disk size for the process
|
||||
int disk_size;
|
||||
|
||||
// Platform used by this program
|
||||
QuakeyPlatform platform;
|
||||
|
||||
} QuakeySpawn;
|
||||
|
||||
typedef unsigned long long QuakeyUInt64;
|
||||
|
||||
// Start a simulation
|
||||
int quakey_init(Quakey **quakey, QuakeyUInt64 seed);
|
||||
|
||||
// Stop a simulation
|
||||
void quakey_free(Quakey *quakey);
|
||||
|
||||
// Add a program to the simulation
|
||||
void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg);
|
||||
|
||||
// Schedule and executes one program until it would block, then returns
|
||||
int quakey_schedule_one(Quakey *quakey);
|
||||
|
||||
// Generate a random u64
|
||||
QuakeyUInt64 quakey_random(void);
|
||||
|
||||
int *mock_errno_ptr(void);
|
||||
|
||||
#ifdef _WIN32
|
||||
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
|
||||
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
|
||||
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
|
||||
BOOL mock_CloseHandle(HANDLE handle);
|
||||
BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
|
||||
BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
|
||||
BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
|
||||
DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
|
||||
BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
|
||||
BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
|
||||
BOOL mock_FlushFileBuffers(HANDLE handle);
|
||||
BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
|
||||
char* mock__fullpath(char *path, char *dst, int cap);
|
||||
HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
|
||||
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
|
||||
BOOL mock_FindClose(HANDLE hFindFile);
|
||||
int mock__mkdir(char *path);
|
||||
#else
|
||||
int mock_socket(int domain, int type, int protocol);
|
||||
int mock_closesocket(int fd);
|
||||
int mock_ioctlsocket(int fd, long cmd, unsigned long *argp);
|
||||
int mock_bind(int fd, void *addr, unsigned long addr_len);
|
||||
int mock_connect(int fd, void *addr, unsigned long addr_len);
|
||||
int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
|
||||
int mock_listen(int fd, int backlog);
|
||||
int mock_accept(int fd, void *addr, socklen_t *addr_len);
|
||||
int mock_recv(int fd, char *dst, int len, int flags);
|
||||
int mock_send(int fd, char *src, int len, int flags);
|
||||
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
|
||||
int mock_open(char *path, int flags, int mode);
|
||||
int mock_fcntl(int fd, int cmd, int flags);
|
||||
int mock_close(int fd);
|
||||
int mock_fstat(int fd, struct stat *buf);
|
||||
int mock_read(int fd, char *dst, int len);
|
||||
int mock_write(int fd, char *src, int len);
|
||||
off_t mock_lseek(int fd, off_t offset, int whence);
|
||||
int mock_flock(int fd, int op);
|
||||
int mock_fsync(int fd);
|
||||
int mock_mkstemp(char *path);
|
||||
int mock_mkdir(char *path, mode_t mode);
|
||||
int mock_remove(char *path);
|
||||
int mock_rename(char *oldpath, char *newpath);
|
||||
char* mock_realpath(char *path, char *dst);
|
||||
DIR* mock_opendir(char *name);
|
||||
struct dirent* mock_readdir(DIR *dirp);
|
||||
int mock_closedir(DIR *dirp);
|
||||
#endif
|
||||
|
||||
void *mock_malloc(size_t size);
|
||||
void *mock_realloc(void *ptr, size_t size);
|
||||
void mock_free(void *ptr);
|
||||
|
||||
#ifdef QUAKEY_ENABLE_MOCKS
|
||||
|
||||
#undef errno
|
||||
#define errno (*mock_errno_ptr())
|
||||
|
||||
#define malloc mock_malloc
|
||||
#define realloc mock_realloc
|
||||
#define free mock_free
|
||||
#define socket mock_socket
|
||||
#define closesocket mock_closesocket
|
||||
#define ioctlsocket mock_ioctlsocket
|
||||
#define bind mock_bind
|
||||
#define connect mock_connect
|
||||
#define getsockopt mock_getsockopt
|
||||
#define listen mock_listen
|
||||
#define accept mock_accept
|
||||
#define recv mock_recv
|
||||
#define send mock_send
|
||||
#define clock_gettime mock_clock_gettime
|
||||
#define QueryPerformanceCounter mock_QueryPerformanceCounter
|
||||
#define QueryPerformanceFrequency mock_QueryPerformanceFrequency
|
||||
#define open mock_open
|
||||
#define fcntl mock_fcntl
|
||||
#define close mock_close
|
||||
#define CreateFileW mock_CreateFileW
|
||||
#define CloseHandle mock_CloseHandle
|
||||
#define ReadFile mock_ReadFile
|
||||
#define WriteFile mock_WriteFile
|
||||
#define read mock_read
|
||||
#define write mock_write
|
||||
#define fstat mock_fstat
|
||||
#define GetFileSizeEx mock_GetFileSizeEx
|
||||
#define lseek mock_lseek
|
||||
#define SetFilePointer mock_SetFilePointer
|
||||
#define flock mock_flock
|
||||
#define LockFile mock_LockFile
|
||||
#define UnlockFile mock_UnlockFile
|
||||
#define fsync mock_fsync
|
||||
#define FlushFileBuffers mock_FlushFileBuffers
|
||||
#define mkstemp mock_mkstemp
|
||||
#define mkdir mock_mkdir
|
||||
#define _mkdir mock__mkdir
|
||||
#define remove mock_remove
|
||||
#define rename mock_rename
|
||||
#define MoveFileExW mock_MoveFileExW
|
||||
#define realpath mock_realpath
|
||||
#define _fullpath mock__fullpath
|
||||
#define opendir mock_opendir
|
||||
#define readdir mock_readdir
|
||||
#define closedir mock_closedir
|
||||
#define FindFirstFileA mock_FindFirstFileA
|
||||
#define FindNextFileA mock_FindNextFileA
|
||||
#define FindClose mock_FindClose
|
||||
#endif
|
||||
|
||||
#endif // QUAKEY_INCLUDED
|
||||
+6548
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,801 @@
|
||||
/*
|
||||
* The little filesystem
|
||||
*
|
||||
* Copyright (c) 2022, The littlefs authors.
|
||||
* Copyright (c) 2017, Arm Limited. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
#ifndef LFS_H
|
||||
#define LFS_H
|
||||
|
||||
#include "lfs_util.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
|
||||
/// Version info ///
|
||||
|
||||
// Software library version
|
||||
// Major (top-nibble), incremented on backwards incompatible changes
|
||||
// Minor (bottom-nibble), incremented on feature additions
|
||||
#define LFS_VERSION 0x0002000b
|
||||
#define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
|
||||
#define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0))
|
||||
|
||||
// Version of On-disk data structures
|
||||
// Major (top-nibble), incremented on backwards incompatible changes
|
||||
// Minor (bottom-nibble), incremented on feature additions
|
||||
#define LFS_DISK_VERSION 0x00020001
|
||||
#define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16))
|
||||
#define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0))
|
||||
|
||||
|
||||
/// Definitions ///
|
||||
|
||||
// Type definitions
|
||||
typedef uint32_t lfs_size_t;
|
||||
typedef uint32_t lfs_off_t;
|
||||
|
||||
typedef int32_t lfs_ssize_t;
|
||||
typedef int32_t lfs_soff_t;
|
||||
|
||||
typedef uint32_t lfs_block_t;
|
||||
|
||||
// Maximum name size in bytes, may be redefined to reduce the size of the
|
||||
// info struct. Limited to <= 1022. Stored in superblock and must be
|
||||
// respected by other littlefs drivers.
|
||||
#ifndef LFS_NAME_MAX
|
||||
#define LFS_NAME_MAX 255
|
||||
#endif
|
||||
|
||||
// Maximum size of a file in bytes, may be redefined to limit to support other
|
||||
// drivers. Limited on disk to <= 2147483647. Stored in superblock and must be
|
||||
// respected by other littlefs drivers.
|
||||
#ifndef LFS_FILE_MAX
|
||||
#define LFS_FILE_MAX 2147483647
|
||||
#endif
|
||||
|
||||
// Maximum size of custom attributes in bytes, may be redefined, but there is
|
||||
// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored
|
||||
// in superblock and must be respected by other littlefs drivers.
|
||||
#ifndef LFS_ATTR_MAX
|
||||
#define LFS_ATTR_MAX 1022
|
||||
#endif
|
||||
|
||||
// Possible error codes, these are negative to allow
|
||||
// valid positive return values
|
||||
enum lfs_error {
|
||||
LFS_ERR_OK = 0, // No error
|
||||
LFS_ERR_IO = -5, // Error during device operation
|
||||
LFS_ERR_CORRUPT = -84, // Corrupted
|
||||
LFS_ERR_NOENT = -2, // No directory entry
|
||||
LFS_ERR_EXIST = -17, // Entry already exists
|
||||
LFS_ERR_NOTDIR = -20, // Entry is not a dir
|
||||
LFS_ERR_ISDIR = -21, // Entry is a dir
|
||||
LFS_ERR_NOTEMPTY = -39, // Dir is not empty
|
||||
LFS_ERR_BADF = -9, // Bad file number
|
||||
LFS_ERR_FBIG = -27, // File too large
|
||||
LFS_ERR_INVAL = -22, // Invalid parameter
|
||||
LFS_ERR_NOSPC = -28, // No space left on device
|
||||
LFS_ERR_NOMEM = -12, // No more memory available
|
||||
LFS_ERR_NOATTR = -61, // No data/attr available
|
||||
LFS_ERR_NAMETOOLONG = -36, // File name too long
|
||||
};
|
||||
|
||||
// File types
|
||||
enum lfs_type {
|
||||
// file types
|
||||
LFS_TYPE_REG = 0x001,
|
||||
LFS_TYPE_DIR = 0x002,
|
||||
|
||||
// internally used types
|
||||
LFS_TYPE_SPLICE = 0x400,
|
||||
LFS_TYPE_NAME = 0x000,
|
||||
LFS_TYPE_STRUCT = 0x200,
|
||||
LFS_TYPE_USERATTR = 0x300,
|
||||
LFS_TYPE_FROM = 0x100,
|
||||
LFS_TYPE_TAIL = 0x600,
|
||||
LFS_TYPE_GLOBALS = 0x700,
|
||||
LFS_TYPE_CRC = 0x500,
|
||||
|
||||
// internally used type specializations
|
||||
LFS_TYPE_CREATE = 0x401,
|
||||
LFS_TYPE_DELETE = 0x4ff,
|
||||
LFS_TYPE_SUPERBLOCK = 0x0ff,
|
||||
LFS_TYPE_DIRSTRUCT = 0x200,
|
||||
LFS_TYPE_CTZSTRUCT = 0x202,
|
||||
LFS_TYPE_INLINESTRUCT = 0x201,
|
||||
LFS_TYPE_SOFTTAIL = 0x600,
|
||||
LFS_TYPE_HARDTAIL = 0x601,
|
||||
LFS_TYPE_MOVESTATE = 0x7ff,
|
||||
LFS_TYPE_CCRC = 0x500,
|
||||
LFS_TYPE_FCRC = 0x5ff,
|
||||
|
||||
// internal chip sources
|
||||
LFS_FROM_NOOP = 0x000,
|
||||
LFS_FROM_MOVE = 0x101,
|
||||
LFS_FROM_USERATTRS = 0x102,
|
||||
};
|
||||
|
||||
// File open flags
|
||||
enum lfs_open_flags {
|
||||
// open flags
|
||||
LFS_O_RDONLY = 1, // Open a file as read only
|
||||
#ifndef LFS_READONLY
|
||||
LFS_O_WRONLY = 2, // Open a file as write only
|
||||
LFS_O_RDWR = 3, // Open a file as read and write
|
||||
LFS_O_CREAT = 0x0100, // Create a file if it does not exist
|
||||
LFS_O_EXCL = 0x0200, // Fail if a file already exists
|
||||
LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size
|
||||
LFS_O_APPEND = 0x0800, // Move to end of file on every write
|
||||
#endif
|
||||
|
||||
// internally used flags
|
||||
#ifndef LFS_READONLY
|
||||
LFS_F_DIRTY = 0x010000, // File does not match storage
|
||||
LFS_F_WRITING = 0x020000, // File has been written since last flush
|
||||
#endif
|
||||
LFS_F_READING = 0x040000, // File has been read since last flush
|
||||
#ifndef LFS_READONLY
|
||||
LFS_F_ERRED = 0x080000, // An error occurred during write
|
||||
#endif
|
||||
LFS_F_INLINE = 0x100000, // Currently inlined in directory entry
|
||||
};
|
||||
|
||||
// File seek flags
|
||||
enum lfs_whence_flags {
|
||||
LFS_SEEK_SET = 0, // Seek relative to an absolute position
|
||||
LFS_SEEK_CUR = 1, // Seek relative to the current file position
|
||||
LFS_SEEK_END = 2, // Seek relative to the end of the file
|
||||
};
|
||||
|
||||
|
||||
// Configuration provided during initialization of the littlefs
|
||||
struct lfs_config {
|
||||
// Opaque user provided context that can be used to pass
|
||||
// information to the block device operations
|
||||
void *context;
|
||||
|
||||
// Read a region in a block. Negative error codes are propagated
|
||||
// to the user.
|
||||
int (*read)(const struct lfs_config *c, lfs_block_t block,
|
||||
lfs_off_t off, void *buffer, lfs_size_t size);
|
||||
|
||||
// Program a region in a block. The block must have previously
|
||||
// been erased. Negative error codes are propagated to the user.
|
||||
// May return LFS_ERR_CORRUPT if the block should be considered bad.
|
||||
int (*prog)(const struct lfs_config *c, lfs_block_t block,
|
||||
lfs_off_t off, const void *buffer, lfs_size_t size);
|
||||
|
||||
// Erase a block. A block must be erased before being programmed.
|
||||
// The state of an erased block is undefined. Negative error codes
|
||||
// are propagated to the user.
|
||||
// May return LFS_ERR_CORRUPT if the block should be considered bad.
|
||||
int (*erase)(const struct lfs_config *c, lfs_block_t block);
|
||||
|
||||
// Sync the state of the underlying block device. Negative error codes
|
||||
// are propagated to the user.
|
||||
int (*sync)(const struct lfs_config *c);
|
||||
|
||||
#ifdef LFS_THREADSAFE
|
||||
// Lock the underlying block device. Negative error codes
|
||||
// are propagated to the user.
|
||||
int (*lock)(const struct lfs_config *c);
|
||||
|
||||
// Unlock the underlying block device. Negative error codes
|
||||
// are propagated to the user.
|
||||
int (*unlock)(const struct lfs_config *c);
|
||||
#endif
|
||||
|
||||
// Minimum size of a block read in bytes. All read operations will be a
|
||||
// multiple of this value.
|
||||
lfs_size_t read_size;
|
||||
|
||||
// Minimum size of a block program in bytes. All program operations will be
|
||||
// a multiple of this value.
|
||||
lfs_size_t prog_size;
|
||||
|
||||
// Size of an erasable block in bytes. This does not impact ram consumption
|
||||
// and may be larger than the physical erase size. However, non-inlined
|
||||
// files take up at minimum one block. Must be a multiple of the read and
|
||||
// program sizes.
|
||||
lfs_size_t block_size;
|
||||
|
||||
// Number of erasable blocks on the device. Defaults to block_count stored
|
||||
// on disk when zero.
|
||||
lfs_size_t block_count;
|
||||
|
||||
// Number of erase cycles before littlefs evicts metadata logs and moves
|
||||
// the metadata to another block. Suggested values are in the
|
||||
// range 100-1000, with large values having better performance at the cost
|
||||
// of less consistent wear distribution.
|
||||
//
|
||||
// Set to -1 to disable block-level wear-leveling.
|
||||
int32_t block_cycles;
|
||||
|
||||
// Size of block caches in bytes. Each cache buffers a portion of a block in
|
||||
// RAM. The littlefs needs a read cache, a program cache, and one additional
|
||||
// cache per file. Larger caches can improve performance by storing more
|
||||
// data and reducing the number of disk accesses. Must be a multiple of the
|
||||
// read and program sizes, and a factor of the block size.
|
||||
lfs_size_t cache_size;
|
||||
|
||||
// Size of the lookahead buffer in bytes. A larger lookahead buffer
|
||||
// increases the number of blocks found during an allocation pass. The
|
||||
// lookahead buffer is stored as a compact bitmap, so each byte of RAM
|
||||
// can track 8 blocks.
|
||||
lfs_size_t lookahead_size;
|
||||
|
||||
// Threshold for metadata compaction during lfs_fs_gc in bytes. Metadata
|
||||
// pairs that exceed this threshold will be compacted during lfs_fs_gc.
|
||||
// Defaults to ~88% block_size when zero, though the default may change
|
||||
// in the future.
|
||||
//
|
||||
// Note this only affects lfs_fs_gc. Normal compactions still only occur
|
||||
// when full.
|
||||
//
|
||||
// Set to -1 to disable metadata compaction during lfs_fs_gc.
|
||||
lfs_size_t compact_thresh;
|
||||
|
||||
// Optional statically allocated read buffer. Must be cache_size.
|
||||
// By default lfs_malloc is used to allocate this buffer.
|
||||
void *read_buffer;
|
||||
|
||||
// Optional statically allocated program buffer. Must be cache_size.
|
||||
// By default lfs_malloc is used to allocate this buffer.
|
||||
void *prog_buffer;
|
||||
|
||||
// Optional statically allocated lookahead buffer. Must be lookahead_size.
|
||||
// By default lfs_malloc is used to allocate this buffer.
|
||||
void *lookahead_buffer;
|
||||
|
||||
// Optional upper limit on length of file names in bytes. No downside for
|
||||
// larger names except the size of the info struct which is controlled by
|
||||
// the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX or name_max stored on
|
||||
// disk when zero.
|
||||
lfs_size_t name_max;
|
||||
|
||||
// Optional upper limit on files in bytes. No downside for larger files
|
||||
// but must be <= LFS_FILE_MAX. Defaults to LFS_FILE_MAX or file_max stored
|
||||
// on disk when zero.
|
||||
lfs_size_t file_max;
|
||||
|
||||
// Optional upper limit on custom attributes in bytes. No downside for
|
||||
// larger attributes size but must be <= LFS_ATTR_MAX. Defaults to
|
||||
// LFS_ATTR_MAX or attr_max stored on disk when zero.
|
||||
lfs_size_t attr_max;
|
||||
|
||||
// Optional upper limit on total space given to metadata pairs in bytes. On
|
||||
// devices with large blocks (e.g. 128kB) setting this to a low size (2-8kB)
|
||||
// can help bound the metadata compaction time. Must be <= block_size.
|
||||
// Defaults to block_size when zero.
|
||||
lfs_size_t metadata_max;
|
||||
|
||||
// Optional upper limit on inlined files in bytes. Inlined files live in
|
||||
// metadata and decrease storage requirements, but may be limited to
|
||||
// improve metadata-related performance. Must be <= cache_size, <=
|
||||
// attr_max, and <= block_size/8. Defaults to the largest possible
|
||||
// inline_max when zero.
|
||||
//
|
||||
// Set to -1 to disable inlined files.
|
||||
lfs_size_t inline_max;
|
||||
|
||||
#ifdef LFS_MULTIVERSION
|
||||
// On-disk version to use when writing in the form of 16-bit major version
|
||||
// + 16-bit minor version. This limiting metadata to what is supported by
|
||||
// older minor versions. Note that some features will be lost. Defaults to
|
||||
// to the most recent minor version when zero.
|
||||
uint32_t disk_version;
|
||||
#endif
|
||||
};
|
||||
|
||||
// File info structure
|
||||
struct lfs_info {
|
||||
// Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR
|
||||
uint8_t type;
|
||||
|
||||
// Size of the file, only valid for REG files. Limited to 32-bits.
|
||||
lfs_size_t size;
|
||||
|
||||
// Name of the file stored as a null-terminated string. Limited to
|
||||
// LFS_NAME_MAX+1, which can be changed by redefining LFS_NAME_MAX to
|
||||
// reduce RAM. LFS_NAME_MAX is stored in superblock and must be
|
||||
// respected by other littlefs drivers.
|
||||
char name[LFS_NAME_MAX+1];
|
||||
};
|
||||
|
||||
// Filesystem info structure
|
||||
struct lfs_fsinfo {
|
||||
// On-disk version.
|
||||
uint32_t disk_version;
|
||||
|
||||
// Size of a logical block in bytes.
|
||||
lfs_size_t block_size;
|
||||
|
||||
// Number of logical blocks in filesystem.
|
||||
lfs_size_t block_count;
|
||||
|
||||
// Upper limit on the length of file names in bytes.
|
||||
lfs_size_t name_max;
|
||||
|
||||
// Upper limit on the size of files in bytes.
|
||||
lfs_size_t file_max;
|
||||
|
||||
// Upper limit on the size of custom attributes in bytes.
|
||||
lfs_size_t attr_max;
|
||||
};
|
||||
|
||||
// Custom attribute structure, used to describe custom attributes
|
||||
// committed atomically during file writes.
|
||||
struct lfs_attr {
|
||||
// 8-bit type of attribute, provided by user and used to
|
||||
// identify the attribute
|
||||
uint8_t type;
|
||||
|
||||
// Pointer to buffer containing the attribute
|
||||
void *buffer;
|
||||
|
||||
// Size of attribute in bytes, limited to LFS_ATTR_MAX
|
||||
lfs_size_t size;
|
||||
};
|
||||
|
||||
// Optional configuration provided during lfs_file_opencfg
|
||||
struct lfs_file_config {
|
||||
// Optional statically allocated file buffer. Must be cache_size.
|
||||
// By default lfs_malloc is used to allocate this buffer.
|
||||
void *buffer;
|
||||
|
||||
// Optional list of custom attributes related to the file. If the file
|
||||
// is opened with read access, these attributes will be read from disk
|
||||
// during the open call. If the file is opened with write access, the
|
||||
// attributes will be written to disk every file sync or close. This
|
||||
// write occurs atomically with update to the file's contents.
|
||||
//
|
||||
// Custom attributes are uniquely identified by an 8-bit type and limited
|
||||
// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller
|
||||
// than the buffer, it will be padded with zeros. If the stored attribute
|
||||
// is larger, then it will be silently truncated. If the attribute is not
|
||||
// found, it will be created implicitly.
|
||||
struct lfs_attr *attrs;
|
||||
|
||||
// Number of custom attributes in the list
|
||||
lfs_size_t attr_count;
|
||||
};
|
||||
|
||||
|
||||
/// internal littlefs data structures ///
|
||||
typedef struct lfs_cache {
|
||||
lfs_block_t block;
|
||||
lfs_off_t off;
|
||||
lfs_size_t size;
|
||||
uint8_t *buffer;
|
||||
} lfs_cache_t;
|
||||
|
||||
typedef struct lfs_mdir {
|
||||
lfs_block_t pair[2];
|
||||
uint32_t rev;
|
||||
lfs_off_t off;
|
||||
uint32_t etag;
|
||||
uint16_t count;
|
||||
bool erased;
|
||||
bool split;
|
||||
lfs_block_t tail[2];
|
||||
} lfs_mdir_t;
|
||||
|
||||
// littlefs directory type
|
||||
typedef struct lfs_dir {
|
||||
struct lfs_dir *next;
|
||||
uint16_t id;
|
||||
uint8_t type;
|
||||
lfs_mdir_t m;
|
||||
|
||||
lfs_off_t pos;
|
||||
lfs_block_t head[2];
|
||||
} lfs_dir_t;
|
||||
|
||||
// littlefs file type
|
||||
typedef struct lfs_file {
|
||||
struct lfs_file *next;
|
||||
uint16_t id;
|
||||
uint8_t type;
|
||||
lfs_mdir_t m;
|
||||
|
||||
struct lfs_ctz {
|
||||
lfs_block_t head;
|
||||
lfs_size_t size;
|
||||
} ctz;
|
||||
|
||||
uint32_t flags;
|
||||
lfs_off_t pos;
|
||||
lfs_block_t block;
|
||||
lfs_off_t off;
|
||||
lfs_cache_t cache;
|
||||
|
||||
const struct lfs_file_config *cfg;
|
||||
} lfs_file_t;
|
||||
|
||||
typedef struct lfs_superblock {
|
||||
uint32_t version;
|
||||
lfs_size_t block_size;
|
||||
lfs_size_t block_count;
|
||||
lfs_size_t name_max;
|
||||
lfs_size_t file_max;
|
||||
lfs_size_t attr_max;
|
||||
} lfs_superblock_t;
|
||||
|
||||
typedef struct lfs_gstate {
|
||||
uint32_t tag;
|
||||
lfs_block_t pair[2];
|
||||
} lfs_gstate_t;
|
||||
|
||||
// The littlefs filesystem type
|
||||
typedef struct lfs {
|
||||
lfs_cache_t rcache;
|
||||
lfs_cache_t pcache;
|
||||
|
||||
lfs_block_t root[2];
|
||||
struct lfs_mlist {
|
||||
struct lfs_mlist *next;
|
||||
uint16_t id;
|
||||
uint8_t type;
|
||||
lfs_mdir_t m;
|
||||
} *mlist;
|
||||
uint32_t seed;
|
||||
|
||||
lfs_gstate_t gstate;
|
||||
lfs_gstate_t gdisk;
|
||||
lfs_gstate_t gdelta;
|
||||
|
||||
struct lfs_lookahead {
|
||||
lfs_block_t start;
|
||||
lfs_block_t size;
|
||||
lfs_block_t next;
|
||||
lfs_block_t ckpoint;
|
||||
uint8_t *buffer;
|
||||
} lookahead;
|
||||
|
||||
const struct lfs_config *cfg;
|
||||
lfs_size_t block_count;
|
||||
lfs_size_t name_max;
|
||||
lfs_size_t file_max;
|
||||
lfs_size_t attr_max;
|
||||
lfs_size_t inline_max;
|
||||
|
||||
#ifdef LFS_MIGRATE
|
||||
struct lfs1 *lfs1;
|
||||
#endif
|
||||
} lfs_t;
|
||||
|
||||
|
||||
/// Filesystem functions ///
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Format a block device with the littlefs
|
||||
//
|
||||
// Requires a littlefs object and config struct. This clobbers the littlefs
|
||||
// object, and does not leave the filesystem mounted. The config struct must
|
||||
// be zeroed for defaults and backwards compatibility.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_format(lfs_t *lfs, const struct lfs_config *config);
|
||||
#endif
|
||||
|
||||
// Mounts a littlefs
|
||||
//
|
||||
// Requires a littlefs object and config struct. Multiple filesystems
|
||||
// may be mounted simultaneously with multiple littlefs objects. Both
|
||||
// lfs and config must be allocated while mounted. The config struct must
|
||||
// be zeroed for defaults and backwards compatibility.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_mount(lfs_t *lfs, const struct lfs_config *config);
|
||||
|
||||
// Unmounts a littlefs
|
||||
//
|
||||
// Does nothing besides releasing any allocated resources.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_unmount(lfs_t *lfs);
|
||||
|
||||
/// General operations ///
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Removes a file or directory
|
||||
//
|
||||
// If removing a directory, the directory must be empty.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_remove(lfs_t *lfs, const char *path);
|
||||
#endif
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Rename or move a file or directory
|
||||
//
|
||||
// If the destination exists, it must match the source in type.
|
||||
// If the destination is a directory, the directory must be empty.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
|
||||
#endif
|
||||
|
||||
// Find info about a file or directory
|
||||
//
|
||||
// Fills out the info structure, based on the specified file or directory.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
|
||||
|
||||
// Get a custom attribute
|
||||
//
|
||||
// Custom attributes are uniquely identified by an 8-bit type and limited
|
||||
// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than
|
||||
// the buffer, it will be padded with zeros. If the stored attribute is larger,
|
||||
// then it will be silently truncated. If no attribute is found, the error
|
||||
// LFS_ERR_NOATTR is returned and the buffer is filled with zeros.
|
||||
//
|
||||
// Returns the size of the attribute, or a negative error code on failure.
|
||||
// Note, the returned size is the size of the attribute on disk, irrespective
|
||||
// of the size of the buffer. This can be used to dynamically allocate a buffer
|
||||
// or check for existence.
|
||||
lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
|
||||
uint8_t type, void *buffer, lfs_size_t size);
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Set custom attributes
|
||||
//
|
||||
// Custom attributes are uniquely identified by an 8-bit type and limited
|
||||
// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be
|
||||
// implicitly created.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_setattr(lfs_t *lfs, const char *path,
|
||||
uint8_t type, const void *buffer, lfs_size_t size);
|
||||
#endif
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Removes a custom attribute
|
||||
//
|
||||
// If an attribute is not found, nothing happens.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
|
||||
#endif
|
||||
|
||||
|
||||
/// File operations ///
|
||||
|
||||
#ifndef LFS_NO_MALLOC
|
||||
// Open a file
|
||||
//
|
||||
// The mode that the file is opened in is determined by the flags, which
|
||||
// are values from the enum lfs_open_flags that are bitwise-ored together.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
|
||||
const char *path, int flags);
|
||||
|
||||
// if LFS_NO_MALLOC is defined, lfs_file_open() will fail with LFS_ERR_NOMEM
|
||||
// thus use lfs_file_opencfg() with config.buffer set.
|
||||
#endif
|
||||
|
||||
// Open a file with extra configuration
|
||||
//
|
||||
// The mode that the file is opened in is determined by the flags, which
|
||||
// are values from the enum lfs_open_flags that are bitwise-ored together.
|
||||
//
|
||||
// The config struct provides additional config options per file as described
|
||||
// above. The config struct must remain allocated while the file is open, and
|
||||
// the config struct must be zeroed for defaults and backwards compatibility.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
|
||||
const char *path, int flags,
|
||||
const struct lfs_file_config *config);
|
||||
|
||||
// Close a file
|
||||
//
|
||||
// Any pending writes are written out to storage as though
|
||||
// sync had been called and releases any allocated resources.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_close(lfs_t *lfs, lfs_file_t *file);
|
||||
|
||||
// Synchronize a file on storage
|
||||
//
|
||||
// Any pending writes are written out to storage.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_sync(lfs_t *lfs, lfs_file_t *file);
|
||||
|
||||
// Read data from file
|
||||
//
|
||||
// Takes a buffer and size indicating where to store the read data.
|
||||
// Returns the number of bytes read, or a negative error code on failure.
|
||||
lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
|
||||
void *buffer, lfs_size_t size);
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Write data to file
|
||||
//
|
||||
// Takes a buffer and size indicating the data to write. The file will not
|
||||
// actually be updated on the storage until either sync or close is called.
|
||||
//
|
||||
// Returns the number of bytes written, or a negative error code on failure.
|
||||
lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
|
||||
const void *buffer, lfs_size_t size);
|
||||
#endif
|
||||
|
||||
// Change the position of the file
|
||||
//
|
||||
// The change in position is determined by the offset and whence flag.
|
||||
// Returns the new position of the file, or a negative error code on failure.
|
||||
lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
|
||||
lfs_soff_t off, int whence);
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Truncates the size of the file to the specified size
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size);
|
||||
#endif
|
||||
|
||||
// Return the position of the file
|
||||
//
|
||||
// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR)
|
||||
// Returns the position of the file, or a negative error code on failure.
|
||||
lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file);
|
||||
|
||||
// Change the position of the file to the beginning of the file
|
||||
//
|
||||
// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_SET)
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file);
|
||||
|
||||
// Return the size of the file
|
||||
//
|
||||
// Similar to lfs_file_seek(lfs, file, 0, LFS_SEEK_END)
|
||||
// Returns the size of the file, or a negative error code on failure.
|
||||
lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file);
|
||||
|
||||
|
||||
/// Directory operations ///
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Create a directory
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_mkdir(lfs_t *lfs, const char *path);
|
||||
#endif
|
||||
|
||||
// Open a directory
|
||||
//
|
||||
// Once open a directory can be used with read to iterate over files.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path);
|
||||
|
||||
// Close a directory
|
||||
//
|
||||
// Releases any allocated resources.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir);
|
||||
|
||||
// Read an entry in the directory
|
||||
//
|
||||
// Fills out the info structure, based on the specified file or directory.
|
||||
// Returns a positive value on success, 0 at the end of directory,
|
||||
// or a negative error code on failure.
|
||||
int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info);
|
||||
|
||||
// Change the position of the directory
|
||||
//
|
||||
// The new off must be a value previous returned from tell and specifies
|
||||
// an absolute offset in the directory seek.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off);
|
||||
|
||||
// Return the position of the directory
|
||||
//
|
||||
// The returned offset is only meant to be consumed by seek and may not make
|
||||
// sense, but does indicate the current position in the directory iteration.
|
||||
//
|
||||
// Returns the position of the directory, or a negative error code on failure.
|
||||
lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir);
|
||||
|
||||
// Change the position of the directory to the beginning of the directory
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir);
|
||||
|
||||
|
||||
/// Filesystem-level filesystem operations
|
||||
|
||||
// Find on-disk info about the filesystem
|
||||
//
|
||||
// Fills out the fsinfo structure based on the filesystem found on-disk.
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo);
|
||||
|
||||
// Finds the current size of the filesystem
|
||||
//
|
||||
// Note: Result is best effort. If files share COW structures, the returned
|
||||
// size may be larger than the filesystem actually is.
|
||||
//
|
||||
// Returns the number of allocated blocks, or a negative error code on failure.
|
||||
lfs_ssize_t lfs_fs_size(lfs_t *lfs);
|
||||
|
||||
// Traverse through all blocks in use by the filesystem
|
||||
//
|
||||
// The provided callback will be called with each block address that is
|
||||
// currently in use by the filesystem. This can be used to determine which
|
||||
// blocks are in use or how much of the storage is available.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Attempt to make the filesystem consistent and ready for writing
|
||||
//
|
||||
// Calling this function is not required, consistency will be implicitly
|
||||
// enforced on the first operation that writes to the filesystem, but this
|
||||
// function allows the work to be performed earlier and without other
|
||||
// filesystem changes.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_fs_mkconsistent(lfs_t *lfs);
|
||||
#endif
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Attempt any janitorial work
|
||||
//
|
||||
// This currently:
|
||||
// 1. Calls mkconsistent if not already consistent
|
||||
// 2. Compacts metadata > compact_thresh
|
||||
// 3. Populates the block allocator
|
||||
//
|
||||
// Though additional janitorial work may be added in the future.
|
||||
//
|
||||
// Calling this function is not required, but may allow the offloading of
|
||||
// expensive janitorial work to a less time-critical code path.
|
||||
//
|
||||
// Returns a negative error code on failure. Accomplishing nothing is not
|
||||
// an error.
|
||||
int lfs_fs_gc(lfs_t *lfs);
|
||||
#endif
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
// Grows the filesystem to a new size, updating the superblock with the new
|
||||
// block count.
|
||||
//
|
||||
// If LFS_SHRINKNONRELOCATING is defined, this function will also accept
|
||||
// block_counts smaller than the current configuration, after checking
|
||||
// that none of the blocks that are being removed are in use.
|
||||
// Note that littlefs's pseudorandom block allocation means that
|
||||
// this is very unlikely to work in the general case.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count);
|
||||
#endif
|
||||
|
||||
#ifndef LFS_READONLY
|
||||
#ifdef LFS_MIGRATE
|
||||
// Attempts to migrate a previous version of littlefs
|
||||
//
|
||||
// Behaves similarly to the lfs_format function. Attempts to mount
|
||||
// the previous version of littlefs and update the filesystem so it can be
|
||||
// mounted with the current version of littlefs.
|
||||
//
|
||||
// Requires a littlefs object and config struct. This clobbers the littlefs
|
||||
// object, and does not leave the filesystem mounted. The config struct must
|
||||
// be zeroed for defaults and backwards compatibility.
|
||||
//
|
||||
// Returns a negative error code on failure.
|
||||
int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* lfs util functions
|
||||
*
|
||||
* Copyright (c) 2022, The littlefs authors.
|
||||
* Copyright (c) 2017, Arm Limited. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
#include "lfs_util.h"
|
||||
|
||||
// Only compile if user does not provide custom config
|
||||
#ifndef LFS_CONFIG
|
||||
|
||||
|
||||
// If user provides their own CRC impl we don't need this
|
||||
#ifndef LFS_CRC
|
||||
// Software CRC implementation with small lookup table
|
||||
uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) {
|
||||
static const uint32_t rtable[16] = {
|
||||
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac,
|
||||
0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
|
||||
0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c,
|
||||
0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c,
|
||||
};
|
||||
|
||||
const uint8_t *data = buffer;
|
||||
|
||||
for (size_t i = 0; i < size; i++) {
|
||||
crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 0)) & 0xf];
|
||||
crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 4)) & 0xf];
|
||||
}
|
||||
|
||||
return crc;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,277 @@
|
||||
/*
|
||||
* lfs utility functions
|
||||
*
|
||||
* Copyright (c) 2022, The littlefs authors.
|
||||
* Copyright (c) 2017, Arm Limited. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
#ifndef LFS_UTIL_H
|
||||
#define LFS_UTIL_H
|
||||
|
||||
#define LFS_STRINGIZE(x) LFS_STRINGIZE2(x)
|
||||
#define LFS_STRINGIZE2(x) #x
|
||||
|
||||
// Users can override lfs_util.h with their own configuration by defining
|
||||
// LFS_CONFIG as a header file to include (-DLFS_CONFIG=lfs_config.h).
|
||||
//
|
||||
// If LFS_CONFIG is used, none of the default utils will be emitted and must be
|
||||
// provided by the config file. To start, I would suggest copying lfs_util.h
|
||||
// and modifying as needed.
|
||||
#ifdef LFS_CONFIG
|
||||
#include LFS_STRINGIZE(LFS_CONFIG)
|
||||
#else
|
||||
|
||||
// Alternatively, users can provide a header file which defines
|
||||
// macros and other things consumed by littlefs.
|
||||
//
|
||||
// For example, provide my_defines.h, which contains
|
||||
// something like:
|
||||
//
|
||||
// #include <stddef.h>
|
||||
// extern void *my_malloc(size_t sz);
|
||||
// #define LFS_MALLOC(sz) my_malloc(sz)
|
||||
//
|
||||
// And build littlefs with the header by defining LFS_DEFINES.
|
||||
// (-DLFS_DEFINES=my_defines.h)
|
||||
|
||||
#ifdef LFS_DEFINES
|
||||
#include LFS_STRINGIZE(LFS_DEFINES)
|
||||
#endif
|
||||
|
||||
// System includes
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
#define LFS_NO_DEBUG
|
||||
#define LFS_NO_WARN
|
||||
#define LFS_NO_ERROR
|
||||
|
||||
#ifndef LFS_NO_MALLOC
|
||||
#include <quakey.h>
|
||||
#endif
|
||||
#ifndef LFS_NO_ASSERT
|
||||
#include <assert.h>
|
||||
#endif
|
||||
#if !defined(LFS_NO_DEBUG) || \
|
||||
!defined(LFS_NO_WARN) || \
|
||||
!defined(LFS_NO_ERROR) || \
|
||||
defined(LFS_YES_TRACE)
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
|
||||
// Macros, may be replaced by system specific wrappers. Arguments to these
|
||||
// macros must not have side-effects as the macros can be removed for a smaller
|
||||
// code footprint
|
||||
|
||||
// Logging functions
|
||||
#ifndef LFS_TRACE
|
||||
#ifdef LFS_YES_TRACE
|
||||
#define LFS_TRACE_(fmt, ...) \
|
||||
printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
|
||||
#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
|
||||
#else
|
||||
#define LFS_TRACE(...)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef LFS_DEBUG
|
||||
#ifndef LFS_NO_DEBUG
|
||||
#define LFS_DEBUG_(fmt, ...) \
|
||||
printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
|
||||
#define LFS_DEBUG(...) LFS_DEBUG_(__VA_ARGS__, "")
|
||||
#else
|
||||
#define LFS_DEBUG(...)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef LFS_WARN
|
||||
#ifndef LFS_NO_WARN
|
||||
#define LFS_WARN_(fmt, ...) \
|
||||
printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
|
||||
#define LFS_WARN(...) LFS_WARN_(__VA_ARGS__, "")
|
||||
#else
|
||||
#define LFS_WARN(...)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef LFS_ERROR
|
||||
#ifndef LFS_NO_ERROR
|
||||
#define LFS_ERROR_(fmt, ...) \
|
||||
printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
|
||||
#define LFS_ERROR(...) LFS_ERROR_(__VA_ARGS__, "")
|
||||
#else
|
||||
#define LFS_ERROR(...)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Runtime assertions
|
||||
#ifndef LFS_ASSERT
|
||||
#ifndef LFS_NO_ASSERT
|
||||
#define LFS_ASSERT(test) assert(test)
|
||||
#else
|
||||
#define LFS_ASSERT(test)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
// Builtin functions, these may be replaced by more efficient
|
||||
// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more
|
||||
// expensive basic C implementation for debugging purposes
|
||||
|
||||
// Min/max functions for unsigned 32-bit numbers
|
||||
static inline uint32_t lfs_max(uint32_t a, uint32_t b) {
|
||||
return (a > b) ? a : b;
|
||||
}
|
||||
|
||||
static inline uint32_t lfs_min(uint32_t a, uint32_t b) {
|
||||
return (a < b) ? a : b;
|
||||
}
|
||||
|
||||
// Align to nearest multiple of a size
|
||||
static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) {
|
||||
return a - (a % alignment);
|
||||
}
|
||||
|
||||
static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) {
|
||||
return lfs_aligndown(a + alignment-1, alignment);
|
||||
}
|
||||
|
||||
// Find the smallest power of 2 greater than or equal to a
|
||||
static inline uint32_t lfs_npw2(uint32_t a) {
|
||||
#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
|
||||
return 32 - __builtin_clz(a-1);
|
||||
#else
|
||||
uint32_t r = 0;
|
||||
uint32_t s;
|
||||
a -= 1;
|
||||
s = (a > 0xffff) << 4; a >>= s; r |= s;
|
||||
s = (a > 0xff ) << 3; a >>= s; r |= s;
|
||||
s = (a > 0xf ) << 2; a >>= s; r |= s;
|
||||
s = (a > 0x3 ) << 1; a >>= s; r |= s;
|
||||
return (r | (a >> 1)) + 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Count the number of trailing binary zeros in a
|
||||
// lfs_ctz(0) may be undefined
|
||||
static inline uint32_t lfs_ctz(uint32_t a) {
|
||||
#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__)
|
||||
return __builtin_ctz(a);
|
||||
#else
|
||||
return lfs_npw2((a & -a) + 1) - 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Count the number of binary ones in a
|
||||
static inline uint32_t lfs_popc(uint32_t a) {
|
||||
#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
|
||||
return __builtin_popcount(a);
|
||||
#else
|
||||
a = a - ((a >> 1) & 0x55555555);
|
||||
a = (a & 0x33333333) + ((a >> 2) & 0x33333333);
|
||||
return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Find the sequence comparison of a and b, this is the distance
|
||||
// between a and b ignoring overflow
|
||||
static inline int lfs_scmp(uint32_t a, uint32_t b) {
|
||||
return (int)(unsigned)(a - b);
|
||||
}
|
||||
|
||||
// Convert between 32-bit little-endian and native order
|
||||
static inline uint32_t lfs_fromle32(uint32_t a) {
|
||||
#if (defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return a;
|
||||
#elif !defined(LFS_NO_INTRINSICS) && ( \
|
||||
(defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
|
||||
return __builtin_bswap32(a);
|
||||
#else
|
||||
return ((uint32_t)((uint8_t*)&a)[0] << 0) |
|
||||
((uint32_t)((uint8_t*)&a)[1] << 8) |
|
||||
((uint32_t)((uint8_t*)&a)[2] << 16) |
|
||||
((uint32_t)((uint8_t*)&a)[3] << 24);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uint32_t lfs_tole32(uint32_t a) {
|
||||
return lfs_fromle32(a);
|
||||
}
|
||||
|
||||
// Convert between 32-bit big-endian and native order
|
||||
static inline uint32_t lfs_frombe32(uint32_t a) {
|
||||
#if !defined(LFS_NO_INTRINSICS) && ( \
|
||||
(defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
|
||||
return __builtin_bswap32(a);
|
||||
#elif (defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \
|
||||
(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
|
||||
return a;
|
||||
#else
|
||||
return ((uint32_t)((uint8_t*)&a)[0] << 24) |
|
||||
((uint32_t)((uint8_t*)&a)[1] << 16) |
|
||||
((uint32_t)((uint8_t*)&a)[2] << 8) |
|
||||
((uint32_t)((uint8_t*)&a)[3] << 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uint32_t lfs_tobe32(uint32_t a) {
|
||||
return lfs_frombe32(a);
|
||||
}
|
||||
|
||||
// Calculate CRC-32 with polynomial = 0x04c11db7
|
||||
#ifdef LFS_CRC
|
||||
static inline uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) {
|
||||
return LFS_CRC(crc, buffer, size);
|
||||
}
|
||||
#else
|
||||
uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
|
||||
#endif
|
||||
|
||||
// Allocate memory, only used if buffers are not provided to littlefs
|
||||
//
|
||||
// littlefs current has no alignment requirements, as it only allocates
|
||||
// byte-level buffers.
|
||||
static inline void *lfs_malloc(size_t size) {
|
||||
#if defined(LFS_MALLOC)
|
||||
return LFS_MALLOC(size);
|
||||
#elif !defined(LFS_NO_MALLOC)
|
||||
return malloc(size);
|
||||
#else
|
||||
(void)size;
|
||||
return NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Deallocate memory, only used if buffers are not provided to littlefs
|
||||
static inline void lfs_free(void *p) {
|
||||
#if defined(LFS_FREE)
|
||||
LFS_FREE(p);
|
||||
#elif !defined(LFS_NO_MALLOC)
|
||||
free(p);
|
||||
#else
|
||||
(void)p;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif
|
||||
#endif
|
||||
+4304
File diff suppressed because it is too large
Load Diff
@@ -1,85 +0,0 @@
|
||||
# Scripts
|
||||
|
||||
This directory contains utility scripts for ToastyFS development and testing.
|
||||
|
||||
## Cluster Demo Script
|
||||
|
||||
The `cluster_demo.sh` script allows you to easily spawn and manage a ToastyFS cluster for demo and testing purposes.
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Start a cluster with 3 chunk servers (default)
|
||||
./scripts/cluster_demo.sh start
|
||||
|
||||
# Start a cluster with a custom number of chunk servers
|
||||
./scripts/cluster_demo.sh start 5
|
||||
|
||||
# Check cluster status
|
||||
./scripts/cluster_demo.sh status
|
||||
|
||||
# Stop the cluster
|
||||
./scripts/cluster_demo.sh stop
|
||||
|
||||
# Clean all cluster data and logs
|
||||
./scripts/cluster_demo.sh clean
|
||||
```
|
||||
|
||||
### What it does
|
||||
|
||||
The script:
|
||||
- Builds ToastyFS if needed
|
||||
- Starts one metadata server (leader) on port 8080
|
||||
- Starts the specified number of chunk servers on ports 8081, 8082, etc.
|
||||
- Stores all process IDs for easy management
|
||||
- Captures logs to `cluster_logs/` directory
|
||||
- Stores data in `cluster_data/` directory
|
||||
|
||||
All servers run in the background, making it easy to test the cluster with client applications.
|
||||
|
||||
## Branch Coverage Measurement
|
||||
|
||||
### Prerequisites
|
||||
|
||||
The HTML report generation requires `lcov`:
|
||||
|
||||
```bash
|
||||
sudo apt-get install lcov
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Quick text summary (5 second simulation)
|
||||
```bash
|
||||
make coverage-report
|
||||
```
|
||||
|
||||
### HTML report with branch details (5 second simulation)
|
||||
```bash
|
||||
make coverage-html
|
||||
```
|
||||
|
||||
### Custom duration
|
||||
```bash
|
||||
./scripts/measure_coverage.sh 10 # 10 second run, text output
|
||||
./scripts/measure_coverage.sh 10 --html # 10 second run, HTML output
|
||||
```
|
||||
|
||||
## What gets measured
|
||||
|
||||
The coverage tool:
|
||||
- Builds the test binary with coverage instrumentation (`--coverage` flag)
|
||||
- Runs the random simulation for the specified duration
|
||||
- Generates reports showing which branches were executed
|
||||
|
||||
## Output
|
||||
|
||||
**Text report**: Shows per-file and total branch coverage percentages
|
||||
|
||||
**HTML report**: Interactive report generated by lcov/genhtml showing:
|
||||
- Overall coverage summary
|
||||
- Per-file coverage breakdown
|
||||
- Source code with execution counts
|
||||
- Branch coverage details
|
||||
|
||||
The HTML report is generated in `coverage_report/index.html`
|
||||
@@ -1,314 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# ToastyFS Cluster Demo Script
|
||||
# ===========================
|
||||
# This script allows you to easily spawn and manage a ToastyFS cluster for demo purposes.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/cluster_demo.sh start [num_chunk_servers] - Start a cluster
|
||||
# ./scripts/cluster_demo.sh stop - Stop the cluster
|
||||
# ./scripts/cluster_demo.sh status - Show cluster status
|
||||
# ./scripts/cluster_demo.sh clean - Clean data and log files
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
PID_FILE="$PROJECT_DIR/.cluster_demo.pids"
|
||||
LOG_DIR="$PROJECT_DIR/cluster_logs"
|
||||
DATA_DIR="$PROJECT_DIR/cluster_data"
|
||||
|
||||
METADATA_PORT=8080
|
||||
CHUNK_SERVER_BASE_PORT=8081
|
||||
WEB_SERVER_PORT=8090
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
print_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
build_if_needed() {
|
||||
local need_build=false
|
||||
|
||||
if [ ! -f "$PROJECT_DIR/toastyfs.out" ] && [ ! -f "$PROJECT_DIR/toastyfs.exe" ]; then
|
||||
need_build=true
|
||||
fi
|
||||
|
||||
if [ ! -f "$PROJECT_DIR/toastyfs_web.out" ] && [ ! -f "$PROJECT_DIR/toastyfs_web.exe" ]; then
|
||||
need_build=true
|
||||
fi
|
||||
|
||||
if [ "$need_build" = true ]; then
|
||||
print_info "Binaries not found. Building ToastyFS..."
|
||||
cd "$PROJECT_DIR"
|
||||
make
|
||||
print_success "Build complete"
|
||||
fi
|
||||
}
|
||||
|
||||
get_binary() {
|
||||
if [ -f "$PROJECT_DIR/toastyfs.out" ]; then
|
||||
echo "$PROJECT_DIR/toastyfs.out"
|
||||
elif [ -f "$PROJECT_DIR/toastyfs.exe" ]; then
|
||||
echo "$PROJECT_DIR/toastyfs.exe"
|
||||
else
|
||||
print_error "ToastyFS binary not found"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
get_web_binary() {
|
||||
if [ -f "$PROJECT_DIR/toastyfs_web.out" ]; then
|
||||
echo "$PROJECT_DIR/toastyfs_web.out"
|
||||
elif [ -f "$PROJECT_DIR/toastyfs_web.exe" ]; then
|
||||
echo "$PROJECT_DIR/toastyfs_web.exe"
|
||||
else
|
||||
print_error "ToastyFS web binary not found"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
start_cluster() {
|
||||
local num_chunk_servers=${1:-3}
|
||||
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
print_error "Cluster already running (PID file exists)"
|
||||
print_info "Run '$0 stop' first or remove $PID_FILE if the cluster is not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build_if_needed
|
||||
local binary=$(get_binary)
|
||||
|
||||
# Create directories
|
||||
mkdir -p "$LOG_DIR"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
print_info "Starting ToastyFS cluster..."
|
||||
print_info " Metadata server: 127.0.0.1:$METADATA_PORT"
|
||||
print_info " Chunk servers: $num_chunk_servers"
|
||||
print_info " Web server: 127.0.0.1:$WEB_SERVER_PORT"
|
||||
echo
|
||||
|
||||
# Start metadata server
|
||||
print_info "Starting metadata server on port $METADATA_PORT..."
|
||||
local metadata_wal="$DATA_DIR/metadata.wal"
|
||||
"$binary" --leader \
|
||||
--addr 127.0.0.1 \
|
||||
--port $METADATA_PORT \
|
||||
--wal-file "$metadata_wal" \
|
||||
> "$LOG_DIR/metadata.log" 2>&1 &
|
||||
|
||||
local metadata_pid=$!
|
||||
echo "$metadata_pid" > "$PID_FILE"
|
||||
print_success "Metadata server started (PID: $metadata_pid)"
|
||||
|
||||
# Wait a bit for metadata server to start
|
||||
sleep 1
|
||||
|
||||
# Start chunk servers
|
||||
for i in $(seq 1 $num_chunk_servers); do
|
||||
local port=$((CHUNK_SERVER_BASE_PORT + i - 1))
|
||||
local data_path="$DATA_DIR/chunk_server_$i/"
|
||||
|
||||
print_info "Starting chunk server $i on port $port..."
|
||||
mkdir -p "$data_path"
|
||||
|
||||
"$binary" \
|
||||
--addr 127.0.0.1 \
|
||||
--port $port \
|
||||
--path "$data_path" \
|
||||
--remote-addr 127.0.0.1 \
|
||||
--remote-port $METADATA_PORT \
|
||||
> "$LOG_DIR/chunk_server_$i.log" 2>&1 &
|
||||
|
||||
local chunk_pid=$!
|
||||
echo "$chunk_pid" >> "$PID_FILE"
|
||||
print_success "Chunk server $i started (PID: $chunk_pid)"
|
||||
|
||||
# Small delay between starting servers
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
# Start web server
|
||||
local web_binary=$(get_web_binary)
|
||||
print_info "Starting web server on port $WEB_SERVER_PORT..."
|
||||
"$web_binary" \
|
||||
--upstream-addr 127.0.0.1 \
|
||||
--upstream-port $METADATA_PORT \
|
||||
--local-addr 127.0.0.1 \
|
||||
--local-port $WEB_SERVER_PORT \
|
||||
> "$LOG_DIR/web_server.log" 2>&1 &
|
||||
|
||||
local web_pid=$!
|
||||
echo "$web_pid" >> "$PID_FILE"
|
||||
print_success "Web server started (PID: $web_pid)"
|
||||
|
||||
echo
|
||||
print_success "Cluster started successfully!"
|
||||
echo
|
||||
print_info "Connect to the cluster using:"
|
||||
print_info " ToastyFS Protocol: 127.0.0.1:$METADATA_PORT"
|
||||
print_info " HTTP Interface: http://127.0.0.1:$WEB_SERVER_PORT"
|
||||
echo
|
||||
print_info "View logs at: $LOG_DIR/"
|
||||
print_info "Data stored at: $DATA_DIR/"
|
||||
echo
|
||||
print_info "To stop the cluster, run: $0 stop"
|
||||
}
|
||||
|
||||
stop_cluster() {
|
||||
if [ ! -f "$PID_FILE" ]; then
|
||||
print_warning "No PID file found. Cluster may not be running."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_info "Stopping ToastyFS cluster..."
|
||||
|
||||
local stopped=0
|
||||
local failed=0
|
||||
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ]; then
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null && {
|
||||
print_success "Stopped process $pid"
|
||||
stopped=$((stopped + 1))
|
||||
} || {
|
||||
print_error "Failed to stop process $pid"
|
||||
failed=$((failed + 1))
|
||||
}
|
||||
else
|
||||
print_warning "Process $pid not running"
|
||||
fi
|
||||
fi
|
||||
done < "$PID_FILE"
|
||||
|
||||
rm -f "$PID_FILE"
|
||||
|
||||
echo
|
||||
print_success "Cluster stopped (Stopped: $stopped, Failed: $failed)"
|
||||
}
|
||||
|
||||
show_status() {
|
||||
if [ ! -f "$PID_FILE" ]; then
|
||||
print_info "Cluster is NOT running"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_info "Cluster status:"
|
||||
echo
|
||||
|
||||
local running=0
|
||||
local not_running=0
|
||||
local line_num=0
|
||||
local total_lines=$(wc -l < "$PID_FILE")
|
||||
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ]; then
|
||||
line_num=$((line_num + 1))
|
||||
local server_type=""
|
||||
|
||||
if [ $line_num -eq 1 ]; then
|
||||
server_type="Metadata server"
|
||||
elif [ $line_num -eq $total_lines ]; then
|
||||
server_type="Web server"
|
||||
else
|
||||
server_type="Chunk server $((line_num - 1))"
|
||||
fi
|
||||
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
echo -e " ${GREEN}●${NC} $server_type (PID: $pid) - ${GREEN}running${NC}"
|
||||
running=$((running + 1))
|
||||
else
|
||||
echo -e " ${RED}●${NC} $server_type (PID: $pid) - ${RED}not running${NC}"
|
||||
not_running=$((not_running + 1))
|
||||
fi
|
||||
fi
|
||||
done < "$PID_FILE"
|
||||
|
||||
echo
|
||||
if [ $not_running -eq 0 ]; then
|
||||
print_success "All $running servers are running"
|
||||
else
|
||||
print_warning "$running running, $not_running not running"
|
||||
fi
|
||||
|
||||
if [ -d "$LOG_DIR" ]; then
|
||||
echo
|
||||
print_info "Log directory: $LOG_DIR"
|
||||
print_info "Recent log files:"
|
||||
ls -lht "$LOG_DIR" | head -n 6 | tail -n 5 | awk '{print " " $9 " (" $5 ")"}'
|
||||
fi
|
||||
}
|
||||
|
||||
clean_data() {
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
print_error "Cluster is running. Stop it first with: $0 stop"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_warning "This will delete all cluster data and logs!"
|
||||
read -p "Are you sure? (y/N) " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Cleaning data and logs..."
|
||||
rm -rf "$DATA_DIR"
|
||||
rm -rf "$LOG_DIR"
|
||||
print_success "Cleaned successfully"
|
||||
else
|
||||
print_info "Cancelled"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main command dispatcher
|
||||
case "${1:-}" in
|
||||
start)
|
||||
start_cluster "${2:-3}"
|
||||
;;
|
||||
stop)
|
||||
stop_cluster
|
||||
;;
|
||||
status)
|
||||
show_status
|
||||
;;
|
||||
clean)
|
||||
clean_data
|
||||
;;
|
||||
*)
|
||||
echo "ToastyFS Cluster Demo Script"
|
||||
echo
|
||||
echo "Usage:"
|
||||
echo " $0 start [num_chunk_servers] - Start a cluster (default: 3 chunk servers)"
|
||||
echo " $0 stop - Stop the cluster"
|
||||
echo " $0 status - Show cluster status"
|
||||
echo " $0 clean - Clean data and log files"
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo " $0 start # Start with 3 chunk servers"
|
||||
echo " $0 start 5 # Start with 5 chunk servers"
|
||||
echo " $0 status # Check if cluster is running"
|
||||
echo " $0 stop # Stop all servers"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Fix line endings for coverage scripts
|
||||
|
||||
echo "Fixing line endings for coverage scripts..."
|
||||
|
||||
# Convert CRLF to LF
|
||||
dos2unix scripts/measure_coverage.sh 2>/dev/null || sed -i 's/\r$//' scripts/measure_coverage.sh
|
||||
dos2unix scripts/generate_coverage_html.sh 2>/dev/null || sed -i 's/\r$//' scripts/generate_coverage_html.sh
|
||||
|
||||
# Ensure execute permissions
|
||||
chmod +x scripts/measure_coverage.sh
|
||||
chmod +x scripts/generate_coverage_html.sh
|
||||
|
||||
echo "Done! Line endings fixed and execute permissions set."
|
||||
echo ""
|
||||
echo "You can now run:"
|
||||
echo " make coverage-report"
|
||||
echo " make coverage-html"
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Generate HTML coverage report using lcov
|
||||
|
||||
set -e
|
||||
|
||||
OUTPUT_DIR="coverage_report"
|
||||
|
||||
# Check if lcov is installed
|
||||
if ! command -v lcov &> /dev/null; then
|
||||
echo "Error: lcov is not installed"
|
||||
echo "Please install it with: sudo apt-get install lcov"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Capture coverage data with branch coverage enabled
|
||||
echo "Capturing coverage data..."
|
||||
lcov --capture --directory . --output-file coverage.info --rc lcov_branch_coverage=1
|
||||
|
||||
# Filter out system headers if any exist
|
||||
echo "Filtering coverage data..."
|
||||
lcov --remove coverage.info '/usr/*' --output-file coverage.info --ignore-errors unused --rc lcov_branch_coverage=1 || cp coverage.info coverage.info.bak
|
||||
|
||||
# Generate HTML report
|
||||
echo "Generating HTML report..."
|
||||
genhtml coverage.info --output-directory "$OUTPUT_DIR" --branch-coverage --rc lcov_branch_coverage=1
|
||||
|
||||
# Clean up
|
||||
rm -f coverage.info
|
||||
|
||||
echo "HTML report generated in $OUTPUT_DIR/"
|
||||
echo "Open $OUTPUT_DIR/index.html in your browser"
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Script to measure branch coverage of the random simulation
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Branch Coverage Measurement Tool${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo
|
||||
|
||||
# Clean previous coverage data
|
||||
echo -e "${YELLOW}Cleaning previous coverage data...${NC}"
|
||||
rm -f src/*.gcda src/*.gcno *.gcda *.gcno
|
||||
make clean > /dev/null 2>&1
|
||||
|
||||
# Build with coverage
|
||||
echo -e "${YELLOW}Building with coverage instrumentation...${NC}"
|
||||
make coverage
|
||||
|
||||
# Run the simulation for a limited time
|
||||
echo -e "${YELLOW}Running simulation...${NC}"
|
||||
SIMULATION_TIME=${1:-5} # Default to 5 seconds if not specified
|
||||
echo "Running for ${SIMULATION_TIME} seconds..."
|
||||
|
||||
# Run simulation in background and kill it after specified time
|
||||
timeout ${SIMULATION_TIME}s ./toastyfs_random_test_coverage.out || true
|
||||
|
||||
# Generate coverage reports
|
||||
echo
|
||||
echo -e "${YELLOW}Generating coverage reports...${NC}"
|
||||
|
||||
# Find all .gcda files and generate coverage reports
|
||||
GCDA_FILES=$(find . -name "*.gcda")
|
||||
|
||||
TOTAL_BRANCHES=0
|
||||
TAKEN_BRANCHES=0
|
||||
|
||||
# Generate gcov reports from the coverage data files
|
||||
for gcda_file in $GCDA_FILES; do
|
||||
# Extract the source file name from the gcda filename
|
||||
# Files are named like: toastyfs_random_test_coverage.out-basic.gcda
|
||||
basename=$(basename "$gcda_file" .gcda)
|
||||
source_name=${basename#*-} # Remove prefix up to and including '-'
|
||||
|
||||
# Run gcov to generate the .gcov file
|
||||
gcov -b "$gcda_file" > /dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Parse gcov output to count branches
|
||||
echo
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Branch Coverage Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo
|
||||
|
||||
# Process each .gcov file
|
||||
for gcov_file in *.c.gcov; do
|
||||
if [ -f "$gcov_file" ]; then
|
||||
# Extract branch statistics from the gcov file
|
||||
branches=$(grep -c "^branch" "$gcov_file" 2>/dev/null)
|
||||
if [ -z "$branches" ]; then branches=0; fi
|
||||
|
||||
if [ "$branches" -gt 0 ]; then
|
||||
taken_count=$(grep "^branch" "$gcov_file" 2>/dev/null | grep -c "taken [1-9]" 2>/dev/null || echo "0")
|
||||
# Clean up the value - remove any whitespace/newlines
|
||||
taken_count=$(echo "$taken_count" | tr -d '[:space:]')
|
||||
if [ -z "$taken_count" ] || [ "$taken_count" = "" ]; then taken_count=0; fi
|
||||
|
||||
TOTAL_BRANCHES=$((TOTAL_BRANCHES + branches))
|
||||
TAKEN_BRANCHES=$((TAKEN_BRANCHES + taken_count))
|
||||
percentage=$((taken_count * 100 / branches))
|
||||
filename=$(echo "$gcov_file" | sed 's/.gcov$//')
|
||||
printf "%-40s %5d / %5d branches (%3d%%)\n" "$filename" "$taken_count" "$branches" "$percentage"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
if [ "$TOTAL_BRANCHES" -gt 0 ]; then
|
||||
COVERAGE_PERCENT=$((TAKEN_BRANCHES * 100 / TOTAL_BRANCHES))
|
||||
echo -e "${GREEN}Total: $TAKEN_BRANCHES / $TOTAL_BRANCHES branches reached (${COVERAGE_PERCENT}%)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}No branch coverage data found${NC}"
|
||||
fi
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo
|
||||
|
||||
# Generate HTML report if requested
|
||||
if [ "$2" == "--html" ]; then
|
||||
echo -e "${YELLOW}Generating HTML coverage report...${NC}"
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
"$SCRIPT_DIR/generate_coverage_html.sh"
|
||||
echo -e "${GREEN}HTML report generated in coverage_report/index.html${NC}"
|
||||
echo "Open with: firefox coverage_report/index.html (or your preferred browser)"
|
||||
echo
|
||||
fi
|
||||
|
||||
# Clean up gcov files unless HTML was requested
|
||||
if [ "$2" != "--html" ] && [ "$2" != "--detailed" ]; then
|
||||
echo -e "${YELLOW}Cleaning up coverage files...${NC}"
|
||||
rm -f *.gcov
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}Coverage measurement complete!${NC}"
|
||||
+17
-9
@@ -1,7 +1,9 @@
|
||||
#include <string.h>
|
||||
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
#include "basic.h"
|
||||
#include "system.h"
|
||||
|
||||
bool streq(string s1, string s2)
|
||||
{
|
||||
@@ -24,10 +26,10 @@ Time get_current_time(void)
|
||||
int64_t freq;
|
||||
int ok;
|
||||
|
||||
ok = sys_QueryPerformanceCounter((LARGE_INTEGER*) &count);
|
||||
ok = QueryPerformanceCounter((LARGE_INTEGER*) &count);
|
||||
if (!ok) return INVALID_TIME;
|
||||
|
||||
ok = sys_QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
|
||||
ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq);
|
||||
if (!ok) return INVALID_TIME;
|
||||
|
||||
uint64_t res = 1000000000 * (double) count / freq;
|
||||
@@ -37,7 +39,7 @@ Time get_current_time(void)
|
||||
{
|
||||
struct timespec time;
|
||||
|
||||
if (sys_clock_gettime(CLOCK_REALTIME, &time))
|
||||
if (clock_gettime(CLOCK_REALTIME, &time))
|
||||
return INVALID_TIME;
|
||||
|
||||
uint64_t res;
|
||||
@@ -99,11 +101,17 @@ int getargi(int argc, char **argv, char *name, int fallback)
|
||||
if (i == argc)
|
||||
break;
|
||||
|
||||
int tmp = atoi(argv[i]);
|
||||
if (tmp == 0 && argv[i][0] != '0') // best effort
|
||||
errno = 0;
|
||||
char *end;
|
||||
long val = strtol(argv[i], &end, 10);
|
||||
|
||||
if (end == argv[i] || *end != '\0' || errno == ERANGE)
|
||||
break;
|
||||
|
||||
return tmp;
|
||||
if (val < INT_MIN || val > INT_MAX)
|
||||
break;
|
||||
|
||||
return (int) val;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
+12
-9
@@ -1,8 +1,11 @@
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <stdint.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "system.h"
|
||||
#include "byte_queue.h"
|
||||
|
||||
// This is the implementation of a byte queue useful
|
||||
@@ -32,12 +35,12 @@ void byte_queue_free(ByteQueue *queue)
|
||||
{
|
||||
if (queue->read_target) {
|
||||
if (queue->read_target != queue->data)
|
||||
sys_free(queue->read_target);
|
||||
free(queue->read_target);
|
||||
queue->read_target = NULL;
|
||||
queue->read_target_size = 0;
|
||||
}
|
||||
|
||||
sys_free(queue->data);
|
||||
free(queue->data);
|
||||
queue->data = NULL;
|
||||
}
|
||||
|
||||
@@ -99,7 +102,7 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num)
|
||||
|
||||
if (queue->read_target) {
|
||||
if (queue->read_target != queue->data)
|
||||
sys_free(queue->read_target);
|
||||
free(queue->read_target);
|
||||
queue->read_target = NULL;
|
||||
queue->read_target_size = 0;
|
||||
}
|
||||
@@ -224,7 +227,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
|
||||
if (size > queue->limit)
|
||||
size = queue->limit;
|
||||
|
||||
uint8_t *data = sys_malloc(size);
|
||||
uint8_t *data = malloc(size);
|
||||
if (!data) {
|
||||
queue->flags |= BYTE_QUEUE_ERROR;
|
||||
return 0;
|
||||
@@ -234,7 +237,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap)
|
||||
memcpy(data, queue->data + queue->head, queue->used);
|
||||
|
||||
if (queue->read_target != queue->data)
|
||||
sys_free(queue->data);
|
||||
free(queue->data);
|
||||
|
||||
queue->data = data;
|
||||
queue->head = 0;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#ifndef BYTE_QUEUE_INCLUDED
|
||||
#define BYTE_QUEUE_INCLUDED
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "basic.h"
|
||||
|
||||
typedef struct {
|
||||
|
||||
+71
-37
@@ -1,14 +1,18 @@
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "metadata_server.h"
|
||||
#include "sha256.h"
|
||||
#include "message.h"
|
||||
#include "file_system.h"
|
||||
#include "hash_set.h"
|
||||
#include "config.h"
|
||||
#include "tcp.h"
|
||||
#include "message.h"
|
||||
#include "byte_queue.h"
|
||||
#include "chunk_server.h"
|
||||
|
||||
static string hash2path(ChunkServer *state, SHA256 hash, char *out)
|
||||
@@ -99,12 +103,12 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
|
||||
return -1;
|
||||
|
||||
if (patch_off > SIZE_MAX - patch.len) {
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (patch_off + (size_t) patch.len > (size_t) data.len) {
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -112,11 +116,11 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk,
|
||||
|
||||
ret = store_chunk(state, data, new_hash);
|
||||
if (ret < 0) {
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return -1;
|
||||
}
|
||||
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -129,7 +133,7 @@ static void download_targets_init(DownloadTargets *targets)
|
||||
|
||||
static void download_targets_free(DownloadTargets *targets)
|
||||
{
|
||||
sys_free(targets->items);
|
||||
free(targets->items);
|
||||
}
|
||||
|
||||
static void download_targets_remove(DownloadTargets *targets,
|
||||
@@ -161,13 +165,13 @@ static int download_targets_push(DownloadTargets *targets,
|
||||
else
|
||||
new_capacity = 2 * targets->capacity;
|
||||
|
||||
DownloadTarget *new_items = sys_malloc(new_capacity * sizeof(DownloadTarget));
|
||||
DownloadTarget *new_items = malloc(new_capacity * sizeof(DownloadTarget));
|
||||
if (new_items == NULL)
|
||||
return -1;
|
||||
|
||||
if (targets->capacity > 0) {
|
||||
memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0]));
|
||||
sys_free(targets->items);
|
||||
free(targets->items);
|
||||
}
|
||||
|
||||
targets->items = new_items;
|
||||
@@ -563,7 +567,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
|
||||
if (binary_read(&reader, NULL, 1))
|
||||
return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message"));
|
||||
|
||||
char *mem = sys_malloc(chunk_size);
|
||||
char *mem = malloc(chunk_size);
|
||||
if (mem == NULL)
|
||||
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory"));
|
||||
|
||||
@@ -579,7 +583,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg)
|
||||
SHA256 dummy;
|
||||
int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy);
|
||||
|
||||
sys_free(mem);
|
||||
free(mem);
|
||||
|
||||
if (ret < 0)
|
||||
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error"));
|
||||
@@ -694,7 +698,7 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
|
||||
string slice;
|
||||
if (full == 0) {
|
||||
if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) {
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range"));
|
||||
}
|
||||
slice = (string) { data.ptr + target_off, target_len };
|
||||
@@ -710,11 +714,11 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg)
|
||||
message_write(&writer, &target_len, sizeof(target_len));
|
||||
message_write(&writer, slice.ptr, slice.len);
|
||||
if (!message_writer_free(&writer)) {
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return -1;
|
||||
}
|
||||
|
||||
sys_free(data.ptr);
|
||||
free(data.ptr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -812,8 +816,12 @@ static int send_sync_message(ChunkServer *state)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout)
|
||||
int chunk_server_init(void *state_, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout)
|
||||
{
|
||||
ChunkServer *state = state_;
|
||||
|
||||
string addr = getargs(argc, argv, "--addr", "127.0.0.1");
|
||||
int port = getargi(argc, argv, "--port", 8081);
|
||||
string path = getargs(argc, argv, "--path", "chunk_server_data/");
|
||||
@@ -821,34 +829,45 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1");
|
||||
int remote_port = getargi(argc, argv, "--remote-port", 8080);
|
||||
|
||||
if (port <= 0 || port >= 1<<16)
|
||||
if (port <= 0 || port >= 1<<16) {
|
||||
fprintf(stderr, "chunk server :: Invalid port\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (remote_port <= 0 || remote_port >= 1<<16)
|
||||
if (remote_port <= 0 || remote_port >= 1<<16) {
|
||||
fprintf(stderr, "chunk server :: Invalid remote port\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
Time current_time = get_current_time();
|
||||
if (current_time == INVALID_TIME)
|
||||
if (current_time == INVALID_TIME) {
|
||||
fprintf(stderr, "chunk server :: Couldn't read the time\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
state->trace = trace;
|
||||
state->reconnect_delay = 1; // 1 second
|
||||
|
||||
if (tcp_context_init(&state->tcp) < 0)
|
||||
if (tcp_context_init(&state->tcp) < 0) {
|
||||
fprintf(stderr, "chunk server :: Couldn't setup the TCP context\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
int ret = tcp_listen(&state->tcp, addr, port);
|
||||
if (ret < 0) {
|
||||
fprintf(stderr, "chunk server :: Couldn't setup the TCP listener\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (create_dir(path) && errno != EEXIST) {
|
||||
fprintf(stderr, "chunk server :: Couldn't create chunk folder\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (get_full_path(path, state->path) < 0) {
|
||||
fprintf(stderr, "chunk server :: Couldn't convert path to absolute\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
@@ -861,6 +880,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
|
||||
char tmp[1<<10];
|
||||
if (addr.len >= (int) sizeof(tmp)) {
|
||||
fprintf(stderr, "chunk server :: Address is too long\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
@@ -868,6 +888,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
tmp[addr.len] = '\0';
|
||||
state->local_addr.is_ipv4 = true;
|
||||
if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) {
|
||||
fprintf(stderr, "chunk server :: Couldn't parse address\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
@@ -876,6 +897,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
// Initialize metadata server address
|
||||
// // TODO: This should also support IPv6
|
||||
if (remote_addr.len >= (int) sizeof(tmp)) {
|
||||
fprintf(stderr, "chunk server :: Remote address is too long\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
@@ -883,6 +905,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
tmp[remote_addr.len] = '\0';
|
||||
state->remote_addr.is_ipv4 = true;
|
||||
if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) {
|
||||
fprintf(stderr, "chunk server :: Couldn't parse remote address\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
@@ -904,23 +927,20 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts
|
||||
);
|
||||
|
||||
*timeout = 0;
|
||||
return tcp_register_events(&state->tcp, contexts, polled);
|
||||
}
|
||||
|
||||
int chunk_server_free(ChunkServer *state)
|
||||
{
|
||||
download_targets_free(&state->download_targets);
|
||||
timed_hash_set_free(&state->cs_rem_list);
|
||||
hash_set_free(&state->cs_lst_list);
|
||||
hash_set_free(&state->cs_add_list);
|
||||
tcp_context_free(&state->tcp);
|
||||
if (pcap < TCP_POLL_CAPACITY) {
|
||||
fprintf(stderr, "chunk server :: Capacity isn't large enough\n");
|
||||
return -1;
|
||||
}
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
|
||||
int chunk_server_tick(void *state_, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
|
||||
{
|
||||
ChunkServer *state = state_;
|
||||
Event events[TCP_EVENT_CAPACITY];
|
||||
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
|
||||
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
|
||||
|
||||
Time current_time = get_current_time();
|
||||
if (current_time == INVALID_TIME)
|
||||
@@ -1031,5 +1051,19 @@ int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled
|
||||
}
|
||||
|
||||
*timeout = deadline_to_timeout(deadline, current_time);
|
||||
return tcp_register_events(&state->tcp, contexts, polled);
|
||||
if (pcap < TCP_POLL_CAPACITY)
|
||||
return -1;
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chunk_server_free(void *state_)
|
||||
{
|
||||
ChunkServer *state = state_;
|
||||
download_targets_free(&state->download_targets);
|
||||
timed_hash_set_free(&state->cs_rem_list);
|
||||
hash_set_free(&state->cs_lst_list);
|
||||
hash_set_free(&state->cs_add_list);
|
||||
tcp_context_free(&state->tcp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
+11
-9
@@ -1,12 +1,6 @@
|
||||
#ifndef CHUNK_SERVER_INCLUDED
|
||||
#define CHUNK_SERVER_INCLUDED
|
||||
|
||||
#include <limits.h>
|
||||
|
||||
#include "basic.h"
|
||||
#include "metadata_server.h"
|
||||
#include "tcp.h"
|
||||
|
||||
#define TAG_METADATA_SERVER 1
|
||||
#define TAG_CHUNK_SERVER 2
|
||||
|
||||
@@ -57,8 +51,16 @@ typedef struct {
|
||||
|
||||
} ChunkServer;
|
||||
|
||||
int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout);
|
||||
int chunk_server_free(ChunkServer *state);
|
||||
int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
|
||||
struct pollfd;
|
||||
|
||||
int chunk_server_init(void *state, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int chunk_server_tick(void *state, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int chunk_server_free(void *state);
|
||||
|
||||
#endif // CHUNK_SERVER_INCLUDED
|
||||
+42
-40
@@ -1,22 +1,24 @@
|
||||
#include <assert.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
# define POLL WSAPoll
|
||||
#else
|
||||
# define POLL poll
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#define POLL WSAPoll
|
||||
typedef CRITICAL_SECTION Mutex;
|
||||
#else
|
||||
#include <pthread.h>
|
||||
#include <arpa/inet.h>
|
||||
#define POLL poll
|
||||
typedef pthread_mutex_t Mutex;
|
||||
#endif
|
||||
|
||||
#include "tcp.h"
|
||||
#include "system.h"
|
||||
#include "config.h"
|
||||
#include "message.h"
|
||||
#include "file_tree.h"
|
||||
@@ -322,7 +324,7 @@ static int handle_to_operation(ToastyFS *toasty, ToastyHandle handle)
|
||||
|
||||
ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
|
||||
{
|
||||
ToastyFS *toasty = sys_malloc(sizeof(ToastyFS));
|
||||
ToastyFS *toasty = malloc(sizeof(ToastyFS));
|
||||
if (toasty == NULL)
|
||||
return NULL;
|
||||
|
||||
@@ -330,7 +332,7 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
|
||||
{
|
||||
char tmp[128];
|
||||
if (addr.len >= (int) sizeof(tmp)) {
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(tmp, addr.ptr, addr.len);
|
||||
@@ -339,26 +341,26 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port)
|
||||
addr2.is_ipv4 = true;
|
||||
addr2.port = port;
|
||||
if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) {
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (mutex_init(&toasty->mutex) < 0) {
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (tcp_context_init(&toasty->tcp) < 0) {
|
||||
mutex_free(&toasty->mutex);
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) {
|
||||
tcp_context_free(&toasty->tcp);
|
||||
mutex_free(&toasty->mutex);
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -435,7 +437,7 @@ void toasty_disconnect(ToastyFS *toasty)
|
||||
{
|
||||
tcp_context_free(&toasty->tcp);
|
||||
mutex_free(&toasty->mutex);
|
||||
sys_free(toasty);
|
||||
free(toasty);
|
||||
}
|
||||
|
||||
static bool
|
||||
@@ -947,7 +949,7 @@ static void process_event_for_list(ToastyFS *toasty,
|
||||
return;
|
||||
}
|
||||
|
||||
ToastyListingEntry *entities = sys_malloc(item_count * sizeof(ToastyListingEntry));
|
||||
ToastyListingEntry *entities = malloc(item_count * sizeof(ToastyListingEntry));
|
||||
if (entities == NULL) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
@@ -959,28 +961,28 @@ static void process_event_for_list(ToastyFS *toasty,
|
||||
uint64_t gen;
|
||||
if (!binary_read(&reader, &gen, sizeof(gen))) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t is_dir;
|
||||
if (!binary_read(&reader, &is_dir, sizeof(is_dir))) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
|
||||
uint16_t name_len;
|
||||
if (!binary_read(&reader, &name_len, sizeof(name_len))) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
|
||||
char *name = (char*) reader.src + reader.cur;
|
||||
if (!binary_read(&reader, NULL, name_len)) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -989,7 +991,7 @@ static void process_event_for_list(ToastyFS *toasty,
|
||||
|
||||
if (name_len > sizeof(entities[i].name)-1) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
memcpy(entities[i].name, name, name_len);
|
||||
@@ -999,7 +1001,7 @@ static void process_event_for_list(ToastyFS *toasty,
|
||||
// Check there is nothing else to read
|
||||
if (binary_read(&reader, NULL, 1)) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user };
|
||||
sys_free(entities);
|
||||
free(entities);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1089,7 +1091,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
}
|
||||
|
||||
// Allocate ranges
|
||||
Range *ranges = sys_malloc(num_chunks_needed * sizeof(Range));
|
||||
Range *ranges = malloc(num_chunks_needed * sizeof(Range));
|
||||
if (ranges == NULL) {
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
@@ -1104,7 +1106,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
// Read hash
|
||||
SHA256 hash;
|
||||
if (!binary_read(&reader, &hash, sizeof(hash))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1112,7 +1114,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
// Read number of servers
|
||||
uint32_t num_servers;
|
||||
if (!binary_read(&reader, &num_servers, sizeof(num_servers))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1120,7 +1122,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
// Parse IPv4 addresses
|
||||
uint32_t num_ipv4;
|
||||
if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1134,7 +1136,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
uint16_t port;
|
||||
if (!binary_read(&reader, &ipv4, sizeof(ipv4)) ||
|
||||
!binary_read(&reader, &port, sizeof(port))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1149,21 +1151,21 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
// Skip IPv6 addresses
|
||||
uint32_t num_ipv6;
|
||||
if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
for (uint32_t j = 0; j < num_ipv6; j++) {
|
||||
if (!binary_read(&reader, NULL, sizeof(IPv6)) ||
|
||||
!binary_read(&reader, NULL, sizeof(uint16_t))) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1208,7 +1210,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
Range *r = &ranges[0];
|
||||
int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL);
|
||||
if (cs_idx < 0) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1216,7 +1218,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
|
||||
if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk,
|
||||
r->length_within_chunk, opidx, 0) < 0) {
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user };
|
||||
return;
|
||||
}
|
||||
@@ -1225,7 +1227,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
toasty->operations[opidx].ranges_head = 1;
|
||||
} else {
|
||||
// No chunks to download
|
||||
sys_free(ranges);
|
||||
free(ranges);
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
|
||||
}
|
||||
|
||||
@@ -1299,7 +1301,7 @@ static void process_event_for_read(ToastyFS *toasty,
|
||||
|
||||
// Check if done
|
||||
if (toasty->operations[opidx].num_pending == 0) {
|
||||
sys_free(toasty->operations[opidx].ranges);
|
||||
free(toasty->operations[opidx].ranges);
|
||||
toasty->operations[opidx].ranges = NULL;
|
||||
toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes };
|
||||
}
|
||||
@@ -1424,7 +1426,7 @@ static int schedule_upload(ToastyFS *toasty, int opidx, UploadSchedule upload)
|
||||
else
|
||||
new_cap_uploads = 2 * o->cap_uploads;
|
||||
|
||||
UploadSchedule *uploads = sys_malloc(new_cap_uploads * sizeof(UploadSchedule));
|
||||
UploadSchedule *uploads = malloc(new_cap_uploads * sizeof(UploadSchedule));
|
||||
if (uploads == NULL)
|
||||
return -1;
|
||||
|
||||
@@ -1534,7 +1536,7 @@ static void process_event_for_write(ToastyFS *toasty,
|
||||
|
||||
toasty->operations[opidx].num_chunks = num_all_hasehs;
|
||||
toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow
|
||||
toasty->operations[opidx].hashes = sys_malloc(num_hashes * sizeof(SHA256));
|
||||
toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256));
|
||||
if (toasty->operations[opidx].hashes == NULL) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
@@ -1957,7 +1959,7 @@ static void process_event_for_write(ToastyFS *toasty,
|
||||
} ChunkUploadResult;
|
||||
|
||||
int num_upload_results = toasty->operations[opidx].num_chunks;
|
||||
ChunkUploadResult *upload_results = sys_malloc(num_upload_results * sizeof(ChunkUploadResult));
|
||||
ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult));
|
||||
if (upload_results == NULL) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
@@ -2471,7 +2473,7 @@ int toasty_list(ToastyFS *toasty, ToastyString path,
|
||||
|
||||
void toasty_free_listing(ToastyListing *listing)
|
||||
{
|
||||
sys_free(listing->items);
|
||||
free(listing->items);
|
||||
}
|
||||
|
||||
int toasty_read(ToastyFS *toasty, ToastyString path,
|
||||
|
||||
@@ -1,433 +0,0 @@
|
||||
#if defined(__linux__) && defined(__x86_64__)
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include <elf.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <signal.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdbool.h>
|
||||
#include <sys/reg.h>
|
||||
#include <sys/stat.h>
|
||||
#include <ucontext.h>
|
||||
|
||||
typedef struct {
|
||||
uint64_t base_addr;
|
||||
int count;
|
||||
void *symbols;
|
||||
void *strings;
|
||||
} SymbolTable;
|
||||
|
||||
static int is_hex(char c)
|
||||
{
|
||||
return (c >= '0' && c <= '9') ||
|
||||
(c >= 'a' && c <= 'f') ||
|
||||
(c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
static uint64_t query_base_addr(void)
|
||||
{
|
||||
int fd = open("/proc/self/maps", O_RDONLY);
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
|
||||
char buf[128];
|
||||
int ret = read(fd, buf, sizeof(buf));
|
||||
if (ret < 0) {
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
if (ret == 0 || !is_hex(buf[0]))
|
||||
return -1;
|
||||
|
||||
int i = 0;
|
||||
uint64_t base_addr = 0;
|
||||
for (;;) {
|
||||
char c = buf[i++];
|
||||
|
||||
int d;
|
||||
if (0) {}
|
||||
else if (c >= 'a' && c <= 'f') d = c - 'a' + 10;
|
||||
else if (c >= 'A' && c <= 'F') d = c - 'A' + 10;
|
||||
else d = c - '0';
|
||||
|
||||
if (base_addr > (UINT64_MAX - d) / 16)
|
||||
return -1;
|
||||
base_addr = base_addr * 16 + d;
|
||||
|
||||
if (i == ret)
|
||||
return -1;
|
||||
|
||||
if (buf[i] == '-')
|
||||
break;
|
||||
|
||||
if (!is_hex(buf[i]))
|
||||
return -1;
|
||||
}
|
||||
|
||||
return base_addr;
|
||||
}
|
||||
|
||||
static int current_executable_path(char *dst, int cap)
|
||||
{
|
||||
if (cap == 0)
|
||||
return -1;
|
||||
|
||||
int ret = readlink("/proc/self/exe", dst, cap-1);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
dst[ret] = '\0';
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int load_symbols_from_elf(void *src, int len, SymbolTable *st)
|
||||
{
|
||||
// NOTE: It's assumed is properly aligned
|
||||
assert(((uintptr_t) src & 15) == 0);
|
||||
|
||||
// Check that the file contains a full header
|
||||
if (len < (int) sizeof(Elf64_Ehdr))
|
||||
return -1;
|
||||
Elf64_Ehdr *ehdr = (Elf64_Ehdr*) src;
|
||||
|
||||
// Check that the file contains the full list
|
||||
// of section headers
|
||||
if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > len)
|
||||
return -1;
|
||||
Elf64_Shdr *shdrs = (Elf64_Shdr*) (src + ehdr->e_shoff);
|
||||
|
||||
Elf64_Shdr *shstrtab_hdr = &shdrs[ehdr->e_shstrndx]; // TODO: bounds check
|
||||
char *shstrtab = src + shstrtab_hdr->sh_offset;
|
||||
|
||||
// Iterate over the section headers to find the
|
||||
// one reative to symbols and their strings
|
||||
Elf64_Shdr *symtab_hdr = NULL;
|
||||
Elf64_Shdr *strtab_hdr = NULL;
|
||||
|
||||
for (int i = 0; i < ehdr->e_shnum; i++) {
|
||||
char *section_name = shstrtab + shdrs[i].sh_name;
|
||||
if (0) {}
|
||||
else if (!strcmp(section_name, ".symtab")) symtab_hdr = &shdrs[i];
|
||||
else if (!strcmp(section_name, ".strtab")) strtab_hdr = &shdrs[i];
|
||||
}
|
||||
|
||||
if (symtab_hdr == NULL || strtab_hdr == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *mem = malloc(symtab_hdr->sh_size + strtab_hdr->sh_size);
|
||||
if (mem == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
st->count = symtab_hdr->sh_size / sizeof(Elf64_Sym);
|
||||
st->symbols = mem;
|
||||
st->strings = (char*) st->symbols + symtab_hdr->sh_size;
|
||||
|
||||
memcpy(st->symbols, src + symtab_hdr->sh_offset, symtab_hdr->sh_size);
|
||||
memcpy(st->strings, src + strtab_hdr->sh_offset, strtab_hdr->sh_size);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static char *read_file(char *path, int *len)
|
||||
{
|
||||
int fd = open(path, O_RDONLY);
|
||||
if (fd < 0)
|
||||
return NULL;
|
||||
|
||||
struct stat buf;
|
||||
if (fstat(fd, &buf) < 0) {
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
*len = buf.st_size;
|
||||
|
||||
char *ptr = malloc(*len + 1);
|
||||
if (ptr == NULL) {
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (int num = 0; num < *len; ) {
|
||||
|
||||
int ret = read(fd, ptr + num, *len - num);
|
||||
if (ret <= 0) {
|
||||
free(ptr);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
num += ret;
|
||||
}
|
||||
|
||||
ptr[*len] = '\0';
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static int symbol_table_from_current_process(SymbolTable *st)
|
||||
{
|
||||
uint64_t base_addr = query_base_addr();
|
||||
if (base_addr == (uint64_t) -1)
|
||||
return -1;
|
||||
st->base_addr = base_addr;
|
||||
|
||||
char path[1<<10];
|
||||
if (current_executable_path(path, sizeof(path)) < 0)
|
||||
return -1;
|
||||
|
||||
char *exe_ptr;
|
||||
int exe_len;
|
||||
exe_ptr = read_file(path, &exe_len);
|
||||
if (exe_ptr == NULL)
|
||||
return -1;
|
||||
|
||||
if (load_symbols_from_elf(exe_ptr, exe_len, st) < 0) {
|
||||
free(exe_ptr);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(exe_ptr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void symbol_table_free(SymbolTable *st)
|
||||
{
|
||||
free(st->symbols);
|
||||
}
|
||||
|
||||
static char *symbol_table_find(SymbolTable *st, uint64_t addr)
|
||||
{
|
||||
for (int i = 0; i < st->count; i++) {
|
||||
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
|
||||
|
||||
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
|
||||
continue;
|
||||
|
||||
if (sym->st_value == 0)
|
||||
continue;
|
||||
|
||||
uint64_t sym_beg = st->base_addr + sym->st_value;
|
||||
uint64_t sym_end = st->base_addr + sym->st_value + sym->st_size;
|
||||
|
||||
if (addr >= sym_beg && addr < sym_end)
|
||||
return (char*) st->strings + sym->st_name;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static void symbol_table_dump(SymbolTable *st)
|
||||
{
|
||||
for (int i = 0; i < st->count; i++) {
|
||||
Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i;
|
||||
|
||||
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
|
||||
continue;
|
||||
|
||||
if (sym->st_value == 0)
|
||||
continue;
|
||||
|
||||
char *name = (char*) st->strings + sym->st_name;
|
||||
printf("%s\n", name);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
char *name;
|
||||
uint64_t addr;
|
||||
} StackFrame;
|
||||
|
||||
static int walk_stack(uint64_t rip, uint64_t rbp, SymbolTable *st, StackFrame *frames, int max_frames)
|
||||
{
|
||||
int frame_count = 0;
|
||||
|
||||
if (frame_count < max_frames) {
|
||||
frames[frame_count].addr = rip - st->base_addr;
|
||||
frames[frame_count].name = symbol_table_find(st, rip);
|
||||
frame_count++;
|
||||
}
|
||||
|
||||
while (rbp != 0) {
|
||||
|
||||
if (rbp & 0xF)
|
||||
break;
|
||||
|
||||
uint64_t *frame_ptr = (uint64_t*) rbp;
|
||||
|
||||
uint64_t next_rbp = frame_ptr[0];
|
||||
uint64_t return_addr = frame_ptr[1];
|
||||
|
||||
if (next_rbp != 0 && next_rbp <= rbp)
|
||||
break;
|
||||
|
||||
if (return_addr == 0)
|
||||
break;
|
||||
|
||||
if (frame_count == max_frames)
|
||||
break;
|
||||
frames[frame_count].addr = return_addr - st->base_addr;
|
||||
frames[frame_count].name = symbol_table_find(st, return_addr);
|
||||
frame_count++;
|
||||
|
||||
rbp = next_rbp;
|
||||
}
|
||||
|
||||
return frame_count;
|
||||
}
|
||||
|
||||
static bool crash_logger_symbol_init = false;
|
||||
static char* crash_logger_file_name = NULL;
|
||||
static SymbolTable crash_logger_symbol_table;
|
||||
static char* crash_logger_signal_stack;
|
||||
|
||||
static void crash_handler(int sig, siginfo_t *info, void *ucontext)
|
||||
{
|
||||
(void) info;
|
||||
|
||||
if (crash_logger_symbol_init) {
|
||||
|
||||
// Buffer for evaluating format strings
|
||||
char tmp[1<<9];
|
||||
int len;
|
||||
|
||||
ucontext_t *ctx = (ucontext_t*) ucontext;
|
||||
uint64_t rip = ctx->uc_mcontext.gregs[REG_RIP];
|
||||
uint64_t rbp = ctx->uc_mcontext.gregs[REG_RBP];
|
||||
|
||||
StackFrame frames[64];
|
||||
int count = walk_stack(rip, rbp, &crash_logger_symbol_table, frames, 64);
|
||||
|
||||
int fd = open(crash_logger_file_name, O_WRONLY | O_CREAT, 0666);
|
||||
if (fd < 0)
|
||||
exit(1);
|
||||
|
||||
char *sig_name = "";
|
||||
switch (sig) {
|
||||
case SIGSEGV: sig_name = "Segmentation fault"; break;
|
||||
case SIGBUS : sig_name = "Bus error"; break;
|
||||
case SIGILL : sig_name = "Illegal instruction"; break;
|
||||
case SIGFPE : sig_name = "Floating point exception"; break;
|
||||
case SIGTRAP: sig_name = "Trace trap"; break;
|
||||
case SIGSYS : sig_name = "Bad system call"; break;
|
||||
case SIGABRT: sig_name = "Abort"; break;
|
||||
}
|
||||
if (sig_name[0] == '\0') {
|
||||
len = snprintf(tmp, sizeof(tmp), "(unknown signal %d)\n", sig);
|
||||
write(fd, tmp, len);
|
||||
} else {
|
||||
write(fd, sig_name, strlen(sig_name));
|
||||
write(fd, "\n", 1);
|
||||
}
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
len = snprintf(tmp, sizeof(tmp), " [%d] 0x%lx %s\n", i, frames[i].addr,
|
||||
frames[i].name ? frames[i].name : "?");
|
||||
write(fd, tmp, len);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int crash_logger_init(char *file_name, int file_name_len)
|
||||
{
|
||||
{
|
||||
char *file_name_copy = malloc(file_name_len + 1);
|
||||
if (file_name_copy == NULL)
|
||||
return -1;
|
||||
memcpy(file_name_copy, file_name, file_name_len);
|
||||
file_name_copy[file_name_len] = '\0';
|
||||
crash_logger_file_name = file_name_copy;
|
||||
}
|
||||
|
||||
if (symbol_table_from_current_process(&crash_logger_symbol_table) < 0) {
|
||||
free(crash_logger_file_name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Set up alternate signal stack
|
||||
{
|
||||
crash_logger_signal_stack = malloc(SIGSTKSZ);
|
||||
if (crash_logger_signal_stack == NULL) {
|
||||
symbol_table_free(&crash_logger_symbol_table);
|
||||
free(crash_logger_file_name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
stack_t ss;
|
||||
ss.ss_sp = crash_logger_signal_stack;
|
||||
ss.ss_size = SIGSTKSZ;
|
||||
ss.ss_flags = 0;
|
||||
if (sigaltstack(&ss, NULL) < 0) {
|
||||
free(crash_logger_signal_stack);
|
||||
symbol_table_free(&crash_logger_symbol_table);
|
||||
free(crash_logger_file_name);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Register the crash handler
|
||||
struct sigaction sa;
|
||||
sa.sa_sigaction = crash_handler;
|
||||
sa.sa_flags = SA_SIGINFO | SA_ONSTACK; // Add SA_ONSTACK flag
|
||||
sigemptyset(&sa.sa_mask);
|
||||
|
||||
// Memory errors
|
||||
sigaction(SIGSEGV, &sa, NULL); // Segmentation fault (invalid memory access)
|
||||
sigaction(SIGBUS, &sa, NULL); // Bus error (misaligned access, hardware error)
|
||||
|
||||
// Execution errors
|
||||
sigaction(SIGILL, &sa, NULL); // Illegal instruction
|
||||
sigaction(SIGFPE, &sa, NULL); // Floating point exception
|
||||
sigaction(SIGTRAP, &sa, NULL); // Trace trap
|
||||
|
||||
// System/resource errors
|
||||
sigaction(SIGSYS, &sa, NULL); // Bad system call
|
||||
sigaction(SIGABRT, &sa, NULL); // Abort (from assert, abort(), etc.)
|
||||
|
||||
// Optional: Resource limit violations
|
||||
sigaction(SIGXCPU, &sa, NULL); // CPU time limit exceeded
|
||||
sigaction(SIGXFSZ, &sa, NULL); // File size limit exceeded
|
||||
}
|
||||
|
||||
crash_logger_symbol_init = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void crash_logger_free(void)
|
||||
{
|
||||
if (!crash_logger_symbol_init)
|
||||
return;
|
||||
|
||||
free(crash_logger_signal_stack);
|
||||
symbol_table_free(&crash_logger_symbol_table);
|
||||
free(crash_logger_file_name);
|
||||
|
||||
crash_logger_symbol_init = false;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static int crash_logger_init(char *file_name, int file_name_len)
|
||||
{
|
||||
(void) file_name;
|
||||
(void) file_name_len;
|
||||
return -1;
|
||||
}
|
||||
|
||||
static void crash_logger_free(void)
|
||||
{
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,7 +0,0 @@
|
||||
#ifndef CRASH_LOGGER_INCLUDED
|
||||
#define CRASH_LOGGER_INCLUDED
|
||||
|
||||
int crash_logger_init(char *file_name, int file_name_len);
|
||||
void crash_logger_free(void);
|
||||
|
||||
#endif // CRASH_LOGGER_INCLUDED
|
||||
+41
-40
@@ -1,8 +1,9 @@
|
||||
#include <limits.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
|
||||
#include "system.h"
|
||||
#include "file_system.h"
|
||||
|
||||
int rename_file_or_dir(string oldpath, string newpath);
|
||||
@@ -16,7 +17,7 @@ int file_open(string path, Handle *fd)
|
||||
memcpy(zt, path.ptr, path.len);
|
||||
zt[path.len] = '\0';
|
||||
|
||||
int ret = sys_open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
|
||||
int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
@@ -29,7 +30,7 @@ int file_open(string path, Handle *fd)
|
||||
MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH);
|
||||
wpath[path.len] = L'\0';
|
||||
|
||||
HANDLE h = sys_CreateFileW(
|
||||
HANDLE h = CreateFileW(
|
||||
wpath,
|
||||
GENERIC_WRITE | GENERIC_READ,
|
||||
0,
|
||||
@@ -49,18 +50,18 @@ int file_open(string path, Handle *fd)
|
||||
void file_close(Handle fd)
|
||||
{
|
||||
#ifdef __linux__
|
||||
sys_close((int) fd.data);
|
||||
close((int) fd.data);
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
sys_CloseHandle((HANDLE) fd.data);
|
||||
CloseHandle((HANDLE) fd.data);
|
||||
#endif
|
||||
}
|
||||
|
||||
int file_set_offset(Handle fd, int off)
|
||||
{
|
||||
#ifdef __linux__
|
||||
off_t ret = sys_lseek((int) fd.data, off, SEEK_SET);
|
||||
off_t ret = lseek((int) fd.data, off, SEEK_SET);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
return 0;
|
||||
@@ -69,8 +70,8 @@ int file_set_offset(Handle fd, int off)
|
||||
#ifdef _WIN32
|
||||
LARGE_INTEGER distance;
|
||||
distance.QuadPart = off;
|
||||
if (!sys_SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
|
||||
if (GetLastError() != NO_ERROR)
|
||||
if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN))
|
||||
if (GetLastError() != 0)
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
@@ -79,7 +80,7 @@ int file_set_offset(Handle fd, int off)
|
||||
int file_get_offset(Handle fd, int *off)
|
||||
{
|
||||
#ifdef __linux__
|
||||
off_t ret = sys_lseek((int) fd.data, 0, SEEK_CUR);
|
||||
off_t ret = lseek((int) fd.data, 0, SEEK_CUR);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
*off = (int) ret;
|
||||
@@ -87,8 +88,8 @@ int file_get_offset(Handle fd, int *off)
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
DWORD pos = sys_SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
|
||||
if (pos == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR)
|
||||
DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT);
|
||||
if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0)
|
||||
return -1;
|
||||
*off = (int) pos;
|
||||
return 0;
|
||||
@@ -98,13 +99,13 @@ int file_get_offset(Handle fd, int *off)
|
||||
int file_lock(Handle fd)
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (sys_flock((int) fd.data, LOCK_EX) < 0)
|
||||
if (flock((int) fd.data, LOCK_EX) < 0)
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
if (!sys_LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
|
||||
if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
@@ -113,13 +114,13 @@ int file_lock(Handle fd)
|
||||
int file_unlock(Handle fd)
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (sys_flock((int) fd.data, LOCK_UN) < 0)
|
||||
if (flock((int) fd.data, LOCK_UN) < 0)
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
if (!sys_UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
|
||||
if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD))
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
@@ -128,13 +129,13 @@ int file_unlock(Handle fd)
|
||||
int file_sync(Handle fd)
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (sys_fsync((int) fd.data) < 0)
|
||||
if (fsync((int) fd.data) < 0)
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
if (!sys_FlushFileBuffers((HANDLE) fd.data))
|
||||
if (!FlushFileBuffers((HANDLE) fd.data))
|
||||
return -1;
|
||||
return 0;
|
||||
#endif
|
||||
@@ -143,12 +144,12 @@ int file_sync(Handle fd)
|
||||
int file_read(Handle fd, char *dst, int max)
|
||||
{
|
||||
#ifdef __linux__
|
||||
return sys_read((int) fd.data, dst, max);
|
||||
return read((int) fd.data, dst, max);
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
DWORD num;
|
||||
if (!sys_ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
|
||||
if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL))
|
||||
return -1;
|
||||
if (num > INT_MAX)
|
||||
return -1;
|
||||
@@ -159,12 +160,12 @@ int file_read(Handle fd, char *dst, int max)
|
||||
int file_write(Handle fd, char *src, int len)
|
||||
{
|
||||
#ifdef __linux__
|
||||
return sys_write((int) fd.data, src, len);
|
||||
return write((int) fd.data, src, len);
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
DWORD num;
|
||||
if (!sys_WriteFile((HANDLE) fd.data, src, len, &num, NULL))
|
||||
if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL))
|
||||
return -1;
|
||||
if (num > INT_MAX)
|
||||
return -1;
|
||||
@@ -176,7 +177,7 @@ int file_size(Handle fd, size_t *len)
|
||||
{
|
||||
#ifdef __linux__
|
||||
struct stat buf;
|
||||
if (sys_fstat((int) fd.data, &buf) < 0)
|
||||
if (fstat((int) fd.data, &buf) < 0)
|
||||
return -1;
|
||||
if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX)
|
||||
return -1;
|
||||
@@ -186,7 +187,7 @@ int file_size(Handle fd, size_t *len)
|
||||
|
||||
#ifdef _WIN32
|
||||
LARGE_INTEGER buf;
|
||||
if (!sys_GetFileSizeEx((HANDLE) fd.data, &buf))
|
||||
if (!GetFileSizeEx((HANDLE) fd.data, &buf))
|
||||
return -1;
|
||||
if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX)
|
||||
return -1;
|
||||
@@ -204,10 +205,10 @@ int create_dir(string path)
|
||||
zt[path.len] = '\0';
|
||||
|
||||
#ifdef _WIN32
|
||||
if (sys__mkdir(zt) < 0)
|
||||
if (_mkdir(zt) < 0)
|
||||
return -1;
|
||||
#else
|
||||
if (sys_mkdir(zt, 0766))
|
||||
if (mkdir(zt, 0766))
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
@@ -228,7 +229,7 @@ int rename_file_or_dir(string oldpath, string newpath)
|
||||
memcpy(newpath_zt, newpath.ptr, newpath.len);
|
||||
newpath_zt[newpath.len] = '\0';
|
||||
|
||||
if (sys_rename(oldpath_zt, newpath_zt))
|
||||
if (rename(oldpath_zt, newpath_zt))
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
@@ -241,7 +242,7 @@ int remove_file_or_dir(string path)
|
||||
memcpy(path_zt, path.ptr, path.len);
|
||||
path_zt[path.len] = '\0';
|
||||
|
||||
if (sys_remove(path_zt))
|
||||
if (remove(path_zt))
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
@@ -255,12 +256,12 @@ int get_full_path(string path, char *dst)
|
||||
path_zt[path.len] = '\0';
|
||||
|
||||
#ifdef __linux__
|
||||
if (sys_realpath(path_zt, dst) == NULL)
|
||||
if (realpath(path_zt, dst) == NULL)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
if (sys__fullpath(path_zt, dst, PATH_MAX) == NULL)
|
||||
if (_fullpath(path_zt, dst, PATH_MAX) == NULL)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
@@ -285,7 +286,7 @@ int file_read_all(string path, string *data)
|
||||
return -1;
|
||||
}
|
||||
|
||||
char *dst = sys_malloc(len);
|
||||
char *dst = malloc(len);
|
||||
if (dst == NULL) {
|
||||
file_close(fd);
|
||||
return -1;
|
||||
@@ -295,7 +296,7 @@ int file_read_all(string path, string *data)
|
||||
while ((size_t) copied < len) {
|
||||
ret = file_read(fd, dst + copied, len - copied);
|
||||
if (ret < 0) {
|
||||
sys_free(dst);
|
||||
free(dst);
|
||||
file_close(fd);
|
||||
return -1;
|
||||
}
|
||||
@@ -316,7 +317,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
|
||||
if (ret < 0 || ret >= (int) sizeof(pattern))
|
||||
return -1;
|
||||
|
||||
scanner->handle = sys_FindFirstFileA(pattern, &scanner->find_data);
|
||||
scanner->handle = FindFirstFileA(pattern, &scanner->find_data);
|
||||
if (scanner->handle == INVALID_HANDLE_VALUE) {
|
||||
if (GetLastError() == ERROR_FILE_NOT_FOUND) {
|
||||
scanner->done = true;
|
||||
@@ -336,7 +337,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
|
||||
return 1;
|
||||
|
||||
if (!scanner->first) {
|
||||
BOOL ok = sys_FindNextFileA(scanner->handle, &scanner->find_data);
|
||||
BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data);
|
||||
if (!ok) {
|
||||
scanner->done = true;
|
||||
if (GetLastError() == ERROR_NO_MORE_FILES)
|
||||
@@ -354,7 +355,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
|
||||
|
||||
void directory_scanner_free(DirectoryScanner *scanner)
|
||||
{
|
||||
sys_FindClose(scanner->handle);
|
||||
FindClose(scanner->handle);
|
||||
}
|
||||
|
||||
#else
|
||||
@@ -367,7 +368,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path)
|
||||
memcpy(path_copy, path.ptr, path.len);
|
||||
path_copy[path.len] = '\0';
|
||||
|
||||
scanner->d = sys_opendir(path_copy);
|
||||
scanner->d = opendir(path_copy);
|
||||
if (scanner->d == NULL) {
|
||||
scanner->done = true;
|
||||
return -1;
|
||||
@@ -382,7 +383,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
|
||||
if (scanner->done)
|
||||
return 1;
|
||||
|
||||
scanner->e = sys_readdir(scanner->d);
|
||||
scanner->e = readdir(scanner->d);
|
||||
if (scanner->e == NULL) {
|
||||
scanner->done = true;
|
||||
return 1;
|
||||
@@ -394,7 +395,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name)
|
||||
|
||||
void directory_scanner_free(DirectoryScanner *scanner)
|
||||
{
|
||||
sys_closedir(scanner->d);
|
||||
closedir(scanner->d);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
+5
-7
@@ -1,14 +1,12 @@
|
||||
#ifndef FILE_SYSTEM_INCLUDED
|
||||
#define FILE_SYSTEM_INCLUDED
|
||||
|
||||
#include "basic.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dirent.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
#include "basic.h"
|
||||
|
||||
typedef struct {
|
||||
uint64_t data;
|
||||
|
||||
+18
-16
@@ -1,10 +1,12 @@
|
||||
#include <limits.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <quakey.h>
|
||||
|
||||
#include "basic.h"
|
||||
#include "system.h"
|
||||
#include "file_tree.h"
|
||||
|
||||
static int parse_path(string path, string *comps, int max)
|
||||
@@ -102,7 +104,7 @@ static void dir_free(Dir *d)
|
||||
{
|
||||
for (uint64_t i = 0; i < d->num_children; i++)
|
||||
entity_free(&d->children[i]);
|
||||
sys_free(d->children);
|
||||
free(d->children);
|
||||
}
|
||||
|
||||
static bool gen_match(uint64_t expected_gen, uint64_t entity_gen)
|
||||
@@ -158,7 +160,7 @@ static void file_init(File *f, uint64_t chunk_size)
|
||||
|
||||
static void file_free(File *f)
|
||||
{
|
||||
sys_free(f->chunks);
|
||||
free(f->chunks);
|
||||
f->chunks = NULL;
|
||||
}
|
||||
|
||||
@@ -312,14 +314,14 @@ int file_tree_create_entity(FileTree *ft, string path,
|
||||
if (new_max == 0)
|
||||
new_max = 8;
|
||||
|
||||
Entity *p = sys_malloc(sizeof(Entity) * new_max);
|
||||
Entity *p = malloc(sizeof(Entity) * new_max);
|
||||
if (p == NULL)
|
||||
return FILETREE_NOMEM;
|
||||
|
||||
for (uint64_t i = 0; i < d->num_children; i++)
|
||||
p[i] = d->children[i];
|
||||
|
||||
sys_free(d->children);
|
||||
free(d->children);
|
||||
d->children = p;
|
||||
d->max_children = new_max;
|
||||
}
|
||||
@@ -423,13 +425,13 @@ int file_tree_write(
|
||||
|
||||
if (last_chunk_index >= f->num_chunks) {
|
||||
uint64_t old_num_chunks = f->num_chunks;
|
||||
SHA256 *new_chunks = sys_malloc((last_chunk_index+1) * sizeof(SHA256));
|
||||
SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256));
|
||||
if (new_chunks == NULL)
|
||||
return FILETREE_NOMEM;
|
||||
if (f->chunks) {
|
||||
if (f->num_chunks > 0)
|
||||
memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256));
|
||||
sys_free(f->chunks);
|
||||
free(f->chunks);
|
||||
}
|
||||
f->chunks = new_chunks;
|
||||
f->num_chunks = last_chunk_index+1;
|
||||
@@ -678,13 +680,13 @@ int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *wr
|
||||
sc.write_data = write_data;
|
||||
sc.buffer_used = 0;
|
||||
sc.buffer_size = 1<<10;
|
||||
sc.buffer = sys_malloc(sc.buffer_size);
|
||||
sc.buffer = malloc(sc.buffer_size);
|
||||
sc.error = false;
|
||||
if (sc.buffer == NULL)
|
||||
sc.error = true;
|
||||
entity_serialize(&sc, &ft->root);
|
||||
sc_flush(&sc);
|
||||
sys_free(sc.buffer);
|
||||
free(sc.buffer);
|
||||
if (sc.error)
|
||||
return -1;
|
||||
return 0;
|
||||
@@ -748,7 +750,7 @@ static void file_deserialize(DeserializeContext *dc, File *f)
|
||||
dc_read_u64(dc, &f->num_chunks);
|
||||
dc_read_u64(dc, &f->file_size);
|
||||
|
||||
f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256));
|
||||
f->chunks = malloc(f->num_chunks * sizeof(SHA256));
|
||||
if (f->chunks == NULL) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
@@ -764,7 +766,7 @@ static void dir_deserialize(DeserializeContext *dc, Dir *d)
|
||||
dc_read_u64(dc, &d->num_children);
|
||||
|
||||
d->max_children = d->num_children;
|
||||
d->children = sys_malloc(d->num_children * sizeof(Entity));
|
||||
d->children = malloc(d->num_children * sizeof(Entity));
|
||||
if (d->children == NULL) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
@@ -799,13 +801,13 @@ int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *r
|
||||
dc.buffer_head = 0;
|
||||
dc.buffer_used = 0;
|
||||
dc.buffer_size = 1<<10;
|
||||
dc.buffer = sys_malloc(dc.buffer_size);
|
||||
dc.buffer = malloc(dc.buffer_size);
|
||||
dc.error = false;
|
||||
if (dc.buffer == NULL)
|
||||
dc.error = true;
|
||||
dc.total_read = 0;
|
||||
entity_deserialize(&dc, &ft->root);
|
||||
sys_free(dc.buffer);
|
||||
free(dc.buffer);
|
||||
if (dc.error)
|
||||
return -1;
|
||||
if (dc.total_read > INT_MAX) {
|
||||
|
||||
+10
-9
@@ -1,7 +1,8 @@
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "system.h"
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
#include "hash_set.h"
|
||||
|
||||
void hash_set_init(HashSet *set)
|
||||
@@ -13,7 +14,7 @@ void hash_set_init(HashSet *set)
|
||||
|
||||
void hash_set_free(HashSet *set)
|
||||
{
|
||||
sys_free(set->items);
|
||||
free(set->items);
|
||||
set->items = NULL;
|
||||
}
|
||||
|
||||
@@ -40,7 +41,7 @@ int hash_set_insert(HashSet *set, SHA256 hash)
|
||||
else
|
||||
new_capacity = 2 * set->capacity;
|
||||
|
||||
SHA256 *new_items = sys_realloc(set->items, new_capacity * sizeof(SHA256));
|
||||
SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256));
|
||||
if (new_items == NULL)
|
||||
return -1;
|
||||
|
||||
@@ -109,7 +110,7 @@ void timed_hash_set_init(TimedHashSet *set)
|
||||
|
||||
void timed_hash_set_free(TimedHashSet *set)
|
||||
{
|
||||
sys_free(set->items);
|
||||
free(set->items);
|
||||
set->items = NULL;
|
||||
}
|
||||
|
||||
@@ -137,13 +138,13 @@ int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time)
|
||||
else
|
||||
new_capacity = 2 * set->capacity;
|
||||
|
||||
TimedHash *new_items = sys_malloc(new_capacity * sizeof(TimedHash));
|
||||
TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash));
|
||||
if (new_items == NULL)
|
||||
return -1;
|
||||
|
||||
if (set->capacity > 0) {
|
||||
memcpy(new_items, set->items, set->count * sizeof(set->items[0]));
|
||||
sys_free(set->items);
|
||||
free(set->items);
|
||||
}
|
||||
|
||||
set->items = new_items;
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#else
|
||||
#define POLL_CAPACITY 1024
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
|
||||
#include "metadata_server.h"
|
||||
#include "chunk_server.h"
|
||||
#include "random_client.h"
|
||||
|
||||
#ifdef MAIN_METADATA_SERVER
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret;
|
||||
MetadataServer state;
|
||||
|
||||
void* poll_ctxs[POLL_CAPACITY];
|
||||
struct pollfd poll_array[POLL_CAPACITY];
|
||||
int poll_count;
|
||||
int poll_timeout;
|
||||
|
||||
ret = metadata_server_init(
|
||||
&state,
|
||||
argc,
|
||||
argv,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
for (;;) {
|
||||
|
||||
#ifdef _WIN32
|
||||
WSAPoll(poll_array, poll_count, poll_timeout);
|
||||
#else
|
||||
poll(poll_array, poll_count, poll_timeout);
|
||||
#endif
|
||||
|
||||
ret = metadata_server_tick(
|
||||
&state,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
metadata_server_free(&state);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MAIN_CHUNK_SERVER
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret;
|
||||
ChunkServer state;
|
||||
|
||||
void* poll_ctxs[POLL_CAPACITY];
|
||||
struct pollfd poll_array[POLL_CAPACITY];
|
||||
int poll_count;
|
||||
int poll_timeout;
|
||||
|
||||
ret = chunk_server_init(
|
||||
&state,
|
||||
argc,
|
||||
argv,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
for (;;) {
|
||||
|
||||
#ifdef _WIN32
|
||||
WSAPoll(poll_array, poll_count, poll_timeout);
|
||||
#else
|
||||
poll(poll_array, poll_count, poll_timeout);
|
||||
#endif
|
||||
|
||||
ret = chunk_server_tick(
|
||||
&state,
|
||||
poll_ctxs,
|
||||
poll_array,
|
||||
POLL_CAPACITY,
|
||||
&poll_count,
|
||||
&poll_timeout
|
||||
);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
chunk_server_free(&state);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MAIN_SIMULATION
|
||||
int main(void)
|
||||
{
|
||||
Quakey *quakey;
|
||||
int ret = quakey_init(&quakey, 1);
|
||||
if (ret < 0)
|
||||
return -1;
|
||||
|
||||
// Client
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.state_size = sizeof(RandomClient),
|
||||
.init_func = random_client_init,
|
||||
.tick_func = random_client_tick,
|
||||
.free_func = random_client_free,
|
||||
.addrs = (char*[]) { "127.0.0.2" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 1<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
quakey_spawn(quakey, config, "cs --addr 127.0.0.2");
|
||||
}
|
||||
|
||||
// Metadata Server
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.state_size = sizeof(MetadataServer),
|
||||
.init_func = metadata_server_init,
|
||||
.tick_func = metadata_server_tick,
|
||||
.free_func = metadata_server_free,
|
||||
.addrs = (char*[]) { "127.0.0.3" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 1<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
quakey_spawn(quakey, config, "ms --addr 127.0.0.3");
|
||||
}
|
||||
|
||||
// Chunk Server
|
||||
{
|
||||
QuakeySpawn config = {
|
||||
.state_size = sizeof(ChunkServer),
|
||||
.init_func = chunk_server_init,
|
||||
.tick_func = chunk_server_tick,
|
||||
.free_func = chunk_server_free,
|
||||
.addrs = (char*[]) { "127.0.0.4" },
|
||||
.num_addrs = 1,
|
||||
.disk_size = 1<<20,
|
||||
.platform = QUAKEY_LINUX,
|
||||
};
|
||||
quakey_spawn(quakey, config, "cs --addr 127.0.0.4");
|
||||
}
|
||||
|
||||
for (;;)
|
||||
quakey_schedule_one(quakey);
|
||||
|
||||
quakey_free(quakey);
|
||||
return 0;
|
||||
}
|
||||
#endif // MAIN_SIMULATION
|
||||
@@ -1,132 +0,0 @@
|
||||
#ifdef BUILD_SERVER
|
||||
|
||||
#include <time.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <winsock2.h>
|
||||
#define POLL WSAPoll
|
||||
#else
|
||||
#include <poll.h>
|
||||
#define POLL poll
|
||||
#endif
|
||||
|
||||
#include "crash_logger.h"
|
||||
#include "chunk_server.h"
|
||||
#include "metadata_server.h"
|
||||
|
||||
int metadata_server_main(int argc, char **argv)
|
||||
{
|
||||
{
|
||||
time_t now = time(NULL);
|
||||
struct tm unpacked_now;
|
||||
#ifdef _WIN32
|
||||
if (gmtime_s(&unpacked_now, &now) != 0)
|
||||
return -1;
|
||||
#else
|
||||
if (gmtime_r(&now, &unpacked_now) == NULL)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
char time[sizeof("YYYYMMDDthhmmssz")];
|
||||
int ret = strftime(time, sizeof(time),
|
||||
"%Y%m%dT%H%M%SZ", &unpacked_now);
|
||||
if (ret != sizeof(time)-1)
|
||||
return -1;
|
||||
|
||||
char path[1<<10];
|
||||
ret = snprintf(path, sizeof(path), "crash_%s_MS_%d.txt", time, getpid());
|
||||
if (ret < 0 || ret >= (int) sizeof(path)) {
|
||||
return -1;
|
||||
}
|
||||
int path_len = ret;
|
||||
|
||||
crash_logger_init(path, path_len);
|
||||
}
|
||||
|
||||
void *contexts[TCP_POLL_CAPACITY];
|
||||
struct pollfd polled[TCP_POLL_CAPACITY];
|
||||
int num_polled;
|
||||
int timeout = -1;
|
||||
|
||||
MetadataServer state;
|
||||
num_polled = metadata_server_init(
|
||||
&state, argc, argv, contexts, polled, &timeout);
|
||||
if (num_polled < 0) return -1;
|
||||
|
||||
for (;;) {
|
||||
POLL(polled, num_polled, timeout);
|
||||
|
||||
timeout = -1;
|
||||
num_polled = metadata_server_step(
|
||||
&state, contexts, polled, num_polled, &timeout);
|
||||
if (num_polled < 0) return -1;
|
||||
}
|
||||
metadata_server_free(&state);
|
||||
crash_logger_free();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chunk_server_main(int argc, char **argv)
|
||||
{
|
||||
{
|
||||
time_t now = time(NULL);
|
||||
struct tm unpacked_now;
|
||||
#ifdef _WIN32
|
||||
if (gmtime_s(&unpacked_now, &now) != 0)
|
||||
return -1;
|
||||
#else
|
||||
if (gmtime_r(&now, &unpacked_now) == NULL)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
char time[sizeof("YYYYMMDDthhmmssz")];
|
||||
int ret = strftime(time, sizeof(time),
|
||||
"%Y%m%dT%H%M%SZ", &unpacked_now);
|
||||
if (ret != sizeof(time)-1)
|
||||
return -1;
|
||||
|
||||
char path[1<<10];
|
||||
ret = snprintf(path, sizeof(path), "crash_%s_CS_%d.txt", time, getpid());
|
||||
if (ret < 0 || ret >= (int) sizeof(path)) {
|
||||
return -1;
|
||||
}
|
||||
int path_len = ret;
|
||||
|
||||
crash_logger_init(path, path_len);
|
||||
}
|
||||
|
||||
void *contexts[TCP_POLL_CAPACITY];
|
||||
struct pollfd polled[TCP_POLL_CAPACITY];
|
||||
int num_polled;
|
||||
int timeout = -1;
|
||||
|
||||
ChunkServer state;
|
||||
num_polled = chunk_server_init(
|
||||
&state, argc, argv, contexts, polled, &timeout);
|
||||
if (num_polled < 0) return -1;
|
||||
|
||||
for (;;) {
|
||||
|
||||
POLL(polled, num_polled, timeout);
|
||||
|
||||
timeout = -1;
|
||||
num_polled = chunk_server_step(
|
||||
&state, contexts, polled, num_polled, &timeout);
|
||||
if (num_polled < 0) return -1;
|
||||
}
|
||||
|
||||
chunk_server_free(&state);
|
||||
crash_logger_free();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (getargb(argc, argv, "--leader"))
|
||||
return metadata_server_main(argc, argv);
|
||||
else
|
||||
return chunk_server_main(argc, argv);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,57 +0,0 @@
|
||||
#ifdef BUILD_TEST
|
||||
|
||||
#include <stdio.h>
|
||||
#include <signal.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "system.h"
|
||||
|
||||
static sig_atomic_t simulation_should_stop = false;
|
||||
|
||||
static void signal_handler(int signum)
|
||||
{
|
||||
(void)signum;
|
||||
simulation_should_stop = true;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
|
||||
// Set up signal handlers for clean shutdown
|
||||
signal(SIGINT, signal_handler);
|
||||
signal(SIGTERM, signal_handler);
|
||||
|
||||
startup_simulation(2);
|
||||
|
||||
// Spawn metadata server (leader)
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8080 --leader");
|
||||
|
||||
// Spawn chunk servers
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8081 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_0/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8082 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_1/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8083 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_2/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8084 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_3/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8085 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_4/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8086 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_5/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8087 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_6/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8088 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_7/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8089 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_8/");
|
||||
spawn_simulated_process("--addr 127.0.0.1 --port 8090 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_9/");
|
||||
|
||||
// Spawn simulation client
|
||||
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
|
||||
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
|
||||
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
|
||||
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
|
||||
spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080");
|
||||
|
||||
while (!simulation_should_stop)
|
||||
update_simulation();
|
||||
|
||||
cleanup_simulation();
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
+4
-7
@@ -1,11 +1,8 @@
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <ws2tcpip.h>
|
||||
#else
|
||||
#include <arpa/inet.h> // inet_ntop
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
#include "message.h"
|
||||
|
||||
bool binary_read(BinaryReader *reader, void *dst, int len)
|
||||
|
||||
+5
-3
@@ -1,9 +1,11 @@
|
||||
#ifndef MESSAGE_INCLUDED
|
||||
#define MESSAGE_INCLUDED
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
#include <stdint.h>
|
||||
#include <quakey.h>
|
||||
#include "basic.h"
|
||||
#include "byte_queue.h"
|
||||
|
||||
|
||||
+46
-18
@@ -1,8 +1,10 @@
|
||||
#define _GNU_SOURCE
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <string.h>
|
||||
#include <quakey.h>
|
||||
#include <stdint.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "message.h"
|
||||
#include "metadata_server.h"
|
||||
@@ -1061,19 +1063,27 @@ static bool is_chunk_server_message_type(uint16_t type)
|
||||
return false;
|
||||
}
|
||||
|
||||
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout)
|
||||
int metadata_server_init(void *state_, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout)
|
||||
{
|
||||
MetadataServer *state = state_;
|
||||
|
||||
string addr = getargs(argc, argv, "--addr", "127.0.0.1");
|
||||
int port = getargi(argc, argv, "--port", 8080);
|
||||
bool trace = getargb(argc, argv, "--trace");
|
||||
string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal");
|
||||
int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit
|
||||
|
||||
if (port <= 0 || port >= 1<<16)
|
||||
if (port <= 0 || port >= 1<<16) {
|
||||
fprintf(stderr, "metadata server :: Invalid port\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (wal_limit < 0)
|
||||
if (wal_limit < 0) {
|
||||
fprintf(stderr, "metadata server :: Invalid WAL limit\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
state->trace = trace;
|
||||
state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro?
|
||||
@@ -1084,22 +1094,27 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
|
||||
for (int i = 0; i < MAX_CHUNK_SERVERS; i++)
|
||||
state->chunk_servers[i].used = false;
|
||||
|
||||
if (tcp_context_init(&state->tcp) < 0)
|
||||
if (tcp_context_init(&state->tcp) < 0) {
|
||||
fprintf(stderr, "metadata server :: Couldn't setup TCP context\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
int ret = tcp_listen(&state->tcp, addr, port);
|
||||
if (ret < 0) {
|
||||
fprintf(stderr, "metadata server :: Couldn't setup TCP listener\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
ret = file_tree_init(&state->file_tree);
|
||||
if (ret < 0) {
|
||||
fprintf(stderr, "metadata server :: Couldn't setup file tree\n");
|
||||
tcp_context_free(&state->tcp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) {
|
||||
fprintf(stderr, "metadata server :: Couldn't setup WAL\n");
|
||||
assert(0); // TODO
|
||||
}
|
||||
|
||||
@@ -1110,21 +1125,21 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co
|
||||
);
|
||||
|
||||
*timeout = -1; // No timeout until we have chunk servers
|
||||
return tcp_register_events(&state->tcp, contexts, polled);
|
||||
}
|
||||
|
||||
int metadata_server_free(MetadataServer *state)
|
||||
{
|
||||
wal_close(&state->wal);
|
||||
file_tree_free(&state->file_tree);
|
||||
tcp_context_free(&state->tcp);
|
||||
if (pcap < TCP_POLL_CAPACITY) {
|
||||
fprintf(stderr, "metadata server :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY);
|
||||
return -1;
|
||||
}
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout)
|
||||
int metadata_server_tick(void *state_, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout)
|
||||
{
|
||||
MetadataServer *state = state_;
|
||||
|
||||
Event events[TCP_EVENT_CAPACITY];
|
||||
int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled);
|
||||
int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum);
|
||||
|
||||
Time current_time = get_current_time();
|
||||
if (current_time == INVALID_TIME)
|
||||
@@ -1243,5 +1258,18 @@ int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *
|
||||
}
|
||||
|
||||
*timeout = deadline_to_timeout(next_wakeup, current_time);
|
||||
return tcp_register_events(&state->tcp, contexts, polled);
|
||||
if (pcap < TCP_POLL_CAPACITY)
|
||||
return -1;
|
||||
*pnum = tcp_register_events(&state->tcp, ctxs, pdata);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int metadata_server_free(void *state_)
|
||||
{
|
||||
MetadataServer *state = state_;
|
||||
|
||||
wal_close(&state->wal);
|
||||
file_tree_free(&state->file_tree);
|
||||
tcp_context_free(&state->tcp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
+10
-3
@@ -52,8 +52,15 @@ typedef struct {
|
||||
|
||||
} MetadataServer;
|
||||
|
||||
int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout);
|
||||
int metadata_server_free(MetadataServer *state);
|
||||
int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout);
|
||||
struct pollfd;
|
||||
|
||||
int metadata_server_init(void *state, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int metadata_server_tick(void *state, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
|
||||
int metadata_server_free(void *state);
|
||||
|
||||
#endif // METADATA_SERVER_INCLUDED
|
||||
@@ -1,12 +1,11 @@
|
||||
#ifdef BUILD_TEST
|
||||
#ifdef MAIN_SIMULATION
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#include <quakey.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "simulation_client.h"
|
||||
#include "tcp.h"
|
||||
#include "random_client.h"
|
||||
|
||||
// Helper function to parse address and port from command line
|
||||
static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port)
|
||||
@@ -19,7 +18,18 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
|
||||
if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) {
|
||||
*addr = argv[i + 1];
|
||||
if (i + 2 < argc) {
|
||||
*port = (uint16_t)atoi(argv[i + 2]);
|
||||
|
||||
errno = 0;
|
||||
char *end;
|
||||
long val = strtol(argv[i+2], &end, 10);
|
||||
|
||||
if (end == argv[i+2] || *end != '\0' || errno == ERANGE)
|
||||
break;
|
||||
|
||||
if (val < 0 || val > UINT16_MAX)
|
||||
break;
|
||||
|
||||
*port = (uint16_t) val;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -27,9 +37,12 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port
|
||||
return true;
|
||||
}
|
||||
|
||||
int simulation_client_init(SimulationClient *client, int argc, char **argv,
|
||||
void **contexts, struct pollfd *polled, int *timeout)
|
||||
int random_client_init(void *state_, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout)
|
||||
{
|
||||
RandomClient *client = state_;
|
||||
|
||||
char *addr;
|
||||
uint16_t port;
|
||||
parse_server_addr(argc, argv, &addr, &port);
|
||||
@@ -43,20 +56,26 @@ int simulation_client_init(SimulationClient *client, int argc, char **argv,
|
||||
printf("Client set up (remote=%s:%d)\n", addr, port);
|
||||
|
||||
*timeout = 0;
|
||||
return toasty_process_events(client->toasty, contexts, polled, 0);
|
||||
if (pcap < TCP_POLL_CAPACITY)
|
||||
return -1;
|
||||
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int random_in_range(int min, int max)
|
||||
{
|
||||
uint64_t n = simulation_random_number();
|
||||
uint64_t n = quakey_random();
|
||||
return min + n % (max - min + 1);
|
||||
}
|
||||
|
||||
int simulation_client_step(SimulationClient *client, void **contexts,
|
||||
struct pollfd *polled, int num_polled, int *timeout)
|
||||
int random_client_tick(void *state_, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout)
|
||||
{
|
||||
RandomClient *client = state_;
|
||||
|
||||
// Process any pending events from the network and get new poll descriptors
|
||||
num_polled = toasty_process_events(client->toasty, contexts, polled, num_polled);
|
||||
*pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum);
|
||||
|
||||
for (int i = 0; i < client->num_pending; i++) {
|
||||
|
||||
@@ -241,12 +260,19 @@ int simulation_client_step(SimulationClient *client, void **contexts,
|
||||
*timeout = 10;
|
||||
else
|
||||
*timeout = -1;
|
||||
return toasty_process_events(client->toasty, contexts, polled, 0);
|
||||
|
||||
if (pcap < TCP_POLL_CAPACITY)
|
||||
return -1;
|
||||
*pnum = toasty_process_events(client->toasty, ctxs, pdata, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void simulation_client_free(SimulationClient *client)
|
||||
int random_client_free(void *state_)
|
||||
{
|
||||
RandomClient *client = state_;
|
||||
|
||||
toasty_disconnect(client->toasty);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // BUILD_TEST
|
||||
#endif // MAIN_SIMULATION
|
||||
@@ -0,0 +1,39 @@
|
||||
#ifndef RANDOM_CLIENT_INCLUDED
|
||||
#define RANDOM_CLIENT_INCLUDED
|
||||
|
||||
#include "ToastyFS.h"
|
||||
|
||||
#define MAX_PENDING_OPERATION 8
|
||||
|
||||
typedef enum {
|
||||
PENDING_OPERATION_CREATE,
|
||||
PENDING_OPERATION_DELETE,
|
||||
PENDING_OPERATION_LIST,
|
||||
PENDING_OPERATION_READ,
|
||||
PENDING_OPERATION_WRITE,
|
||||
} PendingOperationType;
|
||||
|
||||
typedef struct {
|
||||
PendingOperationType type;
|
||||
ToastyHandle handle;
|
||||
void *ptr;
|
||||
} PendingOperation;
|
||||
|
||||
typedef struct {
|
||||
ToastyFS *toasty;
|
||||
int num_pending;
|
||||
PendingOperation pending[MAX_PENDING_OPERATION];
|
||||
} RandomClient;
|
||||
|
||||
struct pollfd;
|
||||
|
||||
int random_client_init(void *state, int argc, char **argv,
|
||||
void **ctxs, struct pollfd *pdata, int pcap, int *pnum,
|
||||
int *timeout);
|
||||
|
||||
int random_client_tick(void *state, void **ctxs,
|
||||
struct pollfd *pdata, int pcap, int *pnum, int *timeout);
|
||||
|
||||
int random_client_free(void *state);
|
||||
|
||||
#endif // RANDOM_CLIENT_INCLUDED
|
||||
@@ -1,43 +0,0 @@
|
||||
#ifndef SIMULATION_CLIENT_INCLUDED
|
||||
#define SIMULATION_CLIENT_INCLUDED
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <winsock2.h>
|
||||
#else
|
||||
#include <poll.h>
|
||||
#endif
|
||||
|
||||
#include "ToastyFS.h"
|
||||
|
||||
#define MAX_PENDING_OPERATION 8
|
||||
|
||||
typedef enum {
|
||||
PENDING_OPERATION_CREATE,
|
||||
PENDING_OPERATION_DELETE,
|
||||
PENDING_OPERATION_LIST,
|
||||
PENDING_OPERATION_READ,
|
||||
PENDING_OPERATION_WRITE,
|
||||
} PendingOperationType;
|
||||
|
||||
typedef struct {
|
||||
PendingOperationType type;
|
||||
ToastyHandle handle;
|
||||
void *ptr;
|
||||
} PendingOperation;
|
||||
|
||||
typedef struct {
|
||||
ToastyFS *toasty;
|
||||
int num_pending;
|
||||
PendingOperation pending[MAX_PENDING_OPERATION];
|
||||
} SimulationClient;
|
||||
|
||||
int simulation_client_init(SimulationClient *client, int argc, char **argv,
|
||||
void **contexts, struct pollfd *polled, int *timeout);
|
||||
int simulation_client_step(SimulationClient *client, void **contexts,
|
||||
struct pollfd *polled, int num_polled, int *timeout);
|
||||
void simulation_client_free(SimulationClient *client);
|
||||
|
||||
#endif // SIMULATION_CLIENT_INCLUDED
|
||||
-2060
File diff suppressed because it is too large
Load Diff
-209
@@ -1,209 +0,0 @@
|
||||
#ifndef SYSTEM_INCLUDED
|
||||
#define SYSTEM_INCLUDED
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
|
||||
#include <direct.h> // _mkdir
|
||||
#include <windows.h>
|
||||
#include <winsock2.h>
|
||||
#include <ws2tcpip.h>
|
||||
|
||||
#else
|
||||
|
||||
#include <poll.h>
|
||||
#include <time.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/random.h>
|
||||
#include <arpa/inet.h>
|
||||
|
||||
#define SOCKET int
|
||||
#define INVALID_SOCKET ((SOCKET) -1)
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef BUILD_TEST
|
||||
|
||||
void startup_simulation(uint64_t seed_);
|
||||
int spawn_simulated_process(char *args);
|
||||
void update_simulation(void);
|
||||
void cleanup_simulation(void);
|
||||
uint64_t simulation_random_number(void);
|
||||
|
||||
void* mock_malloc(size_t len);
|
||||
void* mock_realloc(void *ptr, size_t len);
|
||||
void mock_free(void *ptr);
|
||||
int mock_remove(char *path);
|
||||
int mock_rename(char *oldpath, char *newpath);
|
||||
SOCKET mock_socket(int domain, int type, int protocol);
|
||||
int mock_bind(SOCKET fd, void *addr, size_t addr_len);
|
||||
int mock_listen(SOCKET fd, int backlog);
|
||||
SOCKET mock_accept(SOCKET fd, void *addr, socklen_t *addr_len);
|
||||
int mock_getsockopt(SOCKET fd, int level, int optname, void *optval, socklen_t *optlen);
|
||||
int mock_recv(SOCKET fd, void *dst, int len, int flags);
|
||||
int mock_send(SOCKET fd, void *src, int len, int flags);
|
||||
int mock_connect(SOCKET fd, void *addr, size_t addr_len);
|
||||
|
||||
#ifdef _WIN32
|
||||
int mock_closesocket(SOCKET fd);
|
||||
int mock_ioctlsocket(SOCKET fd, long cmd, u_long *argp);
|
||||
HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile);
|
||||
BOOL mock_CloseHandle(HANDLE handle);
|
||||
BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh);
|
||||
BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh);
|
||||
BOOL mock_FlushFileBuffers(HANDLE handle);
|
||||
BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov);
|
||||
BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov);
|
||||
DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod);
|
||||
BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf);
|
||||
BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount);
|
||||
BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency);
|
||||
char* mock__fullpath(char *path, char *dst, int cap);
|
||||
int mock__mkdir(char *path);
|
||||
HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData);
|
||||
BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData);
|
||||
BOOL mock_FindClose(HANDLE hFindFile);
|
||||
BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags);
|
||||
#else
|
||||
int mock_clock_gettime(clockid_t clockid, struct timespec *tp);
|
||||
int mock_open(char *path, int flags, int mode);
|
||||
int mock_close(int fd);
|
||||
int mock_flock(int fd, int op);
|
||||
int mock_fsync(int fd);
|
||||
int mock_read(int fd, char *dst, int len);
|
||||
int mock_write(int fd, char *src, int len);
|
||||
off_t mock_lseek(int fd, off_t offset, int whence);
|
||||
int mock_fstat(int fd, struct stat *buf);
|
||||
int mock_mkstemp(char *path);
|
||||
char* mock_realpath(char *path, char *dst);
|
||||
int mock_mkdir(char *path, mode_t mode);
|
||||
int mock_fcntl(int fd, int cmd, ...);
|
||||
DIR* mock_opendir(char *name);
|
||||
struct dirent* mock_readdir(DIR *dirp);
|
||||
int mock_closedir(DIR *dirp);
|
||||
#endif
|
||||
|
||||
// Common
|
||||
#define sys_malloc mock_malloc
|
||||
#define sys_realloc mock_realloc
|
||||
#define sys_free mock_free
|
||||
#define sys_remove mock_remove
|
||||
#define sys_rename mock_rename
|
||||
#define sys_socket mock_socket
|
||||
#define sys_bind mock_bind
|
||||
#define sys_listen mock_listen
|
||||
#define sys_accept mock_accept
|
||||
#define sys_getsockopt mock_getsockopt
|
||||
#define sys_recv mock_recv
|
||||
#define sys_send mock_send
|
||||
#define sys_connect mock_connect
|
||||
|
||||
// Windows
|
||||
#define sys__mkdir mock__mkdir
|
||||
#define sys_closesocket mock_closesocket
|
||||
#define sys_ioctlsocket mock_ioctlsocket
|
||||
#define sys_CreateFileW mock_CreateFileW
|
||||
#define sys_CloseHandle mock_CloseHandle
|
||||
#define sys_LockFile mock_LockFile
|
||||
#define sys_UnlockFile mock_UnlockFile
|
||||
#define sys_FlushFileBuffers mock_FlushFileBuffers
|
||||
#define sys_ReadFile mock_ReadFile
|
||||
#define sys_WriteFile mock_WriteFile
|
||||
#define sys_SetFilePointer mock_SetFilePointer
|
||||
#define sys_GetFileSizeEx mock_GetFileSizeEx
|
||||
#define sys__fullpath mock__fullpath
|
||||
#define sys_QueryPerformanceCounter mock_QueryPerformanceCounter
|
||||
#define sys_QueryPerformanceFrequency mock_QueryPerformanceFrequency
|
||||
#define sys_FindFirstFileA mock_FindFirstFileA
|
||||
#define sys_FindNextFileA mock_FindNextFileA
|
||||
#define sys_FindClose mock_FindClose
|
||||
#define sys_MoveFileExW mock_MoveFileExW
|
||||
|
||||
// Linux
|
||||
#define sys_mkdir mock_mkdir
|
||||
#define sys_open mock_open
|
||||
#define sys_close mock_close
|
||||
#define sys_flock mock_flock
|
||||
#define sys_fsync mock_fsync
|
||||
#define sys_read mock_read
|
||||
#define sys_write mock_write
|
||||
#define sys_lseek mock_lseek
|
||||
#define sys_fstat mock_fstat
|
||||
#define sys_mkstemp mock_mkstemp
|
||||
#define sys_realpath mock_realpath
|
||||
#define sys_clock_gettime mock_clock_gettime
|
||||
#define sys_fcntl mock_fcntl
|
||||
#define sys_getrandom mock_getrandom
|
||||
#define sys_opendir mock_opendir
|
||||
#define sys_readdir mock_readdir
|
||||
#define sys_closedir mock_closedir
|
||||
|
||||
#else
|
||||
|
||||
// Common
|
||||
#define sys_malloc malloc
|
||||
#define sys_realloc realloc
|
||||
#define sys_free free
|
||||
#define sys_remove remove
|
||||
#define sys_rename rename
|
||||
#define sys_socket socket
|
||||
#define sys_bind bind
|
||||
#define sys_listen listen
|
||||
#define sys_accept accept
|
||||
#define sys_getsockopt getsockopt
|
||||
#define sys_recv recv
|
||||
#define sys_send send
|
||||
#define sys_connect connect
|
||||
|
||||
// Windows
|
||||
#define sys__mkdir _mkdir
|
||||
#define sys_closesocket closesocket
|
||||
#define sys_ioctlsocket ioctlsocket
|
||||
#define sys_CreateFileW CreateFileW
|
||||
#define sys_CloseHandle CloseHandle
|
||||
#define sys_LockFile LockFile
|
||||
#define sys_UnlockFile UnlockFile
|
||||
#define sys_FlushFileBuffers FlushFileBuffers
|
||||
#define sys_ReadFile ReadFile
|
||||
#define sys_WriteFile WriteFile
|
||||
#define sys_SetFilePointer SetFilePointer
|
||||
#define sys_GetFileSizeEx GetFileSizeEx
|
||||
#define sys__fullpath _fullpath
|
||||
#define sys_QueryPerformanceCounter QueryPerformanceCounter
|
||||
#define sys_QueryPerformanceFrequency QueryPerformanceFrequency
|
||||
#define sys_FindFirstFileA FindFirstFileA
|
||||
#define sys_FindNextFileA FindNextFileA
|
||||
#define sys_FindClose FindClose
|
||||
#define sys_MoveFileExW MoveFileExW
|
||||
|
||||
// Linux
|
||||
#define sys_mkdir mkdir
|
||||
#define sys_open open
|
||||
#define sys_close close
|
||||
#define sys_flock flock
|
||||
#define sys_fsync fsync
|
||||
#define sys_read read
|
||||
#define sys_write write
|
||||
#define sys_lseek lseek
|
||||
#define sys_fstat fstat
|
||||
#define sys_mkstemp mkstemp
|
||||
#define sys_realpath realpath
|
||||
#define sys_clock_gettime clock_gettime
|
||||
#define sys_fcntl fcntl
|
||||
#define sys_opendir opendir
|
||||
#define sys_readdir readdir
|
||||
#define sys_closedir closedir
|
||||
|
||||
#endif
|
||||
#endif // SYSTEM_INCLUDED
|
||||
@@ -1,8 +1,12 @@
|
||||
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "tcp.h"
|
||||
#include "system.h"
|
||||
#include "message.h"
|
||||
|
||||
bool addr_eql(Address a, Address b)
|
||||
@@ -28,15 +32,15 @@ static int set_socket_blocking(SOCKET sock, bool value)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
u_long mode = !value;
|
||||
if (sys_ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
|
||||
if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR)
|
||||
return -1;
|
||||
#else
|
||||
int flags = sys_fcntl(sock, F_GETFL, 0);
|
||||
int flags = fcntl(sock, F_GETFL, 0);
|
||||
if (flags < 0)
|
||||
return -1;
|
||||
if (value) flags &= ~O_NONBLOCK;
|
||||
else flags |= O_NONBLOCK;
|
||||
if (sys_fcntl(sock, F_SETFL, flags) < 0)
|
||||
if (fcntl(sock, F_SETFL, flags) < 0)
|
||||
return -1;
|
||||
#endif
|
||||
|
||||
@@ -45,7 +49,7 @@ static int set_socket_blocking(SOCKET sock, bool value)
|
||||
|
||||
static SOCKET create_listen_socket(string addr, uint16_t port)
|
||||
{
|
||||
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0);
|
||||
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (fd == INVALID_SOCKET)
|
||||
return INVALID_SOCKET;
|
||||
|
||||
@@ -72,13 +76,13 @@ static SOCKET create_listen_socket(string addr, uint16_t port)
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
|
||||
if (sys_bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
|
||||
if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) {
|
||||
CLOSE_SOCKET(fd);
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
|
||||
int backlog = 32;
|
||||
if (sys_listen(fd, backlog) < 0) {
|
||||
if (listen(fd, backlog) < 0) {
|
||||
CLOSE_SOCKET(fd);
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
@@ -319,7 +323,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
|
||||
assert(contexts[i] == NULL);
|
||||
|
||||
if (polled[i].revents & POLLIN) {
|
||||
SOCKET new_fd = sys_accept(tcp->listen_fd, NULL, NULL);
|
||||
SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL);
|
||||
if (new_fd != INVALID_SOCKET) {
|
||||
|
||||
if (set_socket_blocking(new_fd, false) < 0)
|
||||
@@ -347,7 +351,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
|
||||
|
||||
int err = 0;
|
||||
socklen_t len = sizeof(err);
|
||||
if (sys_getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
|
||||
if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0)
|
||||
defer_close = true;
|
||||
else {
|
||||
conn->connecting = false;
|
||||
@@ -360,7 +364,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
|
||||
if (polled[i].revents & POLLIN) {
|
||||
byte_queue_write_setmincap(&conn->input, 1<<9);
|
||||
ByteView buf = byte_queue_write_buf(&conn->input);
|
||||
int num = sys_recv(conn->fd, (char*) buf.ptr, buf.len, 0);
|
||||
int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0);
|
||||
if (num == 0)
|
||||
defer_close = true;
|
||||
else if (num < 0) {
|
||||
@@ -388,7 +392,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd
|
||||
|
||||
if (polled[i].revents & POLLOUT) {
|
||||
ByteView buf = byte_queue_read_buf(&conn->output);
|
||||
int num = sys_send(conn->fd, (char*) buf.ptr, buf.len, 0);
|
||||
int num = send(conn->fd, (char*) buf.ptr, buf.len, 0);
|
||||
if (num < 0) {
|
||||
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN)
|
||||
defer_close = true;
|
||||
@@ -432,7 +436,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
|
||||
return -1;
|
||||
int conn_idx = tcp->num_conns;
|
||||
|
||||
SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0);
|
||||
SOCKET fd = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (fd == INVALID_SOCKET)
|
||||
return -1;
|
||||
|
||||
@@ -447,13 +451,13 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output)
|
||||
buf.sin_family = AF_INET;
|
||||
buf.sin_port = htons(addr.port);
|
||||
memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4));
|
||||
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf));
|
||||
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
|
||||
} else {
|
||||
struct sockaddr_in6 buf;
|
||||
buf.sin6_family = AF_INET6;
|
||||
buf.sin6_port = htons(addr.port);
|
||||
memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6));
|
||||
ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf));
|
||||
ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf));
|
||||
}
|
||||
|
||||
bool connecting;
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
#ifndef TCP_INCLUDED
|
||||
#define TCP_INCLUDED
|
||||
|
||||
#include <stdbool.h>
|
||||
#ifdef MAIN_SIMULATION
|
||||
# define QUAKEY_ENABLE_MOCKS
|
||||
# include <quakey.h>
|
||||
#else
|
||||
# ifdef _WIN32
|
||||
# include <winsock2.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#include "system.h"
|
||||
#include "byte_queue.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#define CLOSE_SOCKET sys_closesocket
|
||||
#define CLOSE_SOCKET closesocket
|
||||
#else
|
||||
#define CLOSE_SOCKET sys_close
|
||||
#define SOCKET int
|
||||
#define INVALID_SOCKET -1
|
||||
#define CLOSE_SOCKET close
|
||||
#endif
|
||||
|
||||
#ifndef TCP_CONNECTION_LIMIT
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
#include <stddef.h>
|
||||
#include <limits.h>
|
||||
|
||||
#ifdef MAIN_SIMULATION
|
||||
#define QUAKEY_ENABLE_MOCKS
|
||||
#endif
|
||||
|
||||
#include <quakey.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "wal.h"
|
||||
#include "file_system.h"
|
||||
#include "file_tree.h"
|
||||
#include "system.h"
|
||||
|
||||
#define WAL_MAGIC 0xcafebebe
|
||||
#define WAL_VERSION 1
|
||||
@@ -190,7 +192,7 @@ static int swap_file(WAL *wal)
|
||||
}
|
||||
|
||||
// MOVEFILE_REPLACE_EXISTING allows atomic overwrite
|
||||
if (!sys_MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
|
||||
if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) {
|
||||
file_unlock(temp_handle);
|
||||
file_close(temp_handle);
|
||||
remove_file_or_dir(temp_path);
|
||||
@@ -267,12 +269,12 @@ static int next_entry(Handle handle, WALEntry *entry)
|
||||
return -1;
|
||||
|
||||
// Dynamically allocate path buffer
|
||||
char *path_buffer = sys_malloc(path_len);
|
||||
char *path_buffer = malloc(path_len);
|
||||
if (!path_buffer)
|
||||
return -1;
|
||||
|
||||
if (read_exact(handle, path_buffer, path_len) <= 0) {
|
||||
sys_free(path_buffer);
|
||||
free(path_buffer);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -315,13 +317,13 @@ static int next_entry(Handle handle, WALEntry *entry)
|
||||
goto cleanup_error;
|
||||
|
||||
// Dynamically allocate hash buffers
|
||||
SHA256 *hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256));
|
||||
SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256));
|
||||
if (!hashes_buffer) {
|
||||
goto cleanup_error;
|
||||
}
|
||||
|
||||
if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) {
|
||||
sys_free(hashes_buffer);
|
||||
free(hashes_buffer);
|
||||
goto cleanup_error;
|
||||
}
|
||||
|
||||
@@ -337,9 +339,9 @@ static int next_entry(Handle handle, WALEntry *entry)
|
||||
|
||||
cleanup_error:
|
||||
if (entry->path.ptr)
|
||||
sys_free((char*) entry->path.ptr);
|
||||
free((char*) entry->path.ptr);
|
||||
if (entry->hashes)
|
||||
sys_free(entry->hashes);
|
||||
free(entry->hashes);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -351,7 +353,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
|
||||
wal->file_path.ptr = NULL;
|
||||
|
||||
// Copy file_path since the passed string may not have the same lifetime as WAL
|
||||
char *path_copy = sys_malloc(file_path.len);
|
||||
char *path_copy = malloc(file_path.len);
|
||||
if (!path_copy)
|
||||
return -1;
|
||||
memcpy(path_copy, file_path.ptr, file_path.len);
|
||||
@@ -482,9 +484,9 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
|
||||
|
||||
// Free dynamically allocated fields from next_entry
|
||||
if (entry.path.ptr)
|
||||
sys_free((char*) entry.path.ptr);
|
||||
free((char*) entry.path.ptr);
|
||||
if (entry.hashes)
|
||||
sys_free(entry.hashes);
|
||||
free(entry.hashes);
|
||||
|
||||
wal->entry_count++;
|
||||
}
|
||||
@@ -499,7 +501,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit)
|
||||
return 0;
|
||||
|
||||
error_cleanup_path:
|
||||
sys_free(path_copy);
|
||||
free(path_copy);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -508,7 +510,7 @@ void wal_close(WAL *wal)
|
||||
file_unlock(wal->handle);
|
||||
file_close(wal->handle);
|
||||
if (wal->file_path.ptr)
|
||||
sys_free((char*) wal->file_path.ptr);
|
||||
free((char*) wal->file_path.ptr);
|
||||
}
|
||||
|
||||
static int write_u8(Handle handle, uint8_t value)
|
||||
|
||||
-5077
File diff suppressed because it is too large
Load Diff
-1367
File diff suppressed because it is too large
Load Diff
@@ -1,64 +0,0 @@
|
||||
The web interface allows working with the file system cluster using
|
||||
basic HTTP requests. There are many ways to interface with a file system
|
||||
using HTTP (like WebDAV), but for the firsts versions of ToastyFS I
|
||||
opted for some custom minimal semantics. Here are they.
|
||||
|
||||
GET
|
||||
If the path refers to a directory, the listing of its direct children
|
||||
is returned as a newline-separated list of relative paths. Children
|
||||
that are directories have ending slashes, while files don't:
|
||||
|
||||
image.jpg
|
||||
Documents/
|
||||
notes.txt
|
||||
|
||||
If the path refers to a file, its contents are returned. Note that
|
||||
Range requests are not supported.
|
||||
|
||||
Returns 200 on success, 404 if the file/directory doesn't exist, else 5xx.
|
||||
|
||||
HEAD
|
||||
Works like GET but doesn't return the content (The Content-Length is
|
||||
set as if the payload was being send).
|
||||
|
||||
PUT
|
||||
The server considers the resource being uploaded as a directory if the
|
||||
path ends with a slash, and as a file otherwise:
|
||||
|
||||
photos/ directory
|
||||
image.jpg file
|
||||
|
||||
If the path refers to a directory and no file/directory exists at that
|
||||
path, a new one is created and status 201 is returned. If a directory
|
||||
existd already, statis 204 is returned. If a file existed already status
|
||||
??? (TODO) is returned. If the payload the request is not empty, the
|
||||
operation fails with code 400.
|
||||
|
||||
If the path refers to a file, the file is automatically created if it
|
||||
doesn't exist, then the specified contents are written to it, replacing
|
||||
the entire file content (HTTP PUT semantics). This is implemented using
|
||||
two flags:
|
||||
- TOASTY_WRITE_CREATE_IF_MISSING: Creates file if it doesn't exist
|
||||
- TOASTY_WRITE_TRUNCATE_AFTER: Truncates file after the write, discarding
|
||||
any data beyond the uploaded content
|
||||
Files are created with a default chunk size of 4096 bytes.
|
||||
|
||||
If a file or directory already existed at that path, it is overwritten
|
||||
unless the header "X-ToastyFS-Overwrite: no" is sent.
|
||||
(TODO error codes)
|
||||
|
||||
DELETE
|
||||
Deletes the given file or directory. Returns 204 on success, 404 if
|
||||
no file or directory existed at that path, else returns 5xx.
|
||||
|
||||
-----------
|
||||
cases:
|
||||
PUT file / unused path -> OK
|
||||
PUT file / file exists -> overwrite?
|
||||
PUT file / directory exists
|
||||
PUT directory / unused path -> OK
|
||||
PUT directory / file exists
|
||||
PUT directory / directory exists
|
||||
|
||||
Idea: Use patch on the parent directory to rename
|
||||
-----------
|
||||
@@ -1,76 +0,0 @@
|
||||
#include "config.h"
|
||||
|
||||
void parse_config_or_exit(ProxyConfig *config,
|
||||
int argc, char **argv)
|
||||
{
|
||||
config->upstream_addr = TOASTY_STR("127.0.0.1");
|
||||
config->upstream_port = 9000;
|
||||
|
||||
config->local_addr = HTTP_STR("127.0.0.1");
|
||||
config->local_port = 8080;
|
||||
|
||||
config->reuse_addr = false;
|
||||
config->trace_bytes = false;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
|
||||
if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h")) {
|
||||
printf("TODO: print help\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (!strcmp(argv[i], "--upstream-addr")) {
|
||||
|
||||
i++;
|
||||
if (i == argc) {
|
||||
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
|
||||
exit(-1);
|
||||
}
|
||||
config->upstream_addr = (ToastyString) { argv[i], strlen(argv[i]) };
|
||||
|
||||
} else if (!strcmp(argv[i], "--upstream-port")) {
|
||||
|
||||
i++;
|
||||
if (i == argc) {
|
||||
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
|
||||
exit(-1);
|
||||
}
|
||||
int tmp = atoi(argv[i]);
|
||||
if (tmp < 1 || tmp > UINT16_MAX) {
|
||||
fprintf(stderr, "Error: Invalid port %s\n", argv[i]);
|
||||
exit(-1);
|
||||
}
|
||||
config->upstream_port = (uint16_t) tmp;
|
||||
|
||||
} else if (!strcmp(argv[i], "--local-addr")) {
|
||||
|
||||
i++;
|
||||
if (i == argc) {
|
||||
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
|
||||
exit(-1);
|
||||
}
|
||||
config->local_addr = (HTTP_String) { argv[i], strlen(argv[i]) };
|
||||
|
||||
} else if (!strcmp(argv[i], "--local-port")) {
|
||||
|
||||
i++;
|
||||
if (i == argc) {
|
||||
fprintf(stderr, "Error: Missing value after %s\n", argv[i-1]);
|
||||
exit(-1);
|
||||
}
|
||||
int tmp = atoi(argv[i]);
|
||||
if (tmp < 1 || tmp > UINT16_MAX) {
|
||||
fprintf(stderr, "Error: Invalid port %s\n", argv[i]);
|
||||
exit(-1);
|
||||
}
|
||||
config->local_port = (uint16_t) tmp;
|
||||
} else if(!strcmp(argv[i], "--reuse-addr")) {
|
||||
config->reuse_addr = true;
|
||||
} else if(!strcmp(argv[i], "--trace-bytes")) {
|
||||
config->trace_bytes = true;
|
||||
} else {
|
||||
fprintf(stderr, "Error: Invalid option %s\n", argv[i]);
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
#ifndef CONFIG_INCLUDED
|
||||
#define CONFIG_INCLUDED
|
||||
|
||||
#include <chttp.h>
|
||||
#include <ToastyFS.h>
|
||||
|
||||
typedef struct {
|
||||
ToastyString upstream_addr;
|
||||
uint16_t upstream_port;
|
||||
|
||||
HTTP_String local_addr;
|
||||
uint16_t local_port;
|
||||
|
||||
bool reuse_addr;
|
||||
bool trace_bytes;
|
||||
} ProxyConfig;
|
||||
|
||||
void parse_config_or_exit(ProxyConfig *config,
|
||||
int argc, char **argv);
|
||||
|
||||
#endif // CONFIG_INCLUDED
|
||||
@@ -1,84 +0,0 @@
|
||||
#include "event_loop.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#define POLL WSAPoll
|
||||
#else
|
||||
#define POLL poll
|
||||
#endif
|
||||
|
||||
#define POLL_CAPACITY (HTTP_SERVER_POLL_CAPACITY + TOASTY_POLL_CAPACITY)
|
||||
|
||||
void event_loop_init(EventLoop *loop, HTTP_Server *server,
|
||||
ToastyFS *backend)
|
||||
{
|
||||
loop->state = EVENT_LOOP_PROCESSING_COMPLETIONS;
|
||||
loop->server = server;
|
||||
loop->backend = backend;
|
||||
}
|
||||
|
||||
void event_loop_free(EventLoop *loop)
|
||||
{
|
||||
(void) loop;
|
||||
}
|
||||
|
||||
int event_loop_wait(EventLoop *loop, Event *event)
|
||||
{
|
||||
for (;;) {
|
||||
switch (loop->state) {
|
||||
|
||||
int ret;
|
||||
case EVENT_LOOP_PROCESSING_COMPLETIONS:
|
||||
|
||||
ret = toasty_get_result(loop->backend, TOASTY_INVALID, &event->completion);
|
||||
if (ret < 0)
|
||||
return -1; // Error
|
||||
|
||||
if (ret == 0) {
|
||||
// Completed
|
||||
event->type = EVENT_TYPE_COMPLETION;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// fallthrough
|
||||
case EVENT_LOOP_PROCESSING_REQUESTS:
|
||||
|
||||
if (http_server_next_request(loop->server, &event->request, &event->builder)) {
|
||||
// Completed
|
||||
event->type = EVENT_TYPE_REQUEST;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// fallthrough
|
||||
}
|
||||
|
||||
EventRegister reg;
|
||||
void *ptrs[POLL_CAPACITY];
|
||||
struct pollfd polled[POLL_CAPACITY];
|
||||
|
||||
void **http_ptrs = ptrs;
|
||||
struct pollfd *http_polled = polled;
|
||||
|
||||
reg = (EventRegister) { ptrs, polled, 0 };
|
||||
http_server_register_events(loop->server, ®);
|
||||
int num_http_polled = reg.num_polled;
|
||||
|
||||
void **toasty_ptrs = ptrs + num_http_polled;
|
||||
struct pollfd *toasty_polled = polled + num_http_polled;
|
||||
int num_toasty_polled = toasty_process_events(loop->backend, toasty_ptrs, toasty_polled, 0);
|
||||
if (num_toasty_polled < 0)
|
||||
return -1;
|
||||
|
||||
int num_polled = num_http_polled + num_toasty_polled;
|
||||
if (num_http_polled != 0 && num_toasty_polled != 0)
|
||||
POLL(polled, num_polled, -1);
|
||||
|
||||
if (toasty_process_events(loop->backend, toasty_ptrs, toasty_polled, num_toasty_polled) < 0)
|
||||
return -1;
|
||||
|
||||
reg = (EventRegister) { http_ptrs, http_polled, num_http_polled };
|
||||
http_server_process_events(loop->server, reg);
|
||||
}
|
||||
|
||||
// Unreachable
|
||||
return -1;
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
#ifndef EVENT_LOOP_INCLUDED
|
||||
#define EVENT_LOOP_INCLUDED
|
||||
|
||||
#include <ToastyFS.h>
|
||||
#include "chttp.h"
|
||||
|
||||
typedef enum {
|
||||
EVENT_TYPE_REQUEST,
|
||||
EVENT_TYPE_COMPLETION,
|
||||
} EventType;
|
||||
|
||||
typedef struct {
|
||||
EventType type;
|
||||
|
||||
// Completion
|
||||
ToastyResult completion;
|
||||
|
||||
// Request
|
||||
HTTP_Request *request;
|
||||
HTTP_ResponseBuilder builder;
|
||||
} Event;
|
||||
|
||||
|
||||
typedef enum {
|
||||
EVENT_LOOP_PROCESSING_REQUESTS,
|
||||
EVENT_LOOP_PROCESSING_COMPLETIONS,
|
||||
} EventLoopState;
|
||||
|
||||
typedef struct {
|
||||
EventLoopState state;
|
||||
HTTP_Server *server;
|
||||
ToastyFS *backend;
|
||||
} EventLoop;
|
||||
|
||||
void event_loop_init(EventLoop *loop, HTTP_Server *server,
|
||||
ToastyFS *backend);
|
||||
|
||||
void event_loop_free(EventLoop *loop);
|
||||
|
||||
int event_loop_wait(EventLoop *loop, Event *event);
|
||||
|
||||
#endif // EVENT_LOOP_INCLUDED
|
||||
@@ -1,52 +0,0 @@
|
||||
#include "proxy.h"
|
||||
#include "config.h"
|
||||
#include "event_loop.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
ProxyConfig config;
|
||||
parse_config_or_exit(&config, argc, argv);
|
||||
|
||||
ToastyFS *backend = toasty_connect(
|
||||
config.upstream_addr,
|
||||
config.upstream_port);
|
||||
if (backend == NULL) {
|
||||
printf("toasty_connect error\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
HTTP_Server server;
|
||||
if (http_server_init(&server) < 0) {
|
||||
printf("http_server_init error\n");
|
||||
return -1;
|
||||
}
|
||||
http_server_set_reuse_addr(&server, config.reuse_addr);
|
||||
http_server_set_trace_bytes(&server, config.trace_bytes);
|
||||
|
||||
if (http_server_listen_tcp(&server, config.local_addr, config.local_port) < 0) {
|
||||
printf("http_server_listen_tcp error\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
EventLoop loop;
|
||||
event_loop_init(&loop, &server, backend);
|
||||
|
||||
ProxyState proxy;
|
||||
proxy_init(&proxy, backend);
|
||||
|
||||
for (Event event; !event_loop_wait(&loop, &event); ) {
|
||||
|
||||
if (event.type == EVENT_TYPE_REQUEST) {
|
||||
proxy_process_request(&proxy, event.request, event.builder);
|
||||
} else {
|
||||
assert(event.type == EVENT_TYPE_COMPLETION);
|
||||
proxy_process_completion(&proxy, event.completion);
|
||||
}
|
||||
}
|
||||
|
||||
proxy_free(&proxy);
|
||||
event_loop_free(&loop);
|
||||
http_server_free(&server);
|
||||
toasty_disconnect(backend);
|
||||
return 0;
|
||||
}
|
||||
-126
@@ -1,126 +0,0 @@
|
||||
#include "proxy.h"
|
||||
#include "proxy_get.h"
|
||||
#include "proxy_put.h"
|
||||
#include "proxy_delete.h"
|
||||
|
||||
void proxy_init(ProxyState *state, ToastyFS *backend)
|
||||
{
|
||||
state->backend = backend;
|
||||
state->num_operations = 0;
|
||||
for (int i = 0; i < PROXY_CAPACITY; i++)
|
||||
state->operations[i].type = PO_FREE;
|
||||
}
|
||||
|
||||
void proxy_free(ProxyState *state)
|
||||
{
|
||||
for (int i = 0; i < PROXY_CAPACITY; i++) {
|
||||
ProxyOperation *operation = &state->operations[i];
|
||||
if (operation->type != PO_FREE) {
|
||||
http_response_builder_status(operation->builder, 500);
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static ProxyOperation *find_unused_operation(ProxyState *state)
|
||||
{
|
||||
if (state->num_operations == PROXY_CAPACITY)
|
||||
return NULL;
|
||||
|
||||
ProxyOperation *operation = state->operations;
|
||||
while (operation->type != PO_FREE)
|
||||
operation++;
|
||||
|
||||
return operation;
|
||||
}
|
||||
|
||||
void proxy_process_request(ProxyState *state,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder)
|
||||
{
|
||||
ProxyOperation *operation = find_unused_operation(state);
|
||||
|
||||
bool created = false;
|
||||
switch (request->method) {
|
||||
case HTTP_METHOD_GET:
|
||||
if (operation == NULL) {
|
||||
http_response_builder_status(builder, 503); // Service Unavailable
|
||||
http_response_builder_send(builder);
|
||||
} else {
|
||||
created = process_request_get(state, operation, request, builder);
|
||||
}
|
||||
break;
|
||||
case HTTP_METHOD_HEAD:
|
||||
if (operation == NULL) {
|
||||
http_response_builder_status(builder, 503); // Service Unavailable
|
||||
http_response_builder_send(builder);
|
||||
} else {
|
||||
created = process_request_head(state, operation, request, builder);
|
||||
}
|
||||
break;
|
||||
case HTTP_METHOD_PUT:
|
||||
if (operation == NULL) {
|
||||
http_response_builder_status(builder, 503); // Service Unavailable
|
||||
http_response_builder_send(builder);
|
||||
} else {
|
||||
created = process_request_put(state, operation, request, builder);
|
||||
}
|
||||
break;
|
||||
case HTTP_METHOD_DELETE:
|
||||
if (operation == NULL) {
|
||||
http_response_builder_status(builder, 503); // Service Unavailable
|
||||
http_response_builder_send(builder);
|
||||
} else {
|
||||
created = process_request_delete(state, operation, request, builder);
|
||||
}
|
||||
break;
|
||||
case HTTP_METHOD_OPTIONS:
|
||||
http_response_builder_status(builder, 200); // OK
|
||||
http_response_builder_header(builder,
|
||||
HTTP_STR("Allow: GET, HEAD, PUT, DELETE, OPTIONS"));
|
||||
http_response_builder_send(builder);
|
||||
break;
|
||||
default:
|
||||
http_response_builder_status(builder, 405); // Method not allowed
|
||||
http_response_builder_header(builder,
|
||||
HTTP_STR("Allow: GET, HEAD, PUT, DELETE, OPTIONS"));
|
||||
http_response_builder_send(builder);
|
||||
break;
|
||||
}
|
||||
if (created) {
|
||||
state->num_operations++;
|
||||
}
|
||||
}
|
||||
|
||||
void proxy_process_completion(ProxyState *state,
|
||||
ToastyResult completion)
|
||||
{
|
||||
ProxyOperation *operation = completion.user;
|
||||
assert(operation);
|
||||
|
||||
bool completed;
|
||||
switch (operation->type) {
|
||||
case PO_CREATE_DIR:
|
||||
completed = process_completion_create_dir(state, operation, completion);
|
||||
break;
|
||||
case PO_CREATE_FILE:
|
||||
completed = process_completion_create_file(state, operation, completion);
|
||||
break;
|
||||
case PO_DELETE:
|
||||
completed = process_completion_delete(state, operation, completion);
|
||||
break;
|
||||
case PO_READ_DIR:
|
||||
completed = process_completion_read_dir(state, operation, completion);
|
||||
break;
|
||||
case PO_READ_FILE:
|
||||
completed = process_completion_read_file(state, operation, completion);
|
||||
break;
|
||||
case PO_WRITE:
|
||||
completed = process_completion_write(state, operation, completion);
|
||||
break;
|
||||
default:
|
||||
}
|
||||
if (completed) {
|
||||
operation->type = PO_FREE;
|
||||
state->num_operations--;
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
#ifndef PROXY_INCLUDED
|
||||
#define PROXY_INCLUDED
|
||||
|
||||
#include <chttp.h>
|
||||
#include <ToastyFS.h>
|
||||
|
||||
#define PROXY_CAPACITY (1<<10)
|
||||
|
||||
typedef enum {
|
||||
PO_FREE,
|
||||
PO_CREATE_DIR,
|
||||
PO_CREATE_FILE,
|
||||
PO_DELETE,
|
||||
PO_READ_DIR,
|
||||
PO_READ_FILE,
|
||||
PO_WRITE,
|
||||
} ProxyOperationType;
|
||||
|
||||
typedef struct {
|
||||
ProxyOperationType type;
|
||||
HTTP_Request* request;
|
||||
HTTP_ResponseBuilder builder;
|
||||
ToastyHandle handle;
|
||||
|
||||
bool head_only;
|
||||
int transferred;
|
||||
} ProxyOperation;
|
||||
|
||||
typedef struct {
|
||||
ToastyFS *backend;
|
||||
int num_operations;
|
||||
ProxyOperation operations[PROXY_CAPACITY];
|
||||
} ProxyState;
|
||||
|
||||
void proxy_init(ProxyState *state, ToastyFS *backend);
|
||||
void proxy_free(ProxyState *state);
|
||||
|
||||
void proxy_process_request(ProxyState *state,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder);
|
||||
|
||||
void proxy_process_completion(ProxyState *state,
|
||||
ToastyResult completion);
|
||||
|
||||
#endif // PROXY_INCLUDED
|
||||
@@ -1,37 +0,0 @@
|
||||
#include "proxy_delete.h"
|
||||
|
||||
bool process_request_delete(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder)
|
||||
{
|
||||
// TODO: Implement If-Match and If-None-Match headers
|
||||
|
||||
HTTP_String path = request->url.path;
|
||||
ToastyHandle handle = toasty_begin_delete(state->backend,
|
||||
(ToastyString) { path.ptr, path.len }, TOASTY_VERSION_TAG_EMPTY);
|
||||
if (handle == TOASTY_INVALID) {
|
||||
http_response_builder_status(builder, 500); // Internal Server Error
|
||||
http_response_builder_send(builder);
|
||||
return false;
|
||||
}
|
||||
toasty_set_user(state->backend, handle, operation);
|
||||
|
||||
operation->type = PO_DELETE;
|
||||
operation->request = request;
|
||||
operation->builder = builder;
|
||||
operation->handle = handle;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_completion_delete(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
if (completion.type == TOASTY_RESULT_DELETE_SUCCESS) {
|
||||
http_response_builder_status(operation->builder, 204); // No Content
|
||||
http_response_builder_send(operation->builder);
|
||||
} else {
|
||||
// TODO: Should differentiate between error conditions
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
#ifndef PROXY_DELETE_INCLUDED
|
||||
#define PROXY_DELETE_INCLUDED
|
||||
|
||||
#include "proxy.h"
|
||||
|
||||
bool process_request_delete(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder);
|
||||
|
||||
bool process_completion_delete(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
#endif // PROXY_DELETE_INCLUDED
|
||||
@@ -1,143 +0,0 @@
|
||||
#include "proxy_get.h"
|
||||
|
||||
static bool path_refers_to_dir(ToastyString path)
|
||||
{
|
||||
return path.len > 0 && path.ptr[path.len-1] == '/';
|
||||
}
|
||||
|
||||
bool process_request_get(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder)
|
||||
{
|
||||
assert(request->method == HTTP_METHOD_GET
|
||||
|| request->method == HTTP_METHOD_HEAD);
|
||||
|
||||
http_response_builder_status(builder, 200); // OK
|
||||
|
||||
ToastyString path = {
|
||||
request->url.path.ptr,
|
||||
request->url.path.len
|
||||
};
|
||||
|
||||
ToastyHandle handle;
|
||||
if (path_refers_to_dir(path)) {
|
||||
handle = toasty_begin_list(state->backend, path, TOASTY_VERSION_TAG_EMPTY);
|
||||
} else {
|
||||
|
||||
http_response_builder_body_cap(builder, 1<<10); // TODO: pick the prealloc better
|
||||
|
||||
int cap;
|
||||
char *dst = http_response_builder_body_buf(builder, &cap);
|
||||
// TODO: check for NULL dst
|
||||
|
||||
handle = toasty_begin_read(state->backend, path, 0, dst, cap, TOASTY_VERSION_TAG_EMPTY);
|
||||
}
|
||||
if (handle == TOASTY_INVALID) {
|
||||
if (!path_refers_to_dir(path))
|
||||
http_response_builder_body_ack(builder, 0);
|
||||
http_response_builder_status(builder, 500); // Internal Server Error
|
||||
http_response_builder_send(builder);
|
||||
return false;
|
||||
}
|
||||
toasty_set_user(state->backend, handle, operation);
|
||||
|
||||
operation->type = path_refers_to_dir(path) ? PO_READ_DIR : PO_READ_FILE;
|
||||
operation->request = request;
|
||||
operation->builder = builder;
|
||||
operation->handle = handle;
|
||||
operation->head_only = (request->method == HTTP_METHOD_HEAD);
|
||||
operation->transferred = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_request_head(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder)
|
||||
{
|
||||
return process_request_get(state, operation, request, builder);
|
||||
}
|
||||
|
||||
bool process_completion_read_dir(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
if (completion.type == TOASTY_RESULT_LIST_SUCCESS) {
|
||||
http_response_builder_status(operation->builder, 200);
|
||||
for (int i = 0; i < completion.listing.count; i++) {
|
||||
|
||||
ToastyListingEntry *entry = &completion.listing.items[i];
|
||||
|
||||
char vtag[32];
|
||||
int ret = snprintf(vtag, sizeof(vtag), "%lu", entry->vtag);
|
||||
if (ret < 0 || ret >= (int) sizeof(vtag)) {
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
return true;
|
||||
}
|
||||
http_response_builder_body(operation->builder, (HTTP_String) { entry->name, strlen(entry->name) });
|
||||
if (entry->is_dir) {
|
||||
http_response_builder_body(operation->builder, HTTP_STR("/ "));
|
||||
} else {
|
||||
http_response_builder_body(operation->builder, HTTP_STR(" "));
|
||||
}
|
||||
http_response_builder_body(operation->builder, (HTTP_String) { vtag, ret });
|
||||
http_response_builder_body(operation->builder, HTTP_STR("\n"));
|
||||
}
|
||||
http_response_builder_send(operation->builder);
|
||||
toasty_free_listing(&completion.listing);
|
||||
} else {
|
||||
// TODO: Should differentiate between error conditions
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_completion_read_file(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
bool again = false;
|
||||
if (completion.type != TOASTY_RESULT_READ_SUCCESS) {
|
||||
http_response_builder_body_ack(operation->builder, 0);
|
||||
http_response_builder_status(operation->builder, 500);
|
||||
http_response_builder_send(operation->builder);
|
||||
} else {
|
||||
|
||||
// First, ACK the byte we just read, even if it's
|
||||
// just 0 bytes (every bodybuf must by paired with
|
||||
// a bodyack).
|
||||
operation->transferred += completion.bytes_read;
|
||||
int ack = operation->head_only ? 0 : completion.bytes_read;
|
||||
http_response_builder_body_ack(operation->builder, ack);
|
||||
|
||||
// If we didn't reach the end of the file, start
|
||||
// a new read.
|
||||
if (completion.bytes_read > 0) {
|
||||
|
||||
// Make sure there is some free space in the buffer
|
||||
int mincap = 1<<10; // TODO: Choose based on overall file size
|
||||
http_response_builder_body_cap(operation->builder, mincap);
|
||||
|
||||
// Get the location of that buffer
|
||||
int cap;
|
||||
char *dst = http_response_builder_body_buf(operation->builder, &cap);
|
||||
if (dst == NULL) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
|
||||
ToastyString path = {
|
||||
operation->request->url.path.ptr,
|
||||
operation->request->url.path.len,
|
||||
};
|
||||
operation->handle = toasty_begin_read(state->backend, path,
|
||||
operation->transferred, dst, cap, TOASTY_VERSION_TAG_EMPTY);
|
||||
if (operation->handle == TOASTY_INVALID) {
|
||||
assert(0); // TODO
|
||||
}
|
||||
toasty_set_user(state->backend, operation->handle, operation);
|
||||
|
||||
again = true;
|
||||
} else {
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
}
|
||||
|
||||
return !again;
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
#ifndef PROXY_GET_INCLUDED
|
||||
#define PROXY_GET_INCLUDED
|
||||
|
||||
#include "proxy.h"
|
||||
|
||||
bool process_request_get(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder);
|
||||
|
||||
bool process_request_head(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder);
|
||||
|
||||
bool process_completion_read_dir(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
bool process_completion_read_file(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
#endif // PROXY_GET_INCLUDED
|
||||
@@ -1,70 +0,0 @@
|
||||
#include "proxy_put.h"
|
||||
|
||||
bool process_request_put(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder)
|
||||
{
|
||||
ToastyString path = {
|
||||
request->url.path.ptr,
|
||||
request->url.path.len,
|
||||
};
|
||||
// Use both flags for proper HTTP PUT semantics:
|
||||
// - TOASTY_WRITE_CREATE_IF_MISSING: Create file if it doesn't exist
|
||||
// - TOASTY_WRITE_TRUNCATE_AFTER: Replace entire file content (truncate after write)
|
||||
ToastyHandle handle = toasty_begin_write(state->backend, path, 0,
|
||||
request->body.ptr, request->body.len, TOASTY_VERSION_TAG_EMPTY,
|
||||
TOASTY_WRITE_CREATE_IF_MISSING | TOASTY_WRITE_TRUNCATE_AFTER);
|
||||
if (handle == TOASTY_INVALID) {
|
||||
http_response_builder_status(builder, 500); // Internal Server Error
|
||||
http_response_builder_send(builder);
|
||||
return false;
|
||||
}
|
||||
toasty_set_user(state->backend, handle, operation);
|
||||
|
||||
operation->type = PO_WRITE;
|
||||
operation->request = request;
|
||||
operation->builder = builder;
|
||||
operation->handle = handle;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_completion_create_dir(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
if (completion.type == TOASTY_RESULT_CREATE_SUCCESS) {
|
||||
http_response_builder_status(operation->builder, 201); // Created
|
||||
http_response_builder_send(operation->builder);
|
||||
} else {
|
||||
// TODO: Should differentiate between error conditions
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_completion_create_file(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
if (completion.type == TOASTY_RESULT_CREATE_SUCCESS) {
|
||||
http_response_builder_status(operation->builder, 201); // Created
|
||||
http_response_builder_send(operation->builder);
|
||||
} else {
|
||||
// TODO: Should differentiate between error conditions
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool process_completion_write(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion)
|
||||
{
|
||||
if (completion.type == TOASTY_RESULT_WRITE_SUCCESS) {
|
||||
http_response_builder_status(operation->builder, 201); // Created
|
||||
http_response_builder_send(operation->builder);
|
||||
} else {
|
||||
// TODO: Should differentiate between error conditions
|
||||
http_response_builder_status(operation->builder, 500); // Internal Server Error
|
||||
http_response_builder_send(operation->builder);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
#ifndef PROXY_PUT_INCLUDED
|
||||
#define PROXY_PUT_INCLUDED
|
||||
|
||||
#include "proxy.h"
|
||||
|
||||
bool process_request_put(ProxyState *state, ProxyOperation *operation,
|
||||
HTTP_Request *request, HTTP_ResponseBuilder builder);
|
||||
|
||||
bool process_completion_create_dir(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
bool process_completion_create_file(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
bool process_completion_write(ProxyState *state,
|
||||
ProxyOperation *operation, ToastyResult completion);
|
||||
|
||||
#endif // PROXY_PUT_INCLUDED
|
||||
Reference in New Issue
Block a user