From c10493881b3b1afc15e99f154f8efa10cd418cfc Mon Sep 17 00:00:00 2001 From: Francesco Cozzuto Date: Fri, 16 Jan 2026 14:45:09 +0100 Subject: [PATCH] Add Quakey --- .gitattributes | 2 - .gitignore | 22 +- Makefile | 67 - README.md | 24 - app/app.py | 54 - build.sh | 3 + examples/async_api.c | 60 - examples/blocking_api.c | 34 - misc/DESIGN.txt | 233 - misc/PROTOCOL.txt | 281 - misc/SIMTEST.txt | 3 - misc/TODO.txt | 26 - quakey/include/quakey.h | 190 + quakey/src/lfs.c | 6548 ++++++++++++++++++ quakey/src/lfs.h | 801 +++ quakey/src/lfs_util.c | 37 + quakey/src/lfs_util.h | 277 + quakey/src/quakey.c | 4304 ++++++++++++ scripts/README.md | 85 - scripts/cluster_demo.sh | 314 - scripts/fix_line_endings.sh | 18 - scripts/generate_coverage_html.sh | 31 - scripts/measure_coverage.sh | 113 - src/basic.c | 26 +- src/byte_queue.c | 21 +- src/byte_queue.h | 2 + src/chunk_server.c | 108 +- src/chunk_server.h | 22 +- src/client.c | 82 +- src/crash_logger.c | 433 -- src/crash_logger.h | 7 - src/file_system.c | 81 +- src/file_system.h | 14 +- src/file_tree.c | 34 +- src/hash_set.c | 19 +- src/main.c | 170 + src/main_server.c | 132 - src/main_test.c | 57 - src/message.c | 11 +- src/message.h | 8 +- src/metadata_server.c | 64 +- src/metadata_server.h | 15 +- src/{simulation_client.c => random_client.c} | 58 +- src/random_client.h | 39 + src/simulation_client.h | 43 - src/system.c | 2060 ------ src/system.h | 209 - src/tcp.c | 34 +- src/tcp.h | 16 +- src/wal.c | 34 +- web/3p/chttp.c | 5077 -------------- web/3p/chttp.h | 1367 ---- web/DESIGN.txt | 64 - web/src/config.c | 76 - web/src/config.h | 21 - web/src/event_loop.c | 84 - web/src/event_loop.h | 42 - web/src/main.c | 52 - web/src/proxy.c | 126 - web/src/proxy.h | 44 - web/src/proxy_delete.c | 37 - web/src/proxy_delete.h | 12 - web/src/proxy_get.c | 143 - web/src/proxy_get.h | 18 - web/src/proxy_put.c | 70 - web/src/proxy_put.h | 18 - 66 files changed, 12760 insertions(+), 11817 deletions(-) delete mode 100644 .gitattributes delete mode 100644 Makefile delete mode 100644 README.md delete mode 100644 app/app.py create mode 100644 build.sh delete mode 100644 examples/async_api.c delete mode 100644 examples/blocking_api.c delete mode 100644 misc/DESIGN.txt delete mode 100644 misc/PROTOCOL.txt delete mode 100644 misc/SIMTEST.txt delete mode 100644 misc/TODO.txt create mode 100644 quakey/include/quakey.h create mode 100644 quakey/src/lfs.c create mode 100644 quakey/src/lfs.h create mode 100644 quakey/src/lfs_util.c create mode 100644 quakey/src/lfs_util.h create mode 100644 quakey/src/quakey.c delete mode 100644 scripts/README.md delete mode 100755 scripts/cluster_demo.sh delete mode 100755 scripts/fix_line_endings.sh delete mode 100755 scripts/generate_coverage_html.sh delete mode 100755 scripts/measure_coverage.sh delete mode 100644 src/crash_logger.c delete mode 100644 src/crash_logger.h create mode 100644 src/main.c delete mode 100644 src/main_server.c delete mode 100644 src/main_test.c rename src/{simulation_client.c => random_client.c} (86%) create mode 100644 src/random_client.h delete mode 100644 src/simulation_client.h delete mode 100644 src/system.c delete mode 100644 src/system.h delete mode 100644 web/3p/chttp.c delete mode 100644 web/3p/chttp.h delete mode 100644 web/DESIGN.txt delete mode 100644 web/src/config.c delete mode 100644 web/src/config.h delete mode 100644 web/src/event_loop.c delete mode 100644 web/src/event_loop.h delete mode 100644 web/src/main.c delete mode 100644 web/src/proxy.c delete mode 100644 web/src/proxy.h delete mode 100644 web/src/proxy_delete.c delete mode 100644 web/src/proxy_delete.h delete mode 100644 web/src/proxy_get.c delete mode 100644 web/src/proxy_get.h delete mode 100644 web/src/proxy_put.c delete mode 100644 web/src/proxy_put.h diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 7c75d96..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Ensure shell scripts always use LF line endings -*.sh text eol=lf diff --git a/.gitignore b/.gitignore index 089c2c4..a8e4cbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,3 @@ -*.exe -*.out -*.o -*.a -chunk_server_data_* -crash_*.txt - -# Coverage reports -coverage_report/ -*.gcov -*.gcda -*.gcno -*.wal -*.tmp - -# Cluster demo artifacts -.cluster_demo.pids -cluster_logs/ -cluster_data/ +toasty_metadata_server +toasty_chunk_server +toasty_simulation \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 169cb4b..0000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ - -CFLAGS = -Wall -Wextra -ggdb -COVERAGE_CFLAGS = $(CFLAGS) --coverage -COVERAGE_LFLAGS = --coverage - -ifeq ($(OS),Windows_NT) - LFLAGS = -lws2_32 - EXT = .exe -else - LFLAGS = - EXT = .out -endif - -CFILES = $(shell find src -name '*.c') -HFILES = $(shell find src -name '*.h') -OFILES = $(CFILES:.c=.o) - -WEB_CFILES = $(shell find web/src web/3p -name '*.c') -WEB_HFILES = $(shell find web/src web/3p -name '*.h') - -.PHONY: all clean coverage coverage-report coverage-html - -all: toastyfs$(EXT) toastyfs_web$(EXT) toastyfs_random_test$(EXT) example_async_api$(EXT) example_blocking_api$(EXT) libtoastyfs.a - -coverage: toastyfs_random_test_coverage$(EXT) - -coverage-report: - @./scripts/measure_coverage.sh 60 - -coverage-html: - @./scripts/measure_coverage.sh 60 --html - -toastyfs$(EXT): $(CFILES) $(HFILES) - gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_SERVER - -toastyfs_random_test$(EXT): $(CFILES) $(HFILES) - gcc -o $@ $(CFILES) $(CFLAGS) $(LFLAGS) -Iinclude -DBUILD_TEST - -toastyfs_random_test_coverage$(EXT): $(CFILES) $(HFILES) - gcc -o $@ $(CFILES) $(COVERAGE_CFLAGS) $(LFLAGS) $(COVERAGE_LFLAGS) -Iinclude -DBUILD_TEST - -example_async_api$(EXT): libtoastyfs.a examples/async_api.c - gcc -o $@ examples/async_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L. - -example_blocking_api$(EXT): libtoastyfs.a examples/blocking_api.c - gcc -o $@ examples/blocking_api.c $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -L. - -toastyfs_web$(EXT): libtoastyfs.a $(WEB_CFILES) $(WEB_HFILES) - gcc -o $@ $(WEB_CFILES) $(CFLAGS) -ltoastyfs $(LFLAGS) -Iinclude -Iweb/3p -L. - -%.o: %.c $(HFILES) - gcc -c -o $@ $< $(CFLAGS) -Iinclude - -libtoastyfs.a: $(OFILES) - ar rcs $@ $^ - -clean: - rm -f \ - *.exe \ - *.out \ - *.a \ - *.gcov \ - src/*.o \ - src/*.gcda \ - src/*.gcno \ - *.gcda \ - *.gcno diff --git a/README.md b/README.md deleted file mode 100644 index 8ec1667..0000000 --- a/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# ToastyFS -ToastyFS is a distributed file system designed for self-hosting, so it aims to be pragmatic, understandable, and robust. You can use ToastyFS to store your files reliably over multiple machines knowing they will be automatically replicated and healed in case of failures. - -To try it out, run the script `./scripts/cluster_demo.sh`. It will spawn a local cluster with a web interface at `https://127.0.0.1:8090/` which will allow you to PUT, GET, DELETE files. - -⚠️ Note that ToastyFS is still in early development ⚠️ - -🎵 Now let's get toasty 🎵 - -## Features - -- Cross-platform (runs on Windows and Linux) -- Automatic Replication & Self-Healing -- Automatic content deduplication via internal content-addressing -- Configurable file chunk sizes -- Small and understandable - -But ToastyFS is still in early development, so here are the missing features: - -- No master replication -- No authentication or encryption - -## Testing -ToastyFS is tested by running an in-memory simulation of a cluster with many clients running hundreds of random operations in parallel. The test is run for long periods of times under valgrind or compiled with sanitizers. diff --git a/app/app.py b/app/app.py deleted file mode 100644 index 4a4a6b0..0000000 --- a/app/app.py +++ /dev/null @@ -1,54 +0,0 @@ -import asyncio -from pathlib import Path - -import aiohttp - - -async def fetch(session, url): - async with session.get(url) as response: - if response.status != 200: - raise "Didn't work" # TODO - - data = await response.text() - - files = [] - lines = data.splitlines() - for line in lines: - name, vtag = line.split(" ") - - # Process name - is_dir = False - if len(name) > 0 and name[-1] == "/": - name = name[:-1] - is_dir = True - - # Process vtag - vtag = int(vtag) - - files.append({"name": name, "is_dir": is_dir, "vtag": vtag}) - - return files - - -async def main(): - async with aiohttp.ClientSession() as session: - remote = "http://127.0.0.1:8090" # Must not end with a "/" - remote_dir = remote + "/" - local_dir = "local_root" - - local_dir = Path(local_dir) - - while True: - # List of files in the remote root directory - remote_files = await fetch(session, remote_dir) - - # List of files in the local root directory - local_files = list(local_dir.iterdir()) - - await asyncio.sleep(1) - - -try: - asyncio.run(main()) -except KeyboardInterrupt: - pass diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..13aa45c --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c quakey/src/lfs.c quakey/src/lfs_util.c quakey/src/quakey.c -o toasty_simulation -Iquakey/include -Iinclude -Wall -Wextra -ggdb -O0 -DMAIN_SIMULATION +gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_metadata_server -DMAIN_METADATA_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include +gcc src/sha256.c src/basic.c src/file_system.c src/byte_queue.c src/file_tree.c src/message.c src/tcp.c src/wal.c src/hash_set.c src/client.c src/metadata_server.c src/chunk_server.c src/random_client.c src/main.c -o toasty_chunk_server -DMAIN_CHUNK_SERVER -Wall -Wextra -ggdb -O0 -Iinclude -Iquakey/include diff --git a/examples/async_api.c b/examples/async_api.c deleted file mode 100644 index 3735a2d..0000000 --- a/examples/async_api.c +++ /dev/null @@ -1,60 +0,0 @@ -#include -#include - -int main(void) -{ - ToastyString remote_addr = TOASTY_STR("127.0.0.1"); - uint16_t remote_port = 8080; - - ToastyFS *toasty = toasty_connect(remote_addr, remote_port); - if (toasty == NULL) { - printf("Couldn't connect to metadata server"); - return -1; - } - - ToastyString path_1 = TOASTY_STR("/first_file"); - ToastyString path_2 = TOASTY_STR("/second_file"); - - // Begin creation operation. This does not block. - ToastyHandle create_handle_1 = toasty_begin_create_file(toasty, path_1, 1024); - if (create_handle_1 == TOASTY_INVALID) { - printf("Couldn't create file 1"); - return -1; - } - - // This doesn't block either. - ToastyHandle create_handle_2 = toasty_begin_create_file(toasty, path_2, 1024); - if (create_handle_2 == TOASTY_INVALID) { - printf("Couldn't create file 2"); - return -1; - } - - // Now block execution by overlapping the waiting - // times for both file creations. - - ToastyResult result; - int ret = toasty_wait_result(toasty, create_handle_1, &result, -1); - if (ret < 0) { - printf("Couldn't wait for completion\n"); - return -1; - } - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - printf("Couldn't create file 1\n"); - return -1; - } - - ret = toasty_wait_result(toasty, create_handle_2, &result, -1); - if (ret < 0) { - printf("Couldn't wait for completion\n"); - return -1; - } - if (result.type != TOASTY_RESULT_CREATE_SUCCESS) { - printf("Couldn't create file 2\n"); - return -1; - } - - printf("All files were created!\n"); - - toasty_disconnect(toasty); - return 0; -} diff --git a/examples/blocking_api.c b/examples/blocking_api.c deleted file mode 100644 index 0065c2b..0000000 --- a/examples/blocking_api.c +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include - -int main(void) -{ - ToastyString remote_addr = TOASTY_STR("127.0.0.1"); - uint16_t remote_port = 8080; - - ToastyFS *toasty = toasty_connect(remote_addr, remote_port); - if (toasty == NULL) { - printf("Couldn't connect to metadata server"); - return -1; - } - - ToastyString path = TOASTY_STR("/first_file"); - - int ret = toasty_create_file(toasty, path, 1024, NULL); - if (ret < 0) { - printf("Couldn't create file\n"); - toasty_disconnect(toasty); - return -1; - } - - char data[] = "Hello, world!"; - ret = toasty_write(toasty, path, 0, data, sizeof(data)-1, NULL, 0); - if (ret < 0) { - printf("Couldn't write to file\n"); - toasty_disconnect(toasty); - return -1; - } - - toasty_disconnect(toasty); - return 0; -} diff --git a/misc/DESIGN.txt b/misc/DESIGN.txt deleted file mode 100644 index f30c5db..0000000 --- a/misc/DESIGN.txt +++ /dev/null @@ -1,233 +0,0 @@ -Architecture - A ToastyFS instance is composed by a metadata server, a number - of chunk servers, and a number of clients. - - The metadata server stores the full file system hieararchy, - except instead of storing the file contents, it stores an - array of hashes of the chunks of each file. A "chunk" is a - file range that is fixed for a single file but may vary - between files. Chunk servers hold an array of chunks that - are identified by their hash. The metadata server keeps - track of which chunks each chunk server is holding. - - Clients are users of the file system that can read and - write metadata and files. They are assumed to behave - correctly. - - Any read and write operation that doesn't involve file - contents can be performed by clients by talking to the - metadata server directly. Such operations include creating - an empty file or a directory, deleting a file or directory, - listing files. - - If a client wants to read a range of bytes from a file, - it sends the metadata server the file name and range. - The metadata server responds with the chunk size of that - file, the list of hashes for the chunks involved in the - read, and the IP addresses of the chunk servers that hold - each chunk. The metadata server also adds the IP addresses - of three chunk servers any new chunks should be written - to. The client can then download the chunks from the chunk - servers and reassemble the result. - - If a client wants to write at a range of bytes of a file, - it starts by reading that range from the metadata server, - getting the list of hashes it will modify, their locations, - and locations for any new chunks. The client then modifies - the chunk by sending to each chunk server the hash to modify - and the patch (a range of bytes within a chunk plus the new - data). The chunk server creates a new modified chunk and - keeps the old version, then returns the new hash. If all - modifications are successful, the client holds the set of - old hashes and new hashes for that file range. It completes - the write by telling the metadata server to swap the old - hashes with the new ones (optionally including write flags - such as TOASTY_WRITE_CREATE_IF_MISSING). If the old hashes - don't match, another write succeded in the mean time and - touched that range, therefore the write fails. If the old - hashes match, the write succeded. If the client fails to - modify any chunks, it doesn't commit the write with the - metadata server. - - If the file doesn't exist and the TOASTY_WRITE_CREATE_IF_MISSING - flag is set, the metadata server atomically creates the file - with a default chunk size (4096 bytes) and retries the write. - This operation is logged to the WAL to ensure crash consistency. - - If the TOASTY_WRITE_TRUNCATE_AFTER flag is set, the file is - truncated after the write, setting its size to exactly offset+length - and discarding any data beyond that point. This is useful for HTTP - PUT semantics where the entire file content should be replaced. - - Note that write failures may cause chunks to be orphaned - on chunk servers. This is solved by a garbage collection - algorithm implemented by the synchronization messages - between metadata and chunk server. - - Note that clients may cache chunks and index them by their - hash. When they read a file and receive its hashes, they may - avoid reaching for the chunk servers if they already cached - the chunks with those hashes. This allows reading files with - only one round trip at no cost of correctness. If getting - the up-to-date contents is not a concern, clients may also - cache file metadata. - -Metadata and chunk server exchange: - - The metadata server is only aware of each chunk server - as long as they have a TCP connection. When a chunk server - first connects to the metadata server, it authenticates - itself and sends its own IP addresses. If the server is - authentic, the metadata server requests the full list - of chunks the chunk server is holding. Upon receiving the - state of chunk server, the metadata server adds all useful - chunks to the "old_list" and all useless chunks to the - "rem_list", then sends the rem_list to the chunk server - which removes those chunks. - - When writes are committed to the metadata server involving - new chunks to a chunk server, the metadata server adds those - hashes to an "add_list" and any hashes that are not useful - anymore to the rem_list. - - Periodically, the metadata server sends the add_list and - rem_list to the chunk server. These list tell the chunk - server the ideal state it should have from the point of - view of the metadata server. Elements in the add_list should - already be in the chunk servers, and elements from the - rem_list are to be removed. A chunk server marks any chunk - in the rem_list as to be removed and checks that hashes - in the add list are present. If a chunk in the add list - is marked as to be removed, it is unmarked. When a chunk - is marked as to be removed for a certain amount of time, - it is permanently deleted. When the synchronization is - complete, the metadata server merges the add_list into - the old_list and clears the rem_list. If chunks in the - add_list are not present in the chunk server, it responds - with an error message containing the list of missing chunks. - The metadata server then responds with a list of chunk - server addresses where the chunk server with the missing - chunk can download it from. Each chunk server goes - through its download list one at the time downloading - the missing chunks. - - Note that if the chunk server finds that its holding some - chunks that are not in the hash list of the metadata server, - that does not mean they are orphaned. It's possible that - some writes are being performed by clients that have uploaded - chunks to that chunk server but didn't yet acknowledge it - to the metadata server. If all goes well and the write - succeded, the metadata server will add those hashes to the - hash list. Chunk servers should only drop chunks if they - are not referenced by the metadata server for a period of - time (say, 30 minutes). - -Security - All nodes of the system share a secret key and use it to - authenticate each other and encrypt messages. This allows - the server to accept new chunk servers and clients with - no prior setup - -Reliability - The metadata server is a single point of failure. To reduce - the impact of crashes, the metadata server stores all write - operations into a write-ahead log that is replayed any time - the process goes online. - -Chunk Management: - Chunks are added to the system when: - 1. A chunk server connects - 2. A write operation on metadata occurs (adding chunks) - - They are removed when: - 1. A chunk server disconnects - 2. A write operation on metadata occurs (overwriting old chunks) - 3. A delete operation on metadata occurs - 4. A chunk is corrupted or removed forcefully from the chunk server - - The system must make sure that chunks are not over-replicated - or under-replicated. If they are over-replicated, some chunk - servers need to forget some copies. If they are under-replicated, - some chunk servers need to copy chunks from elsewhere. - - Metadata server variables for a chunk server: - ms_old_list: List of chunks that are known to be held by CS - ms_add_list: List of chunks that should be held by CS - ms_rem_list: List of chunks that may be held by CS but should removed from it - - Chunk server variables: - cs_add_list: List of chunks added since the last update - cs_rem_list: List of chunks marked for removal after a timeout - cs_lst_list: List of chunks that were lost due to errors or forceful removals of chunk files - - Metadata change for write: - When clients commit a write by adding new hashes to the metadata, - MS adds those hashes to the ms_add_list for the CS where the client - uploaded the chunks. If a hash was overwritten and became useless, - that hash is added to the ms_rem_list for all CS holding it. - - Metadata change for delete: - All hashes that are no longer reachable by the file tree are added - to the ms_rem_list of their holders - - Chunk upload: - When a chunk is uploaded to a chunk server, its hash is added to - the cs_add_list. - - Periodically on the chunk server: - Elements in the cs_rem_list have a 30 minute timeout after which they are deleted - permanently. - - Periodically: - CS sends cs_add_list to MS - MS may add a subset of cs_add_list to ms_add_list based on the chunk replication and distribution policy - MS sends ms_add_list and ms_rem_list to CS - CS (1) Adds all elements from ms_rem_list to cs_rem_list - (2) Elements in ms_add_list that are not held by the chunk server are added to - a temporary list tmp_list - (3) Removes elements in ms_add_list from cs_add_list and cs_rem_list, then merges cs_add_list into cs_rem_list and clears cs_add_list. - (4) Elements in cs_lst_list are added to tmp_list, then cs_lst_list is cleared. - (6) tmp_list is sent to MS - (7) cs_add_list is cleared - MS (1) Receives tmp_list and sends download locations to CS for those chunks - (2) Merges ms_add_list into ms_old_list, then removes all items in tmp_list from ms_old_list - (3) Sets ms_add_list equal to tmp_list - - Chunk replication and distribution policy: - During an update, when CS reports a new chunk to MS, MS has to decide whether - to allow the CS to keep it or not. - - There are 4 cases: - - The chunk is useless (not referenced by any file) - - The chunk is under-replicated even counting the new copy - - The chunk is properly replicated with the new copy - - The chunk is over-replicated with the new copy - - If the chunk is not referenced by the file tree, do nothing. - - If the chunk is properly replicated or under-replicated, add it to the ms_add_list. - - If the chunk is over-replicated, either don't add it to the ms_add_list or add it to the ms_rem_list of some other holder. - - TODO: The way chunk servers tell about the chunks they are holding - recently changed. Now instead of sending a full chunk list when - connecting, they send batches of chunks to the metadata server - during state updates. It also changed that now chunk servers - initiate updates. - - When a chunk server connects (and authenticates) - it sends alongside the auth message the hash of - the hash list of all chunks. The metadata server - then replies with a message saying whether it - already cached that chunk list or not. If it didn't, the chunk server sends the entire list - in chunks during state updates, with an increased - update frequency. - -Metadata Persistence & Crash Recovery: - The metadata server uses a write-ahead log (WAL) file to store its state on disk. - - Log files start with a full snapshot of the metadata and continues with operation - log entries. - - When a file gets too big, the metadata server creates a new WAL file by writing - a snapshot to it and continuing logging there. diff --git a/misc/PROTOCOL.txt b/misc/PROTOCOL.txt deleted file mode 100644 index 2aa9d3b..0000000 --- a/misc/PROTOCOL.txt +++ /dev/null @@ -1,281 +0,0 @@ -1. Introduction & Notation - -The DESIGN.txt file gives an overview of the system -and how nodes of a cluster interact with each other. -This file documents the specific binary format used -to exchange information between nodes. - -All messages start with a shared header, defined as: - - struct Header { - uint16_t version; - uint16_t type; - uint32_t length; - }; - -1.1 Generation Counter Special Values - -Generation counters (uint64_t) have two special values: - - NO_GENERATION (0): - When used in expect_gen, means "skip generation check entirely". - Files/directories are never assigned this generation value. - - MISSING_FILE_GENERATION (UINT64_MAX): - When used in expect_gen, means "expect file/directory to NOT exist". - - For DELETE: succeeds if file doesn't exist (no-op) - - For WRITE: fails with BADGEN if file exists, fails with NOENT if missing - - For existing operations: causes generation mismatch if file exists - Files/directories are never assigned this generation value. - -2. Client Messages - -2.1 Client to Metadata Server messages - -Let's start from the interactions between a client and -the metadata server: - -[ CREATE | C -> MS ] - Upon file creation, the client sends a "CREATE" message with the following layout: - - struct CreateMessage { - Header header; // type=CREATE - uint16_t path_len; - char path[path_len]; - uint8_t is_dir; - if (is_dir != 0) { - uint32_t chunk_size; - } - }; - - Note that in general only paths up to 65K bytes are - supported and that the layout of fields may depend - on the value of others. - - The server then responds with a CREATE_SUCCESS or CREATE_ERROR message. - -[ DELETE | C -> MS ] - When a client deletes a file, it sends a DELETE - message with the following layout: - - struct DeleteMessage { - Header header; // type=DELETE - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - }; - - The file/directory at the given path is only deleted if its generation counter matches expect_gen. If expect_gen is 0, the file/directory is deleted regardless. - - The server then responds with a DELETE_SUCCESS or DELETE_ERROR message. - -[ LIST | C -> MS ] - When a client requests a directory listing, it sends a LIST message whith the following format: - - struct ListMessage { - Header header; // type=LIST - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - }; - - If the expect_gen field doesn't match the generation counter of the directory, the operation fails. If expect_gen is 0, the check is skipped. - - The server then responds with a LIST_SUCCESS or LIST_ERROR message. - -[ READ | C -> MS ] - When a client requests to read a file, it sends a READ message with the following format: - - struct ReadMessage { - Header header; // type=READ - uint64_t expect_gen; - uint16_t path_len; - char path[path_len]; - uint32_t offset; - uint32_t length; - }; - - The expect_gen field is the expected generation counter for the target resource. If it doesn't match, the operation fails. If expect_gen is 0, the check is skipped. - - The offset and length fields determine the region to be read from the file. - -[ WRITE | C -> MS ] - When a client wants to write to a file, it sends a WRITE message with the following layout: - - struct Location { - uint8_t is_ipv4; - if (is_ipv4) { - uint32_t ipv4; - } else { - uint128_t ipv6; - } - uint16_t port; - }; - - struct WriteChunk { - SHA256 hash; - uint32_t num_locations; - Location locations[num_locations]; - }; - - struct WriteMessage { - Header header; // type=WRITE - uint64_t expect_gen; - uint32_t flags; - uint16_t path_len; - char path[path_len]; - uint32_t offset; - uint32_t length; - uint32_t num_chunks; - WriteChunk chunks[num_chunks]; - }; - - If the expect_gen field doesn't match the generation of the target file, the operation fails. Note that unlike other operations, the expect_gen CAN'T be 0. This is due to the assumption that the chunk size hasn't change for that file since the writer originally retrieved the file's metadata. - - The flags field contains write operation flags. Currently defined flags: - - TOASTY_WRITE_CREATE_IF_MISSING (0x01): If the file doesn't exist, - the metadata server will atomically create it with a default chunk - size of 4096 bytes and retry the write operation. The creation is - logged to the WAL for crash consistency. - - TOASTY_WRITE_TRUNCATE_AFTER (0x02): Truncate the file after the write - operation, setting the file size to exactly offset+length. Any data - beyond this point is discarded. Useful for HTTP PUT semantics where - the entire file content should be replaced. - - The offset and length mark the region that is being written to. - - Then comes an array of num_chunks sections each specifying where a given chunk was written to. Note that the number of chunks is equal to - - num_chunks == length / chunk_size - - Where chunk_size is the one for the target file at the specified generation. - - Each WriteChunk lists the new hashes for the file in the write range and for each one it lists all the chunk servers that are now holding a copy of it. - -2.2 Client to Chunk Server messages - - TODO - -3. Metadata Server messages - -The following is the list of messages the Metadata Server may send to a Client: - -[ CREATE_ERROR | MS -> C ] - When a client sends a CREATE request to the metadata server and the operation fails, the metadata server responds with a CREATE_ERROR message with the following layout: - - struct CreateErrorMessage { - Header header; // type=CREATE_ERROR - uint16_t message_len; - char message[message_len]; - }; - -[ CREATE_SUCCESS | MS -> C ] - When a client sends a CREATE requests to the metadata server which succedes, the metadata server replies with a CREATE_SUCCESS message with the following layout: - - struct CreateSuccessMessage { - Header header; // type=CREATE_SUCCESS - uint64_t gen; - }; - - The gen field is the generation counter given to the file. - -[ DELETE_ERROR | MS -> C ] - See CREATE_ERROR - -[ DELETE_SUCCESS | MS -> C ] - When a client sends a DELETE operation to the metadata server which succedes, a DELETE_SUCCESS message is sent back with the following layout: - - struct DeleteSuccessMessage { - Header header; // type=DELETE_SUCCESS - }; - - It does not store any fields other than the header. - -[ LIST_ERROR | MS -> C ] - When a client sends a LIST request to the metadata server and it fails, the server replies with a LIST_ERROR message with the following layout: - - struct ListErrorMessage { - Header header; // type=LIST_ERROR - uint16_t message_len; - char message[message_len]; - }; - -[ LIST_SUCCESS | MS -> C ] - When a client sends a LIST request to the metadata server and it succedes, the server replies with a LIST_SUCCESSS message with the following layout: - - struct ListItem { - uint64_t gen; - uint8_t is_dir; - uint16_t name_len; - char name[name_len]; - }; - - struct ListSuccessMessage { - Header header; // type=LIST_SUCCESS - uint64_t gen; - uint8_t truncated; - uint32_t item_count; - ListItem items[item_count]; - }; - - The ListSuccessMessage gen field contains the generation counter for the directory, while the gen field in ListItem is the counter for that specific child. - - If the truncated field is non-zero, the actual item count for this directory is greater than the one sent in the message. - -[ READ_ERROR | MS -> C ] - TODO - -[ READ_SUCCESS | MS -> C ] - - struct IPv4AndPort { - uint32_t ipv4; - uint16_t port; - }; - - struct IPv6AndPort { - uint128_t ipv6; - uint16_t port; - }; - - struct ChunkServerAddrList { - uint32_t num_ipv4; - IPv4AndPort ipv4s[num_ipv4]; - uint32_t num_ipv6; - IPv6AndPort ipv6s[num_ipv6]; - }; - - struct ReadChunk { - SHA256 hash; - uint32_t num_holders; - ChunkServerAddrList holders[num_holders]; - }; - - struct ReadSuccessMessage { - Header header; // type=READ_SUCCESS - uint64_t gen; - uint32_t chunk_size; - uint32_t file_length; - uint32_t num_hashes; - ReadChunk chunks[num_hashes]; - uint32_t num_write_locations; - ChunkServerAddrList write_locations[num_write_locations]; - }; - - The message returns general information about the file such as its generation counter, length in bytes, and chunk size. - - The chunks list contains the hashes of the chunks touched by the read and the list of chunk servers that are holding them. - - The write locations are a list of chunk servers that clients may write new chunks to. - -[ WRITE_ERROR | MS -> C ] - See CREATE_ERROR - -[ WRITE_SUCCESS | MS -> C ] - When a client sends a WRITE message which succedes, the metadata server responds with a WRITE_SUCCESS message with the following layout: - - struct WriteSuccessMessage { - Header header; // type=WRITE_SUCCESS - uint64_t gen; - }; - - The gen field is the new generation counter for that file. diff --git a/misc/SIMTEST.txt b/misc/SIMTEST.txt deleted file mode 100644 index d7db9b3..0000000 --- a/misc/SIMTEST.txt +++ /dev/null @@ -1,3 +0,0 @@ -Notes on simulation testing: - - Make sure your program works without fault injections - - Trace the branch coverage of the test and find why some branches are never reached diff --git a/misc/TODO.txt b/misc/TODO.txt deleted file mode 100644 index 3407cca..0000000 --- a/misc/TODO.txt +++ /dev/null @@ -1,26 +0,0 @@ -[X] Implement proper operation handles that allow users to wait for specific events -[X] Implement metadata server WAL -[X] Check that the corner case where read and writes go over the length of the file work correctly -[X] Return the number of bytes read or written in the ToastyFS_Result struct -[X] Add a change counter to FileTree. The counter is initialized to 0 and updated any time the tree is changed. When a new file is created, the current change counter is assigned to it. This makes it possible to verify that the file didn't change in between read and write operations without that clumsy previous hash list and previous chunk size. -[ ] Clean up the read operation for clients -[ ] Make parallel uploads/downloads configurable -[ ] Recalculate next write locations whenever a write occurs, not at each read -[ ] add logging (when chunk servers connect and disconnect to the metadata server) -[ ] Should list scenarios that need testing, like those where chunks would be dropped -[ ] Update DESIGN.txt and the code to remove the chunk list message. The information of chunks held by chunk servers is now transmitted to the metadata server during state updates -[ ] clean up the replication factor business -[ ] Add failt injections to the simulation -[ ] Add authentication -[ ] Add release build and allow switching between debug, release, coverage, sanitizer builds while calling make -[ ] When an operation is committed to the WAL, it may then fail due to not deterministic reasons (out of memory). When the metadata server restarts and replays the log, the operation may succede and cause the system to diverge from the last instance of the process. Is this not a problem? How do we solve it? -[ ] Add notes on who inspired the testing approach -[ ] Add notes on benchmarks and that we don't do batching -[ ] Fix endianess when writing to network and the WAL -[ ] WAL entry checksum -[ ] Check write() errors when appending WAL entries -[ ] When WAL entries are partially written, remove partial data and fail -[ ] Find a way to ensure listing operations of large directories are handled -[ ] What happens if a client adds a chunk such that the hash already exists in the system? The client doesn't know it exists already. Will this lead to over-replication? -[ ] Rename generation counters to version numbers -[ ] Add a flag for write operations to create the file if it doesn't exist alraedy diff --git a/quakey/include/quakey.h b/quakey/include/quakey.h new file mode 100644 index 0000000..17f46b2 --- /dev/null +++ b/quakey/include/quakey.h @@ -0,0 +1,190 @@ +#ifndef QUAKEY_INCLUDED +#define QUAKEY_INCLUDED + +#define _GNU_SOURCE +#include +#include +#ifdef _WIN32 +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct {} Quakey; + +// Function pointers to a simulated program's code +typedef int (*QuakeyInitFunc)(void *state, int argc, char **argv, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout); +typedef int (*QuakeyTickFunc)(void *state, void **ctxs, struct pollfd *pdata, int pcap, int *pnum, int *timeout); +typedef int (*QuakeyFreeFunc)(void *state); + +typedef enum { + QUAKEY_LINUX, + QUAKEY_WINDOWS, +} QuakeyPlatform; + +typedef struct { + + // Size of the opaque state struct + int state_size; + + // Pointers to program code + QuakeyInitFunc init_func; + QuakeyTickFunc tick_func; + QuakeyFreeFunc free_func; + + // Network addresses enabled on the process + char **addrs; + int num_addrs; + + // Disk size for the process + int disk_size; + + // Platform used by this program + QuakeyPlatform platform; + +} QuakeySpawn; + +typedef unsigned long long QuakeyUInt64; + +// Start a simulation +int quakey_init(Quakey **quakey, QuakeyUInt64 seed); + +// Stop a simulation +void quakey_free(Quakey *quakey); + +// Add a program to the simulation +void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg); + +// Schedule and executes one program until it would block, then returns +int quakey_schedule_one(Quakey *quakey); + +// Generate a random u64 +QuakeyUInt64 quakey_random(void); + +int *mock_errno_ptr(void); + +#ifdef _WIN32 +BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount); +BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency); +HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile); +BOOL mock_CloseHandle(HANDLE handle); +BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov); +BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov); +BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf); +DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod); +BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh); +BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh); +BOOL mock_FlushFileBuffers(HANDLE handle); +BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags); +char* mock__fullpath(char *path, char *dst, int cap); +HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData); +BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData); +BOOL mock_FindClose(HANDLE hFindFile); +int mock__mkdir(char *path); +#else +int mock_socket(int domain, int type, int protocol); +int mock_closesocket(int fd); +int mock_ioctlsocket(int fd, long cmd, unsigned long *argp); +int mock_bind(int fd, void *addr, unsigned long addr_len); +int mock_connect(int fd, void *addr, unsigned long addr_len); +int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen); +int mock_listen(int fd, int backlog); +int mock_accept(int fd, void *addr, socklen_t *addr_len); +int mock_recv(int fd, char *dst, int len, int flags); +int mock_send(int fd, char *src, int len, int flags); +int mock_clock_gettime(clockid_t clockid, struct timespec *tp); +int mock_open(char *path, int flags, int mode); +int mock_fcntl(int fd, int cmd, int flags); +int mock_close(int fd); +int mock_fstat(int fd, struct stat *buf); +int mock_read(int fd, char *dst, int len); +int mock_write(int fd, char *src, int len); +off_t mock_lseek(int fd, off_t offset, int whence); +int mock_flock(int fd, int op); +int mock_fsync(int fd); +int mock_mkstemp(char *path); +int mock_mkdir(char *path, mode_t mode); +int mock_remove(char *path); +int mock_rename(char *oldpath, char *newpath); +char* mock_realpath(char *path, char *dst); +DIR* mock_opendir(char *name); +struct dirent* mock_readdir(DIR *dirp); +int mock_closedir(DIR *dirp); +#endif + +void *mock_malloc(size_t size); +void *mock_realloc(void *ptr, size_t size); +void mock_free(void *ptr); + +#ifdef QUAKEY_ENABLE_MOCKS + +#undef errno +#define errno (*mock_errno_ptr()) + +#define malloc mock_malloc +#define realloc mock_realloc +#define free mock_free +#define socket mock_socket +#define closesocket mock_closesocket +#define ioctlsocket mock_ioctlsocket +#define bind mock_bind +#define connect mock_connect +#define getsockopt mock_getsockopt +#define listen mock_listen +#define accept mock_accept +#define recv mock_recv +#define send mock_send +#define clock_gettime mock_clock_gettime +#define QueryPerformanceCounter mock_QueryPerformanceCounter +#define QueryPerformanceFrequency mock_QueryPerformanceFrequency +#define open mock_open +#define fcntl mock_fcntl +#define close mock_close +#define CreateFileW mock_CreateFileW +#define CloseHandle mock_CloseHandle +#define ReadFile mock_ReadFile +#define WriteFile mock_WriteFile +#define read mock_read +#define write mock_write +#define fstat mock_fstat +#define GetFileSizeEx mock_GetFileSizeEx +#define lseek mock_lseek +#define SetFilePointer mock_SetFilePointer +#define flock mock_flock +#define LockFile mock_LockFile +#define UnlockFile mock_UnlockFile +#define fsync mock_fsync +#define FlushFileBuffers mock_FlushFileBuffers +#define mkstemp mock_mkstemp +#define mkdir mock_mkdir +#define _mkdir mock__mkdir +#define remove mock_remove +#define rename mock_rename +#define MoveFileExW mock_MoveFileExW +#define realpath mock_realpath +#define _fullpath mock__fullpath +#define opendir mock_opendir +#define readdir mock_readdir +#define closedir mock_closedir +#define FindFirstFileA mock_FindFirstFileA +#define FindNextFileA mock_FindNextFileA +#define FindClose mock_FindClose +#endif + +#endif // QUAKEY_INCLUDED \ No newline at end of file diff --git a/quakey/src/lfs.c b/quakey/src/lfs.c new file mode 100644 index 0000000..4eb7773 --- /dev/null +++ b/quakey/src/lfs.c @@ -0,0 +1,6548 @@ +/* + * The little filesystem + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include "lfs.h" +#include "lfs_util.h" + + +// some constants used throughout the code +#define LFS_BLOCK_NULL ((lfs_block_t)-1) +#define LFS_BLOCK_INLINE ((lfs_block_t)-2) + +enum { + LFS_OK_RELOCATED = 1, + LFS_OK_DROPPED = 2, + LFS_OK_ORPHANED = 3, +}; + +enum { + LFS_CMP_EQ = 0, + LFS_CMP_LT = 1, + LFS_CMP_GT = 2, +}; + + +/// Caching block device operations /// + +static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) { + // do not zero, cheaper if cache is readonly or only going to be + // written with identical data (during relocates) + (void)lfs; + rcache->block = LFS_BLOCK_NULL; +} + +static inline void lfs_cache_zero(lfs_t *lfs, lfs_cache_t *pcache) { + // zero to avoid information leak + memset(pcache->buffer, 0xff, lfs->cfg->cache_size); + pcache->block = LFS_BLOCK_NULL; +} + +static int lfs_bd_read(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_block_t block, lfs_off_t off, + void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + if (off+size > lfs->cfg->block_size + || (lfs->block_count && block >= lfs->block_count)) { + return LFS_ERR_CORRUPT; + } + + while (size > 0) { + lfs_size_t diff = size; + + if (pcache && block == pcache->block && + off < pcache->off + pcache->size) { + if (off >= pcache->off) { + // is already in pcache? + diff = lfs_min(diff, pcache->size - (off-pcache->off)); + memcpy(data, &pcache->buffer[off-pcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + // pcache takes priority + diff = lfs_min(diff, pcache->off-off); + } + + if (block == rcache->block && + off < rcache->off + rcache->size) { + if (off >= rcache->off) { + // is already in rcache? + diff = lfs_min(diff, rcache->size - (off-rcache->off)); + memcpy(data, &rcache->buffer[off-rcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + // rcache takes priority + diff = lfs_min(diff, rcache->off-off); + } + + if (size >= hint && off % lfs->cfg->read_size == 0 && + size >= lfs->cfg->read_size) { + // bypass cache? + diff = lfs_aligndown(diff, lfs->cfg->read_size); + int err = lfs->cfg->read(lfs->cfg, block, off, data, diff); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + data += diff; + off += diff; + size -= diff; + continue; + } + + // load to cache, first condition can no longer fail + LFS_ASSERT(!lfs->block_count || block < lfs->block_count); + rcache->block = block; + rcache->off = lfs_aligndown(off, lfs->cfg->read_size); + rcache->size = lfs_min( + lfs_min( + lfs_alignup(off+hint, lfs->cfg->read_size), + lfs->cfg->block_size) + - rcache->off, + lfs->cfg->cache_size); + int err = lfs->cfg->read(lfs->cfg, rcache->block, + rcache->off, rcache->buffer, rcache->size); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + } + + return 0; +} + +static int lfs_bd_cmp(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_block_t block, lfs_off_t off, + const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + lfs_size_t diff = 0; + + for (lfs_off_t i = 0; i < size; i += diff) { + uint8_t dat[8]; + + diff = lfs_min(size-i, sizeof(dat)); + int err = lfs_bd_read(lfs, + pcache, rcache, hint-i, + block, off+i, &dat, diff); + if (err) { + return err; + } + + int res = memcmp(dat, data + i, diff); + if (res) { + return res < 0 ? LFS_CMP_LT : LFS_CMP_GT; + } + } + + return LFS_CMP_EQ; +} + +static int lfs_bd_crc(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { + lfs_size_t diff = 0; + + for (lfs_off_t i = 0; i < size; i += diff) { + uint8_t dat[8]; + diff = lfs_min(size-i, sizeof(dat)); + int err = lfs_bd_read(lfs, + pcache, rcache, hint-i, + block, off+i, &dat, diff); + if (err) { + return err; + } + + *crc = lfs_crc(*crc, &dat, diff); + } + + return 0; +} + +#ifndef LFS_READONLY +static int lfs_bd_flush(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { + if (pcache->block != LFS_BLOCK_NULL && pcache->block != LFS_BLOCK_INLINE) { + LFS_ASSERT(pcache->block < lfs->block_count); + lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); + int err = lfs->cfg->prog(lfs->cfg, pcache->block, + pcache->off, pcache->buffer, diff); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + if (validate) { + // check data on disk + lfs_cache_drop(lfs, rcache); + int res = lfs_bd_cmp(lfs, + NULL, rcache, diff, + pcache->block, pcache->off, pcache->buffer, diff); + if (res < 0) { + return res; + } + + if (res != LFS_CMP_EQ) { + return LFS_ERR_CORRUPT; + } + } + + lfs_cache_zero(lfs, pcache); + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_bd_sync(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { + lfs_cache_drop(lfs, rcache); + + int err = lfs_bd_flush(lfs, pcache, rcache, validate); + if (err) { + return err; + } + + err = lfs->cfg->sync(lfs->cfg); + LFS_ASSERT(err <= 0); + return err; +} +#endif + +#ifndef LFS_READONLY +static int lfs_bd_prog(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate, + lfs_block_t block, lfs_off_t off, + const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->block_count); + LFS_ASSERT(off + size <= lfs->cfg->block_size); + + while (size > 0) { + if (block == pcache->block && + off >= pcache->off && + off < pcache->off + lfs->cfg->cache_size) { + // already fits in pcache? + lfs_size_t diff = lfs_min(size, + lfs->cfg->cache_size - (off-pcache->off)); + memcpy(&pcache->buffer[off-pcache->off], data, diff); + + data += diff; + off += diff; + size -= diff; + + pcache->size = lfs_max(pcache->size, off - pcache->off); + if (pcache->size == lfs->cfg->cache_size) { + // eagerly flush out pcache if we fill up + int err = lfs_bd_flush(lfs, pcache, rcache, validate); + if (err) { + return err; + } + } + + continue; + } + + // pcache must have been flushed, either by programming and + // entire block or manually flushing the pcache + LFS_ASSERT(pcache->block == LFS_BLOCK_NULL); + + // prepare pcache, first condition can no longer fail + pcache->block = block; + pcache->off = lfs_aligndown(off, lfs->cfg->prog_size); + pcache->size = 0; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + LFS_ASSERT(block < lfs->block_count); + int err = lfs->cfg->erase(lfs->cfg, block); + LFS_ASSERT(err <= 0); + return err; +} +#endif + + +/// Small type-level utilities /// + +// some operations on paths +static inline lfs_size_t lfs_path_namelen(const char *path) { + return strcspn(path, "/"); +} + +static inline bool lfs_path_islast(const char *path) { + lfs_size_t namelen = lfs_path_namelen(path); + return path[namelen + strspn(path + namelen, "/")] == '\0'; +} + +static inline bool lfs_path_isdir(const char *path) { + return path[lfs_path_namelen(path)] != '\0'; +} + +// operations on block pairs +static inline void lfs_pair_swap(lfs_block_t pair[2]) { + lfs_block_t t = pair[0]; + pair[0] = pair[1]; + pair[1] = t; +} + +static inline bool lfs_pair_isnull(const lfs_block_t pair[2]) { + return pair[0] == LFS_BLOCK_NULL || pair[1] == LFS_BLOCK_NULL; +} + +static inline int lfs_pair_cmp( + const lfs_block_t paira[2], + const lfs_block_t pairb[2]) { + return !(paira[0] == pairb[0] || paira[1] == pairb[1] || + paira[0] == pairb[1] || paira[1] == pairb[0]); +} + +static inline bool lfs_pair_issync( + const lfs_block_t paira[2], + const lfs_block_t pairb[2]) { + return (paira[0] == pairb[0] && paira[1] == pairb[1]) || + (paira[0] == pairb[1] && paira[1] == pairb[0]); +} + +static inline void lfs_pair_fromle32(lfs_block_t pair[2]) { + pair[0] = lfs_fromle32(pair[0]); + pair[1] = lfs_fromle32(pair[1]); +} + +#ifndef LFS_READONLY +static inline void lfs_pair_tole32(lfs_block_t pair[2]) { + pair[0] = lfs_tole32(pair[0]); + pair[1] = lfs_tole32(pair[1]); +} +#endif + +// operations on 32-bit entry tags +typedef uint32_t lfs_tag_t; +typedef int32_t lfs_stag_t; + +#define LFS_MKTAG(type, id, size) \ + (((lfs_tag_t)(type) << 20) | ((lfs_tag_t)(id) << 10) | (lfs_tag_t)(size)) + +#define LFS_MKTAG_IF(cond, type, id, size) \ + ((cond) ? LFS_MKTAG(type, id, size) : LFS_MKTAG(LFS_FROM_NOOP, 0, 0)) + +#define LFS_MKTAG_IF_ELSE(cond, type1, id1, size1, type2, id2, size2) \ + ((cond) ? LFS_MKTAG(type1, id1, size1) : LFS_MKTAG(type2, id2, size2)) + +static inline bool lfs_tag_isvalid(lfs_tag_t tag) { + return !(tag & 0x80000000); +} + +static inline bool lfs_tag_isdelete(lfs_tag_t tag) { + return ((int32_t)(tag << 22) >> 22) == -1; +} + +static inline uint16_t lfs_tag_type1(lfs_tag_t tag) { + return (tag & 0x70000000) >> 20; +} + +static inline uint16_t lfs_tag_type2(lfs_tag_t tag) { + return (tag & 0x78000000) >> 20; +} + +static inline uint16_t lfs_tag_type3(lfs_tag_t tag) { + return (tag & 0x7ff00000) >> 20; +} + +static inline uint8_t lfs_tag_chunk(lfs_tag_t tag) { + return (tag & 0x0ff00000) >> 20; +} + +static inline int8_t lfs_tag_splice(lfs_tag_t tag) { + return (int8_t)lfs_tag_chunk(tag); +} + +static inline uint16_t lfs_tag_id(lfs_tag_t tag) { + return (tag & 0x000ffc00) >> 10; +} + +static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { + return tag & 0x000003ff; +} + +static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { + return sizeof(tag) + lfs_tag_size(tag + lfs_tag_isdelete(tag)); +} + +// operations on attributes in attribute lists +struct lfs_mattr { + lfs_tag_t tag; + const void *buffer; +}; + +struct lfs_diskoff { + lfs_block_t block; + lfs_off_t off; +}; + +#define LFS_MKATTRS(...) \ + (struct lfs_mattr[]){__VA_ARGS__}, \ + sizeof((struct lfs_mattr[]){__VA_ARGS__}) / sizeof(struct lfs_mattr) + +// operations on global state +static inline void lfs_gstate_xor(lfs_gstate_t *a, const lfs_gstate_t *b) { + a->tag ^= b->tag; + a->pair[0] ^= b->pair[0]; + a->pair[1] ^= b->pair[1]; +} + +static inline bool lfs_gstate_iszero(const lfs_gstate_t *a) { + return a->tag == 0 + && a->pair[0] == 0 + && a->pair[1] == 0; +} + +#ifndef LFS_READONLY +static inline bool lfs_gstate_hasorphans(const lfs_gstate_t *a) { + return lfs_tag_size(a->tag); +} + +static inline uint8_t lfs_gstate_getorphans(const lfs_gstate_t *a) { + return lfs_tag_size(a->tag) & 0x1ff; +} + +static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) { + return lfs_tag_type1(a->tag); +} +#endif + +static inline bool lfs_gstate_needssuperblock(const lfs_gstate_t *a) { + return lfs_tag_size(a->tag) >> 9; +} + +static inline bool lfs_gstate_hasmovehere(const lfs_gstate_t *a, + const lfs_block_t *pair) { + return lfs_tag_type1(a->tag) && lfs_pair_cmp(a->pair, pair) == 0; +} + +static inline void lfs_gstate_fromle32(lfs_gstate_t *a) { + a->tag = lfs_fromle32(a->tag); + a->pair[0] = lfs_fromle32(a->pair[0]); + a->pair[1] = lfs_fromle32(a->pair[1]); +} + +#ifndef LFS_READONLY +static inline void lfs_gstate_tole32(lfs_gstate_t *a) { + a->tag = lfs_tole32(a->tag); + a->pair[0] = lfs_tole32(a->pair[0]); + a->pair[1] = lfs_tole32(a->pair[1]); +} +#endif + +// operations on forward-CRCs used to track erased state +struct lfs_fcrc { + lfs_size_t size; + uint32_t crc; +}; + +static void lfs_fcrc_fromle32(struct lfs_fcrc *fcrc) { + fcrc->size = lfs_fromle32(fcrc->size); + fcrc->crc = lfs_fromle32(fcrc->crc); +} + +#ifndef LFS_READONLY +static void lfs_fcrc_tole32(struct lfs_fcrc *fcrc) { + fcrc->size = lfs_tole32(fcrc->size); + fcrc->crc = lfs_tole32(fcrc->crc); +} +#endif + +// other endianness operations +static void lfs_ctz_fromle32(struct lfs_ctz *ctz) { + ctz->head = lfs_fromle32(ctz->head); + ctz->size = lfs_fromle32(ctz->size); +} + +#ifndef LFS_READONLY +static void lfs_ctz_tole32(struct lfs_ctz *ctz) { + ctz->head = lfs_tole32(ctz->head); + ctz->size = lfs_tole32(ctz->size); +} +#endif + +static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { + superblock->version = lfs_fromle32(superblock->version); + superblock->block_size = lfs_fromle32(superblock->block_size); + superblock->block_count = lfs_fromle32(superblock->block_count); + superblock->name_max = lfs_fromle32(superblock->name_max); + superblock->file_max = lfs_fromle32(superblock->file_max); + superblock->attr_max = lfs_fromle32(superblock->attr_max); +} + +#ifndef LFS_READONLY +static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { + superblock->version = lfs_tole32(superblock->version); + superblock->block_size = lfs_tole32(superblock->block_size); + superblock->block_count = lfs_tole32(superblock->block_count); + superblock->name_max = lfs_tole32(superblock->name_max); + superblock->file_max = lfs_tole32(superblock->file_max); + superblock->attr_max = lfs_tole32(superblock->attr_max); +} +#endif + +#ifndef LFS_NO_ASSERT +static bool lfs_mlist_isopen(struct lfs_mlist *head, + struct lfs_mlist *node) { + for (struct lfs_mlist **p = &head; *p; p = &(*p)->next) { + if (*p == (struct lfs_mlist*)node) { + return true; + } + } + + return false; +} +#endif + +static void lfs_mlist_remove(lfs_t *lfs, struct lfs_mlist *mlist) { + for (struct lfs_mlist **p = &lfs->mlist; *p; p = &(*p)->next) { + if (*p == mlist) { + *p = (*p)->next; + break; + } + } +} + +static void lfs_mlist_append(lfs_t *lfs, struct lfs_mlist *mlist) { + mlist->next = lfs->mlist; + lfs->mlist = mlist; +} + +// some other filesystem operations +static uint32_t lfs_fs_disk_version(lfs_t *lfs) { + (void)lfs; +#ifdef LFS_MULTIVERSION + if (lfs->cfg->disk_version) { + return lfs->cfg->disk_version; + } else +#endif + { + return LFS_DISK_VERSION; + } +} + +static uint16_t lfs_fs_disk_version_major(lfs_t *lfs) { + return 0xffff & (lfs_fs_disk_version(lfs) >> 16); + +} + +static uint16_t lfs_fs_disk_version_minor(lfs_t *lfs) { + return 0xffff & (lfs_fs_disk_version(lfs) >> 0); +} + + +/// Internal operations predeclared here /// +#ifndef LFS_READONLY +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, + const struct lfs_mattr *attrs, int attrcount); +static int lfs_dir_compact(lfs_t *lfs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, + lfs_mdir_t *source, uint16_t begin, uint16_t end); +static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size); +static lfs_ssize_t lfs_file_write_(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size); +static int lfs_file_sync_(lfs_t *lfs, lfs_file_t *file); +static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file); +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file); + +static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss); +static int lfs_fs_preporphans(lfs_t *lfs, int8_t orphans); +static void lfs_fs_prepmove(lfs_t *lfs, + uint16_t id, const lfs_block_t pair[2]); +static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *pdir); +static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *parent); +static int lfs_fs_forceconsistency(lfs_t *lfs); +#endif + +static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock); + +#ifdef LFS_MIGRATE +static int lfs1_traverse(lfs_t *lfs, + int (*cb)(void*, lfs_block_t), void *data); +#endif + +static int lfs_dir_rewind_(lfs_t *lfs, lfs_dir_t *dir); + +static lfs_ssize_t lfs_file_flushedread(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size); +static lfs_ssize_t lfs_file_read_(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size); +static int lfs_file_close_(lfs_t *lfs, lfs_file_t *file); +static lfs_soff_t lfs_file_size_(lfs_t *lfs, lfs_file_t *file); + +static lfs_ssize_t lfs_fs_size_(lfs_t *lfs); +static int lfs_fs_traverse_(lfs_t *lfs, + int (*cb)(void *data, lfs_block_t block), void *data, + bool includeorphans); + +static int lfs_deinit(lfs_t *lfs); +static int lfs_unmount_(lfs_t *lfs); + + +/// Block allocator /// + +// allocations should call this when all allocated blocks are committed to +// the filesystem +// +// after a checkpoint, the block allocator may realloc any untracked blocks +static void lfs_alloc_ckpoint(lfs_t *lfs) { + lfs->lookahead.ckpoint = lfs->block_count; +} + +// drop the lookahead buffer, this is done during mounting and failed +// traversals in order to avoid invalid lookahead state +static void lfs_alloc_drop(lfs_t *lfs) { + lfs->lookahead.size = 0; + lfs->lookahead.next = 0; + lfs_alloc_ckpoint(lfs); +} + +#ifndef LFS_READONLY +static int lfs_alloc_lookahead(void *p, lfs_block_t block) { + lfs_t *lfs = (lfs_t*)p; + lfs_block_t off = ((block - lfs->lookahead.start) + + lfs->block_count) % lfs->block_count; + + if (off < lfs->lookahead.size) { + lfs->lookahead.buffer[off / 8] |= 1U << (off % 8); + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_alloc_scan(lfs_t *lfs) { + // move lookahead buffer to the first unused block + // + // note we limit the lookahead buffer to at most the amount of blocks + // checkpointed, this prevents the math in lfs_alloc from underflowing + lfs->lookahead.start = (lfs->lookahead.start + lfs->lookahead.next) + % lfs->block_count; + lfs->lookahead.next = 0; + lfs->lookahead.size = lfs_min( + 8*lfs->cfg->lookahead_size, + lfs->lookahead.ckpoint); + + // find mask of free blocks from tree + memset(lfs->lookahead.buffer, 0, lfs->cfg->lookahead_size); + int err = lfs_fs_traverse_(lfs, lfs_alloc_lookahead, lfs, true); + if (err) { + lfs_alloc_drop(lfs); + return err; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { + while (true) { + // scan our lookahead buffer for free blocks + while (lfs->lookahead.next < lfs->lookahead.size) { + if (!(lfs->lookahead.buffer[lfs->lookahead.next / 8] + & (1U << (lfs->lookahead.next % 8)))) { + // found a free block + *block = (lfs->lookahead.start + lfs->lookahead.next) + % lfs->block_count; + + // eagerly find next free block to maximize how many blocks + // lfs_alloc_ckpoint makes available for scanning + while (true) { + lfs->lookahead.next += 1; + lfs->lookahead.ckpoint -= 1; + + if (lfs->lookahead.next >= lfs->lookahead.size + || !(lfs->lookahead.buffer[lfs->lookahead.next / 8] + & (1U << (lfs->lookahead.next % 8)))) { + return 0; + } + } + } + + lfs->lookahead.next += 1; + lfs->lookahead.ckpoint -= 1; + } + + // In order to keep our block allocator from spinning forever when our + // filesystem is full, we mark points where there are no in-flight + // allocations with a checkpoint before starting a set of allocations. + // + // If we've looked at all blocks since the last checkpoint, we report + // the filesystem as out of storage. + // + if (lfs->lookahead.ckpoint <= 0) { + LFS_ERROR("No more free space 0x%"PRIx32, + (lfs->lookahead.start + lfs->lookahead.next) + % lfs->block_count); + return LFS_ERR_NOSPC; + } + + // No blocks in our lookahead buffer, we need to scan the filesystem for + // unused blocks in the next lookahead window. + int err = lfs_alloc_scan(lfs); + if(err) { + return err; + } + } +} +#endif + +/// Metadata pair and directory operations /// +static lfs_stag_t lfs_dir_getslice(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t gmask, lfs_tag_t gtag, + lfs_off_t goff, void *gbuffer, lfs_size_t gsize) { + lfs_off_t off = dir->off; + lfs_tag_t ntag = dir->etag; + lfs_stag_t gdiff = 0; + + // synthetic moves + if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair) && + lfs_tag_id(gmask) != 0) { + if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(gtag)) { + return LFS_ERR_NOENT; + } else if (lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(gtag)) { + gdiff -= LFS_MKTAG(0, 1, 0); + } + } + + // iterate over dir block backwards (for faster lookups) + while (off >= sizeof(lfs_tag_t) + lfs_tag_dsize(ntag)) { + off -= lfs_tag_dsize(ntag); + lfs_tag_t tag = ntag; + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, sizeof(ntag), + dir->pair[0], off, &ntag, sizeof(ntag)); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + ntag = (lfs_frombe32(ntag) ^ tag) & 0x7fffffff; + + if (lfs_tag_id(gmask) != 0 && + lfs_tag_type1(tag) == LFS_TYPE_SPLICE && + lfs_tag_id(tag) <= lfs_tag_id(gtag - gdiff)) { + if (tag == (LFS_MKTAG(LFS_TYPE_CREATE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & (gtag - gdiff)))) { + // found where we were created + return LFS_ERR_NOENT; + } + + // move around splices + gdiff += LFS_MKTAG(0, lfs_tag_splice(tag), 0); + } + + if ((gmask & tag) == (gmask & (gtag - gdiff))) { + if (lfs_tag_isdelete(tag)) { + return LFS_ERR_NOENT; + } + + lfs_size_t diff = lfs_min(lfs_tag_size(tag), gsize); + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, diff, + dir->pair[0], off+sizeof(tag)+goff, gbuffer, diff); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + memset((uint8_t*)gbuffer + diff, 0, gsize - diff); + + return tag + gdiff; + } + } + + return LFS_ERR_NOENT; +} + +static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t gmask, lfs_tag_t gtag, void *buffer) { + return lfs_dir_getslice(lfs, dir, + gmask, gtag, + 0, buffer, lfs_tag_size(gtag)); +} + +static int lfs_dir_getread(lfs_t *lfs, const lfs_mdir_t *dir, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_tag_t gmask, lfs_tag_t gtag, + lfs_off_t off, void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + if (off+size > lfs->cfg->block_size) { + return LFS_ERR_CORRUPT; + } + + while (size > 0) { + lfs_size_t diff = size; + + if (pcache && pcache->block == LFS_BLOCK_INLINE && + off < pcache->off + pcache->size) { + if (off >= pcache->off) { + // is already in pcache? + diff = lfs_min(diff, pcache->size - (off-pcache->off)); + memcpy(data, &pcache->buffer[off-pcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + // pcache takes priority + diff = lfs_min(diff, pcache->off-off); + } + + if (rcache->block == LFS_BLOCK_INLINE && + off < rcache->off + rcache->size) { + if (off >= rcache->off) { + // is already in rcache? + diff = lfs_min(diff, rcache->size - (off-rcache->off)); + memcpy(data, &rcache->buffer[off-rcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + } + + // load to cache, first condition can no longer fail + rcache->block = LFS_BLOCK_INLINE; + rcache->off = lfs_aligndown(off, lfs->cfg->read_size); + rcache->size = lfs_min(lfs_alignup(off+hint, lfs->cfg->read_size), + lfs->cfg->cache_size); + int err = lfs_dir_getslice(lfs, dir, gmask, gtag, + rcache->off, rcache->buffer, rcache->size); + if (err < 0) { + return err; + } + } + + return 0; +} + +#ifndef LFS_READONLY +static int lfs_dir_traverse_filter(void *p, + lfs_tag_t tag, const void *buffer) { + lfs_tag_t *filtertag = p; + (void)buffer; + + // which mask depends on unique bit in tag structure + uint32_t mask = (tag & LFS_MKTAG(0x100, 0, 0)) + ? LFS_MKTAG(0x7ff, 0x3ff, 0) + : LFS_MKTAG(0x700, 0x3ff, 0); + + // check for redundancy + if ((mask & tag) == (mask & *filtertag) || + lfs_tag_isdelete(*filtertag) || + (LFS_MKTAG(0x7ff, 0x3ff, 0) & tag) == ( + LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & *filtertag))) { + *filtertag = LFS_MKTAG(LFS_FROM_NOOP, 0, 0); + return true; + } + + // check if we need to adjust for created/deleted tags + if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE && + lfs_tag_id(tag) <= lfs_tag_id(*filtertag)) { + *filtertag += LFS_MKTAG(0, lfs_tag_splice(tag), 0); + } + + return false; +} +#endif + +#ifndef LFS_READONLY +// maximum recursive depth of lfs_dir_traverse, the deepest call: +// +// traverse with commit +// '-> traverse with move +// '-> traverse with filter +// +#define LFS_DIR_TRAVERSE_DEPTH 3 + +struct lfs_dir_traverse { + const lfs_mdir_t *dir; + lfs_off_t off; + lfs_tag_t ptag; + const struct lfs_mattr *attrs; + int attrcount; + + lfs_tag_t tmask; + lfs_tag_t ttag; + uint16_t begin; + uint16_t end; + int16_t diff; + + int (*cb)(void *data, lfs_tag_t tag, const void *buffer); + void *data; + + lfs_tag_t tag; + const void *buffer; + struct lfs_diskoff disk; +}; + +static int lfs_dir_traverse(lfs_t *lfs, + const lfs_mdir_t *dir, lfs_off_t off, lfs_tag_t ptag, + const struct lfs_mattr *attrs, int attrcount, + lfs_tag_t tmask, lfs_tag_t ttag, + uint16_t begin, uint16_t end, int16_t diff, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // This function in inherently recursive, but bounded. To allow tool-based + // analysis without unnecessary code-cost we use an explicit stack + struct lfs_dir_traverse stack[LFS_DIR_TRAVERSE_DEPTH-1]; + unsigned sp = 0; + int res; + + // iterate over directory and attrs + lfs_tag_t tag; + const void *buffer; + struct lfs_diskoff disk = {0}; + while (true) { + { + if (off+lfs_tag_dsize(ptag) < dir->off) { + off += lfs_tag_dsize(ptag); + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, sizeof(tag), + dir->pair[0], off, &tag, sizeof(tag)); + if (err) { + return err; + } + + tag = (lfs_frombe32(tag) ^ ptag) | 0x80000000; + disk.block = dir->pair[0]; + disk.off = off+sizeof(lfs_tag_t); + buffer = &disk; + ptag = tag; + } else if (attrcount > 0) { + tag = attrs[0].tag; + buffer = attrs[0].buffer; + attrs += 1; + attrcount -= 1; + } else { + // finished traversal, pop from stack? + res = 0; + break; + } + + // do we need to filter? + lfs_tag_t mask = LFS_MKTAG(0x7ff, 0, 0); + if ((mask & tmask & tag) != (mask & tmask & ttag)) { + continue; + } + + if (lfs_tag_id(tmask) != 0) { + LFS_ASSERT(sp < LFS_DIR_TRAVERSE_DEPTH); + // recurse, scan for duplicates, and update tag based on + // creates/deletes + stack[sp] = (struct lfs_dir_traverse){ + .dir = dir, + .off = off, + .ptag = ptag, + .attrs = attrs, + .attrcount = attrcount, + .tmask = tmask, + .ttag = ttag, + .begin = begin, + .end = end, + .diff = diff, + .cb = cb, + .data = data, + .tag = tag, + .buffer = buffer, + .disk = disk, + }; + sp += 1; + + tmask = 0; + ttag = 0; + begin = 0; + end = 0; + diff = 0; + cb = lfs_dir_traverse_filter; + data = &stack[sp-1].tag; + continue; + } + } + +popped: + // in filter range? + if (lfs_tag_id(tmask) != 0 && + !(lfs_tag_id(tag) >= begin && lfs_tag_id(tag) < end)) { + continue; + } + + // handle special cases for mcu-side operations + if (lfs_tag_type3(tag) == LFS_FROM_NOOP) { + // do nothing + } else if (lfs_tag_type3(tag) == LFS_FROM_MOVE) { + // Without this condition, lfs_dir_traverse can exhibit an + // extremely expensive O(n^3) of nested loops when renaming. + // This happens because lfs_dir_traverse tries to filter tags by + // the tags in the source directory, triggering a second + // lfs_dir_traverse with its own filter operation. + // + // traverse with commit + // '-> traverse with filter + // '-> traverse with move + // '-> traverse with filter + // + // However we don't actually care about filtering the second set of + // tags, since duplicate tags have no effect when filtering. + // + // This check skips this unnecessary recursive filtering explicitly, + // reducing this runtime from O(n^3) to O(n^2). + if (cb == lfs_dir_traverse_filter) { + continue; + } + + // recurse into move + stack[sp] = (struct lfs_dir_traverse){ + .dir = dir, + .off = off, + .ptag = ptag, + .attrs = attrs, + .attrcount = attrcount, + .tmask = tmask, + .ttag = ttag, + .begin = begin, + .end = end, + .diff = diff, + .cb = cb, + .data = data, + .tag = LFS_MKTAG(LFS_FROM_NOOP, 0, 0), + }; + sp += 1; + + uint16_t fromid = lfs_tag_size(tag); + uint16_t toid = lfs_tag_id(tag); + dir = buffer; + off = 0; + ptag = 0xffffffff; + attrs = NULL; + attrcount = 0; + tmask = LFS_MKTAG(0x600, 0x3ff, 0); + ttag = LFS_MKTAG(LFS_TYPE_STRUCT, 0, 0); + begin = fromid; + end = fromid+1; + diff = toid-fromid+diff; + } else if (lfs_tag_type3(tag) == LFS_FROM_USERATTRS) { + for (unsigned i = 0; i < lfs_tag_size(tag); i++) { + const struct lfs_attr *a = buffer; + res = cb(data, LFS_MKTAG(LFS_TYPE_USERATTR + a[i].type, + lfs_tag_id(tag) + diff, a[i].size), a[i].buffer); + if (res < 0) { + return res; + } + + if (res) { + break; + } + } + } else { + res = cb(data, tag + LFS_MKTAG(0, diff, 0), buffer); + if (res < 0) { + return res; + } + + if (res) { + break; + } + } + } + + if (sp > 0) { + // pop from the stack and return, fortunately all pops share + // a destination + dir = stack[sp-1].dir; + off = stack[sp-1].off; + ptag = stack[sp-1].ptag; + attrs = stack[sp-1].attrs; + attrcount = stack[sp-1].attrcount; + tmask = stack[sp-1].tmask; + ttag = stack[sp-1].ttag; + begin = stack[sp-1].begin; + end = stack[sp-1].end; + diff = stack[sp-1].diff; + cb = stack[sp-1].cb; + data = stack[sp-1].data; + tag = stack[sp-1].tag; + buffer = stack[sp-1].buffer; + disk = stack[sp-1].disk; + sp -= 1; + goto popped; + } else { + return res; + } +} +#endif + +static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], + lfs_tag_t fmask, lfs_tag_t ftag, uint16_t *id, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // we can find tag very efficiently during a fetch, since we're already + // scanning the entire directory + lfs_stag_t besttag = -1; + + // if either block address is invalid we return LFS_ERR_CORRUPT here, + // otherwise later writes to the pair could fail + if (lfs->block_count + && (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count)) { + return LFS_ERR_CORRUPT; + } + + // find the block with the most recent revision + uint32_t revs[2] = {0, 0}; + int r = 0; + for (int i = 0; i < 2; i++) { + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, sizeof(revs[i]), + pair[i], 0, &revs[i], sizeof(revs[i])); + revs[i] = lfs_fromle32(revs[i]); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + if (err != LFS_ERR_CORRUPT && + lfs_scmp(revs[i], revs[(i+1)%2]) > 0) { + r = i; + } + } + + dir->pair[0] = pair[(r+0)%2]; + dir->pair[1] = pair[(r+1)%2]; + dir->rev = revs[(r+0)%2]; + dir->off = 0; // nonzero = found some commits + + // now scan tags to fetch the actual dir and find possible match + for (int i = 0; i < 2; i++) { + lfs_off_t off = 0; + lfs_tag_t ptag = 0xffffffff; + + uint16_t tempcount = 0; + lfs_block_t temptail[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + bool tempsplit = false; + lfs_stag_t tempbesttag = besttag; + + // assume not erased until proven otherwise + bool maybeerased = false; + bool hasfcrc = false; + struct lfs_fcrc fcrc; + + dir->rev = lfs_tole32(dir->rev); + uint32_t crc = lfs_crc(0xffffffff, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + + while (true) { + // extract next tag + lfs_tag_t tag; + off += lfs_tag_dsize(ptag); + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off, &tag, sizeof(tag)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + // can't continue? + break; + } + return err; + } + + crc = lfs_crc(crc, &tag, sizeof(tag)); + tag = lfs_frombe32(tag) ^ ptag; + + // next commit not yet programmed? + if (!lfs_tag_isvalid(tag)) { + // we only might be erased if the last tag was a crc + maybeerased = (lfs_tag_type2(ptag) == LFS_TYPE_CCRC); + break; + // out of range? + } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { + break; + } + + ptag = tag; + + if (lfs_tag_type2(tag) == LFS_TYPE_CCRC) { + // check the crc attr + uint32_t dcrc; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } + return err; + } + dcrc = lfs_fromle32(dcrc); + + if (crc != dcrc) { + break; + } + + // reset the next bit if we need to + ptag ^= (lfs_tag_t)(lfs_tag_chunk(tag) & 1U) << 31; + + // toss our crc into the filesystem seed for + // pseudorandom numbers, note we use another crc here + // as a collection function because it is sufficiently + // random and convenient + lfs->seed = lfs_crc(lfs->seed, &crc, sizeof(crc)); + + // update with what's found so far + besttag = tempbesttag; + dir->off = off + lfs_tag_dsize(tag); + dir->etag = ptag; + dir->count = tempcount; + dir->tail[0] = temptail[0]; + dir->tail[1] = temptail[1]; + dir->split = tempsplit; + + // reset crc, hasfcrc + crc = 0xffffffff; + continue; + } + + // crc the entry first, hopefully leaving it in the cache + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + lfs_tag_dsize(tag)-sizeof(tag), &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } + return err; + } + + // directory modification tags? + if (lfs_tag_type1(tag) == LFS_TYPE_NAME) { + // increase count of files if necessary + if (lfs_tag_id(tag) >= tempcount) { + tempcount = lfs_tag_id(tag) + 1; + } + } else if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE) { + tempcount += lfs_tag_splice(tag); + + if (tag == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & tempbesttag))) { + tempbesttag |= 0x80000000; + } else if (tempbesttag != -1 && + lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) { + tempbesttag += LFS_MKTAG(0, lfs_tag_splice(tag), 0); + } + } else if (lfs_tag_type1(tag) == LFS_TYPE_TAIL) { + tempsplit = (lfs_tag_chunk(tag) & 1); + + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), &temptail, 8); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } + return err; + } + lfs_pair_fromle32(temptail); + } else if (lfs_tag_type3(tag) == LFS_TYPE_FCRC) { + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + &fcrc, sizeof(fcrc)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } + return err; + } + + lfs_fcrc_fromle32(&fcrc); + hasfcrc = true; + } + + // found a match for our fetcher? + if ((fmask & tag) == (fmask & ftag)) { + int res = cb(data, tag, &(struct lfs_diskoff){ + dir->pair[0], off+sizeof(tag)}); + if (res < 0) { + if (res == LFS_ERR_CORRUPT) { + break; + } + return res; + } + + if (res == LFS_CMP_EQ) { + // found a match + tempbesttag = tag; + } else if ((LFS_MKTAG(0x7ff, 0x3ff, 0) & tag) == + (LFS_MKTAG(0x7ff, 0x3ff, 0) & tempbesttag)) { + // found an identical tag, but contents didn't match + // this must mean that our besttag has been overwritten + tempbesttag = -1; + } else if (res == LFS_CMP_GT && + lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) { + // found a greater match, keep track to keep things sorted + tempbesttag = tag | 0x80000000; + } + } + } + + // found no valid commits? + if (dir->off == 0) { + // try the other block? + lfs_pair_swap(dir->pair); + dir->rev = revs[(r+1)%2]; + continue; + } + + // did we end on a valid commit? we may have an erased block + dir->erased = false; + if (maybeerased && dir->off % lfs->cfg->prog_size == 0) { + #ifdef LFS_MULTIVERSION + // note versions < lfs2.1 did not have fcrc tags, if + // we're < lfs2.1 treat missing fcrc as erased data + // + // we don't strictly need to do this, but otherwise writing + // to lfs2.0 disks becomes very inefficient + if (lfs_fs_disk_version(lfs) < 0x00020001) { + dir->erased = true; + + } else + #endif + if (hasfcrc) { + // check for an fcrc matching the next prog's erased state, if + // this failed most likely a previous prog was interrupted, we + // need a new erase + uint32_t fcrc_ = 0xffffffff; + int err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], dir->off, fcrc.size, &fcrc_); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + // found beginning of erased part? + dir->erased = (fcrc_ == fcrc.crc); + } + } + + // synthetic move + if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) { + if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) { + besttag |= 0x80000000; + } else if (besttag != -1 && + lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) { + besttag -= LFS_MKTAG(0, 1, 0); + } + } + + // found tag? or found best id? + if (id) { + *id = lfs_min(lfs_tag_id(besttag), dir->count); + } + + if (lfs_tag_isvalid(besttag)) { + return besttag; + } else if (lfs_tag_id(besttag) < dir->count) { + return LFS_ERR_NOENT; + } else { + return 0; + } + } + + LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}", + dir->pair[0], dir->pair[1]); + return LFS_ERR_CORRUPT; +} + +static int lfs_dir_fetch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2]) { + // note, mask=-1, tag=-1 can never match a tag since this + // pattern has the invalid bit set + return (int)lfs_dir_fetchmatch(lfs, dir, pair, + (lfs_tag_t)-1, (lfs_tag_t)-1, NULL, NULL, NULL); +} + +static int lfs_dir_getgstate(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_gstate_t *gstate) { + lfs_gstate_t temp; + lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x7ff, 0, 0), + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0, sizeof(temp)), &temp); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + if (res != LFS_ERR_NOENT) { + // xor together to find resulting gstate + lfs_gstate_fromle32(&temp); + lfs_gstate_xor(gstate, &temp); + } + + return 0; +} + +static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, + uint16_t id, struct lfs_info *info) { + if (id == 0x3ff) { + // special case for root + strcpy(info->name, "/"); + info->type = LFS_TYPE_DIR; + return 0; + } + + lfs_stag_t tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x780, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); + if (tag < 0) { + return (int)tag; + } + + info->type = lfs_tag_type3(tag); + + struct lfs_ctz ctz; + tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + return (int)tag; + } + lfs_ctz_fromle32(&ctz); + + if (lfs_tag_type3(tag) == LFS_TYPE_CTZSTRUCT) { + info->size = ctz.size; + } else if (lfs_tag_type3(tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tag_size(tag); + } + + return 0; +} + +struct lfs_dir_find_match { + lfs_t *lfs; + const void *name; + lfs_size_t size; +}; + +static int lfs_dir_find_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_dir_find_match *name = data; + lfs_t *lfs = name->lfs; + const struct lfs_diskoff *disk = buffer; + + // compare with disk + lfs_size_t diff = lfs_min(name->size, lfs_tag_size(tag)); + int res = lfs_bd_cmp(lfs, + NULL, &lfs->rcache, diff, + disk->block, disk->off, name->name, diff); + if (res != LFS_CMP_EQ) { + return res; + } + + // only equal if our size is still the same + if (name->size != lfs_tag_size(tag)) { + return (name->size < lfs_tag_size(tag)) ? LFS_CMP_LT : LFS_CMP_GT; + } + + // found a match! + return LFS_CMP_EQ; +} + +// lfs_dir_find tries to set path and id even if file is not found +// +// returns: +// - 0 if file is found +// - LFS_ERR_NOENT if file or parent is not found +// - LFS_ERR_NOTDIR if parent is not a dir +static lfs_stag_t lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, + const char **path, uint16_t *id) { + // we reduce path to a single name if we can find it + const char *name = *path; + + // default to root dir + lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); + dir->tail[0] = lfs->root[0]; + dir->tail[1] = lfs->root[1]; + + // empty paths are not allowed + if (*name == '\0') { + return LFS_ERR_INVAL; + } + + while (true) { +nextname: + // skip slashes if we're a directory + if (lfs_tag_type3(tag) == LFS_TYPE_DIR) { + name += strspn(name, "/"); + } + lfs_size_t namelen = strcspn(name, "/"); + + // skip '.' + if (namelen == 1 && memcmp(name, ".", 1) == 0) { + name += namelen; + goto nextname; + } + + // error on unmatched '..', trying to go above root? + if (namelen == 2 && memcmp(name, "..", 2) == 0) { + return LFS_ERR_INVAL; + } + + // skip if matched by '..' in name + const char *suffix = name + namelen; + lfs_size_t sufflen; + int depth = 1; + while (true) { + suffix += strspn(suffix, "/"); + sufflen = strcspn(suffix, "/"); + if (sufflen == 0) { + break; + } + + if (sufflen == 1 && memcmp(suffix, ".", 1) == 0) { + // noop + } else if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { + depth -= 1; + if (depth == 0) { + name = suffix + sufflen; + goto nextname; + } + } else { + depth += 1; + } + + suffix += sufflen; + } + + // found path + if (*name == '\0') { + return tag; + } + + // update what we've found so far + *path = name; + + // only continue if we're a directory + if (lfs_tag_type3(tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + // grab the entry data + if (lfs_tag_id(tag) != 0x3ff) { + lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), dir->tail); + if (res < 0) { + return res; + } + lfs_pair_fromle32(dir->tail); + } + + // find entry matching name + while (true) { + tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, + LFS_MKTAG(0x780, 0, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + id, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, name, namelen}); + if (tag < 0) { + return tag; + } + + if (tag) { + break; + } + + if (!dir->split) { + return LFS_ERR_NOENT; + } + } + + // to next name + name += namelen; + } +} + +// commit logic +struct lfs_commit { + lfs_block_t block; + lfs_off_t off; + lfs_tag_t ptag; + uint32_t crc; + + lfs_off_t begin; + lfs_off_t end; +}; + +#ifndef LFS_READONLY +static int lfs_dir_commitprog(lfs_t *lfs, struct lfs_commit *commit, + const void *buffer, lfs_size_t size) { + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off , + (const uint8_t*)buffer, size); + if (err) { + return err; + } + + commit->crc = lfs_crc(commit->crc, buffer, size); + commit->off += size; + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit, + lfs_tag_t tag, const void *buffer) { + // check if we fit + lfs_size_t dsize = lfs_tag_dsize(tag); + if (commit->off + dsize > commit->end) { + return LFS_ERR_NOSPC; + } + + // write out tag + lfs_tag_t ntag = lfs_tobe32((tag & 0x7fffffff) ^ commit->ptag); + int err = lfs_dir_commitprog(lfs, commit, &ntag, sizeof(ntag)); + if (err) { + return err; + } + + if (!(tag & 0x80000000)) { + // from memory + err = lfs_dir_commitprog(lfs, commit, buffer, dsize-sizeof(tag)); + if (err) { + return err; + } + } else { + // from disk + const struct lfs_diskoff *disk = buffer; + for (lfs_off_t i = 0; i < dsize-sizeof(tag); i++) { + // rely on caching to make this efficient + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, dsize-sizeof(tag)-i, + disk->block, disk->off+i, &dat, 1); + if (err) { + return err; + } + + err = lfs_dir_commitprog(lfs, commit, &dat, 1); + if (err) { + return err; + } + } + } + + commit->ptag = tag & 0x7fffffff; + return 0; +} +#endif + +#ifndef LFS_READONLY + +static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { + // align to program units + // + // this gets a bit complex as we have two types of crcs: + // - 5-word crc with fcrc to check following prog (middle of block) + // - 2-word crc with no following prog (end of block) + const lfs_off_t end = lfs_alignup( + lfs_min(commit->off + 5*sizeof(uint32_t), lfs->cfg->block_size), + lfs->cfg->prog_size); + + lfs_off_t off1 = 0; + uint32_t crc1 = 0; + + // create crc tags to fill up remainder of commit, note that + // padding is not crced, which lets fetches skip padding but + // makes committing a bit more complicated + while (commit->off < end) { + lfs_off_t noff = ( + lfs_min(end - (commit->off+sizeof(lfs_tag_t)), 0x3fe) + + (commit->off+sizeof(lfs_tag_t))); + // too large for crc tag? need padding commits + if (noff < end) { + noff = lfs_min(noff, end - 5*sizeof(uint32_t)); + } + + // space for fcrc? + uint8_t eperturb = (uint8_t)-1; + if (noff >= end && noff <= lfs->cfg->block_size - lfs->cfg->prog_size) { + // first read the leading byte, this always contains a bit + // we can perturb to avoid writes that don't change the fcrc + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->prog_size, + commit->block, noff, &eperturb, 1); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + #ifdef LFS_MULTIVERSION + // unfortunately fcrcs break mdir fetching < lfs2.1, so only write + // these if we're a >= lfs2.1 filesystem + if (lfs_fs_disk_version(lfs) <= 0x00020000) { + // don't write fcrc + } else + #endif + { + // find the expected fcrc, don't bother avoiding a reread + // of the eperturb, it should still be in our cache + struct lfs_fcrc fcrc = { + .size = lfs->cfg->prog_size, + .crc = 0xffffffff + }; + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, lfs->cfg->prog_size, + commit->block, noff, fcrc.size, &fcrc.crc); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + lfs_fcrc_tole32(&fcrc); + err = lfs_dir_commitattr(lfs, commit, + LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)), + &fcrc); + if (err) { + return err; + } + } + } + + // build commit crc + struct { + lfs_tag_t tag; + uint32_t crc; + } ccrc; + lfs_tag_t ntag = LFS_MKTAG( + LFS_TYPE_CCRC + (((uint8_t)~eperturb) >> 7), 0x3ff, + noff - (commit->off+sizeof(lfs_tag_t))); + ccrc.tag = lfs_tobe32(ntag ^ commit->ptag); + commit->crc = lfs_crc(commit->crc, &ccrc.tag, sizeof(lfs_tag_t)); + ccrc.crc = lfs_tole32(commit->crc); + + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off, &ccrc, sizeof(ccrc)); + if (err) { + return err; + } + + // keep track of non-padding checksum to verify + if (off1 == 0) { + off1 = commit->off + sizeof(lfs_tag_t); + crc1 = commit->crc; + } + + commit->off = noff; + // perturb valid bit? + commit->ptag = ntag ^ ((0x80UL & ~eperturb) << 24); + // reset crc for next commit + commit->crc = 0xffffffff; + + // manually flush here since we don't prog the padding, this confuses + // the caching layer + if (noff >= end || noff >= lfs->pcache.off + lfs->cfg->cache_size) { + // flush buffers + int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); + if (err) { + return err; + } + } + } + + // successful commit, check checksums to make sure + // + // note that we don't need to check padding commits, worst + // case if they are corrupted we would have had to compact anyways + lfs_off_t off = commit->begin; + uint32_t crc = 0xffffffff; + int err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, off1+sizeof(uint32_t), + commit->block, off, off1-off, &crc); + if (err) { + return err; + } + + // check non-padding commits against known crc + if (crc != crc1) { + return LFS_ERR_CORRUPT; + } + + // make sure to check crc in case we happen to pick + // up an unrelated crc (frozen block?) + err = lfs_bd_crc(lfs, + NULL, &lfs->rcache, sizeof(uint32_t), + commit->block, off1, sizeof(uint32_t), &crc); + if (err) { + return err; + } + + if (crc != 0) { + return LFS_ERR_CORRUPT; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { + // allocate pair of dir blocks (backwards, so we write block 1 first) + for (int i = 0; i < 2; i++) { + int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); + if (err) { + return err; + } + } + + // zero for reproducibility in case initial block is unreadable + dir->rev = 0; + + // rather than clobbering one of the blocks we just pretend + // the revision may be valid + int err = lfs_bd_read(lfs, + NULL, &lfs->rcache, sizeof(dir->rev), + dir->pair[0], 0, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + // to make sure we don't immediately evict, align the new revision count + // to our block_cycles modulus, see lfs_dir_compact for why our modulus + // is tweaked this way + if (lfs->cfg->block_cycles > 0) { + dir->rev = lfs_alignup(dir->rev, ((lfs->cfg->block_cycles+1)|1)); + } + + // set defaults + dir->off = sizeof(dir->rev); + dir->etag = 0xffffffff; + dir->count = 0; + dir->tail[0] = LFS_BLOCK_NULL; + dir->tail[1] = LFS_BLOCK_NULL; + dir->erased = false; + dir->split = false; + + // don't write out yet, let caller take care of that + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, lfs_mdir_t *tail) { + // steal state + int err = lfs_dir_getgstate(lfs, tail, &lfs->gdelta); + if (err) { + return err; + } + + // steal tail + lfs_pair_tole32(tail->tail); + err = lfs_dir_commit(lfs, dir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + tail->split, 0x3ff, 8), tail->tail})); + lfs_pair_fromle32(tail->tail); + if (err) { + return err; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_split(lfs_t *lfs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, + lfs_mdir_t *source, uint16_t split, uint16_t end) { + // create tail metadata pair + lfs_mdir_t tail; + int err = lfs_dir_alloc(lfs, &tail); + if (err) { + return err; + } + + tail.split = dir->split; + tail.tail[0] = dir->tail[0]; + tail.tail[1] = dir->tail[1]; + + // note we don't care about LFS_OK_RELOCATED + int res = lfs_dir_compact(lfs, &tail, attrs, attrcount, source, split, end); + if (res < 0) { + return res; + } + + dir->tail[0] = tail.pair[0]; + dir->tail[1] = tail.pair[1]; + dir->split = true; + + // update root if needed + if (lfs_pair_cmp(dir->pair, lfs->root) == 0 && split == 0) { + lfs->root[0] = tail.pair[0]; + lfs->root[1] = tail.pair[1]; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_commit_size(void *p, lfs_tag_t tag, const void *buffer) { + lfs_size_t *size = p; + (void)buffer; + + *size += lfs_tag_dsize(tag); + return 0; +} +#endif + +#ifndef LFS_READONLY +struct lfs_dir_commit_commit { + lfs_t *lfs; + struct lfs_commit *commit; +}; +#endif + +#ifndef LFS_READONLY +static int lfs_dir_commit_commit(void *p, lfs_tag_t tag, const void *buffer) { + struct lfs_dir_commit_commit *commit = p; + return lfs_dir_commitattr(commit->lfs, commit->commit, tag, buffer); +} +#endif + +#ifndef LFS_READONLY +static bool lfs_dir_needsrelocation(lfs_t *lfs, lfs_mdir_t *dir) { + // If our revision count == n * block_cycles, we should force a relocation, + // this is how littlefs wear-levels at the metadata-pair level. Note that we + // actually use (block_cycles+1)|1, this is to avoid two corner cases: + // 1. block_cycles = 1, which would prevent relocations from terminating + // 2. block_cycles = 2n, which, due to aliasing, would only ever relocate + // one metadata block in the pair, effectively making this useless + return (lfs->cfg->block_cycles > 0 + && ((dir->rev + 1) % ((lfs->cfg->block_cycles+1)|1) == 0)); +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_compact(lfs_t *lfs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, + lfs_mdir_t *source, uint16_t begin, uint16_t end) { + // save some state in case block is bad + bool relocated = false; + bool tired = lfs_dir_needsrelocation(lfs, dir); + + // increment revision count + dir->rev += 1; + + // do not proactively relocate blocks during migrations, this + // can cause a number of failure states such: clobbering the + // v1 superblock if we relocate root, and invalidating directory + // pointers if we relocate the head of a directory. On top of + // this, relocations increase the overall complexity of + // lfs_migration, which is already a delicate operation. +#ifdef LFS_MIGRATE + if (lfs->lfs1) { + tired = false; + } +#endif + + if (tired && lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) != 0) { + // we're writing too much, time to relocate + goto relocate; + } + + // begin loop to commit compaction to blocks until a compact sticks + while (true) { + { + // setup commit state + struct lfs_commit commit = { + .block = dir->pair[1], + .off = 0, + .ptag = 0xffffffff, + .crc = 0xffffffff, + + .begin = 0, + .end = (lfs->cfg->metadata_max ? + lfs->cfg->metadata_max : lfs->cfg->block_size) - 8, + }; + + // erase block to write to + int err = lfs_bd_erase(lfs, dir->pair[1]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // write out header + dir->rev = lfs_tole32(dir->rev); + err = lfs_dir_commitprog(lfs, &commit, + &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // traverse the directory, this time writing out all unique tags + err = lfs_dir_traverse(lfs, + source, 0, 0xffffffff, attrs, attrcount, + LFS_MKTAG(0x400, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, 0), + begin, end, -begin, + lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ + lfs, &commit}); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // commit tail, which may be new after last size check + if (!lfs_pair_isnull(dir->tail)) { + lfs_pair_tole32(dir->tail); + err = lfs_dir_commitattr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, 8), + dir->tail); + lfs_pair_fromle32(dir->tail); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // bring over gstate? + lfs_gstate_t delta = {0}; + if (!relocated) { + lfs_gstate_xor(&delta, &lfs->gdisk); + lfs_gstate_xor(&delta, &lfs->gstate); + } + lfs_gstate_xor(&delta, &lfs->gdelta); + delta.tag &= ~LFS_MKTAG(0, 0, 0x3ff); + + err = lfs_dir_getgstate(lfs, dir, &delta); + if (err) { + return err; + } + + if (!lfs_gstate_iszero(&delta)) { + lfs_gstate_tole32(&delta); + err = lfs_dir_commitattr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, + sizeof(delta)), &delta); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // complete commit with crc + err = lfs_dir_commitcrc(lfs, &commit); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // successful compaction, swap dir pair to indicate most recent + LFS_ASSERT(commit.off % lfs->cfg->prog_size == 0); + lfs_pair_swap(dir->pair); + dir->count = end - begin; + dir->off = commit.off; + dir->etag = commit.ptag; + // update gstate + lfs->gdelta = (lfs_gstate_t){0}; + if (!relocated) { + lfs->gdisk = lfs->gstate; + } + } + break; + +relocate: + // commit was corrupted, drop caches and prepare to relocate block + relocated = true; + lfs_cache_drop(lfs, &lfs->pcache); + if (!tired) { + LFS_DEBUG("Bad block at 0x%"PRIx32, dir->pair[1]); + } + + // can't relocate superblock, filesystem is now frozen + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { + LFS_WARN("Superblock 0x%"PRIx32" has become unwritable", + dir->pair[1]); + return LFS_ERR_NOSPC; + } + + // relocate half of pair + int err = lfs_alloc(lfs, &dir->pair[1]); + if (err && (err != LFS_ERR_NOSPC || !tired)) { + return err; + } + + tired = false; + continue; + } + + return relocated ? LFS_OK_RELOCATED : 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, + const struct lfs_mattr *attrs, int attrcount, + lfs_mdir_t *source, uint16_t begin, uint16_t end) { + while (true) { + // find size of first split, we do this by halving the split until + // the metadata is guaranteed to fit + // + // Note that this isn't a true binary search, we never increase the + // split size. This may result in poorly distributed metadata but isn't + // worth the extra code size or performance hit to fix. + lfs_size_t split = begin; + while (end - split > 1) { + lfs_size_t size = 0; + int err = lfs_dir_traverse(lfs, + source, 0, 0xffffffff, attrs, attrcount, + LFS_MKTAG(0x400, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, 0), + split, end, -split, + lfs_dir_commit_size, &size); + if (err) { + return err; + } + + // space is complicated, we need room for: + // + // - tail: 4+2*4 = 12 bytes + // - gstate: 4+3*4 = 16 bytes + // - move delete: 4 = 4 bytes + // - crc: 4+4 = 8 bytes + // total = 40 bytes + // + // And we cap at half a block to avoid degenerate cases with + // nearly-full metadata blocks. + // + lfs_size_t metadata_max = (lfs->cfg->metadata_max) + ? lfs->cfg->metadata_max + : lfs->cfg->block_size; + if (end - split < 0xff + && size <= lfs_min( + metadata_max - 40, + lfs_alignup( + metadata_max/2, + lfs->cfg->prog_size))) { + break; + } + + split = split + ((end - split) / 2); + } + + if (split == begin) { + // no split needed + break; + } + + // split into two metadata pairs and continue + int err = lfs_dir_split(lfs, dir, attrs, attrcount, + source, split, end); + if (err && err != LFS_ERR_NOSPC) { + return err; + } + + if (err) { + // we can't allocate a new block, try to compact with degraded + // performance + LFS_WARN("Unable to split {0x%"PRIx32", 0x%"PRIx32"}", + dir->pair[0], dir->pair[1]); + break; + } else { + end = split; + } + } + + if (lfs_dir_needsrelocation(lfs, dir) + && lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { + // oh no! we're writing too much to the superblock, + // should we expand? + lfs_ssize_t size = lfs_fs_size_(lfs); + if (size < 0) { + return size; + } + + // littlefs cannot reclaim expanded superblocks, so expand cautiously + // + // if our filesystem is more than ~88% full, don't expand, this is + // somewhat arbitrary + if (lfs->block_count - size > lfs->block_count/8) { + LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); + int err = lfs_dir_split(lfs, dir, attrs, attrcount, + source, begin, end); + if (err && err != LFS_ERR_NOSPC) { + return err; + } + + if (err) { + // welp, we tried, if we ran out of space there's not much + // we can do, we'll error later if we've become frozen + LFS_WARN("Unable to expand superblock"); + } else { + // duplicate the superblock entry into the new superblock + end = 1; + } + } + } + + return lfs_dir_compact(lfs, dir, attrs, attrcount, source, begin, end); +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_relocatingcommit(lfs_t *lfs, lfs_mdir_t *dir, + const lfs_block_t pair[2], + const struct lfs_mattr *attrs, int attrcount, + lfs_mdir_t *pdir) { + int state = 0; + + // calculate changes to the directory + bool hasdelete = false; + for (int i = 0; i < attrcount; i++) { + if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_CREATE) { + dir->count += 1; + } else if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(dir->count > 0); + dir->count -= 1; + hasdelete = true; + } else if (lfs_tag_type1(attrs[i].tag) == LFS_TYPE_TAIL) { + dir->tail[0] = ((lfs_block_t*)attrs[i].buffer)[0]; + dir->tail[1] = ((lfs_block_t*)attrs[i].buffer)[1]; + dir->split = (lfs_tag_chunk(attrs[i].tag) & 1); + lfs_pair_fromle32(dir->tail); + } + } + + // should we actually drop the directory block? + if (hasdelete && dir->count == 0) { + LFS_ASSERT(pdir); + int err = lfs_fs_pred(lfs, dir->pair, pdir); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err != LFS_ERR_NOENT && pdir->split) { + state = LFS_OK_DROPPED; + goto fixmlist; + } + } + + if (dir->erased && dir->count < 0xff) { + // try to commit + struct lfs_commit commit = { + .block = dir->pair[0], + .off = dir->off, + .ptag = dir->etag, + .crc = 0xffffffff, + + .begin = dir->off, + .end = (lfs->cfg->metadata_max ? + lfs->cfg->metadata_max : lfs->cfg->block_size) - 8, + }; + + // traverse attrs that need to be written out + lfs_pair_tole32(dir->tail); + int err = lfs_dir_traverse(lfs, + dir, dir->off, dir->etag, attrs, attrcount, + 0, 0, 0, 0, 0, + lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ + lfs, &commit}); + lfs_pair_fromle32(dir->tail); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } + + // commit any global diffs if we have any + lfs_gstate_t delta = {0}; + lfs_gstate_xor(&delta, &lfs->gstate); + lfs_gstate_xor(&delta, &lfs->gdisk); + lfs_gstate_xor(&delta, &lfs->gdelta); + delta.tag &= ~LFS_MKTAG(0, 0, 0x3ff); + if (!lfs_gstate_iszero(&delta)) { + err = lfs_dir_getgstate(lfs, dir, &delta); + if (err) { + return err; + } + + lfs_gstate_tole32(&delta); + err = lfs_dir_commitattr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, + sizeof(delta)), &delta); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } + } + + // finalize commit with the crc + err = lfs_dir_commitcrc(lfs, &commit); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } + + // successful commit, update dir + LFS_ASSERT(commit.off % lfs->cfg->prog_size == 0); + dir->off = commit.off; + dir->etag = commit.ptag; + // and update gstate + lfs->gdisk = lfs->gstate; + lfs->gdelta = (lfs_gstate_t){0}; + + goto fixmlist; + } + +compact: + // fall back to compaction + lfs_cache_drop(lfs, &lfs->pcache); + + state = lfs_dir_splittingcompact(lfs, dir, attrs, attrcount, + dir, 0, dir->count); + if (state < 0) { + return state; + } + + goto fixmlist; + +fixmlist:; + // this complicated bit of logic is for fixing up any active + // metadata-pairs that we may have affected + // + // note we have to make two passes since the mdir passed to + // lfs_dir_commit could also be in this list, and even then + // we need to copy the pair so they don't get clobbered if we refetch + // our mdir. + lfs_block_t oldpair[2] = {pair[0], pair[1]}; + for (struct lfs_mlist *d = lfs->mlist; d; d = d->next) { + if (lfs_pair_cmp(d->m.pair, oldpair) == 0) { + d->m = *dir; + if (d->m.pair != pair) { + for (int i = 0; i < attrcount; i++) { + if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_DELETE && + d->id == lfs_tag_id(attrs[i].tag) && + d->type != LFS_TYPE_DIR) { + d->m.pair[0] = LFS_BLOCK_NULL; + d->m.pair[1] = LFS_BLOCK_NULL; + } else if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_DELETE && + d->id > lfs_tag_id(attrs[i].tag)) { + d->id -= 1; + if (d->type == LFS_TYPE_DIR) { + ((lfs_dir_t*)d)->pos -= 1; + } + } else if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_CREATE && + d->id >= lfs_tag_id(attrs[i].tag)) { + d->id += 1; + if (d->type == LFS_TYPE_DIR) { + ((lfs_dir_t*)d)->pos += 1; + } + } + } + } + + while (d->id >= d->m.count && d->m.split) { + // we split and id is on tail now + if (lfs_pair_cmp(d->m.tail, lfs->root) != 0) { + d->id -= d->m.count; + } + int err = lfs_dir_fetch(lfs, &d->m, d->m.tail); + if (err) { + return err; + } + } + } + } + + return state; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_orphaningcommit(lfs_t *lfs, lfs_mdir_t *dir, + const struct lfs_mattr *attrs, int attrcount) { + // check for any inline files that aren't RAM backed and + // forcefully evict them, needed for filesystem consistency + for (lfs_file_t *f = (lfs_file_t*)lfs->mlist; f; f = f->next) { + if (dir != &f->m && lfs_pair_cmp(f->m.pair, dir->pair) == 0 && + f->type == LFS_TYPE_REG && (f->flags & LFS_F_INLINE) && + f->ctz.size > lfs->cfg->cache_size) { + int err = lfs_file_outline(lfs, f); + if (err) { + return err; + } + + err = lfs_file_flush(lfs, f); + if (err) { + return err; + } + } + } + + lfs_block_t lpair[2] = {dir->pair[0], dir->pair[1]}; + lfs_mdir_t ldir = *dir; + lfs_mdir_t pdir; + int state = lfs_dir_relocatingcommit(lfs, &ldir, dir->pair, + attrs, attrcount, &pdir); + if (state < 0) { + return state; + } + + // update if we're not in mlist, note we may have already been + // updated if we are in mlist + if (lfs_pair_cmp(dir->pair, lpair) == 0) { + *dir = ldir; + } + + // commit was successful, but may require other changes in the + // filesystem, these would normally be tail recursive, but we have + // flattened them here avoid unbounded stack usage + + // need to drop? + if (state == LFS_OK_DROPPED) { + // steal state + int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta); + if (err) { + return err; + } + + // steal tail, note that this can't create a recursive drop + lpair[0] = pdir.pair[0]; + lpair[1] = pdir.pair[1]; + lfs_pair_tole32(dir->tail); + state = lfs_dir_relocatingcommit(lfs, &pdir, lpair, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, 8), + dir->tail}), + NULL); + lfs_pair_fromle32(dir->tail); + if (state < 0) { + return state; + } + + ldir = pdir; + } + + // need to relocate? + bool orphans = false; + while (state == LFS_OK_RELOCATED) { + LFS_DEBUG("Relocating {0x%"PRIx32", 0x%"PRIx32"} " + "-> {0x%"PRIx32", 0x%"PRIx32"}", + lpair[0], lpair[1], ldir.pair[0], ldir.pair[1]); + state = 0; + + // update internal root + if (lfs_pair_cmp(lpair, lfs->root) == 0) { + lfs->root[0] = ldir.pair[0]; + lfs->root[1] = ldir.pair[1]; + } + + // update internally tracked dirs + for (struct lfs_mlist *d = lfs->mlist; d; d = d->next) { + if (lfs_pair_cmp(lpair, d->m.pair) == 0) { + d->m.pair[0] = ldir.pair[0]; + d->m.pair[1] = ldir.pair[1]; + } + + if (d->type == LFS_TYPE_DIR && + lfs_pair_cmp(lpair, ((lfs_dir_t*)d)->head) == 0) { + ((lfs_dir_t*)d)->head[0] = ldir.pair[0]; + ((lfs_dir_t*)d)->head[1] = ldir.pair[1]; + } + } + + // find parent + lfs_stag_t tag = lfs_fs_parent(lfs, lpair, &pdir); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; + } + + bool hasparent = (tag != LFS_ERR_NOENT); + if (tag != LFS_ERR_NOENT) { + // note that if we have a parent, we must have a pred, so this will + // always create an orphan + int err = lfs_fs_preporphans(lfs, +1); + if (err) { + return err; + } + + // fix pending move in this pair? this looks like an optimization but + // is in fact _required_ since relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs_gstate_hasmovehere(&lfs->gstate, pdir.pair)) { + moveid = lfs_tag_id(lfs->gstate.tag); + LFS_DEBUG("Fixing move while relocating " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs_fs_prepmove(lfs, 0x3ff, NULL); + if (moveid < lfs_tag_id(tag)) { + tag -= LFS_MKTAG(0, 1, 0); + } + } + + lfs_block_t ppair[2] = {pdir.pair[0], pdir.pair[1]}; + lfs_pair_tole32(ldir.pair); + state = lfs_dir_relocatingcommit(lfs, &pdir, ppair, LFS_MKATTRS( + {LFS_MKTAG_IF(moveid != 0x3ff, + LFS_TYPE_DELETE, moveid, 0), NULL}, + {tag, ldir.pair}), + NULL); + lfs_pair_fromle32(ldir.pair); + if (state < 0) { + return state; + } + + if (state == LFS_OK_RELOCATED) { + lpair[0] = ppair[0]; + lpair[1] = ppair[1]; + ldir = pdir; + orphans = true; + continue; + } + } + + // find pred + int err = lfs_fs_pred(lfs, lpair, &pdir); + if (err && err != LFS_ERR_NOENT) { + return err; + } + LFS_ASSERT(!(hasparent && err == LFS_ERR_NOENT)); + + // if we can't find dir, it must be new + if (err != LFS_ERR_NOENT) { + if (lfs_gstate_hasorphans(&lfs->gstate)) { + // next step, clean up orphans + err = lfs_fs_preporphans(lfs, -(int8_t)hasparent); + if (err) { + return err; + } + } + + // fix pending move in this pair? this looks like an optimization + // but is in fact _required_ since relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs_gstate_hasmovehere(&lfs->gstate, pdir.pair)) { + moveid = lfs_tag_id(lfs->gstate.tag); + LFS_DEBUG("Fixing move while relocating " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs_fs_prepmove(lfs, 0x3ff, NULL); + } + + // replace bad pair, either we clean up desync, or no desync occured + lpair[0] = pdir.pair[0]; + lpair[1] = pdir.pair[1]; + lfs_pair_tole32(ldir.pair); + state = lfs_dir_relocatingcommit(lfs, &pdir, lpair, LFS_MKATTRS( + {LFS_MKTAG_IF(moveid != 0x3ff, + LFS_TYPE_DELETE, moveid, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_TAIL + pdir.split, 0x3ff, 8), + ldir.pair}), + NULL); + lfs_pair_fromle32(ldir.pair); + if (state < 0) { + return state; + } + + ldir = pdir; + } + } + + return orphans ? LFS_OK_ORPHANED : 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, + const struct lfs_mattr *attrs, int attrcount) { + int orphans = lfs_dir_orphaningcommit(lfs, dir, attrs, attrcount); + if (orphans < 0) { + return orphans; + } + + if (orphans) { + // make sure we've removed all orphans, this is a noop if there + // are none, but if we had nested blocks failures we may have + // created some + int err = lfs_fs_deorphan(lfs, false); + if (err) { + return err; + } + } + + return 0; +} +#endif + + +/// Top level directory operations /// +#ifndef LFS_READONLY +static int lfs_mkdir_(lfs_t *lfs, const char *path) { + // deorphan if we haven't yet, needed at most once after poweron + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + struct lfs_mlist cwd; + cwd.next = lfs->mlist; + uint16_t id; + err = lfs_dir_find(lfs, &cwd.m, &path, &id); + if (!(err == LFS_ERR_NOENT && lfs_path_islast(path))) { + return (err < 0) ? err : LFS_ERR_EXIST; + } + + // check that name fits + lfs_size_t nlen = lfs_path_namelen(path); + if (nlen > lfs->name_max) { + return LFS_ERR_NAMETOOLONG; + } + + // build up new directory + lfs_alloc_ckpoint(lfs); + lfs_mdir_t dir; + err = lfs_dir_alloc(lfs, &dir); + if (err) { + return err; + } + + // find end of list + lfs_mdir_t pred = cwd.m; + while (pred.split) { + err = lfs_dir_fetch(lfs, &pred, pred.tail); + if (err) { + return err; + } + } + + // setup dir + lfs_pair_tole32(pred.tail); + err = lfs_dir_commit(lfs, &dir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), pred.tail})); + lfs_pair_fromle32(pred.tail); + if (err) { + return err; + } + + // current block not end of list? + if (cwd.m.split) { + // update tails, this creates a desync + err = lfs_fs_preporphans(lfs, +1); + if (err) { + return err; + } + + // it's possible our predecessor has to be relocated, and if + // our parent is our predecessor's predecessor, this could have + // caused our parent to go out of date, fortunately we can hook + // ourselves into littlefs to catch this + cwd.type = 0; + cwd.id = 0; + lfs->mlist = &cwd; + + lfs_pair_tole32(dir.pair); + err = lfs_dir_commit(lfs, &pred, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), dir.pair})); + lfs_pair_fromle32(dir.pair); + if (err) { + lfs->mlist = cwd.next; + return err; + } + + lfs->mlist = cwd.next; + err = lfs_fs_preporphans(lfs, -1); + if (err) { + return err; + } + } + + // now insert into our parent block + lfs_pair_tole32(dir.pair); + err = lfs_dir_commit(lfs, &cwd.m, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_DIR, id, nlen), path}, + {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, id, 8), dir.pair}, + {LFS_MKTAG_IF(!cwd.m.split, + LFS_TYPE_SOFTTAIL, 0x3ff, 8), dir.pair})); + lfs_pair_fromle32(dir.pair); + if (err) { + return err; + } + + return 0; +} +#endif + +static int lfs_dir_open_(lfs_t *lfs, lfs_dir_t *dir, const char *path) { + lfs_stag_t tag = lfs_dir_find(lfs, &dir->m, &path, NULL); + if (tag < 0) { + return tag; + } + + if (lfs_tag_type3(tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + lfs_block_t pair[2]; + if (lfs_tag_id(tag) == 0x3ff) { + // handle root dir separately + pair[0] = lfs->root[0]; + pair[1] = lfs->root[1]; + } else { + // get dir pair from parent + lfs_stag_t res = lfs_dir_get(lfs, &dir->m, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); + if (res < 0) { + return res; + } + lfs_pair_fromle32(pair); + } + + // fetch first pair + int err = lfs_dir_fetch(lfs, &dir->m, pair); + if (err) { + return err; + } + + // setup entry + dir->head[0] = dir->m.pair[0]; + dir->head[1] = dir->m.pair[1]; + dir->id = 0; + dir->pos = 0; + + // add to list of mdirs + dir->type = LFS_TYPE_DIR; + lfs_mlist_append(lfs, (struct lfs_mlist *)dir); + + return 0; +} + +static int lfs_dir_close_(lfs_t *lfs, lfs_dir_t *dir) { + // remove from list of mdirs + lfs_mlist_remove(lfs, (struct lfs_mlist *)dir); + + return 0; +} + +static int lfs_dir_read_(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { + memset(info, 0, sizeof(*info)); + + // special offset for '.' and '..' + if (dir->pos == 0) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, "."); + dir->pos += 1; + return true; + } else if (dir->pos == 1) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, ".."); + dir->pos += 1; + return true; + } + + while (true) { + if (dir->id == dir->m.count) { + if (!dir->m.split) { + return false; + } + + int err = lfs_dir_fetch(lfs, &dir->m, dir->m.tail); + if (err) { + return err; + } + + dir->id = 0; + } + + int err = lfs_dir_getinfo(lfs, &dir->m, dir->id, info); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + dir->id += 1; + if (err != LFS_ERR_NOENT) { + break; + } + } + + dir->pos += 1; + return true; +} + +static int lfs_dir_seek_(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { + // simply walk from head dir + int err = lfs_dir_rewind_(lfs, dir); + if (err) { + return err; + } + + // first two for ./.. + dir->pos = lfs_min(2, off); + off -= dir->pos; + + // skip superblock entry + dir->id = (off > 0 && lfs_pair_cmp(dir->head, lfs->root) == 0); + + while (off > 0) { + if (dir->id == dir->m.count) { + if (!dir->m.split) { + return LFS_ERR_INVAL; + } + + err = lfs_dir_fetch(lfs, &dir->m, dir->m.tail); + if (err) { + return err; + } + + dir->id = 0; + } + + int diff = lfs_min(dir->m.count - dir->id, off); + dir->id += diff; + dir->pos += diff; + off -= diff; + } + + return 0; +} + +static lfs_soff_t lfs_dir_tell_(lfs_t *lfs, lfs_dir_t *dir) { + (void)lfs; + return dir->pos; +} + +static int lfs_dir_rewind_(lfs_t *lfs, lfs_dir_t *dir) { + // reload the head dir + int err = lfs_dir_fetch(lfs, &dir->m, dir->head); + if (err) { + return err; + } + + dir->id = 0; + dir->pos = 0; + return 0; +} + + +/// File index list operations /// +static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { + lfs_off_t size = *off; + lfs_off_t b = lfs->cfg->block_size - 2*4; + lfs_off_t i = size / b; + if (i == 0) { + return 0; + } + + i = (size - 4*(lfs_popc(i-1)+2)) / b; + *off = size - b*i - 4*lfs_popc(i); + return i; +} + +static int lfs_ctz_find(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, + lfs_block_t head, lfs_size_t size, + lfs_size_t pos, lfs_block_t *block, lfs_off_t *off) { + if (size == 0) { + *block = LFS_BLOCK_NULL; + *off = 0; + return 0; + } + + lfs_off_t current = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); + lfs_off_t target = lfs_ctz_index(lfs, &pos); + + while (current > target) { + lfs_size_t skip = lfs_min( + lfs_npw2(current-target+1) - 1, + lfs_ctz(current)); + + int err = lfs_bd_read(lfs, + pcache, rcache, sizeof(head), + head, 4*skip, &head, sizeof(head)); + head = lfs_fromle32(head); + if (err) { + return err; + } + + current -= 1 << skip; + } + + *block = head; + *off = pos; + return 0; +} + +#ifndef LFS_READONLY +static int lfs_ctz_extend(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, + lfs_block_t head, lfs_size_t size, + lfs_block_t *block, lfs_off_t *off) { + while (true) { + // go ahead and grab a block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { + return err; + } + + { + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + if (size == 0) { + *block = nblock; + *off = 0; + return 0; + } + + lfs_size_t noff = size - 1; + lfs_off_t index = lfs_ctz_index(lfs, &noff); + noff = noff + 1; + + // just copy out the last block if it is incomplete + if (noff != lfs->cfg->block_size) { + for (lfs_off_t i = 0; i < noff; i++) { + uint8_t data; + err = lfs_bd_read(lfs, + NULL, rcache, noff-i, + head, i, &data, 1); + if (err) { + return err; + } + + err = lfs_bd_prog(lfs, + pcache, rcache, true, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + *block = nblock; + *off = noff; + return 0; + } + + // append block + index += 1; + lfs_size_t skips = lfs_ctz(index) + 1; + lfs_block_t nhead = head; + for (lfs_off_t i = 0; i < skips; i++) { + nhead = lfs_tole32(nhead); + err = lfs_bd_prog(lfs, pcache, rcache, true, + nblock, 4*i, &nhead, 4); + nhead = lfs_fromle32(nhead); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + if (i != skips-1) { + err = lfs_bd_read(lfs, + NULL, rcache, sizeof(nhead), + nhead, 4*i, &nhead, sizeof(nhead)); + nhead = lfs_fromle32(nhead); + if (err) { + return err; + } + } + } + + *block = nblock; + *off = 4*skips; + return 0; + } + +relocate: + LFS_DEBUG("Bad block at 0x%"PRIx32, nblock); + + // just clear cache and try a new block + lfs_cache_drop(lfs, pcache); + } +} +#endif + +static int lfs_ctz_traverse(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, + lfs_block_t head, lfs_size_t size, + int (*cb)(void*, lfs_block_t), void *data) { + if (size == 0) { + return 0; + } + + lfs_off_t index = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); + + while (true) { + int err = cb(data, head); + if (err) { + return err; + } + + if (index == 0) { + return 0; + } + + lfs_block_t heads[2]; + int count = 2 - (index & 1); + err = lfs_bd_read(lfs, + pcache, rcache, count*sizeof(head), + head, 0, &heads, count*sizeof(head)); + heads[0] = lfs_fromle32(heads[0]); + heads[1] = lfs_fromle32(heads[1]); + if (err) { + return err; + } + + for (int i = 0; i < count-1; i++) { + err = cb(data, heads[i]); + if (err) { + return err; + } + } + + head = heads[count-1]; + index -= count; + } +} + + +/// Top level file operations /// +static int lfs_file_opencfg_(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags, + const struct lfs_file_config *cfg) { +#ifndef LFS_READONLY + // deorphan if we haven't yet, needed at most once after poweron + if ((flags & LFS_O_WRONLY) == LFS_O_WRONLY) { + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + } +#else + LFS_ASSERT((flags & LFS_O_RDONLY) == LFS_O_RDONLY); +#endif + + // setup simple file details + int err; + file->cfg = cfg; + file->flags = flags; + file->pos = 0; + file->off = 0; + file->cache.buffer = NULL; + + // allocate entry for file if it doesn't exist + lfs_stag_t tag = lfs_dir_find(lfs, &file->m, &path, &file->id); + if (tag < 0 && !(tag == LFS_ERR_NOENT && lfs_path_islast(path))) { + err = tag; + goto cleanup; + } + + // get id, add to list of mdirs to catch update changes + file->type = LFS_TYPE_REG; + lfs_mlist_append(lfs, (struct lfs_mlist *)file); + +#ifdef LFS_READONLY + if (tag == LFS_ERR_NOENT) { + err = LFS_ERR_NOENT; + goto cleanup; +#else + if (tag == LFS_ERR_NOENT) { + if (!(flags & LFS_O_CREAT)) { + err = LFS_ERR_NOENT; + goto cleanup; + } + + // don't allow trailing slashes + if (lfs_path_isdir(path)) { + err = LFS_ERR_NOTDIR; + goto cleanup; + } + + // check that name fits + lfs_size_t nlen = lfs_path_namelen(path); + if (nlen > lfs->name_max) { + err = LFS_ERR_NAMETOOLONG; + goto cleanup; + } + + // get next slot and create entry to remember name + err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), NULL})); + + // it may happen that the file name doesn't fit in the metadata blocks, e.g., a 256 byte file name will + // not fit in a 128 byte block. + err = (err == LFS_ERR_NOSPC) ? LFS_ERR_NAMETOOLONG : err; + if (err) { + goto cleanup; + } + + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, 0); + } else if (flags & LFS_O_EXCL) { + err = LFS_ERR_EXIST; + goto cleanup; +#endif + } else if (lfs_tag_type3(tag) != LFS_TYPE_REG) { + err = LFS_ERR_ISDIR; + goto cleanup; +#ifndef LFS_READONLY + } else if (flags & LFS_O_TRUNC) { + // truncate if requested + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0); + file->flags |= LFS_F_DIRTY; +#endif + } else { + // try to load what's on disk, if it's inlined we'll fix it later + tag = lfs_dir_get(lfs, &file->m, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, 8), &file->ctz); + if (tag < 0) { + err = tag; + goto cleanup; + } + lfs_ctz_fromle32(&file->ctz); + } + + // fetch attrs + for (unsigned i = 0; i < file->cfg->attr_count; i++) { + // if opened for read / read-write operations + if ((file->flags & LFS_O_RDONLY) == LFS_O_RDONLY) { + lfs_stag_t res = lfs_dir_get(lfs, &file->m, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_USERATTR + file->cfg->attrs[i].type, + file->id, file->cfg->attrs[i].size), + file->cfg->attrs[i].buffer); + if (res < 0 && res != LFS_ERR_NOENT) { + err = res; + goto cleanup; + } + } + +#ifndef LFS_READONLY + // if opened for write / read-write operations + if ((file->flags & LFS_O_WRONLY) == LFS_O_WRONLY) { + if (file->cfg->attrs[i].size > lfs->attr_max) { + err = LFS_ERR_NOSPC; + goto cleanup; + } + + file->flags |= LFS_F_DIRTY; + } +#endif + } + + // allocate buffer if needed + if (file->cfg->buffer) { + file->cache.buffer = file->cfg->buffer; + } else { + file->cache.buffer = lfs_malloc(lfs->cfg->cache_size); + if (!file->cache.buffer) { + err = LFS_ERR_NOMEM; + goto cleanup; + } + } + + // zero to avoid information leak + lfs_cache_zero(lfs, &file->cache); + + if (lfs_tag_type3(tag) == LFS_TYPE_INLINESTRUCT) { + // load inline files + file->ctz.head = LFS_BLOCK_INLINE; + file->ctz.size = lfs_tag_size(tag); + file->flags |= LFS_F_INLINE; + file->cache.block = file->ctz.head; + file->cache.off = 0; + file->cache.size = lfs->cfg->cache_size; + + // don't always read (may be new/trunc file) + if (file->ctz.size > 0) { + lfs_stag_t res = lfs_dir_get(lfs, &file->m, + LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, + lfs_min(file->cache.size, 0x3fe)), + file->cache.buffer); + if (res < 0) { + err = res; + goto cleanup; + } + } + } + + return 0; + +cleanup: + // clean up lingering resources +#ifndef LFS_READONLY + file->flags |= LFS_F_ERRED; +#endif + lfs_file_close_(lfs, file); + return err; +} + +#ifndef LFS_NO_MALLOC +static int lfs_file_open_(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags) { + static const struct lfs_file_config defaults = {0}; + int err = lfs_file_opencfg_(lfs, file, path, flags, &defaults); + return err; +} +#endif + +static int lfs_file_close_(lfs_t *lfs, lfs_file_t *file) { +#ifndef LFS_READONLY + int err = lfs_file_sync_(lfs, file); +#else + int err = 0; +#endif + + // remove from list of mdirs + lfs_mlist_remove(lfs, (struct lfs_mlist*)file); + + // clean up memory + if (!file->cfg->buffer) { + lfs_free(file->cache.buffer); + } + + return err; +} + + +#ifndef LFS_READONLY +static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { + while (true) { + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { + return err; + } + + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + if (file->flags & LFS_F_INLINE) { + err = lfs_dir_getread(lfs, &file->m, + // note we evict inline files before they can be dirty + NULL, &file->cache, file->off-i, + LFS_MKTAG(0xfff, 0x1ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), + i, &data, 1); + if (err) { + return err; + } + } else { + err = lfs_bd_read(lfs, + &file->cache, &lfs->rcache, file->off-i, + file->block, i, &data, 1); + if (err) { + return err; + } + } + + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, true, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->cache_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + file->cache.size = lfs->pcache.size; + lfs_cache_zero(lfs, &lfs->pcache); + + file->block = nblock; + file->flags |= LFS_F_WRITING; + return 0; + +relocate: + LFS_DEBUG("Bad block at 0x%"PRIx32, nblock); + + // just clear cache and try a new block + lfs_cache_drop(lfs, &lfs->pcache); + } +} +#endif + +#ifndef LFS_READONLY +static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file) { + file->off = file->pos; + lfs_alloc_ckpoint(lfs); + int err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } + + file->flags &= ~LFS_F_INLINE; + return 0; +} +#endif + +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + if (!(file->flags & LFS_F_INLINE)) { + lfs_cache_drop(lfs, &file->cache); + } + file->flags &= ~LFS_F_READING; + } + +#ifndef LFS_READONLY + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; + + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .ctz.head = file->ctz.head, + .ctz.size = file->ctz.size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; + lfs_cache_drop(lfs, &lfs->rcache); + + while (file->pos < file->ctz.size) { + // copy over a byte at a time, leave it up to caching + // to make this efficient + uint8_t data; + lfs_ssize_t res = lfs_file_flushedread(lfs, &orig, &data, 1); + if (res < 0) { + return res; + } + + res = lfs_file_flushedwrite(lfs, file, &data, 1); + if (res < 0) { + return res; + } + + // keep our reference to the rcache in sync + if (lfs->rcache.block != LFS_BLOCK_NULL) { + lfs_cache_drop(lfs, &orig.cache); + lfs_cache_drop(lfs, &lfs->rcache); + } + } + + // write out what we have + while (true) { + int err = lfs_bd_flush(lfs, &file->cache, &lfs->rcache, true); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + break; + +relocate: + LFS_DEBUG("Bad block at 0x%"PRIx32, file->block); + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } + } + } else { + file->pos = lfs_max(file->pos, file->ctz.size); + } + + // actual file updates + file->ctz.head = file->block; + file->ctz.size = file->pos; + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + + file->pos = pos; + } +#endif + + return 0; +} + +#ifndef LFS_READONLY +static int lfs_file_sync_(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_ERRED) { + // it's not safe to do anything if our file errored + return 0; + } + + int err = lfs_file_flush(lfs, file); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + + if ((file->flags & LFS_F_DIRTY) && + !lfs_pair_isnull(file->m.pair)) { + // before we commit metadata, we need sync the disk to make sure + // data writes don't complete after metadata writes + if (!(file->flags & LFS_F_INLINE)) { + err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); + if (err) { + return err; + } + } + + // update dir entry + uint16_t type; + const void *buffer; + lfs_size_t size; + struct lfs_ctz ctz; + if (file->flags & LFS_F_INLINE) { + // inline the whole file + type = LFS_TYPE_INLINESTRUCT; + buffer = file->cache.buffer; + size = file->ctz.size; + } else { + // update the ctz reference + type = LFS_TYPE_CTZSTRUCT; + // copy ctz so alloc will work during a relocate + ctz = file->ctz; + lfs_ctz_tole32(&ctz); + buffer = &ctz; + size = sizeof(ctz); + } + + // commit file data and attributes + err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS( + {LFS_MKTAG(type, file->id, size), buffer}, + {LFS_MKTAG(LFS_FROM_USERATTRS, file->id, + file->cfg->attr_count), file->cfg->attrs})); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + file->flags &= ~LFS_F_DIRTY; + } + + return 0; +} +#endif + +static lfs_ssize_t lfs_file_flushedread(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + lfs_size_t nsize = size; + + if (file->pos >= file->ctz.size) { + // eof if past end + return 0; + } + + size = lfs_min(size, file->ctz.size - file->pos); + nsize = size; + + while (nsize > 0) { + // check if we need a new block + if (!(file->flags & LFS_F_READING) || + file->off == lfs->cfg->block_size) { + if (!(file->flags & LFS_F_INLINE)) { + int err = lfs_ctz_find(lfs, NULL, &file->cache, + file->ctz.head, file->ctz.size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } + } else { + file->block = LFS_BLOCK_INLINE; + file->off = file->pos; + } + + file->flags |= LFS_F_READING; + } + + // read as much as we can in current block + lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + if (file->flags & LFS_F_INLINE) { + int err = lfs_dir_getread(lfs, &file->m, + NULL, &file->cache, lfs->cfg->block_size, + LFS_MKTAG(0xfff, 0x1ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), + file->off, data, diff); + if (err) { + return err; + } + } else { + int err = lfs_bd_read(lfs, + NULL, &file->cache, lfs->cfg->block_size, + file->block, file->off, data, diff); + if (err) { + return err; + } + } + + file->pos += diff; + file->off += diff; + data += diff; + nsize -= diff; + } + + return size; +} + +static lfs_ssize_t lfs_file_read_(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size) { + LFS_ASSERT((file->flags & LFS_O_RDONLY) == LFS_O_RDONLY); + +#ifndef LFS_READONLY + if (file->flags & LFS_F_WRITING) { + // flush out any writes + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + } +#endif + + return lfs_file_flushedread(lfs, file, buffer, size); +} + + +#ifndef LFS_READONLY +static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + lfs_size_t nsize = size; + + if ((file->flags & LFS_F_INLINE) && + lfs_max(file->pos+nsize, file->ctz.size) > lfs->inline_max) { + // inline file doesn't fit anymore + int err = lfs_file_outline(lfs, file); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + } + + while (nsize > 0) { + // check if we need a new block + if (!(file->flags & LFS_F_WRITING) || + file->off == lfs->cfg->block_size) { + if (!(file->flags & LFS_F_INLINE)) { + if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { + // find out which block we're extending from + int err = lfs_ctz_find(lfs, NULL, &file->cache, + file->ctz.head, file->ctz.size, + file->pos-1, &file->block, &(lfs_off_t){0}); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + // mark cache as dirty since we may have read data into it + lfs_cache_zero(lfs, &file->cache); + } + + // extend file with new blocks + lfs_alloc_ckpoint(lfs); + int err = lfs_ctz_extend(lfs, &file->cache, &lfs->rcache, + file->block, file->pos, + &file->block, &file->off); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + } else { + file->block = LFS_BLOCK_INLINE; + file->off = file->pos; + } + + file->flags |= LFS_F_WRITING; + } + + // program as much as we can in current block + lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + while (true) { + int err = lfs_bd_prog(lfs, &file->cache, &lfs->rcache, true, + file->block, file->off, data, diff); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + file->flags |= LFS_F_ERRED; + return err; + } + + break; +relocate: + err = lfs_file_relocate(lfs, file); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + } + + file->pos += diff; + file->off += diff; + data += diff; + nsize -= diff; + + lfs_alloc_ckpoint(lfs); + } + + return size; +} + +static lfs_ssize_t lfs_file_write_(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size) { + LFS_ASSERT((file->flags & LFS_O_WRONLY) == LFS_O_WRONLY); + + if (file->flags & LFS_F_READING) { + // drop any reads + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + } + + if ((file->flags & LFS_O_APPEND) && file->pos < file->ctz.size) { + file->pos = file->ctz.size; + } + + if (file->pos + size > lfs->file_max) { + // Larger than file limit? + return LFS_ERR_FBIG; + } + + if (!(file->flags & LFS_F_WRITING) && file->pos > file->ctz.size) { + // fill with zeros + lfs_off_t pos = file->pos; + file->pos = file->ctz.size; + + while (file->pos < pos) { + lfs_ssize_t res = lfs_file_flushedwrite(lfs, file, &(uint8_t){0}, 1); + if (res < 0) { + return res; + } + } + } + + lfs_ssize_t nsize = lfs_file_flushedwrite(lfs, file, buffer, size); + if (nsize < 0) { + return nsize; + } + + file->flags &= ~LFS_F_ERRED; + return nsize; +} +#endif + +static lfs_soff_t lfs_file_seek_(lfs_t *lfs, lfs_file_t *file, + lfs_soff_t off, int whence) { + // find new pos + // + // fortunately for us, littlefs is limited to 31-bit file sizes, so we + // don't have to worry too much about integer overflow + lfs_off_t npos = file->pos; + if (whence == LFS_SEEK_SET) { + npos = off; + } else if (whence == LFS_SEEK_CUR) { + npos = file->pos + (lfs_off_t)off; + } else if (whence == LFS_SEEK_END) { + npos = (lfs_off_t)lfs_file_size_(lfs, file) + (lfs_off_t)off; + } + + if (npos > lfs->file_max) { + // file position out of range + return LFS_ERR_INVAL; + } + + if (file->pos == npos) { + // noop - position has not changed + return npos; + } + + // if we're only reading and our new offset is still in the file's cache + // we can avoid flushing and needing to reread the data + if ((file->flags & LFS_F_READING) + && file->off != lfs->cfg->block_size) { + int oindex = lfs_ctz_index(lfs, &(lfs_off_t){file->pos}); + lfs_off_t noff = npos; + int nindex = lfs_ctz_index(lfs, &noff); + if (oindex == nindex + && noff >= file->cache.off + && noff < file->cache.off + file->cache.size) { + file->pos = npos; + file->off = noff; + return npos; + } + } + + // write out everything beforehand, may be noop if rdonly + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + + // update pos + file->pos = npos; + return npos; +} + +#ifndef LFS_READONLY +static int lfs_file_truncate_(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { + LFS_ASSERT((file->flags & LFS_O_WRONLY) == LFS_O_WRONLY); + + if (size > LFS_FILE_MAX) { + return LFS_ERR_INVAL; + } + + lfs_off_t pos = file->pos; + lfs_off_t oldsize = lfs_file_size_(lfs, file); + if (size < oldsize) { + // revert to inline file? + if (size <= lfs->inline_max) { + // flush+seek to head + lfs_soff_t res = lfs_file_seek_(lfs, file, 0, LFS_SEEK_SET); + if (res < 0) { + return (int)res; + } + + // read our data into rcache temporarily + lfs_cache_drop(lfs, &lfs->rcache); + res = lfs_file_flushedread(lfs, file, + lfs->rcache.buffer, size); + if (res < 0) { + return (int)res; + } + + file->ctz.head = LFS_BLOCK_INLINE; + file->ctz.size = size; + file->flags |= LFS_F_DIRTY | LFS_F_READING | LFS_F_INLINE; + file->cache.block = file->ctz.head; + file->cache.off = 0; + file->cache.size = lfs->cfg->cache_size; + memcpy(file->cache.buffer, lfs->rcache.buffer, size); + + } else { + // need to flush since directly changing metadata + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + + // lookup new head in ctz skip list + err = lfs_ctz_find(lfs, NULL, &file->cache, + file->ctz.head, file->ctz.size, + size-1, &file->block, &(lfs_off_t){0}); + if (err) { + return err; + } + + // need to set pos/block/off consistently so seeking back to + // the old position does not get confused + file->pos = size; + file->ctz.head = file->block; + file->ctz.size = size; + file->flags |= LFS_F_DIRTY | LFS_F_READING; + } + } else if (size > oldsize) { + // flush+seek if not already at end + lfs_soff_t res = lfs_file_seek_(lfs, file, 0, LFS_SEEK_END); + if (res < 0) { + return (int)res; + } + + // fill with zeros + while (file->pos < size) { + res = lfs_file_write_(lfs, file, &(uint8_t){0}, 1); + if (res < 0) { + return (int)res; + } + } + } + + // restore pos + lfs_soff_t res = lfs_file_seek_(lfs, file, pos, LFS_SEEK_SET); + if (res < 0) { + return (int)res; + } + + return 0; +} +#endif + +static lfs_soff_t lfs_file_tell_(lfs_t *lfs, lfs_file_t *file) { + (void)lfs; + return file->pos; +} + +static int lfs_file_rewind_(lfs_t *lfs, lfs_file_t *file) { + lfs_soff_t res = lfs_file_seek_(lfs, file, 0, LFS_SEEK_SET); + if (res < 0) { + return (int)res; + } + + return 0; +} + +static lfs_soff_t lfs_file_size_(lfs_t *lfs, lfs_file_t *file) { + (void)lfs; + +#ifndef LFS_READONLY + if (file->flags & LFS_F_WRITING) { + return lfs_max(file->pos, file->ctz.size); + } +#endif + + return file->ctz.size; +} + + +/// General fs operations /// +static int lfs_stat_(lfs_t *lfs, const char *path, struct lfs_info *info) { + lfs_mdir_t cwd; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0) { + return (int)tag; + } + + // only allow trailing slashes on dirs + if (strchr(path, '/') != NULL + && lfs_tag_type3(tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + return lfs_dir_getinfo(lfs, &cwd, lfs_tag_id(tag), info); +} + +#ifndef LFS_READONLY +static int lfs_remove_(lfs_t *lfs, const char *path) { + // deorphan if we haven't yet, needed at most once after poweron + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + lfs_mdir_t cwd; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0 || lfs_tag_id(tag) == 0x3ff) { + return (tag < 0) ? (int)tag : LFS_ERR_INVAL; + } + + struct lfs_mlist dir; + dir.next = lfs->mlist; + if (lfs_tag_type3(tag) == LFS_TYPE_DIR) { + // must be empty before removal + lfs_block_t pair[2]; + lfs_stag_t res = lfs_dir_get(lfs, &cwd, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); + if (res < 0) { + return (int)res; + } + lfs_pair_fromle32(pair); + + err = lfs_dir_fetch(lfs, &dir.m, pair); + if (err) { + return err; + } + + if (dir.m.count > 0 || dir.m.split) { + return LFS_ERR_NOTEMPTY; + } + + // mark fs as orphaned + err = lfs_fs_preporphans(lfs, +1); + if (err) { + return err; + } + + // I know it's crazy but yes, dir can be changed by our parent's + // commit (if predecessor is child) + dir.type = 0; + dir.id = 0; + lfs->mlist = &dir; + } + + // delete the entry + err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0), NULL})); + if (err) { + lfs->mlist = dir.next; + return err; + } + + lfs->mlist = dir.next; + if (lfs_gstate_hasorphans(&lfs->gstate)) { + LFS_ASSERT(lfs_tag_type3(tag) == LFS_TYPE_DIR); + + // fix orphan + err = lfs_fs_preporphans(lfs, -1); + if (err) { + return err; + } + + err = lfs_fs_pred(lfs, dir.m.pair, &cwd); + if (err) { + return err; + } + + err = lfs_dir_drop(lfs, &cwd, &dir.m); + if (err) { + return err; + } + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_rename_(lfs_t *lfs, const char *oldpath, const char *newpath) { + // deorphan if we haven't yet, needed at most once after poweron + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + // find old entry + lfs_mdir_t oldcwd; + lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath, NULL); + if (oldtag < 0 || lfs_tag_id(oldtag) == 0x3ff) { + return (oldtag < 0) ? (int)oldtag : LFS_ERR_INVAL; + } + + // find new entry + lfs_mdir_t newcwd; + uint16_t newid; + lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath, &newid); + if ((prevtag < 0 || lfs_tag_id(prevtag) == 0x3ff) && + !(prevtag == LFS_ERR_NOENT && lfs_path_islast(newpath))) { + return (prevtag < 0) ? (int)prevtag : LFS_ERR_INVAL; + } + + // if we're in the same pair there's a few special cases... + bool samepair = (lfs_pair_cmp(oldcwd.pair, newcwd.pair) == 0); + uint16_t newoldid = lfs_tag_id(oldtag); + + struct lfs_mlist prevdir; + prevdir.next = lfs->mlist; + if (prevtag == LFS_ERR_NOENT) { + // if we're a file, don't allow trailing slashes + if (lfs_path_isdir(newpath) + && lfs_tag_type3(oldtag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + // check that name fits + lfs_size_t nlen = lfs_path_namelen(newpath); + if (nlen > lfs->name_max) { + return LFS_ERR_NAMETOOLONG; + } + + // there is a small chance we are being renamed in the same + // directory/ to an id less than our old id, the global update + // to handle this is a bit messy + if (samepair && newid <= newoldid) { + newoldid += 1; + } + } else if (lfs_tag_type3(prevtag) != lfs_tag_type3(oldtag)) { + return (lfs_tag_type3(prevtag) == LFS_TYPE_DIR) + ? LFS_ERR_ISDIR + : LFS_ERR_NOTDIR; + } else if (samepair && newid == newoldid) { + // we're renaming to ourselves?? + return 0; + } else if (lfs_tag_type3(prevtag) == LFS_TYPE_DIR) { + // must be empty before removal + lfs_block_t prevpair[2]; + lfs_stag_t res = lfs_dir_get(lfs, &newcwd, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); + if (res < 0) { + return (int)res; + } + lfs_pair_fromle32(prevpair); + + // must be empty before removal + err = lfs_dir_fetch(lfs, &prevdir.m, prevpair); + if (err) { + return err; + } + + if (prevdir.m.count > 0 || prevdir.m.split) { + return LFS_ERR_NOTEMPTY; + } + + // mark fs as orphaned + err = lfs_fs_preporphans(lfs, +1); + if (err) { + return err; + } + + // I know it's crazy but yes, dir can be changed by our parent's + // commit (if predecessor is child) + prevdir.type = 0; + prevdir.id = 0; + lfs->mlist = &prevdir; + } + + if (!samepair) { + lfs_fs_prepmove(lfs, newoldid, oldcwd.pair); + } + + // move over all attributes + err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTRS( + {LFS_MKTAG_IF(prevtag != LFS_ERR_NOENT, + LFS_TYPE_DELETE, newid, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0), NULL}, + {LFS_MKTAG(lfs_tag_type3(oldtag), + newid, lfs_path_namelen(newpath)), newpath}, + {LFS_MKTAG(LFS_FROM_MOVE, newid, lfs_tag_id(oldtag)), &oldcwd}, + {LFS_MKTAG_IF(samepair, + LFS_TYPE_DELETE, newoldid, 0), NULL})); + if (err) { + lfs->mlist = prevdir.next; + return err; + } + + // let commit clean up after move (if we're different! otherwise move + // logic already fixed it for us) + if (!samepair && lfs_gstate_hasmove(&lfs->gstate)) { + // prep gstate and delete move id + lfs_fs_prepmove(lfs, 0x3ff, NULL); + err = lfs_dir_commit(lfs, &oldcwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(oldtag), 0), NULL})); + if (err) { + lfs->mlist = prevdir.next; + return err; + } + } + + lfs->mlist = prevdir.next; + if (lfs_gstate_hasorphans(&lfs->gstate)) { + LFS_ASSERT(prevtag != LFS_ERR_NOENT + && lfs_tag_type3(prevtag) == LFS_TYPE_DIR); + + // fix orphan + err = lfs_fs_preporphans(lfs, -1); + if (err) { + return err; + } + + err = lfs_fs_pred(lfs, prevdir.m.pair, &newcwd); + if (err) { + return err; + } + + err = lfs_dir_drop(lfs, &newcwd, &prevdir.m); + if (err) { + return err; + } + } + + return 0; +} +#endif + +static lfs_ssize_t lfs_getattr_(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_mdir_t cwd; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0) { + return tag; + } + + uint16_t id = lfs_tag_id(tag); + if (id == 0x3ff) { + // special case for root + id = 0; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + } + + tag = lfs_dir_get(lfs, &cwd, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_USERATTR + type, + id, lfs_min(size, lfs->attr_max)), + buffer); + if (tag < 0) { + if (tag == LFS_ERR_NOENT) { + return LFS_ERR_NOATTR; + } + + return tag; + } + + return lfs_tag_size(tag); +} + +#ifndef LFS_READONLY +static int lfs_commitattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + lfs_mdir_t cwd; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0) { + return tag; + } + + uint16_t id = lfs_tag_id(tag); + if (id == 0x3ff) { + // special case for root + id = 0; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + } + + return lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_USERATTR + type, id, size), buffer})); +} +#endif + +#ifndef LFS_READONLY +static int lfs_setattr_(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + if (size > lfs->attr_max) { + return LFS_ERR_NOSPC; + } + + return lfs_commitattr(lfs, path, type, buffer, size); +} +#endif + +#ifndef LFS_READONLY +static int lfs_removeattr_(lfs_t *lfs, const char *path, uint8_t type) { + return lfs_commitattr(lfs, path, type, NULL, 0x3ff); +} +#endif + + +/// Filesystem operations /// + +// compile time checks, see lfs.h for why these limits exist +#if LFS_NAME_MAX > 1022 +#error "Invalid LFS_NAME_MAX, must be <= 1022" +#endif + +#if LFS_FILE_MAX > 2147483647 +#error "Invalid LFS_FILE_MAX, must be <= 2147483647" +#endif + +#if LFS_ATTR_MAX > 1022 +#error "Invalid LFS_ATTR_MAX, must be <= 1022" +#endif + +// common filesystem initialization +static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { + lfs->cfg = cfg; + lfs->block_count = cfg->block_count; // May be 0 + int err = 0; + +#ifdef LFS_MULTIVERSION + // this driver only supports minor version < current minor version + LFS_ASSERT(!lfs->cfg->disk_version || ( + (0xffff & (lfs->cfg->disk_version >> 16)) + == LFS_DISK_VERSION_MAJOR + && (0xffff & (lfs->cfg->disk_version >> 0)) + <= LFS_DISK_VERSION_MINOR)); +#endif + + // check that bool is a truthy-preserving type + // + // note the most common reason for this failure is a before-c99 compiler, + // which littlefs currently does not support + LFS_ASSERT((bool)0x80000000); + + // check that the required io functions are provided + LFS_ASSERT(lfs->cfg->read != NULL); +#ifndef LFS_READONLY + LFS_ASSERT(lfs->cfg->prog != NULL); + LFS_ASSERT(lfs->cfg->erase != NULL); + LFS_ASSERT(lfs->cfg->sync != NULL); +#endif + + // validate that the lfs-cfg sizes were initiated properly before + // performing any arithmetic logics with them + LFS_ASSERT(lfs->cfg->read_size != 0); + LFS_ASSERT(lfs->cfg->prog_size != 0); + LFS_ASSERT(lfs->cfg->cache_size != 0); + + // check that block size is a multiple of cache size is a multiple + // of prog and read sizes + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); + LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); + + // check that the block size is large enough to fit all ctz pointers + LFS_ASSERT(lfs->cfg->block_size >= 128); + // this is the exact calculation for all ctz pointers, if this fails + // and the simpler assert above does not, math must be broken + LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) + <= lfs->cfg->block_size); + + // block_cycles = 0 is no longer supported. + // + // block_cycles is the number of erase cycles before littlefs evicts + // metadata logs as a part of wear leveling. Suggested values are in the + // range of 100-1000, or set block_cycles to -1 to disable block-level + // wear-leveling. + LFS_ASSERT(lfs->cfg->block_cycles != 0); + + // check that compact_thresh makes sense + // + // metadata can't be compacted below block_size/2, and metadata can't + // exceed a block_size + LFS_ASSERT(lfs->cfg->compact_thresh == 0 + || lfs->cfg->compact_thresh >= lfs->cfg->block_size/2); + LFS_ASSERT(lfs->cfg->compact_thresh == (lfs_size_t)-1 + || lfs->cfg->compact_thresh <= lfs->cfg->block_size); + + // check that metadata_max is a multiple of read_size and prog_size, + // and a factor of the block_size + LFS_ASSERT(!lfs->cfg->metadata_max + || lfs->cfg->metadata_max % lfs->cfg->read_size == 0); + LFS_ASSERT(!lfs->cfg->metadata_max + || lfs->cfg->metadata_max % lfs->cfg->prog_size == 0); + LFS_ASSERT(!lfs->cfg->metadata_max + || lfs->cfg->block_size % lfs->cfg->metadata_max == 0); + + // setup read cache + if (lfs->cfg->read_buffer) { + lfs->rcache.buffer = lfs->cfg->read_buffer; + } else { + lfs->rcache.buffer = lfs_malloc(lfs->cfg->cache_size); + if (!lfs->rcache.buffer) { + err = LFS_ERR_NOMEM; + goto cleanup; + } + } + + // setup program cache + if (lfs->cfg->prog_buffer) { + lfs->pcache.buffer = lfs->cfg->prog_buffer; + } else { + lfs->pcache.buffer = lfs_malloc(lfs->cfg->cache_size); + if (!lfs->pcache.buffer) { + err = LFS_ERR_NOMEM; + goto cleanup; + } + } + + // zero to avoid information leaks + lfs_cache_zero(lfs, &lfs->rcache); + lfs_cache_zero(lfs, &lfs->pcache); + + // setup lookahead buffer, note mount finishes initializing this after + // we establish a decent pseudo-random seed + LFS_ASSERT(lfs->cfg->lookahead_size > 0); + if (lfs->cfg->lookahead_buffer) { + lfs->lookahead.buffer = lfs->cfg->lookahead_buffer; + } else { + lfs->lookahead.buffer = lfs_malloc(lfs->cfg->lookahead_size); + if (!lfs->lookahead.buffer) { + err = LFS_ERR_NOMEM; + goto cleanup; + } + } + + // check that the size limits are sane + LFS_ASSERT(lfs->cfg->name_max <= LFS_NAME_MAX); + lfs->name_max = lfs->cfg->name_max; + if (!lfs->name_max) { + lfs->name_max = LFS_NAME_MAX; + } + + LFS_ASSERT(lfs->cfg->file_max <= LFS_FILE_MAX); + lfs->file_max = lfs->cfg->file_max; + if (!lfs->file_max) { + lfs->file_max = LFS_FILE_MAX; + } + + LFS_ASSERT(lfs->cfg->attr_max <= LFS_ATTR_MAX); + lfs->attr_max = lfs->cfg->attr_max; + if (!lfs->attr_max) { + lfs->attr_max = LFS_ATTR_MAX; + } + + LFS_ASSERT(lfs->cfg->metadata_max <= lfs->cfg->block_size); + + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= lfs->cfg->cache_size); + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= lfs->attr_max); + LFS_ASSERT(lfs->cfg->inline_max == (lfs_size_t)-1 + || lfs->cfg->inline_max <= ((lfs->cfg->metadata_max) + ? lfs->cfg->metadata_max + : lfs->cfg->block_size)/8); + lfs->inline_max = lfs->cfg->inline_max; + if (lfs->inline_max == (lfs_size_t)-1) { + lfs->inline_max = 0; + } else if (lfs->inline_max == 0) { + lfs->inline_max = lfs_min( + lfs->cfg->cache_size, + lfs_min( + lfs->attr_max, + ((lfs->cfg->metadata_max) + ? lfs->cfg->metadata_max + : lfs->cfg->block_size)/8)); + } + + // setup default state + lfs->root[0] = LFS_BLOCK_NULL; + lfs->root[1] = LFS_BLOCK_NULL; + lfs->mlist = NULL; + lfs->seed = 0; + lfs->gdisk = (lfs_gstate_t){0}; + lfs->gstate = (lfs_gstate_t){0}; + lfs->gdelta = (lfs_gstate_t){0}; +#ifdef LFS_MIGRATE + lfs->lfs1 = NULL; +#endif + + return 0; + +cleanup: + lfs_deinit(lfs); + return err; +} + +static int lfs_deinit(lfs_t *lfs) { + // free allocated memory + if (!lfs->cfg->read_buffer) { + lfs_free(lfs->rcache.buffer); + } + + if (!lfs->cfg->prog_buffer) { + lfs_free(lfs->pcache.buffer); + } + + if (!lfs->cfg->lookahead_buffer) { + lfs_free(lfs->lookahead.buffer); + } + + return 0; +} + + + +#ifndef LFS_READONLY +static int lfs_format_(lfs_t *lfs, const struct lfs_config *cfg) { + int err = 0; + { + err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + LFS_ASSERT(cfg->block_count != 0); + + // create free lookahead + memset(lfs->lookahead.buffer, 0, lfs->cfg->lookahead_size); + lfs->lookahead.start = 0; + lfs->lookahead.size = lfs_min(8*lfs->cfg->lookahead_size, + lfs->block_count); + lfs->lookahead.next = 0; + lfs_alloc_ckpoint(lfs); + + // create root dir + lfs_mdir_t root; + err = lfs_dir_alloc(lfs, &root); + if (err) { + goto cleanup; + } + + // write one superblock + lfs_superblock_t superblock = { + .version = lfs_fs_disk_version(lfs), + .block_size = lfs->cfg->block_size, + .block_count = lfs->block_count, + .name_max = lfs->name_max, + .file_max = lfs->file_max, + .attr_max = lfs->attr_max, + }; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); + if (err) { + goto cleanup; + } + + // force compaction to prevent accidentally mounting any + // older version of littlefs that may live on disk + root.erased = false; + err = lfs_dir_commit(lfs, &root, NULL, 0); + if (err) { + goto cleanup; + } + + // sanity check that fetch works + err = lfs_dir_fetch(lfs, &root, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + } + +cleanup: + lfs_deinit(lfs); + return err; + +} +#endif + +struct lfs_tortoise_t { + lfs_block_t pair[2]; + lfs_size_t i; + lfs_size_t period; +}; + +static int lfs_tortoise_detectcycles( + const lfs_mdir_t *dir, struct lfs_tortoise_t *tortoise) { + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir->tail, tortoise->pair)) { + LFS_WARN("Cycle detected in tail list"); + return LFS_ERR_CORRUPT; + } + if (tortoise->i == tortoise->period) { + tortoise->pair[0] = dir->tail[0]; + tortoise->pair[1] = dir->tail[1]; + tortoise->i = 0; + tortoise->period *= 2; + } + tortoise->i += 1; + + return LFS_ERR_OK; +} + +static int lfs_mount_(lfs_t *lfs, const struct lfs_config *cfg) { + int err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + struct lfs_tortoise_t tortoise = { + .pair = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + while (!lfs_pair_isnull(dir.tail)) { + err = lfs_tortoise_detectcycles(&dir, &tortoise); + if (err < 0) { + goto cleanup; + } + + // fetch next block in tail list + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); + if (tag < 0) { + err = tag; + goto cleanup; + } + + // has superblock? + if (tag && !lfs_tag_isdelete(tag)) { + // update root + lfs->root[0] = dir.pair[0]; + lfs->root[1] = dir.pair[1]; + + // grab superblock + lfs_superblock_t superblock; + tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + err = tag; + goto cleanup; + } + lfs_superblock_fromle32(&superblock); + + // check version + uint16_t major_version = (0xffff & (superblock.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.version >> 0)); + if (major_version != lfs_fs_disk_version_major(lfs) + || minor_version > lfs_fs_disk_version_minor(lfs)) { + LFS_ERROR("Invalid version " + "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + err = LFS_ERR_INVAL; + goto cleanup; + } + + // found older minor version? set an in-device only bit in the + // gstate so we know we need to rewrite the superblock before + // the first write + bool needssuperblock = false; + if (minor_version < lfs_fs_disk_version_minor(lfs)) { + LFS_DEBUG("Found older minor version " + "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + needssuperblock = true; + } + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs_fs_prepsuperblock(lfs, needssuperblock); + + // check superblock configuration + if (superblock.name_max) { + if (superblock.name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", + superblock.name_max, lfs->name_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->name_max = superblock.name_max; + } + + if (superblock.file_max) { + if (superblock.file_max > lfs->file_max) { + LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", + superblock.file_max, lfs->file_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->file_max = superblock.file_max; + } + + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", + superblock.attr_max, lfs->attr_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->attr_max = superblock.attr_max; + + // we also need to update inline_max in case attr_max changed + lfs->inline_max = lfs_min(lfs->inline_max, lfs->attr_max); + } + + // this is where we get the block_count from disk if block_count=0 + if (lfs->cfg->block_count + && superblock.block_count != lfs->cfg->block_count) { + LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", + superblock.block_count, lfs->cfg->block_count); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->block_count = superblock.block_count; + + if (superblock.block_size != lfs->cfg->block_size) { + LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", + superblock.block_size, lfs->cfg->block_size); + err = LFS_ERR_INVAL; + goto cleanup; + } + } + + // has gstate? + err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); + if (err) { + goto cleanup; + } + } + + // update littlefs with gstate + if (!lfs_gstate_iszero(&lfs->gstate)) { + LFS_DEBUG("Found pending gstate 0x%08"PRIx32"%08"PRIx32"%08"PRIx32, + lfs->gstate.tag, + lfs->gstate.pair[0], + lfs->gstate.pair[1]); + } + lfs->gstate.tag += !lfs_tag_isvalid(lfs->gstate.tag); + lfs->gdisk = lfs->gstate; + + // setup free lookahead, to distribute allocations uniformly across + // boots, we start the allocator at a random location + lfs->lookahead.start = lfs->seed % lfs->block_count; + lfs_alloc_drop(lfs); + + return 0; + +cleanup: + lfs_unmount_(lfs); + return err; +} + +static int lfs_unmount_(lfs_t *lfs) { + return lfs_deinit(lfs); +} + + +/// Filesystem filesystem operations /// +static int lfs_fs_stat_(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + // if the superblock is up-to-date, we must be on the most recent + // minor version of littlefs + if (!lfs_gstate_needssuperblock(&lfs->gstate)) { + fsinfo->disk_version = lfs_fs_disk_version(lfs); + + // otherwise we need to read the minor version on disk + } else { + // fetch the superblock + lfs_mdir_t dir; + int err = lfs_dir_fetch(lfs, &dir, lfs->root); + if (err) { + return err; + } + + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + // read the on-disk version + fsinfo->disk_version = superblock.version; + } + + // filesystem geometry + fsinfo->block_size = lfs->cfg->block_size; + fsinfo->block_count = lfs->block_count; + + // other on-disk configuration, we cache all of these for internal use + fsinfo->name_max = lfs->name_max; + fsinfo->file_max = lfs->file_max; + fsinfo->attr_max = lfs->attr_max; + + return 0; +} + +int lfs_fs_traverse_(lfs_t *lfs, + int (*cb)(void *data, lfs_block_t block), void *data, + bool includeorphans) { + // iterate over metadata pairs + lfs_mdir_t dir = {.tail = {0, 1}}; + +#ifdef LFS_MIGRATE + // also consider v1 blocks during migration + if (lfs->lfs1) { + int err = lfs1_traverse(lfs, cb, data); + if (err) { + return err; + } + + dir.tail[0] = lfs->root[0]; + dir.tail[1] = lfs->root[1]; + } +#endif + + struct lfs_tortoise_t tortoise = { + .pair = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS_ERR_OK; + while (!lfs_pair_isnull(dir.tail)) { + err = lfs_tortoise_detectcycles(&dir, &tortoise); + if (err < 0) { + return LFS_ERR_CORRUPT; + } + + for (int i = 0; i < 2; i++) { + int err = cb(data, dir.tail[i]); + if (err) { + return err; + } + } + + // iterate through ids in directory + int err = lfs_dir_fetch(lfs, &dir, dir.tail); + if (err) { + return err; + } + + for (uint16_t id = 0; id < dir.count; id++) { + struct lfs_ctz ctz; + lfs_stag_t tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x700, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + if (tag == LFS_ERR_NOENT) { + continue; + } + return tag; + } + lfs_ctz_fromle32(&ctz); + + if (lfs_tag_type3(tag) == LFS_TYPE_CTZSTRUCT) { + err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache, + ctz.head, ctz.size, cb, data); + if (err) { + return err; + } + } else if (includeorphans && + lfs_tag_type3(tag) == LFS_TYPE_DIRSTRUCT) { + for (int i = 0; i < 2; i++) { + err = cb(data, (&ctz.head)[i]); + if (err) { + return err; + } + } + } + } + } + +#ifndef LFS_READONLY + // iterate over any open files + for (lfs_file_t *f = (lfs_file_t*)lfs->mlist; f; f = f->next) { + if (f->type != LFS_TYPE_REG) { + continue; + } + + if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { + int err = lfs_ctz_traverse(lfs, &f->cache, &lfs->rcache, + f->ctz.head, f->ctz.size, cb, data); + if (err) { + return err; + } + } + + if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { + int err = lfs_ctz_traverse(lfs, &f->cache, &lfs->rcache, + f->block, f->pos, cb, data); + if (err) { + return err; + } + } + } +#endif + + return 0; +} + +#ifndef LFS_READONLY +static int lfs_fs_pred(lfs_t *lfs, + const lfs_block_t pair[2], lfs_mdir_t *pdir) { + // iterate over all directory directory entries + pdir->tail[0] = 0; + pdir->tail[1] = 1; + struct lfs_tortoise_t tortoise = { + .pair = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS_ERR_OK; + while (!lfs_pair_isnull(pdir->tail)) { + err = lfs_tortoise_detectcycles(pdir, &tortoise); + if (err < 0) { + return LFS_ERR_CORRUPT; + } + + if (lfs_pair_cmp(pdir->tail, pair) == 0) { + return 0; + } + + int err = lfs_dir_fetch(lfs, pdir, pdir->tail); + if (err) { + return err; + } + } + + return LFS_ERR_NOENT; +} +#endif + +#ifndef LFS_READONLY +struct lfs_fs_parent_match { + lfs_t *lfs; + const lfs_block_t pair[2]; +}; +#endif + +#ifndef LFS_READONLY +static int lfs_fs_parent_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_fs_parent_match *find = data; + lfs_t *lfs = find->lfs; + const struct lfs_diskoff *disk = buffer; + (void)tag; + + lfs_block_t child[2]; + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + disk->block, disk->off, &child, sizeof(child)); + if (err) { + return err; + } + + lfs_pair_fromle32(child); + return (lfs_pair_cmp(child, find->pair) == 0) ? LFS_CMP_EQ : LFS_CMP_LT; +} +#endif + +#ifndef LFS_READONLY +static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], + lfs_mdir_t *parent) { + // use fetchmatch with callback to find pairs + parent->tail[0] = 0; + parent->tail[1] = 1; + struct lfs_tortoise_t tortoise = { + .pair = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .i = 1, + .period = 1, + }; + int err = LFS_ERR_OK; + while (!lfs_pair_isnull(parent->tail)) { + err = lfs_tortoise_detectcycles(parent, &tortoise); + if (err < 0) { + return err; + } + + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, parent, parent->tail, + LFS_MKTAG(0x7ff, 0, 0x3ff), + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), + NULL, + lfs_fs_parent_match, &(struct lfs_fs_parent_match){ + lfs, {pair[0], pair[1]}}); + if (tag && tag != LFS_ERR_NOENT) { + return tag; + } + } + + return LFS_ERR_NOENT; +} +#endif + +static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock) { + lfs->gstate.tag = (lfs->gstate.tag & ~LFS_MKTAG(0, 0, 0x200)) + | (uint32_t)needssuperblock << 9; +} + +#ifndef LFS_READONLY +static int lfs_fs_preporphans(lfs_t *lfs, int8_t orphans) { + LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) > 0x000 || orphans >= 0); + LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) < 0x1ff || orphans <= 0); + lfs->gstate.tag += orphans; + lfs->gstate.tag = ((lfs->gstate.tag & ~LFS_MKTAG(0x800, 0, 0)) | + ((uint32_t)lfs_gstate_hasorphans(&lfs->gstate) << 31)); + + return 0; +} +#endif + +#ifndef LFS_READONLY +static void lfs_fs_prepmove(lfs_t *lfs, + uint16_t id, const lfs_block_t pair[2]) { + lfs->gstate.tag = ((lfs->gstate.tag & ~LFS_MKTAG(0x7ff, 0x3ff, 0)) | + ((id != 0x3ff) ? LFS_MKTAG(LFS_TYPE_DELETE, id, 0) : 0)); + lfs->gstate.pair[0] = (id != 0x3ff) ? pair[0] : 0; + lfs->gstate.pair[1] = (id != 0x3ff) ? pair[1] : 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_fs_desuperblock(lfs_t *lfs) { + if (!lfs_gstate_needssuperblock(&lfs->gstate)) { + return 0; + } + + LFS_DEBUG("Rewriting superblock {0x%"PRIx32", 0x%"PRIx32"}", + lfs->root[0], + lfs->root[1]); + + lfs_mdir_t root; + int err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // write a new superblock + lfs_superblock_t superblock = { + .version = lfs_fs_disk_version(lfs), + .block_size = lfs->cfg->block_size, + .block_count = lfs->block_count, + .name_max = lfs->name_max, + .file_max = lfs->file_max, + .attr_max = lfs->attr_max, + }; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); + if (err) { + return err; + } + + lfs_fs_prepsuperblock(lfs, false); + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_fs_demove(lfs_t *lfs) { + if (!lfs_gstate_hasmove(&lfs->gdisk)) { + return 0; + } + + // Fix bad moves + LFS_DEBUG("Fixing move {0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16, + lfs->gdisk.pair[0], + lfs->gdisk.pair[1], + lfs_tag_id(lfs->gdisk.tag)); + + // no other gstate is supported at this time, so if we found something else + // something most likely went wrong in gstate calculation + LFS_ASSERT(lfs_tag_type3(lfs->gdisk.tag) == LFS_TYPE_DELETE); + + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs->gdisk.pair); + if (err) { + return err; + } + + // prep gstate and delete move id + uint16_t moveid = lfs_tag_id(lfs->gdisk.tag); + lfs_fs_prepmove(lfs, 0x3ff, NULL); + err = lfs_dir_commit(lfs, &movedir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_DELETE, moveid, 0), NULL})); + if (err) { + return err; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) { + if (!lfs_gstate_hasorphans(&lfs->gstate)) { + return 0; + } + + // Check for orphans in two separate passes: + // - 1 for half-orphans (relocations) + // - 2 for full-orphans (removes/renames) + // + // Two separate passes are needed as half-orphans can contain outdated + // references to full-orphans, effectively hiding them from the deorphan + // search. + // + int pass = 0; + while (pass < 2) { + // Fix any orphans + lfs_mdir_t pdir = {.split = true, .tail = {0, 1}}; + lfs_mdir_t dir; + bool moreorphans = false; + + // iterate over all directory directory entries + while (!lfs_pair_isnull(pdir.tail)) { + int err = lfs_dir_fetch(lfs, &dir, pdir.tail); + if (err) { + return err; + } + + // check head blocks for orphans + if (!pdir.split) { + // check if we have a parent + lfs_mdir_t parent; + lfs_stag_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; + } + + if (pass == 0 && tag != LFS_ERR_NOENT) { + lfs_block_t pair[2]; + lfs_stag_t state = lfs_dir_get(lfs, &parent, + LFS_MKTAG(0x7ff, 0x3ff, 0), tag, pair); + if (state < 0) { + return state; + } + lfs_pair_fromle32(pair); + + if (!lfs_pair_issync(pair, pdir.tail)) { + // we have desynced + LFS_DEBUG("Fixing half-orphan " + "{0x%"PRIx32", 0x%"PRIx32"} " + "-> {0x%"PRIx32", 0x%"PRIx32"}", + pdir.tail[0], pdir.tail[1], pair[0], pair[1]); + + // fix pending move in this pair? this looks like an + // optimization but is in fact _required_ since + // relocating may outdate the move. + uint16_t moveid = 0x3ff; + if (lfs_gstate_hasmovehere(&lfs->gstate, pdir.pair)) { + moveid = lfs_tag_id(lfs->gstate.tag); + LFS_DEBUG("Fixing move while fixing orphans " + "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n", + pdir.pair[0], pdir.pair[1], moveid); + lfs_fs_prepmove(lfs, 0x3ff, NULL); + } + + lfs_pair_tole32(pair); + state = lfs_dir_orphaningcommit(lfs, &pdir, LFS_MKATTRS( + {LFS_MKTAG_IF(moveid != 0x3ff, + LFS_TYPE_DELETE, moveid, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), + pair})); + lfs_pair_fromle32(pair); + if (state < 0) { + return state; + } + + // did our commit create more orphans? + if (state == LFS_OK_ORPHANED) { + moreorphans = true; + } + + // refetch tail + continue; + } + } + + // note we only check for full orphans if we may have had a + // power-loss, otherwise orphans are created intentionally + // during operations such as lfs_mkdir + if (pass == 1 && tag == LFS_ERR_NOENT && powerloss) { + // we are an orphan + LFS_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}", + pdir.tail[0], pdir.tail[1]); + + // steal state + err = lfs_dir_getgstate(lfs, &dir, &lfs->gdelta); + if (err) { + return err; + } + + // steal tail + lfs_pair_tole32(dir.tail); + int state = lfs_dir_orphaningcommit(lfs, &pdir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + dir.split, 0x3ff, 8), + dir.tail})); + lfs_pair_fromle32(dir.tail); + if (state < 0) { + return state; + } + + // did our commit create more orphans? + if (state == LFS_OK_ORPHANED) { + moreorphans = true; + } + + // refetch tail + continue; + } + } + + pdir = dir; + } + + pass = moreorphans ? 0 : pass+1; + } + + // mark orphans as fixed + return lfs_fs_preporphans(lfs, -lfs_gstate_getorphans(&lfs->gstate)); +} +#endif + +#ifndef LFS_READONLY +static int lfs_fs_forceconsistency(lfs_t *lfs) { + int err = lfs_fs_desuperblock(lfs); + if (err) { + return err; + } + + err = lfs_fs_demove(lfs); + if (err) { + return err; + } + + err = lfs_fs_deorphan(lfs, true); + if (err) { + return err; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_fs_mkconsistent_(lfs_t *lfs) { + // lfs_fs_forceconsistency does most of the work here + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + // do we have any pending gstate? + lfs_gstate_t delta = {0}; + lfs_gstate_xor(&delta, &lfs->gdisk); + lfs_gstate_xor(&delta, &lfs->gstate); + if (!lfs_gstate_iszero(&delta)) { + // lfs_dir_commit will implicitly write out any pending gstate + lfs_mdir_t root; + err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + err = lfs_dir_commit(lfs, &root, NULL, 0); + if (err) { + return err; + } + } + + return 0; +} +#endif + +static int lfs_fs_size_count(void *p, lfs_block_t block) { + (void)block; + lfs_size_t *size = p; + *size += 1; + return 0; +} + +static lfs_ssize_t lfs_fs_size_(lfs_t *lfs) { + lfs_size_t size = 0; + int err = lfs_fs_traverse_(lfs, lfs_fs_size_count, &size, false); + if (err) { + return err; + } + + return size; +} + +// explicit garbage collection +#ifndef LFS_READONLY +static int lfs_fs_gc_(lfs_t *lfs) { + // force consistency, even if we're not necessarily going to write, + // because this function is supposed to take care of janitorial work + // isn't it? + int err = lfs_fs_forceconsistency(lfs); + if (err) { + return err; + } + + // try to compact metadata pairs, note we can't really accomplish + // anything if compact_thresh doesn't at least leave a prog_size + // available + if (lfs->cfg->compact_thresh + < lfs->cfg->block_size - lfs->cfg->prog_size) { + // iterate over all mdirs + lfs_mdir_t mdir = {.tail = {0, 1}}; + while (!lfs_pair_isnull(mdir.tail)) { + err = lfs_dir_fetch(lfs, &mdir, mdir.tail); + if (err) { + return err; + } + + // not erased? exceeds our compaction threshold? + if (!mdir.erased || ((lfs->cfg->compact_thresh == 0) + ? mdir.off > lfs->cfg->block_size - lfs->cfg->block_size/8 + : mdir.off > lfs->cfg->compact_thresh)) { + // the easiest way to trigger a compaction is to mark + // the mdir as unerased and add an empty commit + mdir.erased = false; + err = lfs_dir_commit(lfs, &mdir, NULL, 0); + if (err) { + return err; + } + } + } + } + + // try to populate the lookahead buffer, unless it's already full + if (lfs->lookahead.size < lfs_min( + 8 * lfs->cfg->lookahead_size, + lfs->block_count)) { + err = lfs_alloc_scan(lfs); + if (err) { + return err; + } + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +#ifdef LFS_SHRINKNONRELOCATING +static int lfs_shrink_checkblock(void *data, lfs_block_t block) { + lfs_size_t threshold = *((lfs_size_t*)data); + if (block >= threshold) { + return LFS_ERR_NOTEMPTY; + } + return 0; +} +#endif + +static int lfs_fs_grow_(lfs_t *lfs, lfs_size_t block_count) { + int err; + + if (block_count == lfs->block_count) { + return 0; + } + + +#ifndef LFS_SHRINKNONRELOCATING + // shrinking is not supported + LFS_ASSERT(block_count >= lfs->block_count); +#endif +#ifdef LFS_SHRINKNONRELOCATING + if (block_count < lfs->block_count) { + err = lfs_fs_traverse_(lfs, lfs_shrink_checkblock, &block_count, true); + if (err) { + return err; + } + } +#endif + + lfs->block_count = block_count; + + // fetch the root + lfs_mdir_t root; + err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // update the superblock + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + superblock.block_count = lfs->block_count; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {tag, &superblock})); + if (err) { + return err; + } + return 0; +} +#endif + +#ifdef LFS_MIGRATE +////// Migration from littelfs v1 below this ////// + +/// Version info /// + +// Software library version +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS1_VERSION 0x00010007 +#define LFS1_VERSION_MAJOR (0xffff & (LFS1_VERSION >> 16)) +#define LFS1_VERSION_MINOR (0xffff & (LFS1_VERSION >> 0)) + +// Version of On-disk data structures +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS1_DISK_VERSION 0x00010001 +#define LFS1_DISK_VERSION_MAJOR (0xffff & (LFS1_DISK_VERSION >> 16)) +#define LFS1_DISK_VERSION_MINOR (0xffff & (LFS1_DISK_VERSION >> 0)) + + +/// v1 Definitions /// + +// File types +enum lfs1_type { + LFS1_TYPE_REG = 0x11, + LFS1_TYPE_DIR = 0x22, + LFS1_TYPE_SUPERBLOCK = 0x2e, +}; + +typedef struct lfs1 { + lfs_block_t root[2]; +} lfs1_t; + +typedef struct lfs1_entry { + lfs_off_t off; + + struct lfs1_disk_entry { + uint8_t type; + uint8_t elen; + uint8_t alen; + uint8_t nlen; + union { + struct { + lfs_block_t head; + lfs_size_t size; + } file; + lfs_block_t dir[2]; + } u; + } d; +} lfs1_entry_t; + +typedef struct lfs1_dir { + struct lfs1_dir *next; + lfs_block_t pair[2]; + lfs_off_t off; + + lfs_block_t head[2]; + lfs_off_t pos; + + struct lfs1_disk_dir { + uint32_t rev; + lfs_size_t size; + lfs_block_t tail[2]; + } d; +} lfs1_dir_t; + +typedef struct lfs1_superblock { + lfs_off_t off; + + struct lfs1_disk_superblock { + uint8_t type; + uint8_t elen; + uint8_t alen; + uint8_t nlen; + lfs_block_t root[2]; + uint32_t block_size; + uint32_t block_count; + uint32_t version; + char magic[8]; + } d; +} lfs1_superblock_t; + + +/// Low-level wrappers v1->v2 /// +static void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) { + *crc = lfs_crc(*crc, buffer, size); +} + +static int lfs1_bd_read(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + // if we ever do more than writes to alternating pairs, + // this may need to consider pcache + return lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, size, + block, off, buffer, size); +} + +static int lfs1_bd_crc(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, lfs_size_t size, uint32_t *crc) { + for (lfs_off_t i = 0; i < size; i++) { + uint8_t c; + int err = lfs1_bd_read(lfs, block, off+i, &c, 1); + if (err) { + return err; + } + + lfs1_crc(crc, &c, 1); + } + + return 0; +} + + +/// Endian swapping functions /// +static void lfs1_dir_fromle32(struct lfs1_disk_dir *d) { + d->rev = lfs_fromle32(d->rev); + d->size = lfs_fromle32(d->size); + d->tail[0] = lfs_fromle32(d->tail[0]); + d->tail[1] = lfs_fromle32(d->tail[1]); +} + +static void lfs1_dir_tole32(struct lfs1_disk_dir *d) { + d->rev = lfs_tole32(d->rev); + d->size = lfs_tole32(d->size); + d->tail[0] = lfs_tole32(d->tail[0]); + d->tail[1] = lfs_tole32(d->tail[1]); +} + +static void lfs1_entry_fromle32(struct lfs1_disk_entry *d) { + d->u.dir[0] = lfs_fromle32(d->u.dir[0]); + d->u.dir[1] = lfs_fromle32(d->u.dir[1]); +} + +static void lfs1_entry_tole32(struct lfs1_disk_entry *d) { + d->u.dir[0] = lfs_tole32(d->u.dir[0]); + d->u.dir[1] = lfs_tole32(d->u.dir[1]); +} + +static void lfs1_superblock_fromle32(struct lfs1_disk_superblock *d) { + d->root[0] = lfs_fromle32(d->root[0]); + d->root[1] = lfs_fromle32(d->root[1]); + d->block_size = lfs_fromle32(d->block_size); + d->block_count = lfs_fromle32(d->block_count); + d->version = lfs_fromle32(d->version); +} + + +///// Metadata pair and directory operations /// +static inline lfs_size_t lfs1_entry_size(const lfs1_entry_t *entry) { + return 4 + entry->d.elen + entry->d.alen + entry->d.nlen; +} + +static int lfs1_dir_fetch(lfs_t *lfs, + lfs1_dir_t *dir, const lfs_block_t pair[2]) { + // copy out pair, otherwise may be aliasing dir + const lfs_block_t tpair[2] = {pair[0], pair[1]}; + bool valid = false; + + // check both blocks for the most recent revision + for (int i = 0; i < 2; i++) { + struct lfs1_disk_dir test; + int err = lfs1_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); + lfs1_dir_fromle32(&test); + if (err) { + if (err == LFS_ERR_CORRUPT) { + continue; + } + return err; + } + + if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { + continue; + } + + if ((0x7fffffff & test.size) < sizeof(test)+4 || + (0x7fffffff & test.size) > lfs->cfg->block_size) { + continue; + } + + uint32_t crc = 0xffffffff; + lfs1_dir_tole32(&test); + lfs1_crc(&crc, &test, sizeof(test)); + lfs1_dir_fromle32(&test); + err = lfs1_bd_crc(lfs, tpair[i], sizeof(test), + (0x7fffffff & test.size) - sizeof(test), &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + continue; + } + return err; + } + + if (crc != 0) { + continue; + } + + valid = true; + + // setup dir in case it's valid + dir->pair[0] = tpair[(i+0) % 2]; + dir->pair[1] = tpair[(i+1) % 2]; + dir->off = sizeof(dir->d); + dir->d = test; + } + + if (!valid) { + LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}", + tpair[0], tpair[1]); + return LFS_ERR_CORRUPT; + } + + return 0; +} + +static int lfs1_dir_next(lfs_t *lfs, lfs1_dir_t *dir, lfs1_entry_t *entry) { + while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + if (!(0x80000000 & dir->d.size)) { + entry->off = dir->off; + return LFS_ERR_NOENT; + } + + int err = lfs1_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + + dir->off = sizeof(dir->d); + dir->pos += sizeof(dir->d) + 4; + } + + int err = lfs1_bd_read(lfs, dir->pair[0], dir->off, + &entry->d, sizeof(entry->d)); + lfs1_entry_fromle32(&entry->d); + if (err) { + return err; + } + + entry->off = dir->off; + dir->off += lfs1_entry_size(entry); + dir->pos += lfs1_entry_size(entry); + return 0; +} + +/// littlefs v1 specific operations /// +int lfs1_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { + if (lfs_pair_isnull(lfs->lfs1->root)) { + return 0; + } + + // iterate over metadata pairs + lfs1_dir_t dir; + lfs1_entry_t entry; + lfs_block_t cwd[2] = {0, 1}; + + while (true) { + for (int i = 0; i < 2; i++) { + int err = cb(data, cwd[i]); + if (err) { + return err; + } + } + + int err = lfs1_dir_fetch(lfs, &dir, cwd); + if (err) { + return err; + } + + // iterate over contents + while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { + err = lfs1_bd_read(lfs, dir.pair[0], dir.off, + &entry.d, sizeof(entry.d)); + lfs1_entry_fromle32(&entry.d); + if (err) { + return err; + } + + dir.off += lfs1_entry_size(&entry); + if ((0x70 & entry.d.type) == (0x70 & LFS1_TYPE_REG)) { + err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache, + entry.d.u.file.head, entry.d.u.file.size, cb, data); + if (err) { + return err; + } + } + } + + // we also need to check if we contain a threaded v2 directory + lfs_mdir_t dir2 = {.split=true, .tail={cwd[0], cwd[1]}}; + while (dir2.split) { + err = lfs_dir_fetch(lfs, &dir2, dir2.tail); + if (err) { + break; + } + + for (int i = 0; i < 2; i++) { + err = cb(data, dir2.pair[i]); + if (err) { + return err; + } + } + } + + cwd[0] = dir.d.tail[0]; + cwd[1] = dir.d.tail[1]; + + if (lfs_pair_isnull(cwd)) { + break; + } + } + + return 0; +} + +static int lfs1_moved(lfs_t *lfs, const void *e) { + if (lfs_pair_isnull(lfs->lfs1->root)) { + return 0; + } + + // skip superblock + lfs1_dir_t cwd; + int err = lfs1_dir_fetch(lfs, &cwd, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + // iterate over all directory directory entries + lfs1_entry_t entry; + while (!lfs_pair_isnull(cwd.d.tail)) { + err = lfs1_dir_fetch(lfs, &cwd, cwd.d.tail); + if (err) { + return err; + } + + while (true) { + err = lfs1_dir_next(lfs, &cwd, &entry); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err == LFS_ERR_NOENT) { + break; + } + + if (!(0x80 & entry.d.type) && + memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { + return true; + } + } + } + + return false; +} + +/// Filesystem operations /// +static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1, + const struct lfs_config *cfg) { + int err = 0; + { + err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + lfs->lfs1 = lfs1; + lfs->lfs1->root[0] = LFS_BLOCK_NULL; + lfs->lfs1->root[1] = LFS_BLOCK_NULL; + + // setup free lookahead + lfs->lookahead.start = 0; + lfs->lookahead.size = 0; + lfs->lookahead.next = 0; + lfs_alloc_ckpoint(lfs); + + // load superblock + lfs1_dir_t dir; + lfs1_superblock_t superblock; + err = lfs1_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err && err != LFS_ERR_CORRUPT) { + goto cleanup; + } + + if (!err) { + err = lfs1_bd_read(lfs, dir.pair[0], sizeof(dir.d), + &superblock.d, sizeof(superblock.d)); + lfs1_superblock_fromle32(&superblock.d); + if (err) { + goto cleanup; + } + + lfs->lfs1->root[0] = superblock.d.root[0]; + lfs->lfs1->root[1] = superblock.d.root[1]; + } + + if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at {0x%"PRIx32", 0x%"PRIx32"}", + 0, 1); + err = LFS_ERR_CORRUPT; + goto cleanup; + } + + uint16_t major_version = (0xffff & (superblock.d.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.d.version >> 0)); + if ((major_version != LFS1_DISK_VERSION_MAJOR || + minor_version > LFS1_DISK_VERSION_MINOR)) { + LFS_ERROR("Invalid version v%d.%d", major_version, minor_version); + err = LFS_ERR_INVAL; + goto cleanup; + } + + return 0; + } + +cleanup: + lfs_deinit(lfs); + return err; +} + +static int lfs1_unmount(lfs_t *lfs) { + return lfs_deinit(lfs); +} + +/// v1 migration /// +static int lfs_migrate_(lfs_t *lfs, const struct lfs_config *cfg) { + struct lfs1 lfs1; + + // Indeterminate filesystem size not allowed for migration. + LFS_ASSERT(cfg->block_count != 0); + + int err = lfs1_mount(lfs, &lfs1, cfg); + if (err) { + return err; + } + + { + // iterate through each directory, copying over entries + // into new directory + lfs1_dir_t dir1; + lfs_mdir_t dir2; + dir1.d.tail[0] = lfs->lfs1->root[0]; + dir1.d.tail[1] = lfs->lfs1->root[1]; + while (!lfs_pair_isnull(dir1.d.tail)) { + // iterate old dir + err = lfs1_dir_fetch(lfs, &dir1, dir1.d.tail); + if (err) { + goto cleanup; + } + + // create new dir and bind as temporary pretend root + err = lfs_dir_alloc(lfs, &dir2); + if (err) { + goto cleanup; + } + + dir2.rev = dir1.d.rev; + dir1.head[0] = dir1.pair[0]; + dir1.head[1] = dir1.pair[1]; + lfs->root[0] = dir2.pair[0]; + lfs->root[1] = dir2.pair[1]; + + err = lfs_dir_commit(lfs, &dir2, NULL, 0); + if (err) { + goto cleanup; + } + + while (true) { + lfs1_entry_t entry1; + err = lfs1_dir_next(lfs, &dir1, &entry1); + if (err && err != LFS_ERR_NOENT) { + goto cleanup; + } + + if (err == LFS_ERR_NOENT) { + break; + } + + // check that entry has not been moved + if (entry1.d.type & 0x80) { + int moved = lfs1_moved(lfs, &entry1.d.u); + if (moved < 0) { + err = moved; + goto cleanup; + } + + if (moved) { + continue; + } + + entry1.d.type &= ~0x80; + } + + // also fetch name + char name[LFS_NAME_MAX+1]; + memset(name, 0, sizeof(name)); + err = lfs1_bd_read(lfs, dir1.pair[0], + entry1.off + 4+entry1.d.elen+entry1.d.alen, + name, entry1.d.nlen); + if (err) { + goto cleanup; + } + + bool isdir = (entry1.d.type == LFS1_TYPE_DIR); + + // create entry in new dir + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + uint16_t id; + err = lfs_dir_find(lfs, &dir2, &(const char*){name}, &id); + if (!(err == LFS_ERR_NOENT && id != 0x3ff)) { + err = (err < 0) ? err : LFS_ERR_EXIST; + goto cleanup; + } + + lfs1_entry_tole32(&entry1.d); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL}, + {LFS_MKTAG_IF_ELSE(isdir, + LFS_TYPE_DIR, id, entry1.d.nlen, + LFS_TYPE_REG, id, entry1.d.nlen), + name}, + {LFS_MKTAG_IF_ELSE(isdir, + LFS_TYPE_DIRSTRUCT, id, sizeof(entry1.d.u), + LFS_TYPE_CTZSTRUCT, id, sizeof(entry1.d.u)), + &entry1.d.u})); + lfs1_entry_fromle32(&entry1.d); + if (err) { + goto cleanup; + } + } + + if (!lfs_pair_isnull(dir1.d.tail)) { + // find last block and update tail to thread into fs + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + while (dir2.split) { + err = lfs_dir_fetch(lfs, &dir2, dir2.tail); + if (err) { + goto cleanup; + } + } + + lfs_pair_tole32(dir2.pair); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), dir1.d.tail})); + lfs_pair_fromle32(dir2.pair); + if (err) { + goto cleanup; + } + } + + // Copy over first block to thread into fs. Unfortunately + // if this fails there is not much we can do. + LFS_DEBUG("Migrating {0x%"PRIx32", 0x%"PRIx32"} " + "-> {0x%"PRIx32", 0x%"PRIx32"}", + lfs->root[0], lfs->root[1], dir1.head[0], dir1.head[1]); + + err = lfs_bd_erase(lfs, dir1.head[1]); + if (err) { + goto cleanup; + } + + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + for (lfs_off_t i = 0; i < dir2.off; i++) { + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, dir2.off, + dir2.pair[0], i, &dat, 1); + if (err) { + goto cleanup; + } + + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, true, + dir1.head[1], i, &dat, 1); + if (err) { + goto cleanup; + } + } + + err = lfs_bd_flush(lfs, &lfs->pcache, &lfs->rcache, true); + if (err) { + goto cleanup; + } + } + + // Create new superblock. This marks a successful migration! + err = lfs1_dir_fetch(lfs, &dir1, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + + dir2.pair[0] = dir1.pair[0]; + dir2.pair[1] = dir1.pair[1]; + dir2.rev = dir1.d.rev; + dir2.off = sizeof(dir2.rev); + dir2.etag = 0xffffffff; + dir2.count = 0; + dir2.tail[0] = lfs->lfs1->root[0]; + dir2.tail[1] = lfs->lfs1->root[1]; + dir2.erased = false; + dir2.split = true; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = lfs->cfg->block_size, + .block_count = lfs->cfg->block_count, + .name_max = lfs->name_max, + .file_max = lfs->file_max, + .attr_max = lfs->attr_max, + }; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); + if (err) { + goto cleanup; + } + + // sanity check that fetch works + err = lfs_dir_fetch(lfs, &dir2, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + + // force compaction to prevent accidentally mounting v1 + dir2.erased = false; + err = lfs_dir_commit(lfs, &dir2, NULL, 0); + if (err) { + goto cleanup; + } + } + +cleanup: + lfs1_unmount(lfs); + return err; +} + +#endif + + +/// Public API wrappers /// + +// Here we can add tracing/thread safety easily + +// Thread-safe wrappers if enabled +#ifdef LFS_THREADSAFE +#define LFS_LOCK(cfg) cfg->lock(cfg) +#define LFS_UNLOCK(cfg) cfg->unlock(cfg) +#else +#define LFS_LOCK(cfg) ((void)cfg, 0) +#define LFS_UNLOCK(cfg) ((void)cfg) +#endif + +// Public API +#ifndef LFS_READONLY +int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { + int err = LFS_LOCK(cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_format(%p, %p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p, " + ".read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".block_size=%"PRIu32", .block_count=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " + ".lookahead_size=%"PRIu32", .read_buffer=%p, " + ".prog_buffer=%p, .lookahead_buffer=%p, " + ".name_max=%"PRIu32", .file_max=%"PRIu32", " + ".attr_max=%"PRIu32"})", + (void*)lfs, (void*)cfg, cfg->context, + (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, + (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, + cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, + cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, + cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, + cfg->name_max, cfg->file_max, cfg->attr_max); + + err = lfs_format_(lfs, cfg); + + LFS_TRACE("lfs_format -> %d", err); + LFS_UNLOCK(cfg); + return err; +} +#endif + +int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { + int err = LFS_LOCK(cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_mount(%p, %p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p, " + ".read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".block_size=%"PRIu32", .block_count=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " + ".lookahead_size=%"PRIu32", .read_buffer=%p, " + ".prog_buffer=%p, .lookahead_buffer=%p, " + ".name_max=%"PRIu32", .file_max=%"PRIu32", " + ".attr_max=%"PRIu32"})", + (void*)lfs, (void*)cfg, cfg->context, + (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, + (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, + cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, + cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, + cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, + cfg->name_max, cfg->file_max, cfg->attr_max); + + err = lfs_mount_(lfs, cfg); + + LFS_TRACE("lfs_mount -> %d", err); + LFS_UNLOCK(cfg); + return err; +} + +int lfs_unmount(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_unmount(%p)", (void*)lfs); + + err = lfs_unmount_(lfs); + + LFS_TRACE("lfs_unmount -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +#ifndef LFS_READONLY +int lfs_remove(lfs_t *lfs, const char *path) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_remove(%p, \"%s\")", (void*)lfs, path); + + err = lfs_remove_(lfs, path); + + LFS_TRACE("lfs_remove -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_rename(%p, \"%s\", \"%s\")", (void*)lfs, oldpath, newpath); + + err = lfs_rename_(lfs, oldpath, newpath); + + LFS_TRACE("lfs_rename -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_stat(%p, \"%s\", %p)", (void*)lfs, path, (void*)info); + + err = lfs_stat_(lfs, path, info); + + LFS_TRACE("lfs_stat -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_getattr(%p, \"%s\", %"PRIu8", %p, %"PRIu32")", + (void*)lfs, path, type, buffer, size); + + lfs_ssize_t res = lfs_getattr_(lfs, path, type, buffer, size); + + LFS_TRACE("lfs_getattr -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +#ifndef LFS_READONLY +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_setattr(%p, \"%s\", %"PRIu8", %p, %"PRIu32")", + (void*)lfs, path, type, buffer, size); + + err = lfs_setattr_(lfs, path, type, buffer, size); + + LFS_TRACE("lfs_setattr -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_removeattr(%p, \"%s\", %"PRIu8")", (void*)lfs, path, type); + + err = lfs_removeattr_(lfs, path, type); + + LFS_TRACE("lfs_removeattr -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_NO_MALLOC +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_open(%p, %p, \"%s\", %x)", + (void*)lfs, (void*)file, path, (unsigned)flags); + LFS_ASSERT(!lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + err = lfs_file_open_(lfs, file, path, flags); + + LFS_TRACE("lfs_file_open -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags, + const struct lfs_file_config *cfg) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_opencfg(%p, %p, \"%s\", %x, %p {" + ".buffer=%p, .attrs=%p, .attr_count=%"PRIu32"})", + (void*)lfs, (void*)file, path, (unsigned)flags, + (void*)cfg, cfg->buffer, (void*)cfg->attrs, cfg->attr_count); + LFS_ASSERT(!lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + err = lfs_file_opencfg_(lfs, file, path, flags, cfg); + + LFS_TRACE("lfs_file_opencfg -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_close(%p, %p)", (void*)lfs, (void*)file); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + err = lfs_file_close_(lfs, file); + + LFS_TRACE("lfs_file_close -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +#ifndef LFS_READONLY +int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_sync(%p, %p)", (void*)lfs, (void*)file); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + err = lfs_file_sync_(lfs, file); + + LFS_TRACE("lfs_file_sync -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_read(%p, %p, %p, %"PRIu32")", + (void*)lfs, (void*)file, buffer, size); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + lfs_ssize_t res = lfs_file_read_(lfs, file, buffer, size); + + LFS_TRACE("lfs_file_read -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +#ifndef LFS_READONLY +lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_write(%p, %p, %p, %"PRIu32")", + (void*)lfs, (void*)file, buffer, size); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + lfs_ssize_t res = lfs_file_write_(lfs, file, buffer, size); + + LFS_TRACE("lfs_file_write -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} +#endif + +lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, + lfs_soff_t off, int whence) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_seek(%p, %p, %"PRId32", %d)", + (void*)lfs, (void*)file, off, whence); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + lfs_soff_t res = lfs_file_seek_(lfs, file, off, whence); + + LFS_TRACE("lfs_file_seek -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +#ifndef LFS_READONLY +int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_truncate(%p, %p, %"PRIu32")", + (void*)lfs, (void*)file, size); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + err = lfs_file_truncate_(lfs, file, size); + + LFS_TRACE("lfs_file_truncate -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_tell(%p, %p)", (void*)lfs, (void*)file); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + lfs_soff_t res = lfs_file_tell_(lfs, file); + + LFS_TRACE("lfs_file_tell -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_rewind(%p, %p)", (void*)lfs, (void*)file); + + err = lfs_file_rewind_(lfs, file); + + LFS_TRACE("lfs_file_rewind -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_file_size(%p, %p)", (void*)lfs, (void*)file); + LFS_ASSERT(lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)file)); + + lfs_soff_t res = lfs_file_size_(lfs, file); + + LFS_TRACE("lfs_file_size -> %"PRIu32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +#ifndef LFS_READONLY +int lfs_mkdir(lfs_t *lfs, const char *path) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_mkdir(%p, \"%s\")", (void*)lfs, path); + + err = lfs_mkdir_(lfs, path); + + LFS_TRACE("lfs_mkdir -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_open(%p, %p, \"%s\")", (void*)lfs, (void*)dir, path); + LFS_ASSERT(!lfs_mlist_isopen(lfs->mlist, (struct lfs_mlist*)dir)); + + err = lfs_dir_open_(lfs, dir, path); + + LFS_TRACE("lfs_dir_open -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_close(%p, %p)", (void*)lfs, (void*)dir); + + err = lfs_dir_close_(lfs, dir); + + LFS_TRACE("lfs_dir_close -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_read(%p, %p, %p)", + (void*)lfs, (void*)dir, (void*)info); + + err = lfs_dir_read_(lfs, dir, info); + + LFS_TRACE("lfs_dir_read -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_seek(%p, %p, %"PRIu32")", + (void*)lfs, (void*)dir, off); + + err = lfs_dir_seek_(lfs, dir, off); + + LFS_TRACE("lfs_dir_seek -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_tell(%p, %p)", (void*)lfs, (void*)dir); + + lfs_soff_t res = lfs_dir_tell_(lfs, dir); + + LFS_TRACE("lfs_dir_tell -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_dir_rewind(%p, %p)", (void*)lfs, (void*)dir); + + err = lfs_dir_rewind_(lfs, dir); + + LFS_TRACE("lfs_dir_rewind -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_stat(%p, %p)", (void*)lfs, (void*)fsinfo); + + err = lfs_fs_stat_(lfs, fsinfo); + + LFS_TRACE("lfs_fs_stat -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +lfs_ssize_t lfs_fs_size(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_size(%p)", (void*)lfs); + + lfs_ssize_t res = lfs_fs_size_(lfs); + + LFS_TRACE("lfs_fs_size -> %"PRId32, res); + LFS_UNLOCK(lfs->cfg); + return res; +} + +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *, lfs_block_t), void *data) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_traverse(%p, %p, %p)", + (void*)lfs, (void*)(uintptr_t)cb, data); + + err = lfs_fs_traverse_(lfs, cb, data, true); + + LFS_TRACE("lfs_fs_traverse -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + +#ifndef LFS_READONLY +int lfs_fs_mkconsistent(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_mkconsistent(%p)", (void*)lfs); + + err = lfs_fs_mkconsistent_(lfs); + + LFS_TRACE("lfs_fs_mkconsistent -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_fs_gc(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_gc(%p)", (void*)lfs); + + err = lfs_fs_gc_(lfs); + + LFS_TRACE("lfs_fs_gc -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifndef LFS_READONLY +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_grow(%p, %"PRIu32")", (void*)lfs, block_count); + + err = lfs_fs_grow_(lfs, block_count); + + LFS_TRACE("lfs_fs_grow -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + +#ifdef LFS_MIGRATE +int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { + int err = LFS_LOCK(cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_migrate(%p, %p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p, " + ".read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".block_size=%"PRIu32", .block_count=%"PRIu32", " + ".block_cycles=%"PRId32", .cache_size=%"PRIu32", " + ".lookahead_size=%"PRIu32", .read_buffer=%p, " + ".prog_buffer=%p, .lookahead_buffer=%p, " + ".name_max=%"PRIu32", .file_max=%"PRIu32", " + ".attr_max=%"PRIu32"})", + (void*)lfs, (void*)cfg, cfg->context, + (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, + (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, + cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, + cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, + cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, + cfg->name_max, cfg->file_max, cfg->attr_max); + + err = lfs_migrate_(lfs, cfg); + + LFS_TRACE("lfs_migrate -> %d", err); + LFS_UNLOCK(cfg); + return err; +} +#endif diff --git a/quakey/src/lfs.h b/quakey/src/lfs.h new file mode 100644 index 0000000..d1a18a4 --- /dev/null +++ b/quakey/src/lfs.h @@ -0,0 +1,801 @@ +/* + * The little filesystem + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef LFS_H +#define LFS_H + +#include "lfs_util.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + +/// Version info /// + +// Software library version +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS_VERSION 0x0002000b +#define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) +#define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) + +// Version of On-disk data structures +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS_DISK_VERSION 0x00020001 +#define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16)) +#define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0)) + + +/// Definitions /// + +// Type definitions +typedef uint32_t lfs_size_t; +typedef uint32_t lfs_off_t; + +typedef int32_t lfs_ssize_t; +typedef int32_t lfs_soff_t; + +typedef uint32_t lfs_block_t; + +// Maximum name size in bytes, may be redefined to reduce the size of the +// info struct. Limited to <= 1022. Stored in superblock and must be +// respected by other littlefs drivers. +#ifndef LFS_NAME_MAX +#define LFS_NAME_MAX 255 +#endif + +// Maximum size of a file in bytes, may be redefined to limit to support other +// drivers. Limited on disk to <= 2147483647. Stored in superblock and must be +// respected by other littlefs drivers. +#ifndef LFS_FILE_MAX +#define LFS_FILE_MAX 2147483647 +#endif + +// Maximum size of custom attributes in bytes, may be redefined, but there is +// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored +// in superblock and must be respected by other littlefs drivers. +#ifndef LFS_ATTR_MAX +#define LFS_ATTR_MAX 1022 +#endif + +// Possible error codes, these are negative to allow +// valid positive return values +enum lfs_error { + LFS_ERR_OK = 0, // No error + LFS_ERR_IO = -5, // Error during device operation + LFS_ERR_CORRUPT = -84, // Corrupted + LFS_ERR_NOENT = -2, // No directory entry + LFS_ERR_EXIST = -17, // Entry already exists + LFS_ERR_NOTDIR = -20, // Entry is not a dir + LFS_ERR_ISDIR = -21, // Entry is a dir + LFS_ERR_NOTEMPTY = -39, // Dir is not empty + LFS_ERR_BADF = -9, // Bad file number + LFS_ERR_FBIG = -27, // File too large + LFS_ERR_INVAL = -22, // Invalid parameter + LFS_ERR_NOSPC = -28, // No space left on device + LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_NOATTR = -61, // No data/attr available + LFS_ERR_NAMETOOLONG = -36, // File name too long +}; + +// File types +enum lfs_type { + // file types + LFS_TYPE_REG = 0x001, + LFS_TYPE_DIR = 0x002, + + // internally used types + LFS_TYPE_SPLICE = 0x400, + LFS_TYPE_NAME = 0x000, + LFS_TYPE_STRUCT = 0x200, + LFS_TYPE_USERATTR = 0x300, + LFS_TYPE_FROM = 0x100, + LFS_TYPE_TAIL = 0x600, + LFS_TYPE_GLOBALS = 0x700, + LFS_TYPE_CRC = 0x500, + + // internally used type specializations + LFS_TYPE_CREATE = 0x401, + LFS_TYPE_DELETE = 0x4ff, + LFS_TYPE_SUPERBLOCK = 0x0ff, + LFS_TYPE_DIRSTRUCT = 0x200, + LFS_TYPE_CTZSTRUCT = 0x202, + LFS_TYPE_INLINESTRUCT = 0x201, + LFS_TYPE_SOFTTAIL = 0x600, + LFS_TYPE_HARDTAIL = 0x601, + LFS_TYPE_MOVESTATE = 0x7ff, + LFS_TYPE_CCRC = 0x500, + LFS_TYPE_FCRC = 0x5ff, + + // internal chip sources + LFS_FROM_NOOP = 0x000, + LFS_FROM_MOVE = 0x101, + LFS_FROM_USERATTRS = 0x102, +}; + +// File open flags +enum lfs_open_flags { + // open flags + LFS_O_RDONLY = 1, // Open a file as read only +#ifndef LFS_READONLY + LFS_O_WRONLY = 2, // Open a file as write only + LFS_O_RDWR = 3, // Open a file as read and write + LFS_O_CREAT = 0x0100, // Create a file if it does not exist + LFS_O_EXCL = 0x0200, // Fail if a file already exists + LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size + LFS_O_APPEND = 0x0800, // Move to end of file on every write +#endif + + // internally used flags +#ifndef LFS_READONLY + LFS_F_DIRTY = 0x010000, // File does not match storage + LFS_F_WRITING = 0x020000, // File has been written since last flush +#endif + LFS_F_READING = 0x040000, // File has been read since last flush +#ifndef LFS_READONLY + LFS_F_ERRED = 0x080000, // An error occurred during write +#endif + LFS_F_INLINE = 0x100000, // Currently inlined in directory entry +}; + +// File seek flags +enum lfs_whence_flags { + LFS_SEEK_SET = 0, // Seek relative to an absolute position + LFS_SEEK_CUR = 1, // Seek relative to the current file position + LFS_SEEK_END = 2, // Seek relative to the end of the file +}; + + +// Configuration provided during initialization of the littlefs +struct lfs_config { + // Opaque user provided context that can be used to pass + // information to the block device operations + void *context; + + // Read a region in a block. Negative error codes are propagated + // to the user. + int (*read)(const struct lfs_config *c, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size); + + // Program a region in a block. The block must have previously + // been erased. Negative error codes are propagated to the user. + // May return LFS_ERR_CORRUPT if the block should be considered bad. + int (*prog)(const struct lfs_config *c, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size); + + // Erase a block. A block must be erased before being programmed. + // The state of an erased block is undefined. Negative error codes + // are propagated to the user. + // May return LFS_ERR_CORRUPT if the block should be considered bad. + int (*erase)(const struct lfs_config *c, lfs_block_t block); + + // Sync the state of the underlying block device. Negative error codes + // are propagated to the user. + int (*sync)(const struct lfs_config *c); + +#ifdef LFS_THREADSAFE + // Lock the underlying block device. Negative error codes + // are propagated to the user. + int (*lock)(const struct lfs_config *c); + + // Unlock the underlying block device. Negative error codes + // are propagated to the user. + int (*unlock)(const struct lfs_config *c); +#endif + + // Minimum size of a block read in bytes. All read operations will be a + // multiple of this value. + lfs_size_t read_size; + + // Minimum size of a block program in bytes. All program operations will be + // a multiple of this value. + lfs_size_t prog_size; + + // Size of an erasable block in bytes. This does not impact ram consumption + // and may be larger than the physical erase size. However, non-inlined + // files take up at minimum one block. Must be a multiple of the read and + // program sizes. + lfs_size_t block_size; + + // Number of erasable blocks on the device. Defaults to block_count stored + // on disk when zero. + lfs_size_t block_count; + + // Number of erase cycles before littlefs evicts metadata logs and moves + // the metadata to another block. Suggested values are in the + // range 100-1000, with large values having better performance at the cost + // of less consistent wear distribution. + // + // Set to -1 to disable block-level wear-leveling. + int32_t block_cycles; + + // Size of block caches in bytes. Each cache buffers a portion of a block in + // RAM. The littlefs needs a read cache, a program cache, and one additional + // cache per file. Larger caches can improve performance by storing more + // data and reducing the number of disk accesses. Must be a multiple of the + // read and program sizes, and a factor of the block size. + lfs_size_t cache_size; + + // Size of the lookahead buffer in bytes. A larger lookahead buffer + // increases the number of blocks found during an allocation pass. The + // lookahead buffer is stored as a compact bitmap, so each byte of RAM + // can track 8 blocks. + lfs_size_t lookahead_size; + + // Threshold for metadata compaction during lfs_fs_gc in bytes. Metadata + // pairs that exceed this threshold will be compacted during lfs_fs_gc. + // Defaults to ~88% block_size when zero, though the default may change + // in the future. + // + // Note this only affects lfs_fs_gc. Normal compactions still only occur + // when full. + // + // Set to -1 to disable metadata compaction during lfs_fs_gc. + lfs_size_t compact_thresh; + + // Optional statically allocated read buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. + void *read_buffer; + + // Optional statically allocated program buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. + void *prog_buffer; + + // Optional statically allocated lookahead buffer. Must be lookahead_size. + // By default lfs_malloc is used to allocate this buffer. + void *lookahead_buffer; + + // Optional upper limit on length of file names in bytes. No downside for + // larger names except the size of the info struct which is controlled by + // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX or name_max stored on + // disk when zero. + lfs_size_t name_max; + + // Optional upper limit on files in bytes. No downside for larger files + // but must be <= LFS_FILE_MAX. Defaults to LFS_FILE_MAX or file_max stored + // on disk when zero. + lfs_size_t file_max; + + // Optional upper limit on custom attributes in bytes. No downside for + // larger attributes size but must be <= LFS_ATTR_MAX. Defaults to + // LFS_ATTR_MAX or attr_max stored on disk when zero. + lfs_size_t attr_max; + + // Optional upper limit on total space given to metadata pairs in bytes. On + // devices with large blocks (e.g. 128kB) setting this to a low size (2-8kB) + // can help bound the metadata compaction time. Must be <= block_size. + // Defaults to block_size when zero. + lfs_size_t metadata_max; + + // Optional upper limit on inlined files in bytes. Inlined files live in + // metadata and decrease storage requirements, but may be limited to + // improve metadata-related performance. Must be <= cache_size, <= + // attr_max, and <= block_size/8. Defaults to the largest possible + // inline_max when zero. + // + // Set to -1 to disable inlined files. + lfs_size_t inline_max; + +#ifdef LFS_MULTIVERSION + // On-disk version to use when writing in the form of 16-bit major version + // + 16-bit minor version. This limiting metadata to what is supported by + // older minor versions. Note that some features will be lost. Defaults to + // to the most recent minor version when zero. + uint32_t disk_version; +#endif +}; + +// File info structure +struct lfs_info { + // Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR + uint8_t type; + + // Size of the file, only valid for REG files. Limited to 32-bits. + lfs_size_t size; + + // Name of the file stored as a null-terminated string. Limited to + // LFS_NAME_MAX+1, which can be changed by redefining LFS_NAME_MAX to + // reduce RAM. LFS_NAME_MAX is stored in superblock and must be + // respected by other littlefs drivers. + char name[LFS_NAME_MAX+1]; +}; + +// Filesystem info structure +struct lfs_fsinfo { + // On-disk version. + uint32_t disk_version; + + // Size of a logical block in bytes. + lfs_size_t block_size; + + // Number of logical blocks in filesystem. + lfs_size_t block_count; + + // Upper limit on the length of file names in bytes. + lfs_size_t name_max; + + // Upper limit on the size of files in bytes. + lfs_size_t file_max; + + // Upper limit on the size of custom attributes in bytes. + lfs_size_t attr_max; +}; + +// Custom attribute structure, used to describe custom attributes +// committed atomically during file writes. +struct lfs_attr { + // 8-bit type of attribute, provided by user and used to + // identify the attribute + uint8_t type; + + // Pointer to buffer containing the attribute + void *buffer; + + // Size of attribute in bytes, limited to LFS_ATTR_MAX + lfs_size_t size; +}; + +// Optional configuration provided during lfs_file_opencfg +struct lfs_file_config { + // Optional statically allocated file buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. + void *buffer; + + // Optional list of custom attributes related to the file. If the file + // is opened with read access, these attributes will be read from disk + // during the open call. If the file is opened with write access, the + // attributes will be written to disk every file sync or close. This + // write occurs atomically with update to the file's contents. + // + // Custom attributes are uniquely identified by an 8-bit type and limited + // to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller + // than the buffer, it will be padded with zeros. If the stored attribute + // is larger, then it will be silently truncated. If the attribute is not + // found, it will be created implicitly. + struct lfs_attr *attrs; + + // Number of custom attributes in the list + lfs_size_t attr_count; +}; + + +/// internal littlefs data structures /// +typedef struct lfs_cache { + lfs_block_t block; + lfs_off_t off; + lfs_size_t size; + uint8_t *buffer; +} lfs_cache_t; + +typedef struct lfs_mdir { + lfs_block_t pair[2]; + uint32_t rev; + lfs_off_t off; + uint32_t etag; + uint16_t count; + bool erased; + bool split; + lfs_block_t tail[2]; +} lfs_mdir_t; + +// littlefs directory type +typedef struct lfs_dir { + struct lfs_dir *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; + + lfs_off_t pos; + lfs_block_t head[2]; +} lfs_dir_t; + +// littlefs file type +typedef struct lfs_file { + struct lfs_file *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; + + struct lfs_ctz { + lfs_block_t head; + lfs_size_t size; + } ctz; + + uint32_t flags; + lfs_off_t pos; + lfs_block_t block; + lfs_off_t off; + lfs_cache_t cache; + + const struct lfs_file_config *cfg; +} lfs_file_t; + +typedef struct lfs_superblock { + uint32_t version; + lfs_size_t block_size; + lfs_size_t block_count; + lfs_size_t name_max; + lfs_size_t file_max; + lfs_size_t attr_max; +} lfs_superblock_t; + +typedef struct lfs_gstate { + uint32_t tag; + lfs_block_t pair[2]; +} lfs_gstate_t; + +// The littlefs filesystem type +typedef struct lfs { + lfs_cache_t rcache; + lfs_cache_t pcache; + + lfs_block_t root[2]; + struct lfs_mlist { + struct lfs_mlist *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; + } *mlist; + uint32_t seed; + + lfs_gstate_t gstate; + lfs_gstate_t gdisk; + lfs_gstate_t gdelta; + + struct lfs_lookahead { + lfs_block_t start; + lfs_block_t size; + lfs_block_t next; + lfs_block_t ckpoint; + uint8_t *buffer; + } lookahead; + + const struct lfs_config *cfg; + lfs_size_t block_count; + lfs_size_t name_max; + lfs_size_t file_max; + lfs_size_t attr_max; + lfs_size_t inline_max; + +#ifdef LFS_MIGRATE + struct lfs1 *lfs1; +#endif +} lfs_t; + + +/// Filesystem functions /// + +#ifndef LFS_READONLY +// Format a block device with the littlefs +// +// Requires a littlefs object and config struct. This clobbers the littlefs +// object, and does not leave the filesystem mounted. The config struct must +// be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_format(lfs_t *lfs, const struct lfs_config *config); +#endif + +// Mounts a littlefs +// +// Requires a littlefs object and config struct. Multiple filesystems +// may be mounted simultaneously with multiple littlefs objects. Both +// lfs and config must be allocated while mounted. The config struct must +// be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_mount(lfs_t *lfs, const struct lfs_config *config); + +// Unmounts a littlefs +// +// Does nothing besides releasing any allocated resources. +// Returns a negative error code on failure. +int lfs_unmount(lfs_t *lfs); + +/// General operations /// + +#ifndef LFS_READONLY +// Removes a file or directory +// +// If removing a directory, the directory must be empty. +// Returns a negative error code on failure. +int lfs_remove(lfs_t *lfs, const char *path); +#endif + +#ifndef LFS_READONLY +// Rename or move a file or directory +// +// If the destination exists, it must match the source in type. +// If the destination is a directory, the directory must be empty. +// +// Returns a negative error code on failure. +int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); +#endif + +// Find info about a file or directory +// +// Fills out the info structure, based on the specified file or directory. +// Returns a negative error code on failure. +int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); + +// Get a custom attribute +// +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than +// the buffer, it will be padded with zeros. If the stored attribute is larger, +// then it will be silently truncated. If no attribute is found, the error +// LFS_ERR_NOATTR is returned and the buffer is filled with zeros. +// +// Returns the size of the attribute, or a negative error code on failure. +// Note, the returned size is the size of the attribute on disk, irrespective +// of the size of the buffer. This can be used to dynamically allocate a buffer +// or check for existence. +lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size); + +#ifndef LFS_READONLY +// Set custom attributes +// +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be +// implicitly created. +// +// Returns a negative error code on failure. +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size); +#endif + +#ifndef LFS_READONLY +// Removes a custom attribute +// +// If an attribute is not found, nothing happens. +// +// Returns a negative error code on failure. +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); +#endif + + +/// File operations /// + +#ifndef LFS_NO_MALLOC +// Open a file +// +// The mode that the file is opened in is determined by the flags, which +// are values from the enum lfs_open_flags that are bitwise-ored together. +// +// Returns a negative error code on failure. +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags); + +// if LFS_NO_MALLOC is defined, lfs_file_open() will fail with LFS_ERR_NOMEM +// thus use lfs_file_opencfg() with config.buffer set. +#endif + +// Open a file with extra configuration +// +// The mode that the file is opened in is determined by the flags, which +// are values from the enum lfs_open_flags that are bitwise-ored together. +// +// The config struct provides additional config options per file as described +// above. The config struct must remain allocated while the file is open, and +// the config struct must be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags, + const struct lfs_file_config *config); + +// Close a file +// +// Any pending writes are written out to storage as though +// sync had been called and releases any allocated resources. +// +// Returns a negative error code on failure. +int lfs_file_close(lfs_t *lfs, lfs_file_t *file); + +// Synchronize a file on storage +// +// Any pending writes are written out to storage. +// Returns a negative error code on failure. +int lfs_file_sync(lfs_t *lfs, lfs_file_t *file); + +// Read data from file +// +// Takes a buffer and size indicating where to store the read data. +// Returns the number of bytes read, or a negative error code on failure. +lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size); + +#ifndef LFS_READONLY +// Write data to file +// +// Takes a buffer and size indicating the data to write. The file will not +// actually be updated on the storage until either sync or close is called. +// +// Returns the number of bytes written, or a negative error code on failure. +lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size); +#endif + +// Change the position of the file +// +// The change in position is determined by the offset and whence flag. +// Returns the new position of the file, or a negative error code on failure. +lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, + lfs_soff_t off, int whence); + +#ifndef LFS_READONLY +// Truncates the size of the file to the specified size +// +// Returns a negative error code on failure. +int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size); +#endif + +// Return the position of the file +// +// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR) +// Returns the position of the file, or a negative error code on failure. +lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file); + +// Change the position of the file to the beginning of the file +// +// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_SET) +// Returns a negative error code on failure. +int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file); + +// Return the size of the file +// +// Similar to lfs_file_seek(lfs, file, 0, LFS_SEEK_END) +// Returns the size of the file, or a negative error code on failure. +lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); + + +/// Directory operations /// + +#ifndef LFS_READONLY +// Create a directory +// +// Returns a negative error code on failure. +int lfs_mkdir(lfs_t *lfs, const char *path); +#endif + +// Open a directory +// +// Once open a directory can be used with read to iterate over files. +// Returns a negative error code on failure. +int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path); + +// Close a directory +// +// Releases any allocated resources. +// Returns a negative error code on failure. +int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir); + +// Read an entry in the directory +// +// Fills out the info structure, based on the specified file or directory. +// Returns a positive value on success, 0 at the end of directory, +// or a negative error code on failure. +int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info); + +// Change the position of the directory +// +// The new off must be a value previous returned from tell and specifies +// an absolute offset in the directory seek. +// +// Returns a negative error code on failure. +int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off); + +// Return the position of the directory +// +// The returned offset is only meant to be consumed by seek and may not make +// sense, but does indicate the current position in the directory iteration. +// +// Returns the position of the directory, or a negative error code on failure. +lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); + +// Change the position of the directory to the beginning of the directory +// +// Returns a negative error code on failure. +int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); + + +/// Filesystem-level filesystem operations + +// Find on-disk info about the filesystem +// +// Fills out the fsinfo structure based on the filesystem found on-disk. +// Returns a negative error code on failure. +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo); + +// Finds the current size of the filesystem +// +// Note: Result is best effort. If files share COW structures, the returned +// size may be larger than the filesystem actually is. +// +// Returns the number of allocated blocks, or a negative error code on failure. +lfs_ssize_t lfs_fs_size(lfs_t *lfs); + +// Traverse through all blocks in use by the filesystem +// +// The provided callback will be called with each block address that is +// currently in use by the filesystem. This can be used to determine which +// blocks are in use or how much of the storage is available. +// +// Returns a negative error code on failure. +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); + +#ifndef LFS_READONLY +// Attempt to make the filesystem consistent and ready for writing +// +// Calling this function is not required, consistency will be implicitly +// enforced on the first operation that writes to the filesystem, but this +// function allows the work to be performed earlier and without other +// filesystem changes. +// +// Returns a negative error code on failure. +int lfs_fs_mkconsistent(lfs_t *lfs); +#endif + +#ifndef LFS_READONLY +// Attempt any janitorial work +// +// This currently: +// 1. Calls mkconsistent if not already consistent +// 2. Compacts metadata > compact_thresh +// 3. Populates the block allocator +// +// Though additional janitorial work may be added in the future. +// +// Calling this function is not required, but may allow the offloading of +// expensive janitorial work to a less time-critical code path. +// +// Returns a negative error code on failure. Accomplishing nothing is not +// an error. +int lfs_fs_gc(lfs_t *lfs); +#endif + +#ifndef LFS_READONLY +// Grows the filesystem to a new size, updating the superblock with the new +// block count. +// +// If LFS_SHRINKNONRELOCATING is defined, this function will also accept +// block_counts smaller than the current configuration, after checking +// that none of the blocks that are being removed are in use. +// Note that littlefs's pseudorandom block allocation means that +// this is very unlikely to work in the general case. +// +// Returns a negative error code on failure. +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count); +#endif + +#ifndef LFS_READONLY +#ifdef LFS_MIGRATE +// Attempts to migrate a previous version of littlefs +// +// Behaves similarly to the lfs_format function. Attempts to mount +// the previous version of littlefs and update the filesystem so it can be +// mounted with the current version of littlefs. +// +// Requires a littlefs object and config struct. This clobbers the littlefs +// object, and does not leave the filesystem mounted. The config struct must +// be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg); +#endif +#endif + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif \ No newline at end of file diff --git a/quakey/src/lfs_util.c b/quakey/src/lfs_util.c new file mode 100644 index 0000000..87a7824 --- /dev/null +++ b/quakey/src/lfs_util.c @@ -0,0 +1,37 @@ +/* + * lfs util functions + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#include "lfs_util.h" + +// Only compile if user does not provide custom config +#ifndef LFS_CONFIG + + +// If user provides their own CRC impl we don't need this +#ifndef LFS_CRC +// Software CRC implementation with small lookup table +uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) { + static const uint32_t rtable[16] = { + 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, + 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, + 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, + 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c, + }; + + const uint8_t *data = buffer; + + for (size_t i = 0; i < size; i++) { + crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 0)) & 0xf]; + crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 4)) & 0xf]; + } + + return crc; +} +#endif + + +#endif \ No newline at end of file diff --git a/quakey/src/lfs_util.h b/quakey/src/lfs_util.h new file mode 100644 index 0000000..048f0b4 --- /dev/null +++ b/quakey/src/lfs_util.h @@ -0,0 +1,277 @@ +/* + * lfs utility functions + * + * Copyright (c) 2022, The littlefs authors. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ +#ifndef LFS_UTIL_H +#define LFS_UTIL_H + +#define LFS_STRINGIZE(x) LFS_STRINGIZE2(x) +#define LFS_STRINGIZE2(x) #x + +// Users can override lfs_util.h with their own configuration by defining +// LFS_CONFIG as a header file to include (-DLFS_CONFIG=lfs_config.h). +// +// If LFS_CONFIG is used, none of the default utils will be emitted and must be +// provided by the config file. To start, I would suggest copying lfs_util.h +// and modifying as needed. +#ifdef LFS_CONFIG +#include LFS_STRINGIZE(LFS_CONFIG) +#else + +// Alternatively, users can provide a header file which defines +// macros and other things consumed by littlefs. +// +// For example, provide my_defines.h, which contains +// something like: +// +// #include +// extern void *my_malloc(size_t sz); +// #define LFS_MALLOC(sz) my_malloc(sz) +// +// And build littlefs with the header by defining LFS_DEFINES. +// (-DLFS_DEFINES=my_defines.h) + +#ifdef LFS_DEFINES +#include LFS_STRINGIZE(LFS_DEFINES) +#endif + +// System includes +#include +#include +#include +#include + +#define LFS_NO_DEBUG +#define LFS_NO_WARN +#define LFS_NO_ERROR + +#ifndef LFS_NO_MALLOC +#include +#endif +#ifndef LFS_NO_ASSERT +#include +#endif +#if !defined(LFS_NO_DEBUG) || \ + !defined(LFS_NO_WARN) || \ + !defined(LFS_NO_ERROR) || \ + defined(LFS_YES_TRACE) +#include +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + + +// Macros, may be replaced by system specific wrappers. Arguments to these +// macros must not have side-effects as the macros can be removed for a smaller +// code footprint + +// Logging functions +#ifndef LFS_TRACE +#ifdef LFS_YES_TRACE +#define LFS_TRACE_(fmt, ...) \ + printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#else +#define LFS_TRACE(...) +#endif +#endif + +#ifndef LFS_DEBUG +#ifndef LFS_NO_DEBUG +#define LFS_DEBUG_(fmt, ...) \ + printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) +#define LFS_DEBUG(...) LFS_DEBUG_(__VA_ARGS__, "") +#else +#define LFS_DEBUG(...) +#endif +#endif + +#ifndef LFS_WARN +#ifndef LFS_NO_WARN +#define LFS_WARN_(fmt, ...) \ + printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) +#define LFS_WARN(...) LFS_WARN_(__VA_ARGS__, "") +#else +#define LFS_WARN(...) +#endif +#endif + +#ifndef LFS_ERROR +#ifndef LFS_NO_ERROR +#define LFS_ERROR_(fmt, ...) \ + printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__) +#define LFS_ERROR(...) LFS_ERROR_(__VA_ARGS__, "") +#else +#define LFS_ERROR(...) +#endif +#endif + +// Runtime assertions +#ifndef LFS_ASSERT +#ifndef LFS_NO_ASSERT +#define LFS_ASSERT(test) assert(test) +#else +#define LFS_ASSERT(test) +#endif +#endif + + +// Builtin functions, these may be replaced by more efficient +// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more +// expensive basic C implementation for debugging purposes + +// Min/max functions for unsigned 32-bit numbers +static inline uint32_t lfs_max(uint32_t a, uint32_t b) { + return (a > b) ? a : b; +} + +static inline uint32_t lfs_min(uint32_t a, uint32_t b) { + return (a < b) ? a : b; +} + +// Align to nearest multiple of a size +static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) { + return a - (a % alignment); +} + +static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { + return lfs_aligndown(a + alignment-1, alignment); +} + +// Find the smallest power of 2 greater than or equal to a +static inline uint32_t lfs_npw2(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) + return 32 - __builtin_clz(a-1); +#else + uint32_t r = 0; + uint32_t s; + a -= 1; + s = (a > 0xffff) << 4; a >>= s; r |= s; + s = (a > 0xff ) << 3; a >>= s; r |= s; + s = (a > 0xf ) << 2; a >>= s; r |= s; + s = (a > 0x3 ) << 1; a >>= s; r |= s; + return (r | (a >> 1)) + 1; +#endif +} + +// Count the number of trailing binary zeros in a +// lfs_ctz(0) may be undefined +static inline uint32_t lfs_ctz(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__) + return __builtin_ctz(a); +#else + return lfs_npw2((a & -a) + 1) - 1; +#endif +} + +// Count the number of binary ones in a +static inline uint32_t lfs_popc(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) + return __builtin_popcount(a); +#else + a = a - ((a >> 1) & 0x55555555); + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); + return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; +#endif +} + +// Find the sequence comparison of a and b, this is the distance +// between a and b ignoring overflow +static inline int lfs_scmp(uint32_t a, uint32_t b) { + return (int)(unsigned)(a - b); +} + +// Convert between 32-bit little-endian and native order +static inline uint32_t lfs_fromle32(uint32_t a) { +#if (defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return a; +#elif !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return __builtin_bswap32(a); +#else + return ((uint32_t)((uint8_t*)&a)[0] << 0) | + ((uint32_t)((uint8_t*)&a)[1] << 8) | + ((uint32_t)((uint8_t*)&a)[2] << 16) | + ((uint32_t)((uint8_t*)&a)[3] << 24); +#endif +} + +static inline uint32_t lfs_tole32(uint32_t a) { + return lfs_fromle32(a); +} + +// Convert between 32-bit big-endian and native order +static inline uint32_t lfs_frombe32(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return __builtin_bswap32(a); +#elif (defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return a; +#else + return ((uint32_t)((uint8_t*)&a)[0] << 24) | + ((uint32_t)((uint8_t*)&a)[1] << 16) | + ((uint32_t)((uint8_t*)&a)[2] << 8) | + ((uint32_t)((uint8_t*)&a)[3] << 0); +#endif +} + +static inline uint32_t lfs_tobe32(uint32_t a) { + return lfs_frombe32(a); +} + +// Calculate CRC-32 with polynomial = 0x04c11db7 +#ifdef LFS_CRC +static inline uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) { + return LFS_CRC(crc, buffer, size); +} +#else +uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); +#endif + +// Allocate memory, only used if buffers are not provided to littlefs +// +// littlefs current has no alignment requirements, as it only allocates +// byte-level buffers. +static inline void *lfs_malloc(size_t size) { +#if defined(LFS_MALLOC) + return LFS_MALLOC(size); +#elif !defined(LFS_NO_MALLOC) + return malloc(size); +#else + (void)size; + return NULL; +#endif +} + +// Deallocate memory, only used if buffers are not provided to littlefs +static inline void lfs_free(void *p) { +#if defined(LFS_FREE) + LFS_FREE(p); +#elif !defined(LFS_NO_MALLOC) + free(p); +#else + (void)p; +#endif +} + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif +#endif \ No newline at end of file diff --git a/quakey/src/quakey.c b/quakey/src/quakey.c new file mode 100644 index 0000000..ee03b4d --- /dev/null +++ b/quakey/src/quakey.c @@ -0,0 +1,4304 @@ +///////////////////////////////////////////////////////////////// +// Includes + +#include "lfs.h" +#include + +///////////////////////////////////////////////////////////////// +// Utilities + +#define TODO __builtin_trap() + +#ifdef NDEBUG +#define UNREACHABLE {} +#define ASSERT(X) {} +#else +#define UNREACHABLE __builtin_trap() +#define ASSERT(X) {if (!(X)) __builtin_trap();} +#endif + +///////////////////////////////////////////////////////////////// +// Basic Types + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; + +typedef u32 b32; + +typedef u64 Nanos; + +///////////////////////////////////////////////////////////////// +// Network Types + +typedef struct { + u32 data; +} AddrIPv4; + +typedef struct { + u16 data[8]; +} AddrIPv6; + +typedef enum { + ADDR_FAMILY_IPV4, + ADDR_FAMILY_IPV6, +} AddrFamily; + +typedef struct { + AddrFamily family; + union { + AddrIPv4 ipv4; + AddrIPv6 ipv6; + }; +} Addr; + +///////////////////////////////////////////////////////////////// +// Descriptor Type + +typedef struct Desc Desc; +typedef struct Host Host; + +typedef struct { + int head; + int count; + int capacity; + Desc **entries; +} AcceptQueue; + +typedef struct { + char *data; + int head; + int used; + int size; + int refs; +} SocketQueue; + +typedef enum { + // The Desc structure is unused + DESC_EMPTY, + + // The Desc represents a socket created + // with the socket() system call. The + // specific type of socket depends on + // how it's configured + DESC_SOCKET, + + // The Desc represents a listening socket, + // one created with socket() on which listen() + // is used. + DESC_SOCKET_L, + + // The Desc represents a connection socket, + // one created using accept() or with socket() + // and then configured using listen() + DESC_SOCKET_C, + + // The Desc represents an opened file + DESC_FILE, + + // The Desc represent an open directory + DESC_DIRECTORY, +} DescType; + +typedef enum { + CONNECT_STATUS_WAIT, + CONNECT_STATUS_DONE, + CONNECT_STATUS_RESET, + CONNECT_STATUS_CLOSE, + CONNECT_STATUS_NOHOST, +} ConnectStatus; + +struct Desc { + + // Parent host object + // This is set once at startup + Host *host; + + ///////////////////////////////////////// + // General descriptor fields + + DescType type; + b32 non_blocking; + + ///////////////////////////////////////// + // General socket fields + + // True if bind() has been called on this socket + b32 is_explicitly_bound; + + // Address and port bound to this socket (either + // implicitly or explicitly). + // + // An implicit bind occurs when listen() or connect() + // are called on a socket that was never bound to an + // interface using bind(). + // + // The address starts out with family equal to the + // first argument of socket() and address of 0. The + // port starts from 0. + Addr bound_addr; + u16 bound_port; + + ///////////////////////////////////////// + // Listen socket fields + + AcceptQueue accept_queue; + + ///////////////////////////////////////// + // Connection socket fields + + // Status code of the connecting process + ConnectStatus connect_status; + + // These are used when the socket is still connecting + Addr connect_addr; + u16 connect_port; + + // When connected this refers to the peer socket, else it's NULL. + // + // The peer is either a listen socket if the connection wasn't + // accepted yet, or a connection socket if it was. Note that this + // means the references to this descriptor must be removed from + // the peer's accept queue if it's freed abruptly. + Desc *peer; + + // Bytes received from/about to get sent to the peer + SocketQueue *input; + SocketQueue *output; + + ///////////////////////////////////////// + // File fields + + lfs_file_t file; + + ///////////////////////////////////////// + // Directory fields + + lfs_dir_t dir; + + ///////////////////////////////////////// +}; + +///////////////////////////////////////////////////////////////// +// Host Type + +#define HOST_ADDR_LIMIT 2 +#define HOST_DESC_LIMIT 1024 +#define HOST_ARGC_LIMIT 128 + +typedef struct Sim Sim; + +enum { + HOST_ERROR_OTHER = -1, + HOST_ERROR_FULL = -2, + HOST_ERROR_BADIDX = -3, + HOST_ERROR_NOTSOCK = -4, + HOST_ERROR_CANTBIND = -5, + HOST_ERROR_NOTAVAIL = -6, + HOST_ERROR_ADDRUSED = -7, + HOST_ERROR_BADARG = -8, + HOST_ERROR_BADFAM = -9, + HOST_ERROR_RESET = -10, + HOST_ERROR_HANGUP = -11, + HOST_ERROR_NOTCONN = -12, + HOST_ERROR_IO = -13, + HOST_ERROR_ISDIR = -14, + HOST_ERROR_WOULDBLOCK = -15, + HOST_ERROR_NOMEM = -16, + HOST_ERROR_NOENT = -17, + HOST_ERROR_NOTEMPTY = -18, + HOST_ERROR_EXIST = -19, + HOST_ERROR_EXISTS = -19, // Alias for HOST_ERROR_EXIST +}; + +// lseek whence values for host_lseek +enum { + HOST_SEEK_SET = 0, + HOST_SEEK_CUR = 1, + HOST_SEEK_END = 2, +}; + +enum { + HOST_FLAG_NONBLOCK = 1, +}; + +struct Host { + + // Pointer to the parent simulation object + Sim *sim; + + // Platform used by this host + QuakeyPlatform platform; + + // State of the ephimeral port allocation + u16 next_ephemeral_port; + + // Dynamic dopy of the argument string used + // to setup this host. Null bytes were injected + // to terminate each argument + char *arg; + + // Pointers into the argument string to make + // a list of individual arguments + int argc; + char *argv[HOST_ARGC_LIMIT]; + + // Opaque program state + void *state; + int state_size; + + // Pointers to program code + QuakeyInitFunc init_func; + QuakeyTickFunc tick_func; + QuakeyFreeFunc free_func; + + // Descriptor table + int num_desc; + Desc desc[HOST_DESC_LIMIT]; + + // Addresses bound to this host + Addr addrs[HOST_ADDR_LIMIT]; + int num_addrs; + + // Argument for poll() + void* poll_ctxs[HOST_DESC_LIMIT]; + struct pollfd poll_array[HOST_DESC_LIMIT]; + int poll_count; + int poll_timeout; + + b32 timedout; + + // Current error number set by system call mocks + int errno_; + + // Raw disk bytes + int disk_size; + char *disk_data; + + // LittleFS instance managing the disk bytes + lfs_t lfs; + struct lfs_config lfs_cfg; +}; + +typedef struct { + char name[256]; + bool is_dir; +} DirEntry; + +typedef struct { + int64_t size; + bool is_dir; +} FileInfo; + +///////////////////////////////////////////////////////////////// +// Simulation Type + +typedef enum { + EVENT_TYPE_CONNECT, + EVENT_TYPE_DISCONNECT, + EVENT_TYPE_DATA, + EVENT_TYPE_WAKEUP, +} TimeEventType; + +typedef struct { + + TimeEventType type; + + // Time when the event should happen + Nanos time; + + // When type=WAKEUP, refers to the host that needs + // to be woken up + Host *host; + + // When type=CONNECT or type=DATA, refers to the + // descriptor that initiated the connection + Desc *src_desc; + + // When type=DISCONNECT, refers to the descriptor + // that is receiving the disconnection message + Desc *dst_desc; + + // When type=DATA, this refers to the output buffer + // of the socket sending the data. The count variable + // dictates how many bytes can be read from the queue + // + // The queue pointer is reference counted to allow + // descriptors to be deinitialized without creating + // dangling DATA events + int data_count; + SocketQueue *data_queue; + + // If set, the DISCONNECT event is to be intended as + // a forceful shutdown + b32 rst; +} TimeEvent; + +struct Sim { + + uint64_t seed; + + // Current simulated time in nanoseconds + Nanos current_time; + + // List of simulated hosts. It's an array of pointers + // so that host structures are not moved when the array + // is resized. This allows safely holding references to + // hosts. + int num_hosts; + int max_hosts; + Host **hosts; + + // Array of timed events + int num_events; + int max_events; + TimeEvent *events; +}; + +static void time_event_wakeup(Sim *sim, Nanos time, Host *host); +static void time_event_connect(Sim *sim, Nanos time, Desc *desc); +static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst); +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc); + +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// + +///////////////////////////////////////////////////////////////// +// Address Code + +static bool is_digit(char c) +{ + return c >= '0' && c <= '9'; +} + +static bool is_hex_digit(char c) +{ + return (c >= '0' && c <= '9') + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'); +} + +static int parse_ipv4(char *src, int len, int *pcur, AddrIPv4 *ipv4) +{ + int cur = *pcur; + + unsigned int out = 0; + int i = 0; + for (;;) { + + if (cur == len || !is_digit(src[cur])) + return -1; + + int b = 0; + do { + int x = src[cur++] - '0'; + if (b > (UINT8_MAX - x) / 10) + return -1; + b = b * 10 + x; + } while (cur < len && is_digit(src[cur])); + + out <<= 8; + out |= (unsigned char) b; + + i++; + if (i == 4) + break; + + if (cur == len || src[cur] != '.') + return -1; + cur++; + } + + ipv4->data = out; + + *pcur = cur; + return 0; +} + +static int hex_digit_to_int(char c) +{ + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= '0' && c <= '9') return c - '0'; + return -1; +} + +static int parse_ipv6_comp(char *src, int len, int *pcur) +{ + int cur = *pcur; + + unsigned short buf; + if (cur == len || !is_hex_digit(src[cur])) + return -1; + buf = hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + if (cur == len || !is_hex_digit(src[cur])) { + *pcur = cur; + return buf; + } + buf <<= 4; + buf |= hex_digit_to_int(src[cur]); + cur++; + + *pcur = cur; + return (int) buf; +} + +static int parse_ipv6(char *src, int len, int *pcur, AddrIPv6 *ipv6) +{ + int cur = *pcur; + + unsigned short head[8]; + unsigned short tail[8]; + int head_len = 0; + int tail_len = 0; + + if (len - cur > 1 + && src[cur+0] == ':' + && src[cur+1] == ':') + cur += 2; + else { + + for (;;) { + + int ret = parse_ipv6_comp(src, len, &cur); + if (ret < 0) return ret; + + head[head_len++] = (unsigned short) ret; + if (head_len == 8) break; + + if (cur == len || src[cur] != ':') + return -1; + cur++; + + if (cur < len && src[cur] == ':') { + cur++; + break; + } + } + } + + if (head_len < 8) { + while (cur < len && is_hex_digit(src[cur])) { + + int ret = parse_ipv6_comp(src, len, &cur); + if (ret < 0) return ret; + + tail[tail_len++] = (unsigned short) ret; + if (head_len + tail_len == 8) break; + + if (cur == len || src[cur] != ':') + break; + cur++; + } + } + + for (int i = 0; i < head_len; i++) + ipv6->data[i] = head[i]; + + for (int i = 0; i < 8 - head_len - tail_len; i++) + ipv6->data[head_len + i] = 0; + + for (int i = 0; i < tail_len; i++) + ipv6->data[8 - tail_len + i] = tail[i]; + + *pcur = cur; + return 0; +} + +static int addr_parse(char *src, Addr *dst) +{ + int cur = 0; + int len = strlen(src); + + if (parse_ipv4(src, len, &cur, &dst->ipv4) == 0) { + dst->family = ADDR_FAMILY_IPV4; + return 0; + } + + cur = 0; + if (parse_ipv6(src, len, &cur, &dst->ipv6) == 0) { + dst->family = ADDR_FAMILY_IPV6; + return 0; + } + + return -1; +} + +static bool addr_eql(Addr a1, Addr a2) +{ + if (a1.family != a2.family) + return false; + if (a1.family == ADDR_FAMILY_IPV4) + return !memcmp(&a1.ipv4, &a2.ipv4, sizeof(AddrIPv4)); + ASSERT(a1.family == ADDR_FAMILY_IPV6); + return !memcmp(&a1.ipv6, &a2.ipv6, sizeof(AddrIPv6)); +} + +static bool is_zero_addr(Addr addr) +{ + char *p; + int n; + if (addr.family == ADDR_FAMILY_IPV4) { + p = (char*) &addr.ipv4; + n = sizeof(addr.ipv4); + } else { + ASSERT(addr.family == ADDR_FAMILY_IPV6); + p = (char*) &addr.ipv6; + n = sizeof(addr.ipv6); + } + for (int i = 0; i < n; i++) { + if (p[i] != 0) + return false; + } + return true; +} + +///////////////////////////////////////////////////////////////// +// Socket Queue Code + +static SocketQueue *socket_queue_init(int size) +{ + void *mem = malloc(sizeof(SocketQueue) + size); + if (mem == NULL) { + TODO; + } + SocketQueue *queue = mem; + queue->head = 0; + queue->used = 0; + queue->size = size; + queue->data = (char*) (queue + 1); + queue->refs = 1; + return queue; +} + +static SocketQueue *socket_queue_ref(SocketQueue *queue) +{ + queue->refs++; + return queue; +} + +static void socket_queue_unref(SocketQueue *queue) +{ + ASSERT(queue->refs > 0); + queue->refs--; + if (queue->refs == 0) + free(queue); +} + +static char *socket_queue_read_buf(SocketQueue *queue, int *num) +{ + *num = queue->used; + return queue->data + queue->head; +} + +static void socket_queue_read_ack(SocketQueue *queue, int num) +{ + queue->head += num; + queue->used -= num; +} + +static int socket_queue_read(SocketQueue *queue, char *dst, int max) +{ + int num; + char *src = socket_queue_read_buf(queue, &num); + + int copy = max; + if (copy > num) + copy = num; + memcpy(dst, src, copy); + + socket_queue_read_ack(queue, copy); + return copy; +} + +static char *socket_queue_write_buf(SocketQueue *queue, int *cap) +{ + // Only write up to the free space + if (*cap > queue->size - queue->used) + *cap = queue->size - queue->used; + + int last = queue->head + queue->used; + if (queue->size - last < *cap) { + memmove(queue->data, queue->data + queue->head, queue->used); + queue->head = 0; + } + + return queue->data + queue->head + queue->used; +} + +static void socket_queue_write_ack(SocketQueue *queue, int num) +{ + queue->used += num; +} + +static int socket_queue_write(SocketQueue *queue, char *src, int len) +{ + char *dst = socket_queue_write_buf(queue, &len); + memcpy(dst, src, len); + socket_queue_write_ack(queue, len); + return len; +} + +static int socket_queue_move(SocketQueue *dst_queue, SocketQueue *src_queue, int max) +{ + int srclen; + char *src = socket_queue_read_buf(src_queue, &srclen); + + if (srclen > max) + srclen = max; + + int dstlen = srclen; + char *dst = socket_queue_write_buf(dst_queue, &dstlen); + + memcpy(dst, src, dstlen); + + socket_queue_write_ack(dst_queue, dstlen); + socket_queue_read_ack(src_queue, dstlen); + return dstlen; +} + +static b32 socket_queue_full(SocketQueue *queue) +{ + return queue->used == queue->size; +} + +static b32 socket_queue_empty(SocketQueue *queue) +{ + return queue->used == 0; +} + +static b32 socket_queue_used(SocketQueue *queue) +{ + return queue->used; +} + +///////////////////////////////////////////////////////////////// +// Accept Queue Code + +static int accept_queue_init(AcceptQueue *queue, int capacity) +{ + Desc **entries = malloc(capacity * sizeof(Desc*)); + if (entries == NULL) + return -1; + queue->head = 0; + queue->count = 0; + queue->capacity = capacity; + queue->entries = entries; + return 0; +} + +static void accept_queue_free(AcceptQueue *queue) +{ + free(queue->entries); +} + +static Desc **accept_queue_peek(AcceptQueue *queue, int idx) +{ + if (idx >= queue->count) + return NULL; + return &queue->entries[(queue->head + idx) % queue->capacity]; +} + +static void accept_queue_remove(AcceptQueue *queue, Desc *desc) +{ + int i = 0; + while (i < queue->count && desc != *accept_queue_peek(queue, i)) + i++; + + if (i == queue->count) + return; // Not found + + while (i < queue->count-1) { + *accept_queue_peek(queue, i) = *accept_queue_peek(queue, i+1); + i++; + } + + queue->count--; +} + +static int accept_queue_push(AcceptQueue *queue, Desc *desc) +{ + if (queue->count == queue->capacity) + return -1; + int tail = (queue->head + queue->count) % queue->capacity; + queue->entries[tail] = desc; + queue->count++; + return 0; +} + +static int accept_queue_pop(AcceptQueue *queue, Desc **desc) +{ + if (queue->count == 0) + return -1; + *desc = queue->entries[queue->head]; + queue->head = (queue->head + 1) % queue->capacity; + queue->count--; + return 0; +} + +static bool accept_queue_empty(AcceptQueue *queue) +{ + return queue->count == 0; +} + +///////////////////////////////////////////////////////////////// +// Descriptor Code + +// If the descriptor is a connection socket and rst=true, +// the peer connection will be marked as "reset" instead +// of simply closed. +static void desc_free(Desc *desc, lfs_t *lfs, bool rst) +{ + switch (desc->type) { + case DESC_EMPTY: + break; + case DESC_SOCKET: + break; + case DESC_SOCKET_L: + // Update the other ends of the connections waiting to be accepted + for (int i = 0; i < desc->accept_queue.count; i++) { + Desc *peer = *accept_queue_peek(&desc->accept_queue, i); + peer->peer = NULL; + peer->connect_status = CONNECT_STATUS_RESET; + } + accept_queue_free(&desc->accept_queue); + break; + case DESC_SOCKET_C: + if (desc->peer) { + // A connection was previously established. + // We need to update the other end of the connection. + Desc *peer = desc->peer; + if (peer->type == DESC_SOCKET_L) { + // Connection was waiting to be accepted + accept_queue_remove(&peer->accept_queue, desc); + } else { + ASSERT(peer->type == DESC_SOCKET_C); + peer->peer = NULL; + peer->connect_status = rst ? CONNECT_STATUS_RESET : CONNECT_STATUS_CLOSE; + } + } + socket_queue_unref(desc->input); + socket_queue_unref(desc->output); + break; + case DESC_FILE: + lfs_file_close(lfs, &desc->file); + break; + case DESC_DIRECTORY: + lfs_dir_close(lfs, &desc->dir); + break; + default: + UNREACHABLE; + break; + } + desc->type = DESC_EMPTY; +} + +///////////////////////////////////////////////////////////////// +// Host Code + +// Backlog size used by listen() when the provided +// backlog argument is non-positive +#define DEFAULT_BACKLOG 128 + +#define FIRST_EPHEMERAL_PORT 10000 +#define LAST_EPHEMERAL_PORT 50000 + +// Current schedulated host +static Host *host___; + +int errno___; + +static void abort_(char *str) +{ +#ifdef _WIN32 + WriteFile(GetStdHandle((DWORD)-11), str, strlen(str), NULL, NULL); +#else + long ret; + __asm__ volatile ( + "syscall" + : "=a" (ret) + : "a" (1), + "D" (1), + "S" (str), + "d" (strlen(str)) + : "rcx", "r11", "memory" + ); + (void) ret; +#endif + __builtin_trap(); +} + +static int split_args(char *arg, char **argv, int max_argc) +{ + int argc = 0; + for (int cur = 0, len = strlen(arg);; ) { + + while (cur < len && (arg[cur] == ' ' || arg[cur] == '\t')) + cur++; + + if (cur == len) + break; + + int off = cur; + + while (cur < len && arg[cur] != ' ' && arg[cur] != '\t') + cur++; + + if (cur < len) { + arg[cur] = '\0'; + cur++; + } + + if (argc == max_argc) + return -1; + argv[argc++] = arg + off; + } + + return argc; +} + +static int block_device_read(const struct lfs_config *c, + lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) +{ + Host *host = c->context; + + // Block offset + lfs_off_t abs_off = block * c->block_size + off; + + // Bounds check + if (abs_off + size > (lfs_size_t) host->disk_size) + return LFS_ERR_IO; + + // Copy data from disk to buffer + memcpy(buffer, host->disk_data + abs_off, size); + return LFS_ERR_OK; +} + +static int block_device_prog(const struct lfs_config *c, + lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) +{ + Host *host = c->context; + + // Block offset + lfs_off_t abs_off = block * c->block_size + off; + + // Bounds check + if (abs_off + size > (lfs_size_t) host->disk_size) + return LFS_ERR_IO; + + // Copy data from buffer to disk + memcpy(host->disk_data + abs_off, buffer, size); + return LFS_ERR_OK; +} + +static int block_device_erase(const struct lfs_config *c, + lfs_block_t block) +{ + Host *host = c->context; + + // Block offset + lfs_off_t abs_off = block * c->block_size; + + // Bounds check + if (abs_off + c->block_size > (lfs_size_t) host->disk_size) + return LFS_ERR_IO; + + // Erase the block by setting all bytes to 0xFF (typical erased flash state) + memset(host->disk_data + abs_off, 0xFF, c->block_size); + return LFS_ERR_OK; +} + +static int block_device_sync(const struct lfs_config *c) +{ + // No-op for in-memory storage - nothing to flush + (void) c; + return LFS_ERR_OK; +} + +static void host_init(Host *host, Sim *sim, QuakeySpawn config, char *arg) +{ + host->sim = sim; + host->platform = config.platform; + host->next_ephemeral_port = FIRST_EPHEMERAL_PORT; + + int arg_len = strlen(arg); + host->arg = malloc(arg_len+1); + if (host->arg == NULL) { + TODO; + } + memcpy(host->arg, arg, arg_len+1); // +1 for \0 + + host->argc = split_args(host->arg, host->argv, HOST_ARGC_LIMIT); + if (host->argc < 0) { + TODO; + } + + if (config.num_addrs > HOST_ADDR_LIMIT) { + TODO; + } + for (int i = 0; i < config.num_addrs; i++) { + if (addr_parse(config.addrs[i], &host->addrs[i]) < 0) { + TODO; + } + } + host->num_addrs = config.num_addrs; + + host->state_size = config.state_size; + host->state = malloc(config.state_size); + if (host->state == NULL) { + TODO; + } + + host->init_func = config.init_func; + host->tick_func = config.tick_func; + host->free_func = config.free_func; + + host->num_desc = 0; + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + host->desc[i].host = host; + host->desc[i].type = DESC_EMPTY; + } + + host->errno_ = 0; + + host->disk_size = config.disk_size; + host->disk_data = malloc(config.disk_size); + if (host->disk_data == NULL) { + TODO; + } + // Zero out memory to make sure operations are deterministic + memset(host->disk_data, 0, config.disk_size); + + host->lfs_cfg = (struct lfs_config) { + + .context = host, + + // block device operations + .read = block_device_read, + .prog = block_device_prog, + .erase = block_device_erase, + .sync = block_device_sync, + + // block device configuration + .read_size = 16, + .prog_size = 16, + .block_size = 4096, + .block_count = 128, + .cache_size = 16, + .lookahead_size = 16, + .block_cycles = 500, + }; + + int ret = lfs_mount(&host->lfs, &host->lfs_cfg); + if (ret) { + lfs_format(&host->lfs, &host->lfs_cfg); // TODO: can this fail? + ret = lfs_mount(&host->lfs, &host->lfs_cfg); + if (ret) { + TODO; + } + } + + host->timedout = false; + host->poll_count = 0; + host->poll_timeout = -1; + + host___ = host; + ret = config.init_func( + host->state, + host->argc, + host->argv, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + // TODO: if the node fails to initialize the simulation should continue + // running without it + TODO; + } + + if (host->poll_timeout > -1) + time_event_wakeup(host->sim, host->sim->current_time + host->poll_timeout * 1000000, host); +} + +static void host_free(Host *host) +{ + host___ = host; + int ret = host->free_func(host->state); + host___ = NULL; + if (ret < 0) { + TODO; + } + + lfs_unmount(&host->lfs); + free(host->disk_data); + + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (host->desc[i].type != DESC_EMPTY) + desc_free(&host->desc[i], &host->lfs, true); + } + + free(host->state); + free(host->arg); +} + +static b32 host_is_linux(Host *host) +{ + return host->platform == QUAKEY_LINUX; +} + +static b32 host_is_windows(Host *host) +{ + return host->platform == QUAKEY_WINDOWS; +} + +static Nanos host_time(Host *host) +{ + return host->sim->current_time; +} + +static int *host_errno_ptr(Host *host) +{ + return &host->errno_; +} + +static bool is_connected_and_accepted(Desc *desc) +{ + assert(desc->type == DESC_SOCKET_C); + return desc->peer && desc->peer->type != DESC_SOCKET_L; +} + +static bool is_desc_idx_valid(Host *host, int desc_idx); + +static bool host_ready(Host *host) +{ + if (host->timedout) + return true; + + // Check if any polled descriptors have pending events + for (int i = 0; i < host->poll_count; i++) { + + int fd = host->poll_array[i].fd; + int events = host->poll_array[i].events; + + if (!is_desc_idx_valid(host, fd)) + continue; + + Desc *desc = &host->desc[fd]; + + switch (desc->type) { + case DESC_SOCKET_L: + if (events & POLLIN) { + if (!accept_queue_empty(&desc->accept_queue)) + return true; + } + break; + case DESC_SOCKET_C: + if (desc->connect_status == CONNECT_STATUS_NOHOST || + desc->connect_status == CONNECT_STATUS_RESET) + return true; + if (events & POLLIN) { + if (!socket_queue_empty(desc->input) + || desc->connect_status == CONNECT_STATUS_RESET + || desc->connect_status == CONNECT_STATUS_CLOSE) + return true; + } + if (events & POLLOUT) { + if (!socket_queue_full(desc->output) && is_connected_and_accepted(desc)) + return true; + } + break; + case DESC_FILE: + case DESC_DIRECTORY: + // Files and directories are always ready + if (events & (POLLIN | POLLOUT)) + return true; + break; + default: + UNREACHABLE; + break; + } + } + + return false; +} + +static void set_revents_in_poll_array(Host *host) +{ + for (int i = 0; i < host->poll_count; i++) { + + int fd = host->poll_array[i].fd; + int events = host->poll_array[i].events; + + if (!is_desc_idx_valid(host, fd)) { + assert(0); // TODO + } + Desc *desc = &host->desc[fd]; + + int revents = 0; + switch (desc->type) { + case DESC_SOCKET: + // TODO + break; + case DESC_SOCKET_L: + if (events & POLLIN) { + if (!accept_queue_empty(&desc->accept_queue)) + revents = POLLIN; + } + break; + case DESC_SOCKET_C: + if (desc->connect_status == CONNECT_STATUS_NOHOST || + desc->connect_status == CONNECT_STATUS_RESET) + revents |= POLLERR; + if (events & POLLIN) { + // TODO: should report prover events when hup and rst are set + if (!socket_queue_empty(desc->input) || desc->connect_status == CONNECT_STATUS_CLOSE) + revents |= POLLIN; + } + if (events & POLLOUT) { + if (!socket_queue_full(desc->output)) { + if (is_connected_and_accepted(desc)) + revents |= POLLOUT; + } + } + break; + case DESC_FILE: + if (events & POLLIN) { + revents |= POLLIN; + } + if (events & POLLOUT) { + revents |= POLLOUT; + } + break; + case DESC_DIRECTORY: + if (events & POLLIN) { + revents |= POLLIN; + } + if (events & POLLOUT) { + revents |= POLLOUT; + } + break; + default: + UNREACHABLE; + } + + host->poll_array[i].revents = revents; + } +} + +static void host_update(Host *host) +{ + host->timedout = false; + set_revents_in_poll_array(host); + host___ = host; + int ret = host->tick_func( + host->state, + host->poll_ctxs, + host->poll_array, + HOST_DESC_LIMIT, + &host->poll_count, + &host->poll_timeout + ); + host___ = NULL; + if (ret < 0) { + TODO; + } + + if (host->poll_timeout > -1) + time_event_wakeup(host->sim, host->sim->current_time + host->poll_timeout * 1000000, host); +} + +static bool host_has_addr(Host *host, Addr addr) +{ + for (int i = 0; i < host->num_addrs; i++) { + if (addr_eql(host->addrs[i], addr)) + return true; + } + return false; +} + +// Returns true if the descriptor is a socket bound +// (implicitly or explicitly) to an address +static bool is_bound(Desc *desc) +{ + if (desc->type == DESC_SOCKET) + return desc->is_explicitly_bound; + + if (desc->type == DESC_SOCKET_C || + desc->type == DESC_SOCKET_L) + return true; + + return false; +} + +// Note that this is assumed to work on empty +// descriptors too +static bool is_bound_to(Desc *desc, Addr addr, uint16_t port) +{ + if (!is_bound(desc)) + return false; + + if (!is_zero_addr(desc->bound_addr) && !addr_eql(addr, desc->bound_addr)) + return false; + + if (port != desc->bound_port) + return false; + + return true; +} + +static Desc *host_find_desc_bound_to(Host *host, Addr addr, uint16_t port) +{ + for (int i = 0, j = 0; j < host->num_desc; i++) { + + Desc *desc = &host->desc[i]; + if (desc->type == DESC_EMPTY) + continue; + j++; + + if (is_bound_to(desc, addr, port)) + return desc; + } + + return NULL; +} + +static int find_empty_desc_struct(Host *host) +{ + if (host->num_desc == HOST_DESC_LIMIT) + return -1; + + int i = 0; + while (host->desc[i].type != DESC_EMPTY) { + i++; + ASSERT(i < HOST_DESC_LIMIT); + } + + return i; +} + +static int host_create_socket(Host *host, AddrFamily family) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + desc->type = DESC_SOCKET; + desc->non_blocking = false; + desc->is_explicitly_bound = false; + desc->bound_addr = (Addr) { .family=family }; + desc->bound_port = 0; + + host->num_desc++; + return desc_idx; +} + +static bool is_desc_idx_valid(Host *host, int desc_idx) +{ + // Out of bounds + if (desc_idx < 0 || desc_idx >= HOST_DESC_LIMIT) + return false; + + // Not in use + if (host->desc[desc_idx].type == DESC_EMPTY) + return false; + + return true; +} + +static bool is_socket(Desc *desc) +{ + return desc->type == DESC_SOCKET + || desc->type == DESC_SOCKET_L + || desc->type == DESC_SOCKET_C; +} + +static int host_close(Host *host, int desc_idx, bool expect_socket) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + + if (expect_socket) { + if (!is_socket(&host->desc[desc_idx])) + return HOST_ERROR_NOTSOCK; + } + + desc_free(&host->desc[desc_idx], &host->lfs, false); + time_event_disconnect(host->sim, 10000000, &host->desc[desc_idx], false); + + host->num_desc--; + return 0; +} + +static bool interf_exists_locally(Host *host, Addr addr) +{ + for (int i = 0; i < host->num_addrs; i++) + if (addr_eql(host->addrs[i], addr)) + return true; + return false; +} + +static bool addr_in_use(Host *host, Addr addr, uint16_t port) +{ + ASSERT(port != 0); + + if (is_zero_addr(addr)) { + // Any address may conflict with the zero address, + // which means we only need to compare ports. + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (is_socket(&host->desc[i])) { + if (host->desc[i].bound_port == port) + return true; + } + } + } else { + for (int i = 0; i < HOST_DESC_LIMIT; i++) { + if (is_bound_to(&host->desc[i], addr, port)) + return true; + } + } + + return false; +} + +// Returns 0 on error +static uint16_t choose_ephemeral_port(Host *host, Addr addr) +{ + uint16_t first = host->next_ephemeral_port; + uint16_t *next = &host->next_ephemeral_port; + do { + uint16_t port = *next; + if (*next == LAST_EPHEMERAL_PORT) { + *next = FIRST_EPHEMERAL_PORT; + } else { + (*next)++; + } + if (!addr_in_use(host, addr, port)) + return port; + } while (*next != first); + return 0; +} + +static int host_bind(Host *host, int desc_idx, Addr addr, uint16_t port) +{ + ///////////////////////////////////////////////////////// + // Check index + + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + ///////////////////////////////////////////////////////// + // Check descriptor + + if (!is_socket(desc)) + return HOST_ERROR_NOTSOCK; + + if (host->desc[desc_idx].type != DESC_SOCKET) + return HOST_ERROR_CANTBIND; + + if (desc->is_explicitly_bound) + return HOST_ERROR_CANTBIND; + + ///////////////////////////////////////////////////////// + // Check address + + if (addr.family != desc->bound_addr.family) + return HOST_ERROR_BADFAM; + + if (!is_zero_addr(addr)) { + if (!interf_exists_locally(host, addr)) + return HOST_ERROR_NOTAVAIL; + } + + ///////////////////////////////////////////////////////// + // Check port + + if (port == 0) { + port = choose_ephemeral_port(host, addr); + if (port == 0) + return HOST_ERROR_NOTAVAIL; + } else { + if (addr_in_use(host, addr, port)) + return HOST_ERROR_ADDRUSED; + } + + ///////////////////////////////////////////////////////// + // Perform the binding + + desc->is_explicitly_bound = true; + desc->bound_addr = addr; + desc->bound_port = port; + + ///////////////////////////////////////////////////////// + return 0; +} + +static int host_listen(Host *host, int desc_idx, int backlog) +{ + if (backlog <= 0) + backlog = DEFAULT_BACKLOG; + + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->is_explicitly_bound) { + // We need to bind implicitly + // + // The bound_addr field already contains the right + // family and a zero address. The port is 0, which + // is not a valid value. + desc->bound_port = choose_ephemeral_port(host, desc->bound_addr); + if (desc->bound_port == 0) + return HOST_ERROR_ADDRUSED; + } + + if (accept_queue_init(&desc->accept_queue, backlog) < 0) { + TODO; + } + + desc->type = DESC_SOCKET_L; + return 0; +} + +static int host_accept(Host *host, int desc_idx, Addr *addr, uint16_t *port) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET_L) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before accept()\n"); + + int new_desc_idx = find_empty_desc_struct(host); + if (new_desc_idx < 0) + return HOST_ERROR_FULL; + Desc *new_desc = &host->desc[new_desc_idx]; + + Desc *peer; + if (accept_queue_pop(&desc->accept_queue, &peer) < 0) + return HOST_ERROR_WOULDBLOCK; + + *addr = peer->bound_addr; + *port = peer->bound_port; + + Addr local_addr = desc->bound_addr; + if (is_zero_addr(local_addr)) { + assert(host->num_addrs > 0); + local_addr = host->addrs[0]; + } + + new_desc->type = DESC_SOCKET_C; + new_desc->non_blocking = false; + new_desc->bound_addr = local_addr; + new_desc->bound_port = desc->bound_port; + new_desc->connect_addr = peer->bound_addr; + new_desc->connect_port = peer->bound_port; + new_desc->connect_status = CONNECT_STATUS_DONE; + new_desc->peer = peer; + new_desc->input = socket_queue_init(1<<12); + new_desc->output = socket_queue_init(1<<12); + + // Update the peer's end of the connection + peer->peer = new_desc; + + host->num_desc++; + return new_desc_idx; +} + +// TODO: check error codes returned by this function +static int host_connect(Host *host, int desc_idx, + Addr addr, uint16_t port) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before connect()\n"); + + if (!desc->is_explicitly_bound) { + // We need to bind implicitly + // + // The bound_addr field already contains the right + // family and a zero address. The port is 0, which + // is not a valid value. + desc->bound_port = choose_ephemeral_port(host, desc->bound_addr); + if (desc->bound_port == 0) + return HOST_ERROR_ADDRUSED; + } + + // TODO: some percent of times connect() should resolve immediately + + time_event_connect(host->sim, 100000000, desc); + + desc->connect_addr = addr; + desc->connect_port = port; + desc->connect_status = CONNECT_STATUS_WAIT; + desc->peer = NULL; + desc->input = socket_queue_init(1<<12); + desc->output = socket_queue_init(1<<12); + + desc->type = DESC_SOCKET_C; + return 0; +} + +static int host_connect_status(Host *host, int desc_idx, ConnectStatus *status) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_SOCKET_C) { + if (is_socket(desc)) + return HOST_ERROR_BADARG; + return HOST_ERROR_NOTSOCK; + } + + if (status) + *status = desc->connect_status; + return 0; +} + +static int host_open_file(Host *host, char *path, int flags) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + int ret = lfs_file_open(&host->lfs, &desc->file, path, flags); + if (ret < 0) { + TODO; + } + + desc->type = DESC_FILE; + desc->non_blocking = false; + + host->num_desc++; + return desc_idx; +} + +static int host_open_dir(Host *host, char *path) +{ + int desc_idx = find_empty_desc_struct(host); + if (desc_idx < 0) + return HOST_ERROR_FULL; + Desc *desc = &host->desc[desc_idx]; + + int ret = lfs_dir_open(&host->lfs, &desc->dir, path); + if (ret < 0) { + switch (ret) { + case LFS_ERR_NOENT: + return HOST_ERROR_NOENT; + default: + return HOST_ERROR_IO; + } + } + + desc->type = DESC_DIRECTORY; + desc->non_blocking = false; + + host->num_desc++; + return desc_idx; +} + +static int host_read_dir(Host *host, int desc_idx, DirEntry *entry) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_DIRECTORY) + return HOST_ERROR_BADARG; + + struct lfs_info info; + int ret = lfs_dir_read(&host->lfs, &desc->dir, &info); + if (ret < 0) + return HOST_ERROR_IO; + + if (ret == 0) + return 0; // End of directory + + // Copy entry information + // LFS_NAME_MAX is typically 255, and our name buffer is 256 + int i = 0; + while (info.name[i] != '\0' && i < 255) { + entry->name[i] = info.name[i]; + i++; + } + entry->name[i] = '\0'; + entry->is_dir = (info.type == LFS_TYPE_DIR); + + return 1; +} + +static int recv_inner(Desc *desc, char *dst, int len) +{ + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before recv()\n"); + + if (desc->peer == NULL) { + if (desc->connect_status == CONNECT_STATUS_RESET) + return HOST_ERROR_RESET; + if (desc->connect_status == CONNECT_STATUS_CLOSE) + return HOST_ERROR_HANGUP; + return HOST_ERROR_NOTCONN; + } + + int ret = socket_queue_read(desc->input, dst, len); + if (ret == 0) + return HOST_ERROR_WOULDBLOCK; + + return ret; +} + +static int send_inner(Desc *desc, char *src, int len) +{ + if (!desc->non_blocking) + abort_("Socket not configured as non-blocking before send()\n"); + + if (desc->peer == NULL) { + if (desc->connect_status == CONNECT_STATUS_RESET) + return HOST_ERROR_RESET; + if (desc->connect_status == CONNECT_STATUS_CLOSE) + return HOST_ERROR_HANGUP; + return HOST_ERROR_NOTCONN; + } + + int ret = socket_queue_write(desc->output, src, len); + if (ret == 0) + return HOST_ERROR_WOULDBLOCK; + + time_event_send_data(desc->host->sim, 10000000, desc); + return ret; +} + +static int host_read(Host *host, int desc_idx, char *dst, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = recv_inner(desc, dst, len); + } else if (desc->type == DESC_FILE) { + // TODO: what if the file wasn't open for reading? + lfs_ssize_t ret = lfs_file_read(&host->lfs, &desc->file, dst, len); + if (ret < 0) + return HOST_ERROR_IO; + num = ret; + } else { + if (desc->type == DESC_DIRECTORY) + return HOST_ERROR_ISDIR; + return HOST_ERROR_BADARG; + } + + return num; +} + +static int host_write(Host *host, int desc_idx, char *src, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = send_inner(desc, src, len); + } else if (desc->type == DESC_FILE) { + // TODO: what if the file wasn't open for writing? + lfs_ssize_t ret = lfs_file_write(&host->lfs, &desc->file, src, len); + if (ret < 0) + return HOST_ERROR_IO; // TODO: this may be imprecise + num = ret; + } else { + return HOST_ERROR_BADIDX; + } + + return num; +} + +static int host_recv(Host *host, int desc_idx, char *dst, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = recv_inner(desc, dst, len); + } else { + if (!is_socket(desc)) + return HOST_ERROR_NOTSOCK; + TODO; + } + + return num; +} + +static int host_send(Host *host, int desc_idx, char *src, int len) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + int num = 0; + if (desc->type == DESC_SOCKET_C) { + num = send_inner(desc, src, len); + } else { + TODO; + } + + return num; +} + +static int host_mkdir(Host *host, char *path) +{ + int ret = lfs_mkdir(&host->lfs, path); + if (ret < 0) { + switch (ret) { + case LFS_ERR_EXIST: + return HOST_ERROR_EXISTS; + case LFS_ERR_NOENT: + return HOST_ERROR_NOENT; + default: + return HOST_ERROR_IO; + } + } + return 0; +} + +static int host_remove(Host *host, char *path) +{ + int ret = lfs_remove(&host->lfs, path); + if (ret < 0) { + switch (ret) { + case LFS_ERR_NOENT: + return HOST_ERROR_NOENT; + case LFS_ERR_NOTEMPTY: + return HOST_ERROR_NOTEMPTY; + default: + return HOST_ERROR_IO; + } + } + return 0; +} + +static int host_rename(Host *host, char *oldpath, char *newpath) +{ + int ret = lfs_rename(&host->lfs, oldpath, newpath); + if (ret < 0) { + switch (ret) { + case LFS_ERR_NOENT: + return HOST_ERROR_NOENT; + case LFS_ERR_EXIST: + return HOST_ERROR_EXIST; + case LFS_ERR_NOTEMPTY: + return HOST_ERROR_NOTEMPTY; + case LFS_ERR_ISDIR: + return HOST_ERROR_ISDIR; + default: + return HOST_ERROR_IO; + } + } + return 0; +} + +static int host_fileinfo(Host *host, int desc_idx, FileInfo *info) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + switch (desc->type) { + case DESC_FILE: + { + lfs_soff_t size = lfs_file_size(&host->lfs, &desc->file); + if (size < 0) + return HOST_ERROR_IO; + info->size = size; + info->is_dir = false; + } + break; + case DESC_DIRECTORY: + { + info->size = 0; + info->is_dir = true; + } + break; + default: + return HOST_ERROR_BADIDX; + } + + return 0; +} + +static int host_lseek(Host *host, int desc_idx, int64_t offset, int whence) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + + int lfs_whence; + switch (whence) { + case HOST_SEEK_SET: + lfs_whence = LFS_SEEK_SET; + break; + case HOST_SEEK_CUR: + lfs_whence = LFS_SEEK_CUR; + break; + case HOST_SEEK_END: + lfs_whence = LFS_SEEK_END; + break; + default: + return HOST_ERROR_BADARG; + } + + lfs_soff_t ret = lfs_file_seek(&host->lfs, &desc->file, (lfs_soff_t) offset, lfs_whence); + if (ret < 0) + return HOST_ERROR_BADARG; + + return ret; +} + +static int host_fsync(Host *host, int desc_idx) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + if (desc->type != DESC_FILE) + return HOST_ERROR_BADIDX; + + int ret = lfs_file_sync(&host->lfs, &desc->file); + if (ret < 0) + return HOST_ERROR_IO; + + return 0; +} + +static int host_setdescflags(Host *host, int desc_idx, int flags) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + // TODO: check the descriptor type + + desc->non_blocking = (flags & HOST_FLAG_NONBLOCK) != 0; + return 0; +} + +static int host_getdescflags(Host *host, int desc_idx) +{ + if (!is_desc_idx_valid(host, desc_idx)) + return HOST_ERROR_BADIDX; + Desc *desc = &host->desc[desc_idx]; + + // TODO: check the descriptor type + + int flags = 0; + if (desc->non_blocking) + flags |= HOST_FLAG_NONBLOCK; + + return flags; +} + +///////////////////////////////////////////////////////////////// +// Time Event Code + +static void append_event(Sim *sim, TimeEvent event) +{ + if (sim->num_events == sim->max_events) { + int n = 2 * sim->max_events; + if (n == 0) + n = 8; + TimeEvent *p = realloc(sim->events, n * sizeof(TimeEvent)); + if (p == NULL) { + TODO; + } + sim->events = p; + sim->max_events = n; + } + + sim->events[sim->num_events++] = event; +} + +static void time_event_wakeup(Sim *sim, Nanos time, Host *host) +{ + TimeEvent event = { + .type = EVENT_TYPE_WAKEUP, + .time = time, + .host = host, + .src_desc = NULL, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static void time_event_connect(Sim *sim, Nanos time, Desc *desc) +{ + TimeEvent event = { + .type = EVENT_TYPE_CONNECT, + .time = time, + .host = NULL, + .src_desc = desc, + .dst_desc = NULL, + .data_count = 0, + .data_queue = NULL, + .rst = false, + }; + append_event(sim, event); +} + +static b32 remove_connect_event(Sim *sim, Desc *desc) +{ + int i = 0; + while (i < sim->num_events && (sim->events[i].type != EVENT_TYPE_CONNECT || sim->events[i].src_desc != desc)) + i++; + + if (i == sim->num_events) + return false; + + sim->events[i] = sim->events[--sim->num_events]; + return true; +} + +static void time_event_disconnect(Sim *sim, Nanos time, Desc *desc, b32 rst) +{ + if (remove_connect_event(sim, desc)) + return; + + TimeEvent event = { + .type = EVENT_TYPE_DISCONNECT, + .time = time, + .host = NULL, + .src_desc = NULL, + .dst_desc = desc->peer, + .data_count = 0, + .data_queue = NULL, + .rst = rst, + }; + append_event(sim, event); +} + +static void time_event_send_data(Sim *sim, Nanos time, Desc *desc) +{ + TimeEvent event = { + .type = EVENT_TYPE_DATA, + .time = time, + .host = NULL, + .src_desc = NULL, + .dst_desc = desc->peer, + .data_count = socket_queue_used(desc->output), + .data_queue = socket_queue_ref(desc->output), + .rst = false, + }; + append_event(sim, event); +} + +static void time_event_free(TimeEvent *event) +{ + if (event->data_queue) + socket_queue_unref(event->data_queue); +} + +static int sim_find_host(Sim *sim, Addr addr); + +static b32 time_event_process(TimeEvent *event, Sim *sim) +{ + b32 consumed = true; + switch (event->type) { + case EVENT_TYPE_CONNECT: + { + Desc *src_desc = event->src_desc; + assert(event->dst_desc == NULL); + + int idx = sim_find_host(sim, src_desc->connect_addr); + if (idx < 0) { + src_desc->connect_status = CONNECT_STATUS_NOHOST; + break; + } + Host *peer_host = sim->hosts[idx]; + + Desc *peer = host_find_desc_bound_to(peer_host, + src_desc->connect_addr, src_desc->connect_port); + if (peer == NULL) { + // Peer host exists but the port isn't open. Reset the connection. + src_desc->connect_status = CONNECT_STATUS_RESET; + break; + } + + assert(peer->type == DESC_SOCKET_L); + if (accept_queue_push(&peer->accept_queue, src_desc) < 0) { + // Accept queue is full + src_desc->connect_status = CONNECT_STATUS_RESET; + break; + } + + src_desc->connect_status = CONNECT_STATUS_DONE; + src_desc->peer = peer; // Resolved! + } + break; + case EVENT_TYPE_DISCONNECT: + { + Desc *dst_desc = event->dst_desc; + assert(dst_desc->type == DESC_SOCKET_C); + assert(event->src_desc == NULL); + dst_desc->peer = NULL; + dst_desc->connect_status = event->rst + ? CONNECT_STATUS_RESET + : CONNECT_STATUS_CLOSE; + } + break; + case EVENT_TYPE_DATA: + { + Desc *dst_desc = event->dst_desc; + assert(dst_desc->type == DESC_SOCKET_C); + assert(event->src_desc == NULL); + + int num = socket_queue_move(dst_desc->input, event->data_queue, event->data_count); + if (num < 0) { + TODO; + } + + event->data_count -= num; + if (event->data_count == 0) { + socket_queue_unref(event->data_queue); + } else { + consumed = false; + } + } + break; + case EVENT_TYPE_WAKEUP: + event->host->timedout = true; + break; + } + return consumed; +} + +///////////////////////////////////////////////////////////////// +// Sim Code + +static void sim_init(Sim *sim, uint64_t seed) +{ + sim->seed = seed; + sim->current_time = 0; + sim->num_hosts = 0; + sim->max_hosts = 0; + sim->hosts = NULL; + sim->num_events = 0; + sim->max_events = 0; + sim->events = NULL; +} + +static void sim_free(Sim *sim) +{ + for (int i = 0; i < sim->num_hosts; i++) + host_free(sim->hosts[i]); + free(sim->hosts); + + for (int i = 0; i < sim->num_events; i++) + time_event_free(&sim->events[i]); + free(sim->events); +} + +static void sim_spawn(Sim *sim, QuakeySpawn config, char *arg) +{ + if (sim->num_hosts == sim->max_hosts) { + int n = 2 * sim->max_hosts; + if (n == 0) + n = 8; + Host **p = realloc(sim->hosts, n * sizeof(Host*)); + if (p == NULL) { + TODO; + } + sim->hosts = p; + sim->max_hosts = n; + } + + Host *host = malloc(sizeof(Host)); + if (host == NULL) { + TODO; + } + host_init(host, sim, config, arg); + + sim->hosts[sim->num_hosts++] = host; +} + +static int sim_find_host(Sim *sim, Addr addr) +{ + for (int i = 0; i < sim->num_hosts; i++) + if (host_has_addr(sim->hosts[i], addr)) + return i; + return -1; +} + +static void advance_time_to_next_event(Sim *sim) +{ + if (sim->num_events == 0) + return; + + Nanos lowest_time = sim->events[0].time; + for (int i = 1; i < sim->num_events; i++) + if (lowest_time > sim->events[i].time) + lowest_time = sim->events[i].time; + + sim->current_time = lowest_time; +} + +static void process_events_at_current_time(Sim *sim) +{ + bool deferred_disconnect = false; + for (int i = 0; i < sim->num_events; i++) { + if (sim->events[i].time == sim->current_time) { + if (sim->events[i].type == EVENT_TYPE_DISCONNECT) { + deferred_disconnect = true; + } else { + if (time_event_process(&sim->events[i], sim)) + sim->events[i--] = sim->events[--sim->num_events]; + } + } + } + + if (deferred_disconnect) { + for (int i = 0; i < sim->num_events; i++) { + if (sim->events[i].time == sim->current_time) + if (sim->events[i].type == EVENT_TYPE_DISCONNECT) { + if (time_event_process(&sim->events[i], sim)) + sim->events[i--] = sim->events[--sim->num_events]; + } + } + } +} + +static int find_ready_host(Sim *sim) +{ + int i = 0; + while (i < sim->num_hosts && !host_ready(sim->hosts[i])) + i++; + if (i == sim->num_hosts) + return -1; + return i; +} + +static b32 sim_update(Sim *sim) +{ + if (sim->num_hosts == 0) + return false; + + int host_idx; + for (;;) { + + // Schedule the first host that's ready. + // + // If all host are waiting, advance the time to the + // next timed event and try again. + + host_idx = find_ready_host(sim); + if (host_idx > -1) + break; + + if (sim->num_events == 0) + __builtin_trap(); // TODO: remove me + + advance_time_to_next_event(sim); + process_events_at_current_time(sim); + } + + host_update(sim->hosts[host_idx]); + return true; +} + +static uint64_t sim_random(Sim *sim) +{ + uint64_t x = sim->seed; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + sim->seed = x; + return x; +} + +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////// +// Public Interface + +///////////////////////////////////////////////////////////////// +// Quakey Object + +int quakey_init(Quakey **quakey, QuakeyUInt64 seed) +{ + Sim *sim = malloc(sizeof(Sim)); + if (sim == NULL) + return -1; + sim_init(sim, seed); + if (quakey) { + *quakey = (void*) sim; + } else { + sim_free(sim); + free(sim); + } + return 0; +} + +void quakey_free(Quakey *quakey) +{ + if (quakey) { + sim_free((Sim*) quakey); + free(quakey); + } +} + +void quakey_spawn(Quakey *quakey, QuakeySpawn config, char *arg) +{ + return sim_spawn((Sim*) quakey, config, arg); +} + +int quakey_schedule_one(Quakey *quakey) +{ + return sim_update((Sim*) quakey); +} + +QuakeyUInt64 quakey_random(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_errno_ptr() with no node scheduled\n"); + return sim_random(host->sim); +} + +///////////////////////////////////////////////////////////////// +// Mock System Calls + +int *mock_errno_ptr(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_errno_ptr() with no node scheduled\n"); + + return host_errno_ptr(host); +} + +#ifndef _WIN32 + +int mock_socket(int domain, int type, int protocol) +{ + if (domain != AF_INET && domain != AF_INET6) + abort_("Quakey only supports socket() calls with doman=AF_INET or AF_INET6\n"); + + if (type != SOCK_STREAM) + abort_("Quakey only supports socket() calls with type=SOCK_STREAM\n"); + + if (protocol != 0) + abort_("Quakey only supports socket() calls with protocol=0\n"); + + Host *host = host___; + if (host == NULL) + abort_("Call to mock_socket() with no node scheduled\n"); + + AddrFamily family; + switch (domain) { + case AF_INET: + family = ADDR_FAMILY_IPV4; + break; + case AF_INET6: + family = ADDR_FAMILY_IPV6; + break; + default: + UNREACHABLE; + } + + int ret = host_create_socket(host, family); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + int desc_idx = ret; + + return desc_idx; +} + +int mock_close(int fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_close() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_close() not from Linux\n"); + + int desc_idx = fd; + int ret = host_close(host, desc_idx, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +static int convert_addr(void *addr, size_t addr_len, + Addr *converted_addr, uint16_t *converted_port) +{ + int family = ((struct sockaddr*) addr)->sa_family; + switch (family) { + case AF_INET: + { + if (addr_len != sizeof(struct sockaddr_in)) + return -1; + struct sockaddr_in *p = addr; + converted_addr->family = ADDR_FAMILY_IPV4; + converted_addr->ipv4.data = ntohl(((AddrIPv4*) &p->sin_addr)->data); + *converted_port = ntohs(p->sin_port); + } + break; + case AF_INET6: + { + if (addr_len != sizeof(struct sockaddr_in6)) + return -1; + struct sockaddr_in6 *p = addr; + converted_addr->family = ADDR_FAMILY_IPV6; + converted_addr->ipv6 = *(AddrIPv6*) &p->sin6_addr; // TODO: convert to host byte order + *converted_port = ntohs(p->sin6_port); + } + break; + default: + abort_("Quakey only supports the AF_INET and AF_INET6 address families"); + } + return 0; +} + +int mock_bind(int fd, void *addr, unsigned long addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_bind() with no node scheduled\n"); + + Addr converted_addr; + uint16_t converted_port; + int ret = convert_addr(addr, addr_len, &converted_addr, &converted_port); + if (ret < 0) { + *host_errno_ptr(host) = EINVAL; + return ret; + } + + int desc_idx = fd; + ret = host_bind(host, desc_idx, converted_addr, converted_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_CANTBIND: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_BADFAM: + *host_errno_ptr(host) = EAFNOSUPPORT; + return -1; + case HOST_ERROR_NOTAVAIL: + *host_errno_ptr(host) = EADDRNOTAVAIL; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +int mock_listen(int fd, int backlog) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_listen() with no node scheduled\n"); + + int desc_idx = fd; + int ret = host_listen(host, desc_idx, backlog); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return 0; +} + +int mock_connect(int fd, void *addr, unsigned long addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_connect() with no node scheduled\n"); + + Addr converted_addr; + uint16_t converted_port; + int ret = convert_addr(addr, addr_len, &converted_addr, &converted_port); + if (ret < 0) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // TODO: connect() operations are only allowed on non-blocking + // sockets + + int desc_idx = fd; + ret = host_connect(host, desc_idx, converted_addr, converted_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EISCONN; + return -1; + case HOST_ERROR_ADDRUSED: + *host_errno_ptr(host) = EADDRINUSE; + return -1; + default: + break; + } + *host_errno_ptr(host) = EINPROGRESS; + return -1; + } + + *host_errno_ptr(host) = EINPROGRESS; + return -1; +} + +static int convert_linux_open_flags_to_lfs(int flags) +{ + int lfs_flags = 0; + + // Convert access mode (lowest 2 bits) + // Linux: O_RDONLY=0, O_WRONLY=1, O_RDWR=2 + // LFS: LFS_O_RDONLY=1, LFS_O_WRONLY=2, LFS_O_RDWR=3 + int access_mode = flags & 3; + lfs_flags = access_mode + 1; + + // Convert other flags + if (flags & O_CREAT) + lfs_flags |= LFS_O_CREAT; + if (flags & O_EXCL) + lfs_flags |= LFS_O_EXCL; + if (flags & O_TRUNC) + lfs_flags |= LFS_O_TRUNC; + if (flags & O_APPEND) + lfs_flags |= LFS_O_APPEND; + + return lfs_flags; +} + +int mock_open(char *path, int flags, int mode) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_open() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_open() not from Linux\n"); + + int converted_flags = convert_linux_open_flags_to_lfs(flags); + + int ret = host_open_file(host, path, converted_flags); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + default: + break; + } + *host_errno_ptr(host) = ENOENT; + return -1; + } + int desc_idx = ret; + + return desc_idx; +} + +int mock_read(int fd, char *dst, int len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_read() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_read() not from Linux\n"); + + int ret = host_read(host, fd, dst, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = EISDIR; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_write(int fd, char *src, int len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_write() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_write() not from Linux\n"); + + int ret = host_write(host, fd, src, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_IO: + *host_errno_ptr(host) = EIO; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_recv(int fd, char *dst, int len, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_recv() with no node scheduled\n"); + + if (flags) + abort_("Call to mock_recv() with non-zero flags\n"); + + int ret = host_recv(host, fd, dst, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_NOTCONN: + *host_errno_ptr(host) = ENOTCONN; + return -1; + case HOST_ERROR_RESET: + *host_errno_ptr(host) = ECONNRESET; + return -1; + case HOST_ERROR_HANGUP: + *host_errno_ptr(host) = 0; + return 0; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + ASSERT(ret > 0); + return ret; +} + +int mock_send(int fd, char *src, int len, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_send() with no node scheduled\n"); + + if (flags) + abort_("Call to mock_send() with non-zero flags\n"); + + int ret = host_send(host, fd, src, len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_RESET: + *host_errno_ptr(host) = ECONNRESET; + return -1; + case HOST_ERROR_HANGUP: + *host_errno_ptr(host) = EPIPE; + return -1; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + + return ret; +} + +int mock_accept(int fd, void *addr, socklen_t *addr_len) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_accept() with no node scheduled\n"); + + Addr peer_addr; + uint16_t peer_port; + int ret = host_accept(host, fd, &peer_addr, &peer_port); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return -1; + case HOST_ERROR_NOTSOCK: + *host_errno_ptr(host) = ENOTSOCK; + return -1; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EINVAL; + return -1; + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return -1; + case HOST_ERROR_WOULDBLOCK: + *host_errno_ptr(host) = EAGAIN; + return -1; + default: + break; + } + *host_errno_ptr(host) = EIO; + return -1; + } + int new_fd = ret; + + // Fill in the address if provided + if (addr != NULL && addr_len != NULL) { + if (peer_addr.family == ADDR_FAMILY_IPV4) { + struct sockaddr_in *sin = addr; + if (*addr_len >= sizeof(struct sockaddr_in)) { + sin->sin_family = AF_INET; + sin->sin_port = peer_port; + memcpy(&sin->sin_addr, &peer_addr.ipv4, sizeof(peer_addr.ipv4)); + *addr_len = sizeof(struct sockaddr_in); + } + } else { + struct sockaddr_in6 *sin6 = addr; + if (*addr_len >= sizeof(struct sockaddr_in6)) { + sin6->sin6_family = AF_INET6; + sin6->sin6_port = peer_port; + memcpy(&sin6->sin6_addr, &peer_addr.ipv6, sizeof(peer_addr.ipv6)); + *addr_len = sizeof(struct sockaddr_in6); + } + } + } + + return new_fd; +} + +int mock_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen) +{ + if (level != SOL_SOCKET) + abort_("Call to mock_getsockopt() with level other than SOL_SOCKET\n"); + + if (optname != SO_ERROR) + abort_("Call to mock_getsockopt() with option other than SO_ERROR\n"); + + Host *host = host___; + if (host == NULL) + abort_("Call to mock_getsockopt() with no node scheduled\n"); + + ConnectStatus status; + int ret = host_connect_status(host, fd, &status); + if (ret < 0) { + TODO; + } + + int out; + switch (status) { + case CONNECT_STATUS_WAIT: + out = 0; + break; + case CONNECT_STATUS_DONE: + out = 0; + break; + case CONNECT_STATUS_RESET: + out = ECONNRESET; + break; + case CONNECT_STATUS_CLOSE: + out = ECONNRESET; + break; + case CONNECT_STATUS_NOHOST: + out = ETIMEDOUT; + break; + default: + UNREACHABLE; + break; + } + + if (*optlen < sizeof(out)) + memcpy(optval, &out, *optlen); + else { + memcpy(optval, &out, sizeof(out)); + *optlen = sizeof(out); + } + return 0; +} + +int mock_remove(char *path) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_remove() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_remove() not from Linux\n"); + + int ret = host_remove(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ENOTEMPTY; + break; + default: + *host_errno_ptr(host) = EIO; + break; + } + return -1; + } + + return 0; +} + +int mock_rename(char *oldpath, char *newpath) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_rename() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_rename() not from Linux\n"); + + int ret = host_rename(host, oldpath, newpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + break; + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ENOTEMPTY; + break; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = EISDIR; + break; + default: + *host_errno_ptr(host) = EIO; + break; + } + return -1; + } + + return 0; +} + +int mock_clock_gettime(clockid_t clockid, struct timespec *tp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_clock_gettime() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_clock_gettime() not from Linux\n"); + + if (tp == NULL) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // Both CLOCK_REALTIME and CLOCK_MONOTONIC use the same + // simulated time. In simulation, they're equivalent since + // we don't model wall-clock vs monotonic differences. + if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + // Get current time + Nanos now = host_time(host); + + // Convert nanoseconds to timespec + // 1 second = 1,000,000,000 nanoseconds + tp->tv_sec = (time_t) (now / 1000000000ULL); + tp->tv_nsec = (int64_t) (now % 1000000000ULL); + + return 0; +} + +int mock_flock(int fd, int op) +{ + // TODO + return 0; +} + +int mock_fsync(int fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fsync() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fsync() not from Linux\n"); + + int ret = host_fsync(host, fd); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) + *host_errno_ptr(host) = EBADF; + else + *host_errno_ptr(host) = EINVAL; + return -1; + } + + return 0; +} + +off_t mock_lseek(int fd, off_t offset, int whence) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_lseek() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_lseek() not from Linux\n"); + + // Convert POSIX whence to HOST whence + int host_whence; + switch (whence) { + case SEEK_SET: + host_whence = HOST_SEEK_SET; + break; + case SEEK_CUR: + host_whence = HOST_SEEK_CUR; + break; + case SEEK_END: + host_whence = HOST_SEEK_END; + break; + default: + *host_errno_ptr(host) = EINVAL; + return (off_t)-1; + } + + int ret = host_lseek(host, fd, offset, host_whence); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) + *host_errno_ptr(host) = EBADF; + else + *host_errno_ptr(host) = EINVAL; + return (off_t)-1; + } + + return (off_t)ret; +} + +int mock_fstat(int fd, struct stat *buf) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fstat() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fstat() not from Linux\n"); + + if (buf == NULL) { + *host_errno_ptr(host) = EINVAL; + return -1; + } + + FileInfo info; + int ret = host_fileinfo(host, fd, &info); + if (ret < 0) { + if (ret == HOST_ERROR_BADIDX) { + *host_errno_ptr(host) = EBADF; + } else { + *host_errno_ptr(host) = EIO; + } + return -1; + } + + memset(buf, 0, sizeof(*buf)); + + if (info.is_dir) { + buf->st_mode = S_IFDIR | 0755; // Directory with rwxr-xr-x permissions + buf->st_size = 0; + } else { + buf->st_mode = S_IFREG | 0644; // Regular file with rw-r--r-- permissions + buf->st_size = (off_t) info.size; + } + + return 0; +} + +int mock_mkstemp(char *path) +{ + TODO; +} + +char *mock_realpath(char *path, char *dst) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_realpath() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_realpath() not from Linux\n"); + + if (path == NULL) { + *host_errno_ptr(host) = EINVAL; + return NULL; + } + + // Temporary buffer for path normalization + char temp[4096]; + int temp_len = 0; + + // Copy path to temp + for (int i = 0; path[i] != '\0' && temp_len < (int)sizeof(temp) - 1; i++) { + temp[temp_len++] = path[i]; + } + temp[temp_len] = '\0'; + + // Result buffer for the normalized absolute path + char result[4096]; + int result_len = 0; + + // If path doesn't start with '/', prepend '/' (mock has no CWD, uses root) + const char *src = temp; + if (temp[0] != '/') { + result[result_len++] = '/'; + } + + // Parse path components and resolve . and .. + while (*src != '\0') { + // Skip consecutive slashes + while (*src == '/') src++; + + if (*src == '\0') break; + + // Find end of this component + const char *end = src; + while (*end != '\0' && *end != '/') end++; + + int comp_len = (int)(end - src); + + if (comp_len == 1 && src[0] == '.') { + // Current directory - skip it + } else if (comp_len == 2 && src[0] == '.' && src[1] == '.') { + // Parent directory - remove last component from result + if (result_len > 1) { + // Find the last slash before the current position + result_len--; // Move back from current position + while (result_len > 0 && result[result_len - 1] != '/') { + result_len--; + } + if (result_len == 0) { + result_len = 1; // Keep the root slash + } + } + } else { + // Regular component - add it + if (result_len > 1 || (result_len == 1 && result[0] != '/')) { + if (result_len < (int)sizeof(result) - 1) + result[result_len++] = '/'; + } + for (int i = 0; i < comp_len && result_len < (int)sizeof(result) - 1; i++) { + result[result_len++] = src[i]; + } + } + + src = end; + } + + // Ensure we have at least root + if (result_len == 0) { + result[result_len++] = '/'; + } + result[result_len] = '\0'; + + // Unlike _fullpath, realpath requires the path to exist + // Try to open as file first, then as directory + int fd = host_open_file(host, result, LFS_O_RDONLY); + if (fd >= 0) { + host_close(host, fd, false); + } else { + // Try as directory + fd = host_open_dir(host, result); + if (fd >= 0) { + host_close(host, fd, false); + } else { + // Path doesn't exist + *host_errno_ptr(host) = ENOENT; + return NULL; + } + } + + // Allocate buffer if dst is NULL + if (dst == NULL) { + dst = malloc(result_len + 1); + if (dst == NULL) { + *host_errno_ptr(host) = ENOMEM; + return NULL; + } + } + + // Copy result to destination + for (int i = 0; i <= result_len; i++) { + dst[i] = result[i]; + } + + return dst; +} + +int mock_mkdir(char *path, mode_t mode) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_mkdir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_mkdir() not from Linux\n"); + + // LittleFS doesn't use mode, but we accept it for API compatibility + (void) mode; + + int ret = host_mkdir(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + return -1; + case HOST_ERROR_NOENT: + // Parent directory doesn't exist + *host_errno_ptr(host) = ENOENT; + return -1; + default: + *host_errno_ptr(host) = EIO; + return -1; + } + } + + return 0; +} + +int mock_fcntl(int fd, int cmd, int flags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_fcntl() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_fcntl() not from Linux\n"); + + switch (cmd) { + + case F_GETFL: + { + int ret = host_getdescflags(host, fd); + if (ret < 0) { + *host_errno_ptr(host) = EBADF; + return -1; + } + + int flags = 0; + if (ret & HOST_FLAG_NONBLOCK) + flags |= O_NONBLOCK; + + return flags; + } + break; + + case F_SETFL: + { + int host_flags = 0; + if (flags & O_NONBLOCK) + host_flags |= HOST_FLAG_NONBLOCK; + + int ret = host_setdescflags(host, fd, host_flags); + + if (ret < 0) { + *host_errno_ptr(host) = EBADF; + return -1; + } + return 0; + } + break; + + default: + *host_errno_ptr(host) = EINVAL; + return -1; + } +} + +typedef struct { + int fd; // Descriptor index + struct dirent entry; // Current entry (returned by readdir) +} DIR_; + +DIR *mock_opendir(char *name) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_opendir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_opendir() not from Linux\n"); + + int ret = host_open_dir(host, name); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = EMFILE; + return NULL; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ENOENT; + return NULL; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = EIO; + return NULL; + } + } + + // Allocate DIR structure + DIR_ *dirp = malloc(sizeof(DIR_)); + if (dirp == NULL) { + // Close the descriptor since we can't return it + host_close(host, ret, false); + *host_errno_ptr(host) = EMFILE; + return NULL; + } + + dirp->fd = ret; + return (DIR*) dirp; +} + +struct dirent* mock_readdir(DIR *dirp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_readdir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_readdir() not from Linux\n"); + + DIR_ *dirp_ = (DIR_*) dirp; + + if (dirp_ == NULL) { + *host_errno_ptr(host) = EBADF; + return NULL; + } + + DirEntry entry; + int ret = host_read_dir(host, dirp_->fd, &entry); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + return NULL; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = EBADF; + return NULL; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = EIO; + return NULL; + } + } + + if (ret == 0) { + // End of directory - return NULL without setting errno + return NULL; + } + + // Copy to the DIR's entry buffer + int i = 0; + while (entry.name[i] != '\0' && i < 255) { + dirp_->entry.d_name[i] = entry.name[i]; + i++; + } + dirp_->entry.d_name[i] = '\0'; + dirp_->entry.d_type = entry.is_dir ? DT_DIR : DT_REG; + + return &dirp_->entry; +} + +int mock_closedir(DIR *dirp) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_closedir() with no node scheduled\n"); + + if (!host_is_linux(host)) + abort_("Call to mock_closedir() not from Linux\n"); + + DIR_ *dirp_ = (DIR_*) dirp; + + if (dirp_ == NULL) { + *host_errno_ptr(host) = EBADF; + return -1; + } + + int ret = host_close(host, dirp_->fd, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = EBADF; + free(dirp_); + return -1; + default: + *host_errno_ptr(host) = EIO; + free(dirp_); + return -1; + } + } + + free(dirp_); + return 0; +} + +#else + +int mock_GetLastError(void) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_GetLastError() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_GetLastError() not from Windows\n"); + + // Note that technically on windows errno and GetLastError + // are different things. Here we use errno_ to store the + // GetLastError value and assume the user will not access + // errno. + return *host_errno_ptr(host); +} + +int mock_WSAGetLastError(void) +{ + return mock_GetLastError(); +} + +void mock_SetLastError(int err) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_SetLastError() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_SetLastError() not from Windows\n"); + + *host_errno_ptr(host) = err; +} + +void mock_WSASetLastError(int err) +{ + return mock_SetLastError(err); +} + +int mock_closesocket(SOCKET fd) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_closesocket() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_closesocket() not from Windows\n"); + + int desc_idx = fd; + int ret = host_close(host, desc_idx, true); // expect_socket = true + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_NOTSOCK: + // Windows uses WSAGetLastError(), but for simplicity we just return error + return -1; + default: + break; + } + return -1; + } + + return 0; +} + +int mock_ioctlsocket(SOCKET fd, long cmd, unsigned long *argp) +{ + TODO; +} + +// Helper function to convert wide string to narrow string (ASCII subset) +static int wchar_to_char(WCHAR *src, char *dst, int dst_size) +{ + int i = 0; + while (src[i] != 0) { + if (i >= dst_size - 1) + return -1; // Buffer too small + if (src[i] > 127) + return -1; // Non-ASCII character + dst[i] = (char) src[i]; + i++; + } + dst[i] = '\0'; + return i; // Return length +} + +// Convert Windows access flags and creation disposition to LFS flags +static int convert_windows_flags_to_lfs(DWORD dwDesiredAccess, + DWORD dwCreationDisposition, bool *truncate) +{ + int lfs_flags = 0; + + // Convert access mode + if ((dwDesiredAccess & GENERIC_READ) && (dwDesiredAccess & GENERIC_WRITE)) + lfs_flags = LFS_O_RDWR; + else if (dwDesiredAccess & GENERIC_WRITE) + lfs_flags = LFS_O_WRONLY; + else + lfs_flags = LFS_O_RDONLY; + + *truncate = false; + + // Convert creation disposition + switch (dwCreationDisposition) { + case CREATE_NEW: + // Creates a new file, fails if file exists + lfs_flags |= LFS_O_CREAT | LFS_O_EXCL; + break; + case CREATE_ALWAYS: + // Creates a new file, always (truncates if exists) + lfs_flags |= LFS_O_CREAT | LFS_O_TRUNC; + *truncate = true; + break; + case OPEN_EXISTING: + // Opens file only if it exists, fails otherwise + // No extra flags needed - LFS will fail if file doesn't exist + break; + case OPEN_ALWAYS: + // Opens file if it exists, creates if it doesn't + lfs_flags |= LFS_O_CREAT; + break; + case TRUNCATE_EXISTING: + // Opens and truncates, fails if file doesn't exist + lfs_flags |= LFS_O_TRUNC; + *truncate = true; + break; + default: + return -1; // Invalid creation disposition + } + + return lfs_flags; +} + +HANDLE mock_CreateFileW(WCHAR *lpFileName, + DWORD dwDesiredAccess, DWORD dwShareMode, + LPSECURITY_ATTRIBUTES lpSecurityAttributes, + DWORD dwCreationDisposition, + DWORD dwFlagsAndAttributes, HANDLE hTemplateFile) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_CreateFileW() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_CreateFileW() not from Windows\n"); + + // lpSecurityAttributes and hTemplateFile are typically NULL + (void) lpSecurityAttributes; + (void) hTemplateFile; + (void) dwShareMode; // Share mode not implemented in simulation + (void) dwFlagsAndAttributes; // Attributes not implemented in simulation + + // Convert wide string path to narrow string + char path[MAX_PATH]; + if (wchar_to_char(lpFileName, path, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + // Convert Windows flags to LFS flags + bool truncate; + int lfs_flags = convert_windows_flags_to_lfs(dwDesiredAccess, dwCreationDisposition, &truncate); + if (lfs_flags < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + int ret = host_open_file(host, path, lfs_flags); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_EXISTS: + // CREATE_NEW with existing file + *host_errno_ptr(host) = ERROR_FILE_EXISTS; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + return INVALID_HANDLE_VALUE; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + int desc_idx = ret; + + // For OPEN_ALWAYS, if the file already existed, set ERROR_ALREADY_EXISTS + // (but still return success). This is Windows behavior. + if (dwCreationDisposition == OPEN_ALWAYS) { + // We can't easily detect this case here, so we skip it for now + // A full implementation would check if the file was newly created + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (HANDLE)(long long)desc_idx; +} + +BOOL mock_CloseHandle(HANDLE handle) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_CloseHandle() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_CloseHandle() not from Windows\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + // CloseHandle is for file handles, not sockets + // (sockets use closesocket on Windows) + int ret = host_close(host, desc_idx, false); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh) +{ + TODO; +} + +BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh) +{ + TODO; +} + +BOOL mock_FlushFileBuffers(HANDLE handle) +{ + TODO; +} + +BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_ReadFile() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_ReadFile() not from Windows\n"); + + // We don't support overlapped (async) I/O + if (ov != NULL) + abort_("Quakey does not support overlapped I/O in ReadFile\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (dst == NULL && len > 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + int ret = host_read(host, desc_idx, dst, (int)len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_BADARG: + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (num != NULL) + *num = (DWORD)ret; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_WriteFile() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_WriteFile() not from Windows\n"); + + // We don't support overlapped (async) I/O + if (ov != NULL) + abort_("Quakey does not support overlapped I/O in WriteFile\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (src == NULL && len > 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + int ret = host_write(host, desc_idx, src, (int)len); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (num != NULL) + *num = (DWORD)ret; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_SetFilePointer() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_SetFilePointer() not from Windows\n"); + + if (hFile == INVALID_HANDLE_VALUE || hFile == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_SET_FILE_POINTER; + } + + int desc_idx = (int)(long long)hFile; + + // Convert Windows move method to HOST whence + int host_whence; + switch (dwMoveMethod) { + case FILE_BEGIN: + host_whence = HOST_SEEK_SET; + break; + case FILE_CURRENT: + host_whence = HOST_SEEK_CUR; + break; + case FILE_END: + host_whence = HOST_SEEK_END; + break; + default: + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_SET_FILE_POINTER; + } + + // Build 64-bit offset + int64_t offset; + if (lpDistanceToMoveHigh != NULL) { + // 64-bit seek: combine high and low parts + offset = ((int64_t)(*lpDistanceToMoveHigh) << 32) | ((uint32_t)lDistanceToMove); + } else { + // 32-bit seek: use signed extension + offset = (int64_t)lDistanceToMove; + } + + int ret = host_lseek(host, desc_idx, offset, host_whence); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_SET_FILE_POINTER; + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_NEGATIVE_SEEK; + return INVALID_SET_FILE_POINTER; + default: + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_SET_FILE_POINTER; + } + } + + int64_t new_pos = (int64_t)ret; + + // Set high part if requested + if (lpDistanceToMoveHigh != NULL) + *lpDistanceToMoveHigh = (LONG)(new_pos >> 32); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (DWORD)(new_pos & 0xFFFFFFFF); +} + +BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_GetFileSizeEx() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_GetFileSizeEx() not from Windows\n"); + + if (handle == INVALID_HANDLE_VALUE || handle == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + if (buf == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + int desc_idx = (int)(long long)handle; + + FileInfo info; + int ret = host_fileinfo(host, desc_idx, &info); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + buf->QuadPart = (LONGLONG)info.size; + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_QueryPerformanceCounter() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_QueryPerformanceCounter() not from Windows\n"); + + if (lpPerformanceCount == NULL) + return 0; // FALSE + + // Get current time in nanoseconds and convert to performance counter units + // We use nanoseconds directly as the counter value (frequency = 1,000,000,000) + Nanos now = host_time(host); + lpPerformanceCount->QuadPart = (LONGLONG)now; + + return 1; // TRUE +} + +BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_QueryPerformanceFrequency() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_QueryPerformanceFrequency() not from Windows\n"); + + if (lpFrequency == NULL) + return 0; // FALSE + + // Frequency is 1 billion (nanoseconds per second) + // This matches our counter which counts in nanoseconds + lpFrequency->QuadPart = 1000000000LL; + + return 1; // TRUE +} + +char *mock__fullpath(char *path, char *dst, int cap) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock__fullpath() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock__fullpath() not from Windows\n"); + + if (path == NULL) { + *host_errno_ptr(host) = EINVAL; + return NULL; + } + + // Temporary buffer for path normalization + // We'll build the absolute path here + char temp[4096]; + int temp_len = 0; + + // Copy path to temp, converting backslashes to forward slashes + for (int i = 0; path[i] != '\0' && temp_len < (int)sizeof(temp) - 1; i++) { + if (path[i] == '\\') { + temp[temp_len++] = '/'; + } else { + temp[temp_len++] = path[i]; + } + } + temp[temp_len] = '\0'; + + // Result buffer for the normalized absolute path + char result[4096]; + int result_len = 0; + + // If path doesn't start with '/', prepend '/' (mock has no CWD, uses root) + const char *src = temp; + if (temp[0] != '/') { + result[result_len++] = '/'; + } + + // Parse path components and resolve . and .. + while (*src != '\0') { + // Skip consecutive slashes + while (*src == '/') src++; + + if (*src == '\0') break; + + // Find end of this component + const char *end = src; + while (*end != '\0' && *end != '/') end++; + + int comp_len = (int)(end - src); + + if (comp_len == 1 && src[0] == '.') { + // Current directory - skip it + } else if (comp_len == 2 && src[0] == '.' && src[1] == '.') { + // Parent directory - remove last component from result + if (result_len > 1) { + // Find the last slash before the current position + result_len--; // Move back from current position + while (result_len > 0 && result[result_len - 1] != '/') { + result_len--; + } + if (result_len == 0) { + result_len = 1; // Keep the root slash + } + } + } else { + // Regular component - add it + if (result_len > 1 || (result_len == 1 && result[0] != '/')) { + if (result_len < (int)sizeof(result) - 1) + result[result_len++] = '/'; + } + for (int i = 0; i < comp_len && result_len < (int)sizeof(result) - 1; i++) { + result[result_len++] = src[i]; + } + } + + src = end; + } + + // Ensure we have at least root + if (result_len == 0) { + result[result_len++] = '/'; + } + result[result_len] = '\0'; + + // Allocate buffer if dst is NULL + if (dst == NULL) { + dst = malloc(result_len + 1); + if (dst == NULL) { + *host_errno_ptr(host) = ENOMEM; + return NULL; + } + } else { + // Check if result fits in the provided buffer + if (result_len + 1 > cap) { + *host_errno_ptr(host) = ERANGE; + return NULL; + } + } + + // Copy result to destination + for (int i = 0; i <= result_len; i++) { + dst[i] = result[i]; + } + + return dst; +} + +int mock__mkdir(char *path) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock__mkdir() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock__mkdir() not from Windows\n"); + + int ret = host_mkdir(host, path); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = EEXIST; + return -1; + case HOST_ERROR_NOENT: + // Parent directory doesn't exist + *host_errno_ptr(host) = ENOENT; + return -1; + default: + *host_errno_ptr(host) = EIO; + return -1; + } + } + + return 0; +} + +// Structure to track Windows find handle state +typedef struct { + int fd; // Descriptor index for the directory +} FindHandle; + +// Helper function to populate WIN32_FIND_DATAA from a DirEntry +static void populate_find_data(WIN32_FIND_DATAA *data, DirEntry *entry) +{ + // Clear the structure + for (int i = 0; i < (int)sizeof(WIN32_FIND_DATAA); i++) + ((char *)data)[i] = 0; + + // Set file attributes + data->dwFileAttributes = entry->is_dir ? FILE_ATTRIBUTE_DIRECTORY : FILE_ATTRIBUTE_NORMAL; + + // Copy filename + int i = 0; + while (entry->name[i] != '\0' && i < MAX_PATH - 1) { + data->cFileName[i] = entry->name[i]; + i++; + } + data->cFileName[i] = '\0'; +} + +HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindFirstFileA() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindFirstFileA() not from Windows\n"); + + if (lpFileName == NULL || lpFindFileData == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return INVALID_HANDLE_VALUE; + } + + // Extract directory path from the search pattern + // The pattern is typically "path\*" or "path\*.ext" + // We need to find the last path separator and extract the directory + char dirpath[MAX_PATH]; + int len = 0; + while (lpFileName[len] != '\0' && len < MAX_PATH - 1) { + dirpath[len] = lpFileName[len]; + len++; + } + dirpath[len] = '\0'; + + // Find the last path separator (either '/' or '\') + int last_sep = -1; + for (int i = 0; i < len; i++) { + if (dirpath[i] == '/' || dirpath[i] == '\\') + last_sep = i; + } + + // If we found a separator, truncate to get the directory path + // If the pattern is just "*", use "." as the directory + if (last_sep >= 0) { + dirpath[last_sep] = '\0'; + } else { + // No separator found - use current directory + dirpath[0] = '.'; + dirpath[1] = '\0'; + } + + // Open the directory + int ret = host_open_dir(host, dirpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_FULL: + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_PATH_NOT_FOUND; + return INVALID_HANDLE_VALUE; + case HOST_ERROR_IO: + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + // Allocate find handle structure + FindHandle *fh = malloc(sizeof(FindHandle)); + if (fh == NULL) { + host_close(host, ret, false); + *host_errno_ptr(host) = ERROR_NOT_ENOUGH_MEMORY; + return INVALID_HANDLE_VALUE; + } + fh->fd = ret; + + // Read the first entry + DirEntry entry; + int read_ret = host_read_dir(host, fh->fd, &entry); + if (read_ret < 0) { + host_close(host, fh->fd, false); + free(fh); + switch (read_ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return INVALID_HANDLE_VALUE; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return INVALID_HANDLE_VALUE; + } + } + + if (read_ret == 0) { + // Empty directory - no files found + host_close(host, fh->fd, false); + free(fh); + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + return INVALID_HANDLE_VALUE; + } + + // Populate the find data structure + populate_find_data(lpFindFileData, &entry); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return (HANDLE)fh; +} + +BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindNextFileA() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindNextFileA() not from Windows\n"); + + if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL || lpFindFileData == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + FindHandle *fh = (FindHandle *)hFindFile; + + // Read the next entry + DirEntry entry; + int ret = host_read_dir(host, fh->fd, &entry); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_BADIDX: + case HOST_ERROR_BADARG: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + if (ret == 0) { + // No more files + *host_errno_ptr(host) = ERROR_NO_MORE_FILES; + return 0; // FALSE + } + + // Populate the find data structure + populate_find_data(lpFindFileData, &entry); + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_FindClose(HANDLE hFindFile) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_FindClose() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_FindClose() not from Windows\n"); + + if (hFindFile == INVALID_HANDLE_VALUE || hFindFile == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + } + + FindHandle *fh = (FindHandle *)hFindFile; + + int ret = host_close(host, fh->fd, false); + if (ret < 0) { + free(fh); + switch (ret) { + case HOST_ERROR_BADIDX: + *host_errno_ptr(host) = ERROR_INVALID_HANDLE; + return 0; // FALSE + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + return 0; // FALSE + } + } + + free(fh); + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags) +{ + Host *host = host___; + if (host == NULL) + abort_("Call to mock_MoveFileExW() with no node scheduled\n"); + + if (!host_is_windows(host)) + abort_("Call to mock_MoveFileExW() not from Windows\n"); + + // Validate parameters + if (lpExistingFileName == NULL) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // lpNewFileName can be NULL only with MOVEFILE_DELAY_UNTIL_REBOOT + // (marks file for deletion on reboot), but we don't support that + if (lpNewFileName == NULL) { + if (dwFlags & MOVEFILE_DELAY_UNTIL_REBOOT) { + // We don't simulate reboot, so just succeed without doing anything + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE + } + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // Convert wide string paths to narrow strings + char oldpath[MAX_PATH]; + char newpath[MAX_PATH]; + + if (wchar_to_char(lpExistingFileName, oldpath, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + if (wchar_to_char(lpNewFileName, newpath, MAX_PATH) < 0) { + *host_errno_ptr(host) = ERROR_INVALID_PARAMETER; + return 0; // FALSE + } + + // If MOVEFILE_REPLACE_EXISTING is not set and destination exists, fail + // We need to check this before calling host_rename + if (!(dwFlags & MOVEFILE_REPLACE_EXISTING)) { + // Try to check if destination exists by attempting to open it + int check = host_open_file(host, newpath, LFS_O_RDONLY); + if (check >= 0) { + // File exists, close it and return error + host_close(host, check, false); + *host_errno_ptr(host) = ERROR_ALREADY_EXISTS; + return 0; // FALSE + } + } + + int ret = host_rename(host, oldpath, newpath); + if (ret < 0) { + switch (ret) { + case HOST_ERROR_NOENT: + *host_errno_ptr(host) = ERROR_FILE_NOT_FOUND; + break; + case HOST_ERROR_EXIST: + *host_errno_ptr(host) = ERROR_ALREADY_EXISTS; + break; + case HOST_ERROR_NOTEMPTY: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + case HOST_ERROR_ISDIR: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + default: + *host_errno_ptr(host) = ERROR_ACCESS_DENIED; + break; + } + return 0; // FALSE + } + + *host_errno_ptr(host) = ERROR_SUCCESS; + return 1; // TRUE +} + +#endif + +void *mock_malloc(size_t size) +{ + return malloc(size); +} + +void *mock_realloc(void *ptr, size_t size) +{ + return realloc(ptr, size); +} + +void mock_free(void *ptr) +{ + free(ptr); +} diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 5387a44..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Scripts - -This directory contains utility scripts for ToastyFS development and testing. - -## Cluster Demo Script - -The `cluster_demo.sh` script allows you to easily spawn and manage a ToastyFS cluster for demo and testing purposes. - -### Usage - -```bash -# Start a cluster with 3 chunk servers (default) -./scripts/cluster_demo.sh start - -# Start a cluster with a custom number of chunk servers -./scripts/cluster_demo.sh start 5 - -# Check cluster status -./scripts/cluster_demo.sh status - -# Stop the cluster -./scripts/cluster_demo.sh stop - -# Clean all cluster data and logs -./scripts/cluster_demo.sh clean -``` - -### What it does - -The script: -- Builds ToastyFS if needed -- Starts one metadata server (leader) on port 8080 -- Starts the specified number of chunk servers on ports 8081, 8082, etc. -- Stores all process IDs for easy management -- Captures logs to `cluster_logs/` directory -- Stores data in `cluster_data/` directory - -All servers run in the background, making it easy to test the cluster with client applications. - -## Branch Coverage Measurement - -### Prerequisites - -The HTML report generation requires `lcov`: - -```bash -sudo apt-get install lcov -``` - -## Usage - -### Quick text summary (5 second simulation) -```bash -make coverage-report -``` - -### HTML report with branch details (5 second simulation) -```bash -make coverage-html -``` - -### Custom duration -```bash -./scripts/measure_coverage.sh 10 # 10 second run, text output -./scripts/measure_coverage.sh 10 --html # 10 second run, HTML output -``` - -## What gets measured - -The coverage tool: -- Builds the test binary with coverage instrumentation (`--coverage` flag) -- Runs the random simulation for the specified duration -- Generates reports showing which branches were executed - -## Output - -**Text report**: Shows per-file and total branch coverage percentages - -**HTML report**: Interactive report generated by lcov/genhtml showing: -- Overall coverage summary -- Per-file coverage breakdown -- Source code with execution counts -- Branch coverage details - -The HTML report is generated in `coverage_report/index.html` diff --git a/scripts/cluster_demo.sh b/scripts/cluster_demo.sh deleted file mode 100755 index 644abe1..0000000 --- a/scripts/cluster_demo.sh +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash -# -# ToastyFS Cluster Demo Script -# =========================== -# This script allows you to easily spawn and manage a ToastyFS cluster for demo purposes. -# -# Usage: -# ./scripts/cluster_demo.sh start [num_chunk_servers] - Start a cluster -# ./scripts/cluster_demo.sh stop - Stop the cluster -# ./scripts/cluster_demo.sh status - Show cluster status -# ./scripts/cluster_demo.sh clean - Clean data and log files -# - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(dirname "$SCRIPT_DIR")" -PID_FILE="$PROJECT_DIR/.cluster_demo.pids" -LOG_DIR="$PROJECT_DIR/cluster_logs" -DATA_DIR="$PROJECT_DIR/cluster_data" - -METADATA_PORT=8080 -CHUNK_SERVER_BASE_PORT=8081 -WEB_SERVER_PORT=8090 - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -print_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -print_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -build_if_needed() { - local need_build=false - - if [ ! -f "$PROJECT_DIR/toastyfs.out" ] && [ ! -f "$PROJECT_DIR/toastyfs.exe" ]; then - need_build=true - fi - - if [ ! -f "$PROJECT_DIR/toastyfs_web.out" ] && [ ! -f "$PROJECT_DIR/toastyfs_web.exe" ]; then - need_build=true - fi - - if [ "$need_build" = true ]; then - print_info "Binaries not found. Building ToastyFS..." - cd "$PROJECT_DIR" - make - print_success "Build complete" - fi -} - -get_binary() { - if [ -f "$PROJECT_DIR/toastyfs.out" ]; then - echo "$PROJECT_DIR/toastyfs.out" - elif [ -f "$PROJECT_DIR/toastyfs.exe" ]; then - echo "$PROJECT_DIR/toastyfs.exe" - else - print_error "ToastyFS binary not found" - exit 1 - fi -} - -get_web_binary() { - if [ -f "$PROJECT_DIR/toastyfs_web.out" ]; then - echo "$PROJECT_DIR/toastyfs_web.out" - elif [ -f "$PROJECT_DIR/toastyfs_web.exe" ]; then - echo "$PROJECT_DIR/toastyfs_web.exe" - else - print_error "ToastyFS web binary not found" - exit 1 - fi -} - -start_cluster() { - local num_chunk_servers=${1:-3} - - if [ -f "$PID_FILE" ]; then - print_error "Cluster already running (PID file exists)" - print_info "Run '$0 stop' first or remove $PID_FILE if the cluster is not running" - exit 1 - fi - - build_if_needed - local binary=$(get_binary) - - # Create directories - mkdir -p "$LOG_DIR" - mkdir -p "$DATA_DIR" - - print_info "Starting ToastyFS cluster..." - print_info " Metadata server: 127.0.0.1:$METADATA_PORT" - print_info " Chunk servers: $num_chunk_servers" - print_info " Web server: 127.0.0.1:$WEB_SERVER_PORT" - echo - - # Start metadata server - print_info "Starting metadata server on port $METADATA_PORT..." - local metadata_wal="$DATA_DIR/metadata.wal" - "$binary" --leader \ - --addr 127.0.0.1 \ - --port $METADATA_PORT \ - --wal-file "$metadata_wal" \ - > "$LOG_DIR/metadata.log" 2>&1 & - - local metadata_pid=$! - echo "$metadata_pid" > "$PID_FILE" - print_success "Metadata server started (PID: $metadata_pid)" - - # Wait a bit for metadata server to start - sleep 1 - - # Start chunk servers - for i in $(seq 1 $num_chunk_servers); do - local port=$((CHUNK_SERVER_BASE_PORT + i - 1)) - local data_path="$DATA_DIR/chunk_server_$i/" - - print_info "Starting chunk server $i on port $port..." - mkdir -p "$data_path" - - "$binary" \ - --addr 127.0.0.1 \ - --port $port \ - --path "$data_path" \ - --remote-addr 127.0.0.1 \ - --remote-port $METADATA_PORT \ - > "$LOG_DIR/chunk_server_$i.log" 2>&1 & - - local chunk_pid=$! - echo "$chunk_pid" >> "$PID_FILE" - print_success "Chunk server $i started (PID: $chunk_pid)" - - # Small delay between starting servers - sleep 0.5 - done - - # Start web server - local web_binary=$(get_web_binary) - print_info "Starting web server on port $WEB_SERVER_PORT..." - "$web_binary" \ - --upstream-addr 127.0.0.1 \ - --upstream-port $METADATA_PORT \ - --local-addr 127.0.0.1 \ - --local-port $WEB_SERVER_PORT \ - > "$LOG_DIR/web_server.log" 2>&1 & - - local web_pid=$! - echo "$web_pid" >> "$PID_FILE" - print_success "Web server started (PID: $web_pid)" - - echo - print_success "Cluster started successfully!" - echo - print_info "Connect to the cluster using:" - print_info " ToastyFS Protocol: 127.0.0.1:$METADATA_PORT" - print_info " HTTP Interface: http://127.0.0.1:$WEB_SERVER_PORT" - echo - print_info "View logs at: $LOG_DIR/" - print_info "Data stored at: $DATA_DIR/" - echo - print_info "To stop the cluster, run: $0 stop" -} - -stop_cluster() { - if [ ! -f "$PID_FILE" ]; then - print_warning "No PID file found. Cluster may not be running." - exit 0 - fi - - print_info "Stopping ToastyFS cluster..." - - local stopped=0 - local failed=0 - - while IFS= read -r pid; do - if [ -n "$pid" ]; then - if kill -0 "$pid" 2>/dev/null; then - kill "$pid" 2>/dev/null && { - print_success "Stopped process $pid" - stopped=$((stopped + 1)) - } || { - print_error "Failed to stop process $pid" - failed=$((failed + 1)) - } - else - print_warning "Process $pid not running" - fi - fi - done < "$PID_FILE" - - rm -f "$PID_FILE" - - echo - print_success "Cluster stopped (Stopped: $stopped, Failed: $failed)" -} - -show_status() { - if [ ! -f "$PID_FILE" ]; then - print_info "Cluster is NOT running" - exit 0 - fi - - print_info "Cluster status:" - echo - - local running=0 - local not_running=0 - local line_num=0 - local total_lines=$(wc -l < "$PID_FILE") - - while IFS= read -r pid; do - if [ -n "$pid" ]; then - line_num=$((line_num + 1)) - local server_type="" - - if [ $line_num -eq 1 ]; then - server_type="Metadata server" - elif [ $line_num -eq $total_lines ]; then - server_type="Web server" - else - server_type="Chunk server $((line_num - 1))" - fi - - if kill -0 "$pid" 2>/dev/null; then - echo -e " ${GREEN}●${NC} $server_type (PID: $pid) - ${GREEN}running${NC}" - running=$((running + 1)) - else - echo -e " ${RED}●${NC} $server_type (PID: $pid) - ${RED}not running${NC}" - not_running=$((not_running + 1)) - fi - fi - done < "$PID_FILE" - - echo - if [ $not_running -eq 0 ]; then - print_success "All $running servers are running" - else - print_warning "$running running, $not_running not running" - fi - - if [ -d "$LOG_DIR" ]; then - echo - print_info "Log directory: $LOG_DIR" - print_info "Recent log files:" - ls -lht "$LOG_DIR" | head -n 6 | tail -n 5 | awk '{print " " $9 " (" $5 ")"}' - fi -} - -clean_data() { - if [ -f "$PID_FILE" ]; then - print_error "Cluster is running. Stop it first with: $0 stop" - exit 1 - fi - - print_warning "This will delete all cluster data and logs!" - read -p "Are you sure? (y/N) " -n 1 -r - echo - - if [[ $REPLY =~ ^[Yy]$ ]]; then - print_info "Cleaning data and logs..." - rm -rf "$DATA_DIR" - rm -rf "$LOG_DIR" - print_success "Cleaned successfully" - else - print_info "Cancelled" - fi -} - -# Main command dispatcher -case "${1:-}" in - start) - start_cluster "${2:-3}" - ;; - stop) - stop_cluster - ;; - status) - show_status - ;; - clean) - clean_data - ;; - *) - echo "ToastyFS Cluster Demo Script" - echo - echo "Usage:" - echo " $0 start [num_chunk_servers] - Start a cluster (default: 3 chunk servers)" - echo " $0 stop - Stop the cluster" - echo " $0 status - Show cluster status" - echo " $0 clean - Clean data and log files" - echo - echo "Examples:" - echo " $0 start # Start with 3 chunk servers" - echo " $0 start 5 # Start with 5 chunk servers" - echo " $0 status # Check if cluster is running" - echo " $0 stop # Stop all servers" - exit 1 - ;; -esac diff --git a/scripts/fix_line_endings.sh b/scripts/fix_line_endings.sh deleted file mode 100755 index d2af20f..0000000 --- a/scripts/fix_line_endings.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Fix line endings for coverage scripts - -echo "Fixing line endings for coverage scripts..." - -# Convert CRLF to LF -dos2unix scripts/measure_coverage.sh 2>/dev/null || sed -i 's/\r$//' scripts/measure_coverage.sh -dos2unix scripts/generate_coverage_html.sh 2>/dev/null || sed -i 's/\r$//' scripts/generate_coverage_html.sh - -# Ensure execute permissions -chmod +x scripts/measure_coverage.sh -chmod +x scripts/generate_coverage_html.sh - -echo "Done! Line endings fixed and execute permissions set." -echo "" -echo "You can now run:" -echo " make coverage-report" -echo " make coverage-html" diff --git a/scripts/generate_coverage_html.sh b/scripts/generate_coverage_html.sh deleted file mode 100755 index c29788b..0000000 --- a/scripts/generate_coverage_html.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Generate HTML coverage report using lcov - -set -e - -OUTPUT_DIR="coverage_report" - -# Check if lcov is installed -if ! command -v lcov &> /dev/null; then - echo "Error: lcov is not installed" - echo "Please install it with: sudo apt-get install lcov" - exit 1 -fi - -# Capture coverage data with branch coverage enabled -echo "Capturing coverage data..." -lcov --capture --directory . --output-file coverage.info --rc lcov_branch_coverage=1 - -# Filter out system headers if any exist -echo "Filtering coverage data..." -lcov --remove coverage.info '/usr/*' --output-file coverage.info --ignore-errors unused --rc lcov_branch_coverage=1 || cp coverage.info coverage.info.bak - -# Generate HTML report -echo "Generating HTML report..." -genhtml coverage.info --output-directory "$OUTPUT_DIR" --branch-coverage --rc lcov_branch_coverage=1 - -# Clean up -rm -f coverage.info - -echo "HTML report generated in $OUTPUT_DIR/" -echo "Open $OUTPUT_DIR/index.html in your browser" diff --git a/scripts/measure_coverage.sh b/scripts/measure_coverage.sh deleted file mode 100755 index e2755cf..0000000 --- a/scripts/measure_coverage.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -# Script to measure branch coverage of the random simulation - -set -e - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo -e "${BLUE}========================================${NC}" -echo -e "${BLUE}Branch Coverage Measurement Tool${NC}" -echo -e "${BLUE}========================================${NC}" -echo - -# Clean previous coverage data -echo -e "${YELLOW}Cleaning previous coverage data...${NC}" -rm -f src/*.gcda src/*.gcno *.gcda *.gcno -make clean > /dev/null 2>&1 - -# Build with coverage -echo -e "${YELLOW}Building with coverage instrumentation...${NC}" -make coverage - -# Run the simulation for a limited time -echo -e "${YELLOW}Running simulation...${NC}" -SIMULATION_TIME=${1:-5} # Default to 5 seconds if not specified -echo "Running for ${SIMULATION_TIME} seconds..." - -# Run simulation in background and kill it after specified time -timeout ${SIMULATION_TIME}s ./toastyfs_random_test_coverage.out || true - -# Generate coverage reports -echo -echo -e "${YELLOW}Generating coverage reports...${NC}" - -# Find all .gcda files and generate coverage reports -GCDA_FILES=$(find . -name "*.gcda") - -TOTAL_BRANCHES=0 -TAKEN_BRANCHES=0 - -# Generate gcov reports from the coverage data files -for gcda_file in $GCDA_FILES; do - # Extract the source file name from the gcda filename - # Files are named like: toastyfs_random_test_coverage.out-basic.gcda - basename=$(basename "$gcda_file" .gcda) - source_name=${basename#*-} # Remove prefix up to and including '-' - - # Run gcov to generate the .gcov file - gcov -b "$gcda_file" > /dev/null 2>&1 || true -done - -# Parse gcov output to count branches -echo -echo -e "${BLUE}========================================${NC}" -echo -e "${BLUE}Branch Coverage Summary${NC}" -echo -e "${BLUE}========================================${NC}" -echo - -# Process each .gcov file -for gcov_file in *.c.gcov; do - if [ -f "$gcov_file" ]; then - # Extract branch statistics from the gcov file - branches=$(grep -c "^branch" "$gcov_file" 2>/dev/null) - if [ -z "$branches" ]; then branches=0; fi - - if [ "$branches" -gt 0 ]; then - taken_count=$(grep "^branch" "$gcov_file" 2>/dev/null | grep -c "taken [1-9]" 2>/dev/null || echo "0") - # Clean up the value - remove any whitespace/newlines - taken_count=$(echo "$taken_count" | tr -d '[:space:]') - if [ -z "$taken_count" ] || [ "$taken_count" = "" ]; then taken_count=0; fi - - TOTAL_BRANCHES=$((TOTAL_BRANCHES + branches)) - TAKEN_BRANCHES=$((TAKEN_BRANCHES + taken_count)) - percentage=$((taken_count * 100 / branches)) - filename=$(echo "$gcov_file" | sed 's/.gcov$//') - printf "%-40s %5d / %5d branches (%3d%%)\n" "$filename" "$taken_count" "$branches" "$percentage" - fi - fi -done - -echo -echo -e "${BLUE}========================================${NC}" -if [ "$TOTAL_BRANCHES" -gt 0 ]; then - COVERAGE_PERCENT=$((TAKEN_BRANCHES * 100 / TOTAL_BRANCHES)) - echo -e "${GREEN}Total: $TAKEN_BRANCHES / $TOTAL_BRANCHES branches reached (${COVERAGE_PERCENT}%)${NC}" -else - echo -e "${YELLOW}No branch coverage data found${NC}" -fi -echo -e "${BLUE}========================================${NC}" -echo - -# Generate HTML report if requested -if [ "$2" == "--html" ]; then - echo -e "${YELLOW}Generating HTML coverage report...${NC}" - # Get the directory where this script is located - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - "$SCRIPT_DIR/generate_coverage_html.sh" - echo -e "${GREEN}HTML report generated in coverage_report/index.html${NC}" - echo "Open with: firefox coverage_report/index.html (or your preferred browser)" - echo -fi - -# Clean up gcov files unless HTML was requested -if [ "$2" != "--html" ] && [ "$2" != "--detailed" ]; then - echo -e "${YELLOW}Cleaning up coverage files...${NC}" - rm -f *.gcov -fi - -echo -echo -e "${GREEN}Coverage measurement complete!${NC}" diff --git a/src/basic.c b/src/basic.c index 709ce55..364e46e 100644 --- a/src/basic.c +++ b/src/basic.c @@ -1,7 +1,9 @@ -#include - +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif +#include +#include #include "basic.h" -#include "system.h" bool streq(string s1, string s2) { @@ -24,10 +26,10 @@ Time get_current_time(void) int64_t freq; int ok; - ok = sys_QueryPerformanceCounter((LARGE_INTEGER*) &count); + ok = QueryPerformanceCounter((LARGE_INTEGER*) &count); if (!ok) return INVALID_TIME; - ok = sys_QueryPerformanceFrequency((LARGE_INTEGER*) &freq); + ok = QueryPerformanceFrequency((LARGE_INTEGER*) &freq); if (!ok) return INVALID_TIME; uint64_t res = 1000000000 * (double) count / freq; @@ -37,7 +39,7 @@ Time get_current_time(void) { struct timespec time; - if (sys_clock_gettime(CLOCK_REALTIME, &time)) + if (clock_gettime(CLOCK_REALTIME, &time)) return INVALID_TIME; uint64_t res; @@ -99,11 +101,17 @@ int getargi(int argc, char **argv, char *name, int fallback) if (i == argc) break; - int tmp = atoi(argv[i]); - if (tmp == 0 && argv[i][0] != '0') // best effort + errno = 0; + char *end; + long val = strtol(argv[i], &end, 10); + + if (end == argv[i] || *end != '\0' || errno == ERANGE) break; - return tmp; + if (val < INT_MIN || val > INT_MAX) + break; + + return (int) val; } return fallback; } diff --git a/src/byte_queue.c b/src/byte_queue.c index caf2dc6..f382a23 100644 --- a/src/byte_queue.c +++ b/src/byte_queue.c @@ -1,8 +1,11 @@ -#include -#include -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include -#include "system.h" #include "byte_queue.h" // This is the implementation of a byte queue useful @@ -32,12 +35,12 @@ void byte_queue_free(ByteQueue *queue) { if (queue->read_target) { if (queue->read_target != queue->data) - sys_free(queue->read_target); + free(queue->read_target); queue->read_target = NULL; queue->read_target_size = 0; } - sys_free(queue->data); + free(queue->data); queue->data = NULL; } @@ -99,7 +102,7 @@ void byte_queue_read_ack(ByteQueue *queue, uint32_t num) if (queue->read_target) { if (queue->read_target != queue->data) - sys_free(queue->read_target); + free(queue->read_target); queue->read_target = NULL; queue->read_target_size = 0; } @@ -224,7 +227,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap) if (size > queue->limit) size = queue->limit; - uint8_t *data = sys_malloc(size); + uint8_t *data = malloc(size); if (!data) { queue->flags |= BYTE_QUEUE_ERROR; return 0; @@ -234,7 +237,7 @@ int byte_queue_write_setmincap(ByteQueue *queue, uint32_t mincap) memcpy(data, queue->data + queue->head, queue->used); if (queue->read_target != queue->data) - sys_free(queue->data); + free(queue->data); queue->data = data; queue->head = 0; diff --git a/src/byte_queue.h b/src/byte_queue.h index 93649c8..5330dce 100644 --- a/src/byte_queue.h +++ b/src/byte_queue.h @@ -1,6 +1,8 @@ #ifndef BYTE_QUEUE_INCLUDED #define BYTE_QUEUE_INCLUDED +#include + #include "basic.h" typedef struct { diff --git a/src/chunk_server.c b/src/chunk_server.c index 5d543f2..2ef1772 100644 --- a/src/chunk_server.c +++ b/src/chunk_server.c @@ -1,14 +1,18 @@ -#include -#include -#include -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include +#include +#include -#include "config.h" -#include "metadata_server.h" #include "sha256.h" -#include "message.h" #include "file_system.h" +#include "hash_set.h" +#include "config.h" #include "tcp.h" +#include "message.h" +#include "byte_queue.h" #include "chunk_server.h" static string hash2path(ChunkServer *state, SHA256 hash, char *out) @@ -99,12 +103,12 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk, return -1; if (patch_off > SIZE_MAX - patch.len) { - sys_free(data.ptr); + free(data.ptr); return -1; } if (patch_off + (size_t) patch.len > (size_t) data.len) { - sys_free(data.ptr); + free(data.ptr); return -1; } @@ -112,11 +116,11 @@ static int patch_chunk(ChunkServer *state, SHA256 target_chunk, ret = store_chunk(state, data, new_hash); if (ret < 0) { - sys_free(data.ptr); + free(data.ptr); return -1; } - sys_free(data.ptr); + free(data.ptr); return 0; } @@ -129,7 +133,7 @@ static void download_targets_init(DownloadTargets *targets) static void download_targets_free(DownloadTargets *targets) { - sys_free(targets->items); + free(targets->items); } static void download_targets_remove(DownloadTargets *targets, @@ -161,13 +165,13 @@ static int download_targets_push(DownloadTargets *targets, else new_capacity = 2 * targets->capacity; - DownloadTarget *new_items = sys_malloc(new_capacity * sizeof(DownloadTarget)); + DownloadTarget *new_items = malloc(new_capacity * sizeof(DownloadTarget)); if (new_items == NULL) return -1; if (targets->capacity > 0) { memcpy(new_items, targets->items, targets->count * sizeof(targets->items[0])); - sys_free(targets->items); + free(targets->items); } targets->items = new_items; @@ -563,7 +567,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg) if (binary_read(&reader, NULL, 1)) return send_error(&state->tcp, conn_idx, true, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Invalid message")); - char *mem = sys_malloc(chunk_size); + char *mem = malloc(chunk_size); if (mem == NULL) return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("Out of memory")); @@ -579,7 +583,7 @@ process_client_create_chunk(ChunkServer *state, int conn_idx, ByteView msg) SHA256 dummy; int ret = store_chunk(state, (string) { mem, chunk_size }, &dummy); - sys_free(mem); + free(mem); if (ret < 0) return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_CREATE_CHUNK_ERROR, S("I/O error")); @@ -694,7 +698,7 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg) string slice; if (full == 0) { if (target_off >= (size_t) data.len || target_len > (size_t) data.len - target_off) { - sys_free(data.ptr); + free(data.ptr); return send_error(&state->tcp, conn_idx, false, MESSAGE_TYPE_DOWNLOAD_CHUNK_ERROR, S("Invalid range")); } slice = (string) { data.ptr + target_off, target_len }; @@ -710,11 +714,11 @@ process_client_download_chunk(ChunkServer *state, int conn_idx, ByteView msg) message_write(&writer, &target_len, sizeof(target_len)); message_write(&writer, slice.ptr, slice.len); if (!message_writer_free(&writer)) { - sys_free(data.ptr); + free(data.ptr); return -1; } - sys_free(data.ptr); + free(data.ptr); return 0; } @@ -812,8 +816,12 @@ static int send_sync_message(ChunkServer *state) return 0; } -int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout) +int chunk_server_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) { + ChunkServer *state = state_; + string addr = getargs(argc, argv, "--addr", "127.0.0.1"); int port = getargi(argc, argv, "--port", 8081); string path = getargs(argc, argv, "--path", "chunk_server_data/"); @@ -821,34 +829,45 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts string remote_addr = getargs(argc, argv, "--remote-addr", "127.0.0.1"); int remote_port = getargi(argc, argv, "--remote-port", 8080); - if (port <= 0 || port >= 1<<16) + if (port <= 0 || port >= 1<<16) { + fprintf(stderr, "chunk server :: Invalid port\n"); return -1; + } - if (remote_port <= 0 || remote_port >= 1<<16) + if (remote_port <= 0 || remote_port >= 1<<16) { + fprintf(stderr, "chunk server :: Invalid remote port\n"); return -1; + } Time current_time = get_current_time(); - if (current_time == INVALID_TIME) + if (current_time == INVALID_TIME) { + fprintf(stderr, "chunk server :: Couldn't read the time\n"); return -1; + } state->trace = trace; state->reconnect_delay = 1; // 1 second - if (tcp_context_init(&state->tcp) < 0) + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "chunk server :: Couldn't setup the TCP context\n"); return -1; + } int ret = tcp_listen(&state->tcp, addr, port); if (ret < 0) { + fprintf(stderr, "chunk server :: Couldn't setup the TCP listener\n"); tcp_context_free(&state->tcp); return -1; } if (create_dir(path) && errno != EEXIST) { + fprintf(stderr, "chunk server :: Couldn't create chunk folder\n"); tcp_context_free(&state->tcp); return -1; } if (get_full_path(path, state->path) < 0) { + fprintf(stderr, "chunk server :: Couldn't convert path to absolute\n"); tcp_context_free(&state->tcp); return -1; } @@ -861,6 +880,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts char tmp[1<<10]; if (addr.len >= (int) sizeof(tmp)) { + fprintf(stderr, "chunk server :: Address is too long\n"); tcp_context_free(&state->tcp); return -1; } @@ -868,6 +888,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts tmp[addr.len] = '\0'; state->local_addr.is_ipv4 = true; if (inet_pton(AF_INET, tmp, &state->local_addr.ipv4) != 1) { + fprintf(stderr, "chunk server :: Couldn't parse address\n"); tcp_context_free(&state->tcp); return -1; } @@ -876,6 +897,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts // Initialize metadata server address // // TODO: This should also support IPv6 if (remote_addr.len >= (int) sizeof(tmp)) { + fprintf(stderr, "chunk server :: Remote address is too long\n"); tcp_context_free(&state->tcp); return -1; } @@ -883,6 +905,7 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts tmp[remote_addr.len] = '\0'; state->remote_addr.is_ipv4 = true; if (inet_pton(AF_INET, tmp, &state->remote_addr.ipv4) != 1) { + fprintf(stderr, "chunk server :: Couldn't parse remote address\n"); tcp_context_free(&state->tcp); return -1; } @@ -904,23 +927,20 @@ int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts ); *timeout = 0; - return tcp_register_events(&state->tcp, contexts, polled); -} - -int chunk_server_free(ChunkServer *state) -{ - download_targets_free(&state->download_targets); - timed_hash_set_free(&state->cs_rem_list); - hash_set_free(&state->cs_lst_list); - hash_set_free(&state->cs_add_list); - tcp_context_free(&state->tcp); + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "chunk server :: Capacity isn't large enough\n"); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); return 0; } -int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout) +int chunk_server_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) { + ChunkServer *state = state_; Event events[TCP_EVENT_CAPACITY]; - int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled); + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); Time current_time = get_current_time(); if (current_time == INVALID_TIME) @@ -1031,5 +1051,19 @@ int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled } *timeout = deadline_to_timeout(deadline, current_time); - return tcp_register_events(&state->tcp, contexts, polled); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int chunk_server_free(void *state_) +{ + ChunkServer *state = state_; + download_targets_free(&state->download_targets); + timed_hash_set_free(&state->cs_rem_list); + hash_set_free(&state->cs_lst_list); + hash_set_free(&state->cs_add_list); + tcp_context_free(&state->tcp); + return 0; } diff --git a/src/chunk_server.h b/src/chunk_server.h index e91247e..b35f59d 100644 --- a/src/chunk_server.h +++ b/src/chunk_server.h @@ -1,12 +1,6 @@ #ifndef CHUNK_SERVER_INCLUDED #define CHUNK_SERVER_INCLUDED -#include - -#include "basic.h" -#include "metadata_server.h" -#include "tcp.h" - #define TAG_METADATA_SERVER 1 #define TAG_CHUNK_SERVER 2 @@ -57,8 +51,16 @@ typedef struct { } ChunkServer; -int chunk_server_init(ChunkServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout); -int chunk_server_free(ChunkServer *state); -int chunk_server_step(ChunkServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout); +struct pollfd; -#endif // CHUNK_SERVER_INCLUDED +int chunk_server_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int chunk_server_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int chunk_server_free(void *state); + +#endif // CHUNK_SERVER_INCLUDED \ No newline at end of file diff --git a/src/client.c b/src/client.c index ef55fe4..b5a0548 100644 --- a/src/client.c +++ b/src/client.c @@ -1,22 +1,24 @@ -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include #include -#include -#include +#include + +#ifdef _WIN32 +# define POLL WSAPoll +#else +# define POLL poll +#endif #ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#define POLL WSAPoll typedef CRITICAL_SECTION Mutex; #else -#include -#include -#define POLL poll typedef pthread_mutex_t Mutex; #endif #include "tcp.h" -#include "system.h" #include "config.h" #include "message.h" #include "file_tree.h" @@ -322,7 +324,7 @@ static int handle_to_operation(ToastyFS *toasty, ToastyHandle handle) ToastyFS *toasty_connect(ToastyString addr, uint16_t port) { - ToastyFS *toasty = sys_malloc(sizeof(ToastyFS)); + ToastyFS *toasty = malloc(sizeof(ToastyFS)); if (toasty == NULL) return NULL; @@ -330,7 +332,7 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port) { char tmp[128]; if (addr.len >= (int) sizeof(tmp)) { - sys_free(toasty); + free(toasty); return NULL; } memcpy(tmp, addr.ptr, addr.len); @@ -339,26 +341,26 @@ ToastyFS *toasty_connect(ToastyString addr, uint16_t port) addr2.is_ipv4 = true; addr2.port = port; if (inet_pton(AF_INET, tmp, &addr2.ipv4) != 1) { - sys_free(toasty); + free(toasty); return NULL; } } if (mutex_init(&toasty->mutex) < 0) { - sys_free(toasty); + free(toasty); return NULL; } if (tcp_context_init(&toasty->tcp) < 0) { mutex_free(&toasty->mutex); - sys_free(toasty); + free(toasty); return NULL; } if (tcp_connect(&toasty->tcp, addr2, TAG_METADATA_SERVER, NULL) < 0) { tcp_context_free(&toasty->tcp); mutex_free(&toasty->mutex); - sys_free(toasty); + free(toasty); return NULL; } @@ -435,7 +437,7 @@ void toasty_disconnect(ToastyFS *toasty) { tcp_context_free(&toasty->tcp); mutex_free(&toasty->mutex); - sys_free(toasty); + free(toasty); } static bool @@ -947,7 +949,7 @@ static void process_event_for_list(ToastyFS *toasty, return; } - ToastyListingEntry *entities = sys_malloc(item_count * sizeof(ToastyListingEntry)); + ToastyListingEntry *entities = malloc(item_count * sizeof(ToastyListingEntry)); if (entities == NULL) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; return; @@ -959,28 +961,28 @@ static void process_event_for_list(ToastyFS *toasty, uint64_t gen; if (!binary_read(&reader, &gen, sizeof(gen))) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } uint8_t is_dir; if (!binary_read(&reader, &is_dir, sizeof(is_dir))) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } uint16_t name_len; if (!binary_read(&reader, &name_len, sizeof(name_len))) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } char *name = (char*) reader.src + reader.cur; if (!binary_read(&reader, NULL, name_len)) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } @@ -989,7 +991,7 @@ static void process_event_for_list(ToastyFS *toasty, if (name_len > sizeof(entities[i].name)-1) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } memcpy(entities[i].name, name, name_len); @@ -999,7 +1001,7 @@ static void process_event_for_list(ToastyFS *toasty, // Check there is nothing else to read if (binary_read(&reader, NULL, 1)) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_LIST_ERROR, .user=toasty->operations[opidx].user }; - sys_free(entities); + free(entities); return; } @@ -1089,7 +1091,7 @@ static void process_event_for_read(ToastyFS *toasty, } // Allocate ranges - Range *ranges = sys_malloc(num_chunks_needed * sizeof(Range)); + Range *ranges = malloc(num_chunks_needed * sizeof(Range)); if (ranges == NULL) { toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; @@ -1104,7 +1106,7 @@ static void process_event_for_read(ToastyFS *toasty, // Read hash SHA256 hash; if (!binary_read(&reader, &hash, sizeof(hash))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1112,7 +1114,7 @@ static void process_event_for_read(ToastyFS *toasty, // Read number of servers uint32_t num_servers; if (!binary_read(&reader, &num_servers, sizeof(num_servers))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1120,7 +1122,7 @@ static void process_event_for_read(ToastyFS *toasty, // Parse IPv4 addresses uint32_t num_ipv4; if (!binary_read(&reader, &num_ipv4, sizeof(num_ipv4))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1134,7 +1136,7 @@ static void process_event_for_read(ToastyFS *toasty, uint16_t port; if (!binary_read(&reader, &ipv4, sizeof(ipv4)) || !binary_read(&reader, &port, sizeof(port))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1149,21 +1151,21 @@ static void process_event_for_read(ToastyFS *toasty, // Skip IPv6 addresses uint32_t num_ipv6; if (!binary_read(&reader, &num_ipv6, sizeof(num_ipv6))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } for (uint32_t j = 0; j < num_ipv6; j++) { if (!binary_read(&reader, NULL, sizeof(IPv6)) || !binary_read(&reader, NULL, sizeof(uint16_t))) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } } if (!found) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1208,7 +1210,7 @@ static void process_event_for_read(ToastyFS *toasty, Range *r = &ranges[0]; int cs_idx = get_chunk_server(toasty, &r->server_addr, 1, NULL); if (cs_idx < 0) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1216,7 +1218,7 @@ static void process_event_for_read(ToastyFS *toasty, if (send_download_chunk(toasty, cs_idx, r->hash, r->offset_within_chunk, r->length_within_chunk, opidx, 0) < 0) { - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_ERROR, .user=toasty->operations[opidx].user }; return; } @@ -1225,7 +1227,7 @@ static void process_event_for_read(ToastyFS *toasty, toasty->operations[opidx].ranges_head = 1; } else { // No chunks to download - sys_free(ranges); + free(ranges); toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; } @@ -1299,7 +1301,7 @@ static void process_event_for_read(ToastyFS *toasty, // Check if done if (toasty->operations[opidx].num_pending == 0) { - sys_free(toasty->operations[opidx].ranges); + free(toasty->operations[opidx].ranges); toasty->operations[opidx].ranges = NULL; toasty->operations[opidx].result = (ToastyResult) { .type=TOASTY_RESULT_READ_SUCCESS, .user=toasty->operations[opidx].user, .bytes_read=toasty->operations[opidx].actual_bytes }; } @@ -1424,7 +1426,7 @@ static int schedule_upload(ToastyFS *toasty, int opidx, UploadSchedule upload) else new_cap_uploads = 2 * o->cap_uploads; - UploadSchedule *uploads = sys_malloc(new_cap_uploads * sizeof(UploadSchedule)); + UploadSchedule *uploads = malloc(new_cap_uploads * sizeof(UploadSchedule)); if (uploads == NULL) return -1; @@ -1534,7 +1536,7 @@ static void process_event_for_write(ToastyFS *toasty, toasty->operations[opidx].num_chunks = num_all_hasehs; toasty->operations[opidx].num_hashes = num_hashes; // TODO: overflow - toasty->operations[opidx].hashes = sys_malloc(num_hashes * sizeof(SHA256)); + toasty->operations[opidx].hashes = malloc(num_hashes * sizeof(SHA256)); if (toasty->operations[opidx].hashes == NULL) { assert(0); // TODO } @@ -1957,7 +1959,7 @@ static void process_event_for_write(ToastyFS *toasty, } ChunkUploadResult; int num_upload_results = toasty->operations[opidx].num_chunks; - ChunkUploadResult *upload_results = sys_malloc(num_upload_results * sizeof(ChunkUploadResult)); + ChunkUploadResult *upload_results = malloc(num_upload_results * sizeof(ChunkUploadResult)); if (upload_results == NULL) { assert(0); // TODO } @@ -2471,7 +2473,7 @@ int toasty_list(ToastyFS *toasty, ToastyString path, void toasty_free_listing(ToastyListing *listing) { - sys_free(listing->items); + free(listing->items); } int toasty_read(ToastyFS *toasty, ToastyString path, diff --git a/src/crash_logger.c b/src/crash_logger.c deleted file mode 100644 index d9b4366..0000000 --- a/src/crash_logger.c +++ /dev/null @@ -1,433 +0,0 @@ -#if defined(__linux__) && defined(__x86_64__) - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct { - uint64_t base_addr; - int count; - void *symbols; - void *strings; -} SymbolTable; - -static int is_hex(char c) -{ - return (c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -static uint64_t query_base_addr(void) -{ - int fd = open("/proc/self/maps", O_RDONLY); - if (fd < 0) - return -1; - - char buf[128]; - int ret = read(fd, buf, sizeof(buf)); - if (ret < 0) { - close(fd); - return -1; - } - - close(fd); - - if (ret == 0 || !is_hex(buf[0])) - return -1; - - int i = 0; - uint64_t base_addr = 0; - for (;;) { - char c = buf[i++]; - - int d; - if (0) {} - else if (c >= 'a' && c <= 'f') d = c - 'a' + 10; - else if (c >= 'A' && c <= 'F') d = c - 'A' + 10; - else d = c - '0'; - - if (base_addr > (UINT64_MAX - d) / 16) - return -1; - base_addr = base_addr * 16 + d; - - if (i == ret) - return -1; - - if (buf[i] == '-') - break; - - if (!is_hex(buf[i])) - return -1; - } - - return base_addr; -} - -static int current_executable_path(char *dst, int cap) -{ - if (cap == 0) - return -1; - - int ret = readlink("/proc/self/exe", dst, cap-1); - if (ret < 0) - return -1; - dst[ret] = '\0'; - return ret; -} - -static int load_symbols_from_elf(void *src, int len, SymbolTable *st) -{ - // NOTE: It's assumed is properly aligned - assert(((uintptr_t) src & 15) == 0); - - // Check that the file contains a full header - if (len < (int) sizeof(Elf64_Ehdr)) - return -1; - Elf64_Ehdr *ehdr = (Elf64_Ehdr*) src; - - // Check that the file contains the full list - // of section headers - if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > len) - return -1; - Elf64_Shdr *shdrs = (Elf64_Shdr*) (src + ehdr->e_shoff); - - Elf64_Shdr *shstrtab_hdr = &shdrs[ehdr->e_shstrndx]; // TODO: bounds check - char *shstrtab = src + shstrtab_hdr->sh_offset; - - // Iterate over the section headers to find the - // one reative to symbols and their strings - Elf64_Shdr *symtab_hdr = NULL; - Elf64_Shdr *strtab_hdr = NULL; - - for (int i = 0; i < ehdr->e_shnum; i++) { - char *section_name = shstrtab + shdrs[i].sh_name; - if (0) {} - else if (!strcmp(section_name, ".symtab")) symtab_hdr = &shdrs[i]; - else if (!strcmp(section_name, ".strtab")) strtab_hdr = &shdrs[i]; - } - - if (symtab_hdr == NULL || strtab_hdr == NULL) { - return -1; - } - - void *mem = malloc(symtab_hdr->sh_size + strtab_hdr->sh_size); - if (mem == NULL) { - return -1; - } - - st->count = symtab_hdr->sh_size / sizeof(Elf64_Sym); - st->symbols = mem; - st->strings = (char*) st->symbols + symtab_hdr->sh_size; - - memcpy(st->symbols, src + symtab_hdr->sh_offset, symtab_hdr->sh_size); - memcpy(st->strings, src + strtab_hdr->sh_offset, strtab_hdr->sh_size); - - return 0; -} - -static char *read_file(char *path, int *len) -{ - int fd = open(path, O_RDONLY); - if (fd < 0) - return NULL; - - struct stat buf; - if (fstat(fd, &buf) < 0) { - close(fd); - return NULL; - } - *len = buf.st_size; - - char *ptr = malloc(*len + 1); - if (ptr == NULL) { - close(fd); - return NULL; - } - - for (int num = 0; num < *len; ) { - - int ret = read(fd, ptr + num, *len - num); - if (ret <= 0) { - free(ptr); - close(fd); - return NULL; - } - - num += ret; - } - - ptr[*len] = '\0'; - return ptr; -} - -static int symbol_table_from_current_process(SymbolTable *st) -{ - uint64_t base_addr = query_base_addr(); - if (base_addr == (uint64_t) -1) - return -1; - st->base_addr = base_addr; - - char path[1<<10]; - if (current_executable_path(path, sizeof(path)) < 0) - return -1; - - char *exe_ptr; - int exe_len; - exe_ptr = read_file(path, &exe_len); - if (exe_ptr == NULL) - return -1; - - if (load_symbols_from_elf(exe_ptr, exe_len, st) < 0) { - free(exe_ptr); - return -1; - } - - free(exe_ptr); - return 0; -} - -static void symbol_table_free(SymbolTable *st) -{ - free(st->symbols); -} - -static char *symbol_table_find(SymbolTable *st, uint64_t addr) -{ - for (int i = 0; i < st->count; i++) { - Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i; - - if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) - continue; - - if (sym->st_value == 0) - continue; - - uint64_t sym_beg = st->base_addr + sym->st_value; - uint64_t sym_end = st->base_addr + sym->st_value + sym->st_size; - - if (addr >= sym_beg && addr < sym_end) - return (char*) st->strings + sym->st_name; - } - - return NULL; -} - -#if 0 -static void symbol_table_dump(SymbolTable *st) -{ - for (int i = 0; i < st->count; i++) { - Elf64_Sym *sym = (Elf64_Sym*) st->symbols + i; - - if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) - continue; - - if (sym->st_value == 0) - continue; - - char *name = (char*) st->strings + sym->st_name; - printf("%s\n", name); - } -} -#endif - -typedef struct { - char *name; - uint64_t addr; -} StackFrame; - -static int walk_stack(uint64_t rip, uint64_t rbp, SymbolTable *st, StackFrame *frames, int max_frames) -{ - int frame_count = 0; - - if (frame_count < max_frames) { - frames[frame_count].addr = rip - st->base_addr; - frames[frame_count].name = symbol_table_find(st, rip); - frame_count++; - } - - while (rbp != 0) { - - if (rbp & 0xF) - break; - - uint64_t *frame_ptr = (uint64_t*) rbp; - - uint64_t next_rbp = frame_ptr[0]; - uint64_t return_addr = frame_ptr[1]; - - if (next_rbp != 0 && next_rbp <= rbp) - break; - - if (return_addr == 0) - break; - - if (frame_count == max_frames) - break; - frames[frame_count].addr = return_addr - st->base_addr; - frames[frame_count].name = symbol_table_find(st, return_addr); - frame_count++; - - rbp = next_rbp; - } - - return frame_count; -} - -static bool crash_logger_symbol_init = false; -static char* crash_logger_file_name = NULL; -static SymbolTable crash_logger_symbol_table; -static char* crash_logger_signal_stack; - -static void crash_handler(int sig, siginfo_t *info, void *ucontext) -{ - (void) info; - - if (crash_logger_symbol_init) { - - // Buffer for evaluating format strings - char tmp[1<<9]; - int len; - - ucontext_t *ctx = (ucontext_t*) ucontext; - uint64_t rip = ctx->uc_mcontext.gregs[REG_RIP]; - uint64_t rbp = ctx->uc_mcontext.gregs[REG_RBP]; - - StackFrame frames[64]; - int count = walk_stack(rip, rbp, &crash_logger_symbol_table, frames, 64); - - int fd = open(crash_logger_file_name, O_WRONLY | O_CREAT, 0666); - if (fd < 0) - exit(1); - - char *sig_name = ""; - switch (sig) { - case SIGSEGV: sig_name = "Segmentation fault"; break; - case SIGBUS : sig_name = "Bus error"; break; - case SIGILL : sig_name = "Illegal instruction"; break; - case SIGFPE : sig_name = "Floating point exception"; break; - case SIGTRAP: sig_name = "Trace trap"; break; - case SIGSYS : sig_name = "Bad system call"; break; - case SIGABRT: sig_name = "Abort"; break; - } - if (sig_name[0] == '\0') { - len = snprintf(tmp, sizeof(tmp), "(unknown signal %d)\n", sig); - write(fd, tmp, len); - } else { - write(fd, sig_name, strlen(sig_name)); - write(fd, "\n", 1); - } - - for (int i = 0; i < count; i++) { - len = snprintf(tmp, sizeof(tmp), " [%d] 0x%lx %s\n", i, frames[i].addr, - frames[i].name ? frames[i].name : "?"); - write(fd, tmp, len); - } - - close(fd); - } - exit(1); -} - -int crash_logger_init(char *file_name, int file_name_len) -{ - { - char *file_name_copy = malloc(file_name_len + 1); - if (file_name_copy == NULL) - return -1; - memcpy(file_name_copy, file_name, file_name_len); - file_name_copy[file_name_len] = '\0'; - crash_logger_file_name = file_name_copy; - } - - if (symbol_table_from_current_process(&crash_logger_symbol_table) < 0) { - free(crash_logger_file_name); - return -1; - } - - // Set up alternate signal stack - { - crash_logger_signal_stack = malloc(SIGSTKSZ); - if (crash_logger_signal_stack == NULL) { - symbol_table_free(&crash_logger_symbol_table); - free(crash_logger_file_name); - return -1; - } - - stack_t ss; - ss.ss_sp = crash_logger_signal_stack; - ss.ss_size = SIGSTKSZ; - ss.ss_flags = 0; - if (sigaltstack(&ss, NULL) < 0) { - free(crash_logger_signal_stack); - symbol_table_free(&crash_logger_symbol_table); - free(crash_logger_file_name); - return -1; - } - } - - { - // Register the crash handler - struct sigaction sa; - sa.sa_sigaction = crash_handler; - sa.sa_flags = SA_SIGINFO | SA_ONSTACK; // Add SA_ONSTACK flag - sigemptyset(&sa.sa_mask); - - // Memory errors - sigaction(SIGSEGV, &sa, NULL); // Segmentation fault (invalid memory access) - sigaction(SIGBUS, &sa, NULL); // Bus error (misaligned access, hardware error) - - // Execution errors - sigaction(SIGILL, &sa, NULL); // Illegal instruction - sigaction(SIGFPE, &sa, NULL); // Floating point exception - sigaction(SIGTRAP, &sa, NULL); // Trace trap - - // System/resource errors - sigaction(SIGSYS, &sa, NULL); // Bad system call - sigaction(SIGABRT, &sa, NULL); // Abort (from assert, abort(), etc.) - - // Optional: Resource limit violations - sigaction(SIGXCPU, &sa, NULL); // CPU time limit exceeded - sigaction(SIGXFSZ, &sa, NULL); // File size limit exceeded - } - - crash_logger_symbol_init = true; - return 0; -} - -void crash_logger_free(void) -{ - if (!crash_logger_symbol_init) - return; - - free(crash_logger_signal_stack); - symbol_table_free(&crash_logger_symbol_table); - free(crash_logger_file_name); - - crash_logger_symbol_init = false; -} - -#else - -static int crash_logger_init(char *file_name, int file_name_len) -{ - (void) file_name; - (void) file_name_len; - return -1; -} - -static void crash_logger_free(void) -{ -} - -#endif diff --git a/src/crash_logger.h b/src/crash_logger.h deleted file mode 100644 index 948b776..0000000 --- a/src/crash_logger.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef CRASH_LOGGER_INCLUDED -#define CRASH_LOGGER_INCLUDED - -int crash_logger_init(char *file_name, int file_name_len); -void crash_logger_free(void); - -#endif // CRASH_LOGGER_INCLUDED diff --git a/src/file_system.c b/src/file_system.c index 87b0686..63bb836 100644 --- a/src/file_system.c +++ b/src/file_system.c @@ -1,8 +1,9 @@ -#include -#include -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif +#include +#include -#include "system.h" #include "file_system.h" int rename_file_or_dir(string oldpath, string newpath); @@ -16,7 +17,7 @@ int file_open(string path, Handle *fd) memcpy(zt, path.ptr, path.len); zt[path.len] = '\0'; - int ret = sys_open(zt, O_RDWR | O_CREAT | O_APPEND, 0644); + int ret = open(zt, O_RDWR | O_CREAT | O_APPEND, 0644); if (ret < 0) return -1; @@ -29,7 +30,7 @@ int file_open(string path, Handle *fd) MultiByteToWideChar(CP_UTF8, 0, path.ptr, path.len, wpath, MAX_PATH); wpath[path.len] = L'\0'; - HANDLE h = sys_CreateFileW( + HANDLE h = CreateFileW( wpath, GENERIC_WRITE | GENERIC_READ, 0, @@ -49,18 +50,18 @@ int file_open(string path, Handle *fd) void file_close(Handle fd) { #ifdef __linux__ - sys_close((int) fd.data); + close((int) fd.data); #endif #ifdef _WIN32 - sys_CloseHandle((HANDLE) fd.data); + CloseHandle((HANDLE) fd.data); #endif } int file_set_offset(Handle fd, int off) { #ifdef __linux__ - off_t ret = sys_lseek((int) fd.data, off, SEEK_SET); + off_t ret = lseek((int) fd.data, off, SEEK_SET); if (ret < 0) return -1; return 0; @@ -69,8 +70,8 @@ int file_set_offset(Handle fd, int off) #ifdef _WIN32 LARGE_INTEGER distance; distance.QuadPart = off; - if (!sys_SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN)) - if (GetLastError() != NO_ERROR) + if (!SetFilePointer((HANDLE) fd.data, distance.LowPart, &distance.HighPart, FILE_BEGIN)) + if (GetLastError() != 0) return -1; return 0; #endif @@ -79,7 +80,7 @@ int file_set_offset(Handle fd, int off) int file_get_offset(Handle fd, int *off) { #ifdef __linux__ - off_t ret = sys_lseek((int) fd.data, 0, SEEK_CUR); + off_t ret = lseek((int) fd.data, 0, SEEK_CUR); if (ret < 0) return -1; *off = (int) ret; @@ -87,8 +88,8 @@ int file_get_offset(Handle fd, int *off) #endif #ifdef _WIN32 - DWORD pos = sys_SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT); - if (pos == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) + DWORD pos = SetFilePointer((HANDLE) fd.data, 0, NULL, FILE_CURRENT); + if (pos == INVALID_SET_FILE_POINTER && GetLastError() != 0) return -1; *off = (int) pos; return 0; @@ -98,13 +99,13 @@ int file_get_offset(Handle fd, int *off) int file_lock(Handle fd) { #ifdef __linux__ - if (sys_flock((int) fd.data, LOCK_EX) < 0) + if (flock((int) fd.data, LOCK_EX) < 0) return -1; return 0; #endif #ifdef _WIN32 - if (!sys_LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) + if (!LockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) return -1; return 0; #endif @@ -113,13 +114,13 @@ int file_lock(Handle fd) int file_unlock(Handle fd) { #ifdef __linux__ - if (sys_flock((int) fd.data, LOCK_UN) < 0) + if (flock((int) fd.data, LOCK_UN) < 0) return -1; return 0; #endif #ifdef _WIN32 - if (!sys_UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) + if (!UnlockFile((HANDLE) fd.data, 0, 0, MAXDWORD, MAXDWORD)) return -1; return 0; #endif @@ -128,13 +129,13 @@ int file_unlock(Handle fd) int file_sync(Handle fd) { #ifdef __linux__ - if (sys_fsync((int) fd.data) < 0) + if (fsync((int) fd.data) < 0) return -1; return 0; #endif #ifdef _WIN32 - if (!sys_FlushFileBuffers((HANDLE) fd.data)) + if (!FlushFileBuffers((HANDLE) fd.data)) return -1; return 0; #endif @@ -143,12 +144,12 @@ int file_sync(Handle fd) int file_read(Handle fd, char *dst, int max) { #ifdef __linux__ - return sys_read((int) fd.data, dst, max); + return read((int) fd.data, dst, max); #endif #ifdef _WIN32 DWORD num; - if (!sys_ReadFile((HANDLE) fd.data, dst, max, &num, NULL)) + if (!ReadFile((HANDLE) fd.data, dst, max, &num, NULL)) return -1; if (num > INT_MAX) return -1; @@ -159,12 +160,12 @@ int file_read(Handle fd, char *dst, int max) int file_write(Handle fd, char *src, int len) { #ifdef __linux__ - return sys_write((int) fd.data, src, len); + return write((int) fd.data, src, len); #endif #ifdef _WIN32 DWORD num; - if (!sys_WriteFile((HANDLE) fd.data, src, len, &num, NULL)) + if (!WriteFile((HANDLE) fd.data, src, len, &num, NULL)) return -1; if (num > INT_MAX) return -1; @@ -176,7 +177,7 @@ int file_size(Handle fd, size_t *len) { #ifdef __linux__ struct stat buf; - if (sys_fstat((int) fd.data, &buf) < 0) + if (fstat((int) fd.data, &buf) < 0) return -1; if (buf.st_size < 0 || (uint64_t) buf.st_size > SIZE_MAX) return -1; @@ -186,7 +187,7 @@ int file_size(Handle fd, size_t *len) #ifdef _WIN32 LARGE_INTEGER buf; - if (!sys_GetFileSizeEx((HANDLE) fd.data, &buf)) + if (!GetFileSizeEx((HANDLE) fd.data, &buf)) return -1; if (buf.QuadPart < 0 || (uint64_t) buf.QuadPart > SIZE_MAX) return -1; @@ -204,10 +205,10 @@ int create_dir(string path) zt[path.len] = '\0'; #ifdef _WIN32 - if (sys__mkdir(zt) < 0) + if (_mkdir(zt) < 0) return -1; #else - if (sys_mkdir(zt, 0766)) + if (mkdir(zt, 0766)) return -1; #endif @@ -228,7 +229,7 @@ int rename_file_or_dir(string oldpath, string newpath) memcpy(newpath_zt, newpath.ptr, newpath.len); newpath_zt[newpath.len] = '\0'; - if (sys_rename(oldpath_zt, newpath_zt)) + if (rename(oldpath_zt, newpath_zt)) return -1; return 0; } @@ -241,7 +242,7 @@ int remove_file_or_dir(string path) memcpy(path_zt, path.ptr, path.len); path_zt[path.len] = '\0'; - if (sys_remove(path_zt)) + if (remove(path_zt)) return -1; return 0; } @@ -255,12 +256,12 @@ int get_full_path(string path, char *dst) path_zt[path.len] = '\0'; #ifdef __linux__ - if (sys_realpath(path_zt, dst) == NULL) + if (realpath(path_zt, dst) == NULL) return -1; #endif #ifdef _WIN32 - if (sys__fullpath(path_zt, dst, PATH_MAX) == NULL) + if (_fullpath(path_zt, dst, PATH_MAX) == NULL) return -1; #endif @@ -285,7 +286,7 @@ int file_read_all(string path, string *data) return -1; } - char *dst = sys_malloc(len); + char *dst = malloc(len); if (dst == NULL) { file_close(fd); return -1; @@ -295,7 +296,7 @@ int file_read_all(string path, string *data) while ((size_t) copied < len) { ret = file_read(fd, dst + copied, len - copied); if (ret < 0) { - sys_free(dst); + free(dst); file_close(fd); return -1; } @@ -316,7 +317,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path) if (ret < 0 || ret >= (int) sizeof(pattern)) return -1; - scanner->handle = sys_FindFirstFileA(pattern, &scanner->find_data); + scanner->handle = FindFirstFileA(pattern, &scanner->find_data); if (scanner->handle == INVALID_HANDLE_VALUE) { if (GetLastError() == ERROR_FILE_NOT_FOUND) { scanner->done = true; @@ -336,7 +337,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name) return 1; if (!scanner->first) { - BOOL ok = sys_FindNextFileA(scanner->handle, &scanner->find_data); + BOOL ok = FindNextFileA(scanner->handle, &scanner->find_data); if (!ok) { scanner->done = true; if (GetLastError() == ERROR_NO_MORE_FILES) @@ -354,7 +355,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name) void directory_scanner_free(DirectoryScanner *scanner) { - sys_FindClose(scanner->handle); + FindClose(scanner->handle); } #else @@ -367,7 +368,7 @@ int directory_scanner_init(DirectoryScanner *scanner, string path) memcpy(path_copy, path.ptr, path.len); path_copy[path.len] = '\0'; - scanner->d = sys_opendir(path_copy); + scanner->d = opendir(path_copy); if (scanner->d == NULL) { scanner->done = true; return -1; @@ -382,7 +383,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name) if (scanner->done) return 1; - scanner->e = sys_readdir(scanner->d); + scanner->e = readdir(scanner->d); if (scanner->e == NULL) { scanner->done = true; return 1; @@ -394,7 +395,7 @@ int directory_scanner_next(DirectoryScanner *scanner, string *name) void directory_scanner_free(DirectoryScanner *scanner) { - sys_closedir(scanner->d); + closedir(scanner->d); } #endif diff --git a/src/file_system.h b/src/file_system.h index 894732d..d12ce5e 100644 --- a/src/file_system.h +++ b/src/file_system.h @@ -1,14 +1,12 @@ #ifndef FILE_SYSTEM_INCLUDED #define FILE_SYSTEM_INCLUDED -#include "basic.h" - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#else -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS #endif +#include +#include +#include "basic.h" typedef struct { uint64_t data; @@ -50,4 +48,4 @@ int directory_scanner_init(DirectoryScanner *scanner, string path); int directory_scanner_next(DirectoryScanner *scanner, string *name); void directory_scanner_free(DirectoryScanner *scanner); -#endif // FILE_SYSTEM_INCLUDED +#endif // FILE_SYSTEM_INCLUDED \ No newline at end of file diff --git a/src/file_tree.c b/src/file_tree.c index 244c4f7..6bf36b3 100644 --- a/src/file_tree.c +++ b/src/file_tree.c @@ -1,10 +1,12 @@ -#include +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include #include -#include -#include +#include #include "basic.h" -#include "system.h" #include "file_tree.h" static int parse_path(string path, string *comps, int max) @@ -102,7 +104,7 @@ static void dir_free(Dir *d) { for (uint64_t i = 0; i < d->num_children; i++) entity_free(&d->children[i]); - sys_free(d->children); + free(d->children); } static bool gen_match(uint64_t expected_gen, uint64_t entity_gen) @@ -158,7 +160,7 @@ static void file_init(File *f, uint64_t chunk_size) static void file_free(File *f) { - sys_free(f->chunks); + free(f->chunks); f->chunks = NULL; } @@ -312,14 +314,14 @@ int file_tree_create_entity(FileTree *ft, string path, if (new_max == 0) new_max = 8; - Entity *p = sys_malloc(sizeof(Entity) * new_max); + Entity *p = malloc(sizeof(Entity) * new_max); if (p == NULL) return FILETREE_NOMEM; for (uint64_t i = 0; i < d->num_children; i++) p[i] = d->children[i]; - sys_free(d->children); + free(d->children); d->children = p; d->max_children = new_max; } @@ -423,13 +425,13 @@ int file_tree_write( if (last_chunk_index >= f->num_chunks) { uint64_t old_num_chunks = f->num_chunks; - SHA256 *new_chunks = sys_malloc((last_chunk_index+1) * sizeof(SHA256)); + SHA256 *new_chunks = malloc((last_chunk_index+1) * sizeof(SHA256)); if (new_chunks == NULL) return FILETREE_NOMEM; if (f->chunks) { if (f->num_chunks > 0) memcpy(new_chunks, f->chunks, f->num_chunks * sizeof(SHA256)); - sys_free(f->chunks); + free(f->chunks); } f->chunks = new_chunks; f->num_chunks = last_chunk_index+1; @@ -678,13 +680,13 @@ int file_tree_serialize(FileTree *ft, int (*write_fn)(char*,int,void*), void *wr sc.write_data = write_data; sc.buffer_used = 0; sc.buffer_size = 1<<10; - sc.buffer = sys_malloc(sc.buffer_size); + sc.buffer = malloc(sc.buffer_size); sc.error = false; if (sc.buffer == NULL) sc.error = true; entity_serialize(&sc, &ft->root); sc_flush(&sc); - sys_free(sc.buffer); + free(sc.buffer); if (sc.error) return -1; return 0; @@ -748,7 +750,7 @@ static void file_deserialize(DeserializeContext *dc, File *f) dc_read_u64(dc, &f->num_chunks); dc_read_u64(dc, &f->file_size); - f->chunks = sys_malloc(f->num_chunks * sizeof(SHA256)); + f->chunks = malloc(f->num_chunks * sizeof(SHA256)); if (f->chunks == NULL) { assert(0); // TODO } @@ -764,7 +766,7 @@ static void dir_deserialize(DeserializeContext *dc, Dir *d) dc_read_u64(dc, &d->num_children); d->max_children = d->num_children; - d->children = sys_malloc(d->num_children * sizeof(Entity)); + d->children = malloc(d->num_children * sizeof(Entity)); if (d->children == NULL) { assert(0); // TODO } @@ -799,13 +801,13 @@ int file_tree_deserialize(FileTree *ft, int (*read_fn)(char*,int,void*), void *r dc.buffer_head = 0; dc.buffer_used = 0; dc.buffer_size = 1<<10; - dc.buffer = sys_malloc(dc.buffer_size); + dc.buffer = malloc(dc.buffer_size); dc.error = false; if (dc.buffer == NULL) dc.error = true; dc.total_read = 0; entity_deserialize(&dc, &ft->root); - sys_free(dc.buffer); + free(dc.buffer); if (dc.error) return -1; if (dc.total_read > INT_MAX) { diff --git a/src/hash_set.c b/src/hash_set.c index a2ea582..80b7ae2 100644 --- a/src/hash_set.c +++ b/src/hash_set.c @@ -1,7 +1,8 @@ -#include -#include - -#include "system.h" +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif +#include +#include #include "hash_set.h" void hash_set_init(HashSet *set) @@ -13,7 +14,7 @@ void hash_set_init(HashSet *set) void hash_set_free(HashSet *set) { - sys_free(set->items); + free(set->items); set->items = NULL; } @@ -40,7 +41,7 @@ int hash_set_insert(HashSet *set, SHA256 hash) else new_capacity = 2 * set->capacity; - SHA256 *new_items = sys_realloc(set->items, new_capacity * sizeof(SHA256)); + SHA256 *new_items = realloc(set->items, new_capacity * sizeof(SHA256)); if (new_items == NULL) return -1; @@ -109,7 +110,7 @@ void timed_hash_set_init(TimedHashSet *set) void timed_hash_set_free(TimedHashSet *set) { - sys_free(set->items); + free(set->items); set->items = NULL; } @@ -137,13 +138,13 @@ int timed_hash_set_insert(TimedHashSet *set, SHA256 hash, Time time) else new_capacity = 2 * set->capacity; - TimedHash *new_items = sys_malloc(new_capacity * sizeof(TimedHash)); + TimedHash *new_items = malloc(new_capacity * sizeof(TimedHash)); if (new_items == NULL) return -1; if (set->capacity > 0) { memcpy(new_items, set->items, set->count * sizeof(set->items[0])); - sys_free(set->items); + free(set->items); } set->items = new_items; diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..3172127 --- /dev/null +++ b/src/main.c @@ -0,0 +1,170 @@ +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#else +#define POLL_CAPACITY 1024 +#endif +#include +#include + +#include "metadata_server.h" +#include "chunk_server.h" +#include "random_client.h" + +#ifdef MAIN_METADATA_SERVER +int main(int argc, char **argv) +{ + int ret; + MetadataServer state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = metadata_server_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = metadata_server_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + metadata_server_free(&state); + return 0; +} +#endif + +#ifdef MAIN_CHUNK_SERVER +int main(int argc, char **argv) +{ + int ret; + ChunkServer state; + + void* poll_ctxs[POLL_CAPACITY]; + struct pollfd poll_array[POLL_CAPACITY]; + int poll_count; + int poll_timeout; + + ret = chunk_server_init( + &state, + argc, + argv, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + + for (;;) { + +#ifdef _WIN32 + WSAPoll(poll_array, poll_count, poll_timeout); +#else + poll(poll_array, poll_count, poll_timeout); +#endif + + ret = chunk_server_tick( + &state, + poll_ctxs, + poll_array, + POLL_CAPACITY, + &poll_count, + &poll_timeout + ); + if (ret < 0) + return -1; + } + + chunk_server_free(&state); + return 0; +} +#endif + +#ifdef MAIN_SIMULATION +int main(void) +{ + Quakey *quakey; + int ret = quakey_init(&quakey, 1); + if (ret < 0) + return -1; + + // Client + { + QuakeySpawn config = { + .state_size = sizeof(RandomClient), + .init_func = random_client_init, + .tick_func = random_client_tick, + .free_func = random_client_free, + .addrs = (char*[]) { "127.0.0.2" }, + .num_addrs = 1, + .disk_size = 1<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "cs --addr 127.0.0.2"); + } + + // Metadata Server + { + QuakeySpawn config = { + .state_size = sizeof(MetadataServer), + .init_func = metadata_server_init, + .tick_func = metadata_server_tick, + .free_func = metadata_server_free, + .addrs = (char*[]) { "127.0.0.3" }, + .num_addrs = 1, + .disk_size = 1<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "ms --addr 127.0.0.3"); + } + + // Chunk Server + { + QuakeySpawn config = { + .state_size = sizeof(ChunkServer), + .init_func = chunk_server_init, + .tick_func = chunk_server_tick, + .free_func = chunk_server_free, + .addrs = (char*[]) { "127.0.0.4" }, + .num_addrs = 1, + .disk_size = 1<<20, + .platform = QUAKEY_LINUX, + }; + quakey_spawn(quakey, config, "cs --addr 127.0.0.4"); + } + + for (;;) + quakey_schedule_one(quakey); + + quakey_free(quakey); + return 0; +} +#endif // MAIN_SIMULATION diff --git a/src/main_server.c b/src/main_server.c deleted file mode 100644 index 6c1dd1b..0000000 --- a/src/main_server.c +++ /dev/null @@ -1,132 +0,0 @@ -#ifdef BUILD_SERVER - -#include -#include - -#ifdef _WIN32 -#include -#define POLL WSAPoll -#else -#include -#define POLL poll -#endif - -#include "crash_logger.h" -#include "chunk_server.h" -#include "metadata_server.h" - -int metadata_server_main(int argc, char **argv) -{ - { - time_t now = time(NULL); - struct tm unpacked_now; -#ifdef _WIN32 - if (gmtime_s(&unpacked_now, &now) != 0) - return -1; -#else - if (gmtime_r(&now, &unpacked_now) == NULL) - return -1; -#endif - - char time[sizeof("YYYYMMDDthhmmssz")]; - int ret = strftime(time, sizeof(time), - "%Y%m%dT%H%M%SZ", &unpacked_now); - if (ret != sizeof(time)-1) - return -1; - - char path[1<<10]; - ret = snprintf(path, sizeof(path), "crash_%s_MS_%d.txt", time, getpid()); - if (ret < 0 || ret >= (int) sizeof(path)) { - return -1; - } - int path_len = ret; - - crash_logger_init(path, path_len); - } - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled; - int timeout = -1; - - MetadataServer state; - num_polled = metadata_server_init( - &state, argc, argv, contexts, polled, &timeout); - if (num_polled < 0) return -1; - - for (;;) { - POLL(polled, num_polled, timeout); - - timeout = -1; - num_polled = metadata_server_step( - &state, contexts, polled, num_polled, &timeout); - if (num_polled < 0) return -1; - } - metadata_server_free(&state); - crash_logger_free(); - return 0; -} - -int chunk_server_main(int argc, char **argv) -{ - { - time_t now = time(NULL); - struct tm unpacked_now; -#ifdef _WIN32 - if (gmtime_s(&unpacked_now, &now) != 0) - return -1; -#else - if (gmtime_r(&now, &unpacked_now) == NULL) - return -1; -#endif - - char time[sizeof("YYYYMMDDthhmmssz")]; - int ret = strftime(time, sizeof(time), - "%Y%m%dT%H%M%SZ", &unpacked_now); - if (ret != sizeof(time)-1) - return -1; - - char path[1<<10]; - ret = snprintf(path, sizeof(path), "crash_%s_CS_%d.txt", time, getpid()); - if (ret < 0 || ret >= (int) sizeof(path)) { - return -1; - } - int path_len = ret; - - crash_logger_init(path, path_len); - } - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled; - int timeout = -1; - - ChunkServer state; - num_polled = chunk_server_init( - &state, argc, argv, contexts, polled, &timeout); - if (num_polled < 0) return -1; - - for (;;) { - - POLL(polled, num_polled, timeout); - - timeout = -1; - num_polled = chunk_server_step( - &state, contexts, polled, num_polled, &timeout); - if (num_polled < 0) return -1; - } - - chunk_server_free(&state); - crash_logger_free(); - return 0; -} - -int main(int argc, char **argv) -{ - if (getargb(argc, argv, "--leader")) - return metadata_server_main(argc, argv); - else - return chunk_server_main(argc, argv); -} - -#endif diff --git a/src/main_test.c b/src/main_test.c deleted file mode 100644 index 17d0cfb..0000000 --- a/src/main_test.c +++ /dev/null @@ -1,57 +0,0 @@ -#ifdef BUILD_TEST - -#include -#include -#include - -#include "system.h" - -static sig_atomic_t simulation_should_stop = false; - -static void signal_handler(int signum) -{ - (void)signum; - simulation_should_stop = true; -} - -int main(int argc, char **argv) -{ - (void)argc; - (void)argv; - - // Set up signal handlers for clean shutdown - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - startup_simulation(2); - - // Spawn metadata server (leader) - spawn_simulated_process("--addr 127.0.0.1 --port 8080 --leader"); - - // Spawn chunk servers - spawn_simulated_process("--addr 127.0.0.1 --port 8081 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_0/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8082 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_1/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8083 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_2/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8084 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_3/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8085 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_4/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8086 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_5/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8087 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_6/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8088 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_7/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8089 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_8/"); - spawn_simulated_process("--addr 127.0.0.1 --port 8090 --remote-addr 127.0.0.1 --remote-port 8080 --path chunk_server_data_9/"); - - // Spawn simulation client - spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080"); - spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080"); - spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080"); - spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080"); - spawn_simulated_process("--client --remote-addr 127.0.0.1 --remote-port 8080"); - - while (!simulation_should_stop) - update_simulation(); - - cleanup_simulation(); - return 0; -} - -#endif diff --git a/src/message.c b/src/message.c index cd948c4..67b7d1c 100644 --- a/src/message.c +++ b/src/message.c @@ -1,11 +1,8 @@ -#include - -#ifdef _WIN32 -#include -#else -#include // inet_ntop +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS #endif - +#include +#include #include "message.h" bool binary_read(BinaryReader *reader, void *dst, int len) diff --git a/src/message.h b/src/message.h index 3740ad2..9d1c220 100644 --- a/src/message.h +++ b/src/message.h @@ -1,9 +1,11 @@ #ifndef MESSAGE_INCLUDED #define MESSAGE_INCLUDED -#include -#include - +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif +#include +#include #include "basic.h" #include "byte_queue.h" diff --git a/src/metadata_server.c b/src/metadata_server.c index b139140..8e2d6b9 100644 --- a/src/metadata_server.c +++ b/src/metadata_server.c @@ -1,8 +1,10 @@ -#define _GNU_SOURCE +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif -#include +#include +#include #include -#include #include "message.h" #include "metadata_server.h" @@ -1061,19 +1063,27 @@ static bool is_chunk_server_message_type(uint16_t type) return false; } -int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout) +int metadata_server_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) { + MetadataServer *state = state_; + string addr = getargs(argc, argv, "--addr", "127.0.0.1"); int port = getargi(argc, argv, "--port", 8080); bool trace = getargb(argc, argv, "--trace"); string wal_file = getargs(argc, argv, "--wal-file", "metadata.wal"); int wal_limit = getargi(argc, argv, "--wal-limit", 1000); // TODO: Choose a good default limit - if (port <= 0 || port >= 1<<16) + if (port <= 0 || port >= 1<<16) { + fprintf(stderr, "metadata server :: Invalid port\n"); return -1; + } - if (wal_limit < 0) + if (wal_limit < 0) { + fprintf(stderr, "metadata server :: Invalid WAL limit\n"); return -1; + } state->trace = trace; state->replication_factor = 3; // TODO: what about the REPLICATION_FACTOR macro? @@ -1084,22 +1094,27 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co for (int i = 0; i < MAX_CHUNK_SERVERS; i++) state->chunk_servers[i].used = false; - if (tcp_context_init(&state->tcp) < 0) + if (tcp_context_init(&state->tcp) < 0) { + fprintf(stderr, "metadata server :: Couldn't setup TCP context\n"); return -1; + } int ret = tcp_listen(&state->tcp, addr, port); if (ret < 0) { + fprintf(stderr, "metadata server :: Couldn't setup TCP listener\n"); tcp_context_free(&state->tcp); return -1; } ret = file_tree_init(&state->file_tree); if (ret < 0) { + fprintf(stderr, "metadata server :: Couldn't setup file tree\n"); tcp_context_free(&state->tcp); return -1; } if (wal_open(&state->wal, &state->file_tree, wal_file, wal_limit) < 0) { + fprintf(stderr, "metadata server :: Couldn't setup WAL\n"); assert(0); // TODO } @@ -1110,21 +1125,21 @@ int metadata_server_init(MetadataServer *state, int argc, char **argv, void **co ); *timeout = -1; // No timeout until we have chunk servers - return tcp_register_events(&state->tcp, contexts, polled); -} - -int metadata_server_free(MetadataServer *state) -{ - wal_close(&state->wal); - file_tree_free(&state->file_tree); - tcp_context_free(&state->tcp); + if (pcap < TCP_POLL_CAPACITY) { + fprintf(stderr, "metadata server :: Not enough poll() capacity (got %d, needed %d)\n", pcap, TCP_POLL_CAPACITY); + return -1; + } + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); return 0; } -int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout) +int metadata_server_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout) { + MetadataServer *state = state_; + Event events[TCP_EVENT_CAPACITY]; - int num_events = tcp_translate_events(&state->tcp, events, contexts, polled, num_polled); + int num_events = tcp_translate_events(&state->tcp, events, ctxs, pdata, *pnum); Time current_time = get_current_time(); if (current_time == INVALID_TIME) @@ -1243,5 +1258,18 @@ int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd * } *timeout = deadline_to_timeout(next_wakeup, current_time); - return tcp_register_events(&state->tcp, contexts, polled); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = tcp_register_events(&state->tcp, ctxs, pdata); + return 0; +} + +int metadata_server_free(void *state_) +{ + MetadataServer *state = state_; + + wal_close(&state->wal); + file_tree_free(&state->file_tree); + tcp_context_free(&state->tcp); + return 0; } diff --git a/src/metadata_server.h b/src/metadata_server.h index acf5c9f..dad2c68 100644 --- a/src/metadata_server.h +++ b/src/metadata_server.h @@ -52,8 +52,15 @@ typedef struct { } MetadataServer; -int metadata_server_init(MetadataServer *state, int argc, char **argv, void **contexts, struct pollfd *polled, int *timeout); -int metadata_server_free(MetadataServer *state); -int metadata_server_step(MetadataServer *state, void **contexts, struct pollfd *polled, int num_polled, int *timeout); +struct pollfd; -#endif // METADATA_SERVER_INCLUDED +int metadata_server_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int metadata_server_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int metadata_server_free(void *state); + +#endif // METADATA_SERVER_INCLUDED \ No newline at end of file diff --git a/src/simulation_client.c b/src/random_client.c similarity index 86% rename from src/simulation_client.c rename to src/random_client.c index a7ab665..4e1d549 100644 --- a/src/simulation_client.c +++ b/src/random_client.c @@ -1,12 +1,11 @@ -#ifdef BUILD_TEST +#ifdef MAIN_SIMULATION -#include -#include -#include +#define QUAKEY_ENABLE_MOCKS +#include #include -#include "simulation_client.h" #include "tcp.h" +#include "random_client.h" // Helper function to parse address and port from command line static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port) @@ -19,7 +18,18 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port if (!strcmp(argv[i], "--server") || !strcmp(argv[i], "-s")) { *addr = argv[i + 1]; if (i + 2 < argc) { - *port = (uint16_t)atoi(argv[i + 2]); + + errno = 0; + char *end; + long val = strtol(argv[i+2], &end, 10); + + if (end == argv[i+2] || *end != '\0' || errno == ERANGE) + break; + + if (val < 0 || val > UINT16_MAX) + break; + + *port = (uint16_t) val; return true; } } @@ -27,9 +37,12 @@ static bool parse_server_addr(int argc, char **argv, char **addr, uint16_t *port return true; } -int simulation_client_init(SimulationClient *client, int argc, char **argv, - void **contexts, struct pollfd *polled, int *timeout) +int random_client_init(void *state_, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout) { + RandomClient *client = state_; + char *addr; uint16_t port; parse_server_addr(argc, argv, &addr, &port); @@ -43,20 +56,26 @@ int simulation_client_init(SimulationClient *client, int argc, char **argv, printf("Client set up (remote=%s:%d)\n", addr, port); *timeout = 0; - return toasty_process_events(client->toasty, contexts, polled, 0); + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); + return 0; } static int random_in_range(int min, int max) { - uint64_t n = simulation_random_number(); + uint64_t n = quakey_random(); return min + n % (max - min + 1); } -int simulation_client_step(SimulationClient *client, void **contexts, - struct pollfd *polled, int num_polled, int *timeout) +int random_client_tick(void *state_, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, + int *timeout) { + RandomClient *client = state_; + // Process any pending events from the network and get new poll descriptors - num_polled = toasty_process_events(client->toasty, contexts, polled, num_polled); + *pnum = toasty_process_events(client->toasty, ctxs, pdata, *pnum); for (int i = 0; i < client->num_pending; i++) { @@ -241,12 +260,19 @@ int simulation_client_step(SimulationClient *client, void **contexts, *timeout = 10; else *timeout = -1; - return toasty_process_events(client->toasty, contexts, polled, 0); + + if (pcap < TCP_POLL_CAPACITY) + return -1; + *pnum = toasty_process_events(client->toasty, ctxs, pdata, 0); + return 0; } -void simulation_client_free(SimulationClient *client) +int random_client_free(void *state_) { + RandomClient *client = state_; + toasty_disconnect(client->toasty); + return 0; } -#endif // BUILD_TEST +#endif // MAIN_SIMULATION \ No newline at end of file diff --git a/src/random_client.h b/src/random_client.h new file mode 100644 index 0000000..157af23 --- /dev/null +++ b/src/random_client.h @@ -0,0 +1,39 @@ +#ifndef RANDOM_CLIENT_INCLUDED +#define RANDOM_CLIENT_INCLUDED + +#include "ToastyFS.h" + +#define MAX_PENDING_OPERATION 8 + +typedef enum { + PENDING_OPERATION_CREATE, + PENDING_OPERATION_DELETE, + PENDING_OPERATION_LIST, + PENDING_OPERATION_READ, + PENDING_OPERATION_WRITE, +} PendingOperationType; + +typedef struct { + PendingOperationType type; + ToastyHandle handle; + void *ptr; +} PendingOperation; + +typedef struct { + ToastyFS *toasty; + int num_pending; + PendingOperation pending[MAX_PENDING_OPERATION]; +} RandomClient; + +struct pollfd; + +int random_client_init(void *state, int argc, char **argv, + void **ctxs, struct pollfd *pdata, int pcap, int *pnum, + int *timeout); + +int random_client_tick(void *state, void **ctxs, + struct pollfd *pdata, int pcap, int *pnum, int *timeout); + +int random_client_free(void *state); + +#endif // RANDOM_CLIENT_INCLUDED \ No newline at end of file diff --git a/src/simulation_client.h b/src/simulation_client.h deleted file mode 100644 index d6a5c2d..0000000 --- a/src/simulation_client.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef SIMULATION_CLIENT_INCLUDED -#define SIMULATION_CLIENT_INCLUDED - -#include -#include - -#ifdef _WIN32 -#include -#else -#include -#endif - -#include "ToastyFS.h" - -#define MAX_PENDING_OPERATION 8 - -typedef enum { - PENDING_OPERATION_CREATE, - PENDING_OPERATION_DELETE, - PENDING_OPERATION_LIST, - PENDING_OPERATION_READ, - PENDING_OPERATION_WRITE, -} PendingOperationType; - -typedef struct { - PendingOperationType type; - ToastyHandle handle; - void *ptr; -} PendingOperation; - -typedef struct { - ToastyFS *toasty; - int num_pending; - PendingOperation pending[MAX_PENDING_OPERATION]; -} SimulationClient; - -int simulation_client_init(SimulationClient *client, int argc, char **argv, - void **contexts, struct pollfd *polled, int *timeout); -int simulation_client_step(SimulationClient *client, void **contexts, - struct pollfd *polled, int num_polled, int *timeout); -void simulation_client_free(SimulationClient *client); - -#endif // SIMULATION_CLIENT_INCLUDED diff --git a/src/system.c b/src/system.c deleted file mode 100644 index 8807fd3..0000000 --- a/src/system.c +++ /dev/null @@ -1,2060 +0,0 @@ -#include "basic.h" -#include "tcp.h" -#ifdef BUILD_TEST - -#include -#include -#include -#include - -#include "system.h" -#include "chunk_server.h" -#include "metadata_server.h" -#include "simulation_client.h" - -#ifdef _WIN32 -#define NATIVE_HANDLE HANDLE -#else -#define NATIVE_HANDLE int -#endif - -#define MAX_DESCRIPTORS 1024 -#define MAX_ALLOCATIONS 128 -#define MAX_PROCESSES 32 - -#define DATA_QUEUE_SIZE (1<<9) - -typedef struct Process Process; - -typedef enum { - DESC_EMPTY, - DESC_FILE, - DESC_SOCKET, - DESC_LISTEN_SOCKET, - DESC_CONNECTION_SOCKET, - DESC_DIRECTORY, -} DescriptorType; - -typedef enum { - DESC_ADDR_VOID, - DESC_ADDR_IPV4, - DESC_ADDR_IPV6, -} DescriptorAddressType; - -typedef struct { - DescriptorAddressType type; - union { - struct sockaddr_in ipv4; - struct sockaddr_in6 ipv6; - }; -} DescriptorAddress; - -typedef struct { - Process *process; - int descriptor_index; - uint32_t generation; -} DescriptorHandle; - -typedef struct { - int head; - int used; - int size; - DescriptorHandle *items; -} AcceptQueue; - -typedef struct { - int size; - int used; - char *data; -} DataQueue; - -typedef enum { - CONNECTION_DELAYED, - CONNECTION_QUEUED, - CONNECTION_ESTABLISHED, - CONNECTION_FAILED, -} ConnectionState; - -typedef struct { - - // ------ Common ---------------- - - DescriptorType type; - uint32_t generation; - - // ------ File ------------------ - - NATIVE_HANDLE real_fd; - - // ------ Directory ------------- - -#ifdef _WIN32 - HANDLE real_d; -#else - DIR *real_d; -#endif - - // ------ Socket ---------------- - - // Events reported by the last "poll" call - // for this descriptor - int events; - - // Events triggered since the last "poll" - // call. Note that these may include events - // not present in the "events" set. - int revents; - - // Context for this descriptor, set by the - // last "poll" call. - void *context; - - // Whether this socket is configured as non-blocking - bool is_nonblocking; - - // Address bound to this descriptor by the - // "bind" call. - DescriptorAddress address; - - // ------ Listen socket --------- - - AcceptQueue accept_queue; - - // ------ Connection socket ----- - - ConnectionState connection_state; - - // When QUEUED, this refers to the peer listener - // socket. When ESTABLISHED, this refers to the - // peer connection socket. - DescriptorHandle connection_peer; - - // Address of the last connect() call - // on this socket if it's still in the - // "DELAYED" state. - DescriptorAddress connect_address; - - // Error number when the connection is FAILED - int connect_errno; - - // Data written to this descriptor using "write" - // or "send". - DataQueue output_data; - - // ------------------------------ - -} Descriptor; - -typedef struct { - void *ptr; - size_t len; - char *file; - int line; -} Allocation; - -typedef enum { - PROCESS_TYPE_METADATA_SERVER, - PROCESS_TYPE_CHUNK_SERVER, - PROCESS_TYPE_CLIENT, -} ProcessType; - -struct Process { - - int num_desc; - Descriptor desc[MAX_DESCRIPTORS]; - - int num_allocs; - Allocation allocs[MAX_ALLOCATIONS]; - - Time wakeup_time; - - ProcessType type; - union { - ChunkServer chunk_server; - MetadataServer metadata_server; - SimulationClient simulation_client; - }; -}; - -static int num_processes = 0; -static Process *processes[MAX_PROCESSES]; -static Process *current_process = NULL; -static uint64_t current_time = 1; -static uint64_t seed; - -// Helper to set socket errors correctly on Windows vs Linux -#ifdef _WIN32 -#define SET_SOCKET_ERROR(err) WSASetLastError(err) -#define SOCKET_ERROR_WOULDBLOCK WSAEWOULDBLOCK -#define SOCKET_ERROR_AFNOSUPPORT WSAEAFNOSUPPORT -#define SOCKET_ERROR_MFILE WSAEMFILE -#define SOCKET_ERROR_BADF WSAEBADF -#define SOCKET_ERROR_NOTSOCK WSAENOTSOCK -#define SOCKET_ERROR_INVAL WSAEINVAL -#define SOCKET_ERROR_ADDRINUSE WSAEADDRINUSE -#define SOCKET_ERROR_DESTADDRREQ WSAEDESTADDRREQ -#define SOCKET_ERROR_CONNABORTED WSAECONNABORTED -#define SOCKET_ERROR_ISCONN WSAEISCONN -#define SOCKET_ERROR_INPROGRESS WSAEINPROGRESS -#define SOCKET_ERROR_OPNOTSUPP WSAEOPNOTSUPP -#define SOCKET_ERROR_NOTCONN WSAENOTCONN -#define SOCKET_ERROR_CONNRESET WSAECONNRESET -#define SOCKET_ERROR_PROTOOPT WSAENOPROTOOPT -#define SOCKET_ERROR_PIPE WSAESHUTDOWN // Closest to EPIPE on Windows -#else -#define SET_SOCKET_ERROR(err) (errno = (err)) -#define SOCKET_ERROR_WOULDBLOCK EWOULDBLOCK -#define SOCKET_ERROR_AFNOSUPPORT EAFNOSUPPORT -#define SOCKET_ERROR_MFILE EMFILE -#define SOCKET_ERROR_BADF EBADF -#define SOCKET_ERROR_NOTSOCK ENOTSOCK -#define SOCKET_ERROR_INVAL EINVAL -#define SOCKET_ERROR_ADDRINUSE EADDRINUSE -#define SOCKET_ERROR_DESTADDRREQ EDESTADDRREQ -#define SOCKET_ERROR_CONNABORTED ECONNABORTED -#define SOCKET_ERROR_ISCONN EISCONN -#define SOCKET_ERROR_INPROGRESS EINPROGRESS -#define SOCKET_ERROR_OPNOTSUPP EOPNOTSUPP -#define SOCKET_ERROR_NOTCONN ENOTCONN -#define SOCKET_ERROR_CONNRESET ECONNRESET -#define SOCKET_ERROR_PROTOOPT ENOPROTOOPT -#define SOCKET_ERROR_PIPE EPIPE -#endif - -static int count_non_empty_desc(Process *p) -{ - int n = 0; - for (int i = 0; i < MAX_DESCRIPTORS; i++) - if (p->desc[i].type != DESC_EMPTY) - n++; - return n; -} -#define CHECK_NON_EMPTY_DESC_INVARIANT assert(count_non_empty_desc(current_process) == current_process->num_desc); - -static void process_poll_array(Process *process, - void **contexts, struct pollfd *polled, int num_polled) -{ - for (int i = 0, j = 0; j < process->num_desc; i++) { - - assert(i < MAX_DESCRIPTORS); - Descriptor *desc = &process->desc[i]; - if (desc->type == DESC_EMPTY) - continue; - j++; - - desc->events = 0; - desc->revents = 0; - desc->context = NULL; - } - - for (int i = 0; i < num_polled; i++) { - - SOCKET fd = polled[i].fd; - if (fd == INVALID_SOCKET) - continue; - - int idx = (int) fd; - process->desc[idx].events = polled[i].events; - process->desc[idx].revents = 0; - process->desc[idx].context = contexts[i]; - } -} - -#define MAX_ARGS 128 - -static bool is_space(char c) -{ - return c == ' ' || c == '\t' || c == '\r' || c == '\n'; -} - -void startup_simulation(uint64_t seed_) -{ - seed = seed_; - if (seed == 0) - seed = 1; - - num_processes = 0; - current_process = NULL; -} - -uint64_t simulation_random_number(void) -{ - uint64_t x = seed; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - seed = x; - return x; -} - -int spawn_simulated_process(char *args) -{ - if (num_processes == MAX_PROCESSES) - return -1; - - char mem[1<<10]; - int args_len = strlen(args); - if (args_len >= (int) sizeof(mem)) - return -1; - memcpy(mem, args, args_len); - mem[args_len] = '\0'; - args = mem; - - int argc = 0; - char *argv[MAX_ARGS]; - for (int cur = 0;;) { - - while (cur < args_len && is_space(args[cur])) - cur++; - - if (cur == args_len || argc == MAX_ARGS) - break; - - argv[argc++] = args + cur; - - while (cur < args_len && !is_space(args[cur])) - cur++; - - args[cur] = '\0'; - if (cur < args_len) - cur++; - } - - bool leader = getargb(argc, argv, "--leader"); - bool client = getargb(argc, argv, "--client"); - - Process *process = malloc(sizeof(Process)); - if (process == NULL) - return -1; - - // Determine process type - if (client) { - process->type = PROCESS_TYPE_CLIENT; - } else if (leader) { - process->type = PROCESS_TYPE_METADATA_SERVER; - } else { - process->type = PROCESS_TYPE_CHUNK_SERVER; - } - - process->num_desc = 0; - process->num_allocs = 0; - - for (int i = 0; i < MAX_DESCRIPTORS; i++) { - process->desc[i].type = DESC_EMPTY; - process->desc[i].generation = 0; - } - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled; - int timeout = -1; - - current_process = process; - - switch (process->type) { - case PROCESS_TYPE_METADATA_SERVER: - num_polled = metadata_server_init(&process->metadata_server, argc, argv, contexts, polled, &timeout); - break; - case PROCESS_TYPE_CHUNK_SERVER: - num_polled = chunk_server_init(&process->chunk_server, argc, argv, contexts, polled, &timeout); - break; - case PROCESS_TYPE_CLIENT: - num_polled = simulation_client_init(&process->simulation_client, argc, argv, contexts, polled, &timeout); - break; - default: - num_polled = -1; - break; - } - - current_process = NULL; - if (num_polled < 0) { - assert(0); // TODO - } - - if (timeout < 0) { - process->wakeup_time = INVALID_TIME; - } else { - process->wakeup_time = current_time + timeout * 1000000; - } - - process_poll_array(process, contexts, polled, num_polled); - - processes[num_processes++] = process; - return 0; -} - -static void free_process(Process *process) -{ - switch (process->type) { - case PROCESS_TYPE_METADATA_SERVER: - metadata_server_free(&process->metadata_server); - break; - case PROCESS_TYPE_CHUNK_SERVER: - chunk_server_free(&process->chunk_server); - break; - case PROCESS_TYPE_CLIENT: - simulation_client_free(&process->simulation_client); - break; - } - free(process); -} - -void cleanup_simulation(void) -{ - for (int i = 0; i < num_processes; i++) { - current_process = processes[i]; - free_process(processes[i]); - current_process = NULL; - } -} - -static bool addr_eql_2(DescriptorAddress a, DescriptorAddress b) -{ - if (a.type != b.type) - return false; - - if (a.type == DESC_ADDR_IPV4) { - return a.ipv4.sin_family == b.ipv4.sin_family - && a.ipv4.sin_port == b.ipv4.sin_port - && !memcmp(&a.ipv4.sin_addr, &a.ipv4.sin_addr, sizeof(a.ipv4.sin_addr)); - } else { - return a.ipv6.sin6_family == b.ipv6.sin6_family - && a.ipv6.sin6_port == b.ipv6.sin6_port - && !memcmp(&a.ipv6.sin6_addr, &a.ipv6.sin6_addr, sizeof(a.ipv6.sin6_addr)); - } -} - -static bool find_peer_by_address(DescriptorAddress address, DescriptorHandle *handle) -{ - for (int i = 0; i < num_processes; i++) { - - for (int j = 0, k = 0; k < processes[i]->num_desc; j++) { - - assert(j < MAX_DESCRIPTORS); - Descriptor *desc = &processes[i]->desc[j]; - if (desc->type == DESC_EMPTY) - continue; - k++; - - if (desc->type == DESC_LISTEN_SOCKET && - addr_eql_2(address, desc->address)) { - *handle = (DescriptorHandle) { processes[i], j, desc->generation }; - return true; - } - } - } - - return false; -} - -static Descriptor *handle_to_desc(DescriptorHandle handle) -{ - if (handle.process == NULL - || handle.descriptor_index < 0 - || handle.descriptor_index >= MAX_DESCRIPTORS) - return NULL; - Process *process = handle.process; - Descriptor *desc = &process->desc[handle.descriptor_index]; - if (desc->type == DESC_EMPTY || desc->generation != handle.generation) - return NULL; - return desc; -} - -static void accept_queue_init(AcceptQueue *accept_queue, int size) -{ - accept_queue->head = 0; - accept_queue->used = 0; - accept_queue->size = size; - accept_queue->items = malloc(size * sizeof(DescriptorHandle)); - if (accept_queue->items == NULL) { - assert(0); // TODO - } -} - -static void accept_queue_free(AcceptQueue *accept_queue) -{ - free(accept_queue->items); -} - -static bool accept_queue_push(AcceptQueue *accept_queue, DescriptorHandle handle) -{ - if (accept_queue->used == accept_queue->size) - return false; - int tail = (accept_queue->head + accept_queue->used) % accept_queue->size; - accept_queue->items[tail] = handle; - accept_queue->used++; - return true; -} - -static bool accept_queue_pop(AcceptQueue *accept_queue, DescriptorHandle *item) -{ - if (accept_queue->used == 0) - return false; - *item = accept_queue->items[accept_queue->head]; - accept_queue->head = (accept_queue->head + 1) % accept_queue->size; - accept_queue->used--; - return true; -} - -static void accept_queue_remove(AcceptQueue *queue, DescriptorHandle handle) -{ - int i = 0; - while (i < queue->used && ( - queue->items[i].process != handle.process || - queue->items[i].descriptor_index != handle.descriptor_index || - queue->items[i].generation != handle.generation)) - i++; - - if (i == queue->used) - return; - - for (; i < queue->used-1; i++) { - int u = (queue->head + i + 0) % queue->size; - int v = (queue->head + i + 1) % queue->size; - queue->items[u] = queue->items[v]; - } -} - -static bool accept_queue_empty(AcceptQueue *queue) -{ - return queue->used == 0; -} - -static void data_queue_init(DataQueue *queue, int size) -{ - queue->used = 0; - queue->size = size; - queue->data = malloc(size * sizeof(char)); - if (queue->data == NULL) { - assert(0); // TODO - } -} - -static void data_queue_free(DataQueue *queue) -{ - free(queue->data); -} - -static int data_queue_read(DataQueue *queue, char *dst, int max) -{ - int num = max; - if (num > queue->used) - num = queue->used; - - if (num > 0) { - memcpy(dst, queue->data, num); - memmove(queue->data, queue->data + num, queue->used - num); - queue->used -= num; - } - - return num; -} - -static int data_queue_write(DataQueue *queue, char *src, int len) -{ - int num = len; - if (num > queue->size - queue->used) - num = queue->size - queue->used; - - memcpy(queue->data + queue->used, src, num); - queue->used += num; - - return num; -} - -static bool data_queue_empty(DataQueue *queue) -{ - return queue->used == 0; -} - -static bool data_queue_full(DataQueue *queue) -{ - return queue->used == queue->size; -} - -static int setup_poll_array(void **contexts, struct pollfd *polled) -{ - int num_polled = 0; - - for (int j = 0, k = 0; k < current_process->num_desc; j++) { - - assert(j < MAX_DESCRIPTORS); - Descriptor *desc = ¤t_process->desc[j]; - if (desc->type == DESC_EMPTY) - continue; - k++; - - int revents = 0; - switch (desc->type) { - - case DESC_FILE: - // Ignore - break; - - case DESC_SOCKET: - // Ignore - break; - - case DESC_LISTEN_SOCKET: - if (!accept_queue_empty(&desc->accept_queue)) - revents |= POLLIN; - break; - - case DESC_CONNECTION_SOCKET: - switch (desc->connection_state) { - - case CONNECTION_DELAYED: - break; - - case CONNECTION_QUEUED: - break; - - case CONNECTION_ESTABLISHED: - { - Descriptor *peer = handle_to_desc(desc->connection_peer); - if (peer == NULL) { - revents |= POLLIN; - } else { - if (!data_queue_full(&desc->output_data)) - revents |= POLLOUT; - if (!data_queue_empty(&peer->output_data)) - revents |= POLLIN; - } - } - break; - - case CONNECTION_FAILED: - assert(0); // TODO - break; - } - break; - - default: - UNREACHABLE; - } - - revents &= desc->events; - if (revents) { - polled[num_polled].fd = (SOCKET) j; - polled[num_polled].events = desc->events; - polled[num_polled].revents = revents; - contexts[num_polled] = desc->context; - num_polled++; - } - } - - return num_polled; -} - -static void update_network(void) -{ - for (int i = 0; i < num_processes; i++) { - - for (int j = 0, k = 0; k < processes[i]->num_desc; j++) { - - Descriptor *desc = &processes[i]->desc[j]; - if (desc->type == DESC_EMPTY) - continue; - k++; - - if (desc->type != DESC_CONNECTION_SOCKET) - continue; - - switch (desc->connection_state) { - - case CONNECTION_DELAYED: - { - DescriptorHandle peer_handle; - if (!find_peer_by_address(desc->connect_address, &peer_handle)) { - desc->revents |= POLLOUT; - desc->connection_state = CONNECTION_FAILED; - desc->connect_errno = EHOSTUNREACH; // TODO: This only works on Linux, not Windows - break; - } - Descriptor *peer = handle_to_desc(peer_handle); - - DescriptorHandle self_handle = { processes[i], j, desc->generation }; - if (!accept_queue_push(&peer->accept_queue, self_handle)) { - desc->revents |= POLLOUT; - desc->connection_state = CONNECTION_FAILED; - desc->connect_errno = ECONNREFUSED; // TODO: This only works on Linux, not Windows - break; - } - - peer->revents |= POLLIN; - - desc->connection_state = CONNECTION_QUEUED; - desc->connection_peer = peer_handle; - } - break; - - case CONNECTION_QUEUED: - { - if (handle_to_desc(desc->connection_peer) == NULL) { - // Listener closed before accepting - desc->revents |= POLLOUT; - desc->connection_state = CONNECTION_FAILED; - desc->connect_errno = ECONNREFUSED; // TODO: This only works on Linux, not Windows - break; - } - } - break; - - default: - break; - } - } - } -} - -void update_simulation(void) -{ - for (;;) { - int num_ready = 0; - for (int i = 0; i < num_processes; i++) { - - current_process = processes[i]; - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled = setup_poll_array(contexts, polled); - - if (num_polled > 0) { - - num_ready++; - - int timeout = -1; - switch (current_process->type) { - case PROCESS_TYPE_METADATA_SERVER: - num_polled = metadata_server_step( - ¤t_process->metadata_server, - contexts, - polled, - num_polled, - &timeout - ); - break; - case PROCESS_TYPE_CHUNK_SERVER: - num_polled = chunk_server_step( - ¤t_process->chunk_server, - contexts, - polled, - num_polled, - &timeout - ); - break; - case PROCESS_TYPE_CLIENT: - num_polled = simulation_client_step( - ¤t_process->simulation_client, - contexts, - polled, - num_polled, - &timeout - ); - break; - } - - if (num_polled < 0) { - assert(0); // TODO - } - - if (timeout < 0) { - current_process->wakeup_time = INVALID_TIME; - } else { - current_process->wakeup_time = current_time + (Time) timeout * 1000000; - } - - process_poll_array(current_process, contexts, polled, num_polled); - } - - current_process = NULL; - - update_network(); - } - - if (num_ready == 0) - break; - } - - Process *next_process = NULL; - for (int i = 0; i < num_processes; i++) - if (processes[i]->wakeup_time != INVALID_TIME) - if (next_process == NULL || processes[i]->wakeup_time < next_process->wakeup_time) - next_process = processes[i]; - - if (next_process == NULL) { - assert(0); // Nothing to schedule next - } - - assert(current_time <= next_process->wakeup_time); - current_time = next_process->wakeup_time; - - current_process = next_process; - - void *contexts[TCP_POLL_CAPACITY]; - struct pollfd polled[TCP_POLL_CAPACITY]; - int num_polled = setup_poll_array(contexts, polled); - - int timeout = -1; - switch (current_process->type) { - case PROCESS_TYPE_METADATA_SERVER: - num_polled = metadata_server_step( - ¤t_process->metadata_server, - contexts, - polled, - num_polled, - &timeout - ); - break; - case PROCESS_TYPE_CHUNK_SERVER: - num_polled = chunk_server_step( - ¤t_process->chunk_server, - contexts, - polled, - num_polled, - &timeout - ); - break; - case PROCESS_TYPE_CLIENT: - num_polled = simulation_client_step( - ¤t_process->simulation_client, - contexts, - polled, - num_polled, - &timeout - ); - break; - } - - if (num_polled < 0) { - assert(0); // TODO - } - - if (timeout < 0) { - current_process->wakeup_time = INVALID_TIME; - } else { - current_process->wakeup_time = current_time + (Time) timeout * 1000000; - } - - process_poll_array(current_process, contexts, polled, num_polled); - - current_process = NULL; - - update_network(); -} - -void *mock_malloc(size_t len) -{ - return malloc(len); -} - -void *mock_realloc(void *ptr, size_t len) -{ - return realloc(ptr, len); -} - -void mock_free(void *ptr) -{ - free(ptr); -} - -int mock_remove(char *path) -{ - return remove(path); -} - -int mock_rename(char *oldpath, char *newpath) -{ - return rename(oldpath, newpath); -} - -SOCKET mock_socket(int domain, int type, int protocol) -{ - if (domain != AF_INET || type != SOCK_STREAM || protocol != 0) { - SET_SOCKET_ERROR(SOCKET_ERROR_AFNOSUPPORT); // Address family not supported - return INVALID_SOCKET; - } - - if (current_process->num_desc == MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_MFILE); // Too many open files - return INVALID_SOCKET; - } - - int idx = 0; - while (current_process->desc[idx].type != DESC_EMPTY) { - idx++; - assert(idx < MAX_DESCRIPTORS); - } - - Descriptor *desc = ¤t_process->desc[idx]; - desc->type = DESC_SOCKET; - desc->events = 0; - desc->revents = 0; - desc->context = NULL; - desc->is_nonblocking = false; - desc->address = (DescriptorAddress) { .type=DESC_ADDR_VOID }; - - current_process->num_desc++; - - CHECK_NON_EMPTY_DESC_INVARIANT; - return (SOCKET) idx; -} - -static DescriptorAddress convert_address(void *addr, size_t addr_len) -{ - int family = ((struct sockaddr*) addr)->sa_family; - - if (family == AF_INET && addr_len == sizeof(struct sockaddr_in)) - return (DescriptorAddress) { .type=DESC_ADDR_IPV4, .ipv4=*(struct sockaddr_in*) addr }; - - if (family == AF_INET6 && addr_len != sizeof(struct sockaddr_in6)) - return (DescriptorAddress) { .type=DESC_ADDR_IPV6, .ipv6=*(struct sockaddr_in6*) addr }; - - return (DescriptorAddress) { .type=DESC_ADDR_VOID }; -} - -int mock_bind(SOCKET fd, void *addr, size_t addr_len) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTSOCK); // Socket operation on non-socket - return -1; - } - - DescriptorAddress address = convert_address(addr, addr_len); - if (address.type == DESC_ADDR_VOID) { - SET_SOCKET_ERROR(SOCKET_ERROR_INVAL); // Invalid argument - return -1; - } - - // Check if address is already in use by another socket - for (int i = 0; i < current_process->num_desc; i++) { - Descriptor *other = ¤t_process->desc[i]; - if (other->type != DESC_EMPTY && i != idx) { - if (other->address.type == address.type) { - if (address.type == DESC_ADDR_IPV4 && - other->address.ipv4.sin_port == address.ipv4.sin_port && - other->address.ipv4.sin_addr.s_addr == address.ipv4.sin_addr.s_addr) { - SET_SOCKET_ERROR(SOCKET_ERROR_ADDRINUSE); // Address already in use - return -1; - } - } - } - } - - desc->address = address; - return 0; -} - -int mock_listen(SOCKET fd, int backlog) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTSOCK); // Socket operation on non-socket - return -1; - } - - if (desc->address.type == DESC_ADDR_VOID) { - SET_SOCKET_ERROR(SOCKET_ERROR_DESTADDRREQ); // Destination address required (socket not bound) - return -1; - } - - desc->type = DESC_LISTEN_SOCKET; - accept_queue_init(&desc->accept_queue, backlog); - - return 0; -} - -SOCKET mock_accept(SOCKET fd, void *addr, socklen_t *addr_len) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return INVALID_SOCKET; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_LISTEN_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_INVAL); // Invalid argument (not a listening socket) - return INVALID_SOCKET; - } - - DescriptorHandle peer_handle; - if (!accept_queue_pop(&desc->accept_queue, &peer_handle)) { - // Socket would block - abort if not configured as non-blocking - if (!desc->is_nonblocking) { - fprintf(stderr, "SIMULATION ERROR: accept() would block on socket %d but socket is not configured as non-blocking\n", (int)fd); - abort(); - } - SET_SOCKET_ERROR(SOCKET_ERROR_WOULDBLOCK); // Would block (no pending connections) - return INVALID_SOCKET; - } - - Descriptor *peer = handle_to_desc(peer_handle); - if (peer == NULL) { - // Peer closed without removing itself from the accept queue! - SET_SOCKET_ERROR(SOCKET_ERROR_CONNABORTED); // Connection aborted - return INVALID_SOCKET; - } - - if (current_process->num_desc == MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_MFILE); // Too many open files - return INVALID_SOCKET; - } - int new_idx = 0; - while (current_process->desc[new_idx].type != DESC_EMPTY) { - new_idx++; - assert(new_idx < MAX_DESCRIPTORS); - } - Descriptor *new_desc = ¤t_process->desc[new_idx]; - new_desc->type = DESC_CONNECTION_SOCKET; - new_desc->events = 0; - new_desc->revents = 0; - new_desc->context = NULL; - new_desc->is_nonblocking = false; - new_desc->address = (DescriptorAddress) { .type=DESC_ADDR_VOID }; - new_desc->connection_state = CONNECTION_ESTABLISHED; - new_desc->connection_peer = peer_handle; - data_queue_init(&new_desc->output_data, DATA_QUEUE_SIZE); - - peer->connection_peer = (DescriptorHandle) { current_process, new_idx, new_desc->generation }; - peer->connection_state = CONNECTION_ESTABLISHED; - peer->revents |= POLLOUT; - data_queue_init(&peer->output_data, DATA_QUEUE_SIZE); - - current_process->num_desc++; - CHECK_NON_EMPTY_DESC_INVARIANT; - return (SOCKET) new_idx; -} - -int mock_getsockopt(SOCKET fd, int level, int optname, void *optval, socklen_t *optlen) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type == DESC_EMPTY) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); - return -1; - } - - // Only support SOL_SOCKET level for now - if (level != SOL_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_PROTOOPT); // Protocol not available - return -1; - } - - // Support SO_ERROR option - if (optname == SO_ERROR) { - if (*optlen < sizeof(int)) { - SET_SOCKET_ERROR(SOCKET_ERROR_INVAL); - return -1; - } - - int val; - switch (desc->type) { - - case DESC_EMPTY: - assert(0); - break; - - case DESC_FILE: - SET_SOCKET_ERROR(SOCKET_ERROR_NOTSOCK); - return -1; - - case DESC_SOCKET: - val = 0; // No error - break; - - case DESC_LISTEN_SOCKET: - val = 0; // No error - break; - - case DESC_CONNECTION_SOCKET: - switch (desc->connection_state) { - - case CONNECTION_DELAYED: - val = 0; // No error - break; - - case CONNECTION_QUEUED: - val = 0; // No error - break; - - case CONNECTION_ESTABLISHED: - val = 0; // No error - break; - - case CONNECTION_FAILED: - val = desc->connect_errno; - break; - } - break; - - default: - UNREACHABLE; - } - *(int*)optval = val; - *optlen = sizeof(int); - return 0; - } - - SET_SOCKET_ERROR(SOCKET_ERROR_PROTOOPT); - return -1; -} - -int mock_recv(SOCKET fd, void *dst, int len, int flags) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - if (flags != 0) { - SET_SOCKET_ERROR(SOCKET_ERROR_OPNOTSUPP); // Operation not supported - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_CONNECTION_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTCONN); // Transport endpoint is not connected - return -1; - } - - if (desc->connection_state != CONNECTION_ESTABLISHED) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTCONN); - return -1; - } - - Descriptor *peer = handle_to_desc(desc->connection_peer); - if (peer == NULL) { - // Peer closed - return 0 to indicate orderly shutdown - return 0; - } - - DataQueue *input_data = &peer->output_data; - int bytes_read = data_queue_read(input_data, dst, len); - - // If no data available, would block - if (bytes_read == 0) { - // Socket would block - abort if not configured as non-blocking - if (!desc->is_nonblocking) { - fprintf(stderr, "SIMULATION ERROR: recv() would block on socket %d but socket is not configured as non-blocking\n", (int)fd); - abort(); - } - SET_SOCKET_ERROR(SOCKET_ERROR_WOULDBLOCK); - return -1; - } - - return bytes_read; -} - -int mock_send(SOCKET fd, void *src, int len, int flags) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - if (flags != 0) { - SET_SOCKET_ERROR(SOCKET_ERROR_OPNOTSUPP); // Operation not supported - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_CONNECTION_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTCONN); // Transport endpoint is not connected - return -1; - } - - if (desc->connection_state != CONNECTION_ESTABLISHED) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTCONN); - return -1; - } - - // Check if peer is still connected - Descriptor *peer = handle_to_desc(desc->connection_peer); - if (peer == NULL) { - SET_SOCKET_ERROR(SOCKET_ERROR_PIPE); // Broken pipe / connection shutdown - return -1; - } - - // Write data to output queue - int bytes_written = data_queue_write(&desc->output_data, src, len); - - // If queue is full, we would block - if (bytes_written < len) { - // Socket would block - abort if not configured as non-blocking - if (!desc->is_nonblocking && bytes_written == 0) { - fprintf(stderr, "SIMULATION ERROR: send() would block on socket %d but socket is not configured as non-blocking\n", (int)fd); - abort(); - } - SET_SOCKET_ERROR(SOCKET_ERROR_WOULDBLOCK); - return bytes_written > 0 ? bytes_written : -1; - } - - return bytes_written; -} - -int mock_connect(SOCKET fd, void *addr, size_t addr_len) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); // Bad file descriptor - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_ISCONN); // Transport endpoint is already connected - return -1; - } - - desc->type = DESC_CONNECTION_SOCKET; - desc->connection_state = CONNECTION_DELAYED; - desc->connect_address = convert_address(addr, addr_len); - if (desc->connect_address.type == DESC_ADDR_VOID) { - SET_SOCKET_ERROR(SOCKET_ERROR_INVAL); // Invalid argument - return -1; - } - - // Return EINPROGRESS/WSAEWOULDBLOCK to indicate non-blocking connection in progress - SET_SOCKET_ERROR(SOCKET_ERROR_INPROGRESS); - return -1; -} - -static int -wrap_native_file_into_desc(NATIVE_HANDLE handle) -{ - if (current_process->num_desc == MAX_DESCRIPTORS) { - // TODO - return -1; - } - - int idx = 0; - while (current_process->desc[idx].type != DESC_EMPTY) { - idx++; - assert(idx < MAX_DESCRIPTORS); - } - - Descriptor *desc = ¤t_process->desc[idx]; - - desc->type = DESC_FILE; - desc->real_fd = handle; - - current_process->num_desc++; - CHECK_NON_EMPTY_DESC_INVARIANT; - return idx; -} - -static void close_desc(Descriptor *desc) -{ - switch (desc->type) { - - case DESC_EMPTY: - assert(0); // TODO - break; - - case DESC_FILE: -#ifdef _WIN32 - CloseHandle(desc->real_fd); -#else - close(desc->real_fd); -#endif - break; - - case DESC_SOCKET: - // TODO - break; - - case DESC_LISTEN_SOCKET: - accept_queue_free(&desc->accept_queue); - break; - - case DESC_CONNECTION_SOCKET: - switch (desc->connection_state) { - - case CONNECTION_DELAYED: - // TODO - break; - - case CONNECTION_QUEUED: - { - Descriptor *peer = handle_to_desc(desc->connection_peer); - if (peer == NULL) break; - - DescriptorHandle self_handle = { current_process, desc - current_process->desc, desc->generation }; - accept_queue_remove(&peer->accept_queue, self_handle); - } - break; - - case CONNECTION_ESTABLISHED: - data_queue_free(&desc->output_data); - break; - - case CONNECTION_FAILED: - // TODO - break; - } - // TODO - break; - - case DESC_DIRECTORY: -#ifdef _WIN32 - FindClose(desc->real_d); -#else - closedir(desc->real_d); -#endif - break; - } - desc->type = DESC_EMPTY; - desc->generation++; -} - -#ifdef _WIN32 - -int mock_closesocket(SOCKET fd) -{ - if (fd == INVALID_SOCKET) { - // TODO: set error - return -1; - } - int idx = (int) fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_SOCKET && - desc->type != DESC_LISTEN_SOCKET && - desc->type != DESC_CONNECTION_SOCKET) { - // TODO: set error - return -1; - } - - close_desc(desc); - current_process->num_desc--; - CHECK_NON_EMPTY_DESC_INVARIANT; - return 0; -} - -HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, - DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, - DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, - HANDLE hTemplateFile) -{ - HANDLE handle = CreateFileW(lpFileName, - dwDesiredAccess, dwShareMode, - lpSecurityAttributes, dwCreationDisposition, - dwFlagsAndAttributes, hTemplateFile); - if (handle == INVALID_HANDLE_VALUE) - return INVALID_HANDLE_VALUE; - int fd = wrap_native_file_into_desc(handle); - if (fd < 0) { - CloseHandle(handle); - return INVALID_HANDLE_VALUE; - } - return (HANDLE) fd; -} - -BOOL mock_CloseHandle(HANDLE handle) -{ - if (handle == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) handle; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - close_desc(desc); - current_process->num_desc--; - CHECK_NON_EMPTY_DESC_INVARIANT; - return TRUE; -} - -BOOL mock_LockFile(HANDLE hFile, - DWORD dwFileOffsetLow, - DWORD dwFileOffsetHigh, - DWORD nNumberOfBytesToLockLow, - DWORD nNumberOfBytesToLockHigh) -{ - if (hFile == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) hFile; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real LockFile, last error is set by the real call - return LockFile( - desc->real_fd, - dwFileOffsetLow, - dwFileOffsetHigh, - nNumberOfBytesToLockLow, - nNumberOfBytesToLockHigh); -} - -BOOL mock_UnlockFile( - HANDLE hFile, - DWORD dwFileOffsetLow, - DWORD dwFileOffsetHigh, - DWORD nNumberOfBytesToUnlockLow, - DWORD nNumberOfBytesToUnlockHigh) -{ - if (hFile == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) hFile; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real UnlockFile, last error is set by the real call - return UnlockFile( - desc->real_fd, - dwFileOffsetLow, - dwFileOffsetHigh, - nNumberOfBytesToUnlockLow, - nNumberOfBytesToUnlockHigh); -} - -BOOL mock_FlushFileBuffers(HANDLE handle) -{ - if (handle == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) handle; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real FlushFileBuffers, last error is set by the real call - return FlushFileBuffers(desc->real_fd); -} - -BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov) -{ - if (handle == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) handle; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real ReadFile, last error is set by the real call - return ReadFile(desc->real_fd, dst, len, num, ov); -} - -BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov) -{ - if (handle == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) handle; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real WriteFile, last error is set by the real call - return WriteFile(desc->real_fd, src, len, num, ov); -} - -DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod) -{ - if (hFile == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return INVALID_SET_FILE_POINTER; - } - - int idx = (int) hFile; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return INVALID_SET_FILE_POINTER; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return INVALID_SET_FILE_POINTER; - } - - // Forward to real SetFilePointer, last error is set by the real call - return SetFilePointer(desc->real_fd, lDistanceToMove, lpDistanceToMoveHigh, dwMoveMethod); -} - -BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf) -{ - if (handle == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) handle; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real GetFileSizeEx, last error is set by the real call - return GetFileSizeEx(desc->real_fd, buf); -} - -char *mock__fullpath(char *path, char *dst, int cap) -{ - return _fullpath(path, dst, cap); -} - -int mock__mkdir(char *path) -{ - return _mkdir(path); -} - -BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount) -{ - if (lpPerformanceCount == NULL) { - SetLastError(ERROR_INVALID_PARAMETER); - return FALSE; - } - - // Use current_time (in nanoseconds) to generate deterministic performance counter - // Frequency is 10 MHz (10,000,000 counts per second) - const LONGLONG frequency = 10000000LL; - - // Convert nanoseconds to performance counter ticks - LONGLONG count = (LONGLONG)(current_time * frequency) / 1000000000LL; - - lpPerformanceCount->QuadPart = count; - return TRUE; -} - -BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency) -{ - if (lpFrequency == NULL) { - SetLastError(ERROR_INVALID_PARAMETER); - return FALSE; - } - - // Return fixed frequency of 10 MHz for deterministic behavior - // This is a common frequency on modern systems - lpFrequency->QuadPart = 10000000LL; // 10 million counts per second - return TRUE; -} - -int mock_ioctlsocket(SOCKET fd, long cmd, u_long *argp) -{ - if (fd == INVALID_SOCKET || (int)fd < 0 || (int)fd >= MAX_DESCRIPTORS) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); - return -1; - } - - int idx = (int) fd; - Descriptor *desc = ¤t_process->desc[idx]; - - if (desc->type == DESC_EMPTY) { - SET_SOCKET_ERROR(SOCKET_ERROR_BADF); - return -1; - } - - if (desc->type != DESC_SOCKET && - desc->type != DESC_LISTEN_SOCKET && - desc->type != DESC_CONNECTION_SOCKET) { - SET_SOCKET_ERROR(SOCKET_ERROR_NOTSOCK); - return -1; - } - - if ((long unsigned int) cmd == FIONBIO) { - desc->is_nonblocking = (*argp != 0); - return 0; - } - - SET_SOCKET_ERROR(SOCKET_ERROR_INVAL); - return -1; -} - -HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData) -{ - HANDLE handle = FindFirstFileA(lpFileName, lpFindFileData); - if (handle == INVALID_HANDLE_VALUE) - return INVALID_HANDLE_VALUE; - - if (current_process->num_desc == MAX_DESCRIPTORS) { - FindClose(handle); - SetLastError(ERROR_TOO_MANY_OPEN_FILES); - return INVALID_HANDLE_VALUE; - } - - int idx = 0; - while (current_process->desc[idx].type != DESC_EMPTY) { - idx++; - assert(idx < MAX_DESCRIPTORS); - } - - Descriptor *desc = ¤t_process->desc[idx]; - desc->type = DESC_DIRECTORY; - desc->real_d = handle; - - current_process->num_desc++; - CHECK_NON_EMPTY_DESC_INVARIANT; - return (HANDLE) idx; -} - -BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData) -{ - if (hFindFile == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) hFindFile; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_DIRECTORY) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - // Forward to real FindNextFileA, last error is set by the real call - return FindNextFileA(desc->real_d, lpFindFileData); -} - -BOOL mock_FindClose(HANDLE hFindFile) -{ - if (hFindFile == INVALID_HANDLE_VALUE) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - int idx = (int) hFindFile; - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_DIRECTORY) { - SetLastError(ERROR_INVALID_HANDLE); - return FALSE; - } - - close_desc(desc); - current_process->num_desc--; - CHECK_NON_EMPTY_DESC_INVARIANT; - return TRUE; -} - -BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags) -{ - // Forward to real MoveFileExW, last error is set by the real call - return MoveFileExW(lpExistingFileName, lpNewFileName, dwFlags); -} - -#else - -int mock_clock_gettime(clockid_t clockid, struct timespec *tp) -{ - if (tp == NULL) { - errno = EFAULT; // Bad address - return -1; - } - - // Only support CLOCK_REALTIME and CLOCK_MONOTONIC for now - if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC) { - errno = EINVAL; // Invalid clock ID - return -1; - } - - // Convert current_time (nanoseconds) to timespec for deterministic behavior - tp->tv_sec = current_time / 1000000000ULL; - tp->tv_nsec = current_time % 1000000000ULL; - return 0; -} - -int mock_open(char *path, int flags, int mode) -{ - int fd = open(path, flags, mode); - if (fd < 0) return -1; - - int wrapped_fd = wrap_native_file_into_desc(fd); - if (wrapped_fd < 0) { - close(fd); - return -1; - } - - return wrapped_fd; -} - -int mock_close(int fd) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = (int) fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type == DESC_EMPTY) { - errno = EBADF; - return -1; - } - - close_desc(desc); - current_process->num_desc--; - CHECK_NON_EMPTY_DESC_INVARIANT; - return 0; -} - -int mock_flock(int fd, int op) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - errno = EBADF; // Not a file descriptor - return -1; - } - - // Forward to real flock, errno is set by the real call - return flock(desc->real_fd, op); -} - -int mock_fsync(int fd) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - errno = EINVAL; // Invalid argument (not a file) - return -1; - } - - // Forward to real fsync, errno is set by the real call - return fsync(desc->real_fd); -} - -int mock_read(int fd, char *dst, int len) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type == DESC_EMPTY) { - errno = EBADF; - return -1; - } - - if (desc->type == DESC_FILE) { - // Forward to real read, errno is set by the real call - return read(desc->real_fd, dst, len); - } else { - // Socket read - return mock_recv(fd, dst, len, 0); - } -} - -int mock_write(int fd, char *src, int len) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type == DESC_EMPTY) { - errno = EBADF; - return -1; - } - - if (desc->type == DESC_FILE) { - // Forward to real write, errno is set by the real call - return write(desc->real_fd, src, len); - } else { - // Socket write - return mock_send(fd, src, len, 0); - } -} - -off_t mock_lseek(int fd, off_t offset, int whence) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - errno = EBADF; // Not a file descriptor - return -1; - } - - // Forward to real lseek, errno is set by the real call - return lseek(desc->real_fd, offset, whence); -} - -int mock_fstat(int fd, struct stat *buf) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = fd; - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_FILE) { - errno = EBADF; // Not a file descriptor - return -1; - } - - // Forward to real fstat, errno is set by the real call - return fstat(desc->real_fd, buf); -} - -int mock_mkstemp(char *path) -{ - int fd = mkstemp(path); - if (fd < 0) return fd; - - int wrapped_fd = wrap_native_file_into_desc(fd); - if (wrapped_fd < 0) { - close(fd); - return -1; - } - - return wrapped_fd; -} - -char* mock_realpath(char *path, char *dst) -{ - return realpath(path, dst); -} - -int mock_mkdir(char *path, mode_t mode) -{ - return mkdir(path, mode); -} - -int mock_fcntl(int fd, int cmd, ...) -{ - if (fd < 0 || fd >= MAX_DESCRIPTORS) { - errno = EBADF; - return -1; - } - - int idx = fd; - Descriptor *desc = ¤t_process->desc[idx]; - - if (desc->type == DESC_EMPTY) { - errno = EBADF; - return -1; - } - - if (cmd == F_GETFL) { - // Return flags - only O_NONBLOCK is tracked - if (desc->type == DESC_FILE) { - // For files, forward to real fcntl - return fcntl(desc->real_fd, cmd); - } - return desc->is_nonblocking ? O_NONBLOCK : 0; - } - - if (cmd == F_SETFL) { - if (desc->type == DESC_FILE) { - // For files, forward to real fcntl - va_list args; - va_start(args, cmd); - int flags = va_arg(args, int); - va_end(args); - return fcntl(desc->real_fd, cmd, flags); - } - - if (desc->type != DESC_SOCKET && - desc->type != DESC_LISTEN_SOCKET && - desc->type != DESC_CONNECTION_SOCKET) { - errno = EBADF; - return -1; - } - - va_list args; - va_start(args, cmd); - int flags = va_arg(args, int); - va_end(args); - - desc->is_nonblocking = (flags & O_NONBLOCK) != 0; - return 0; - } - - errno = EINVAL; - return -1; -} - -DIR *mock_opendir(char *name) -{ - DIR *dir = opendir(name); - if (dir == NULL) - return NULL; - - if (current_process->num_desc == MAX_DESCRIPTORS) { - closedir(dir); - errno = EMFILE; // Too many open files - return NULL; - } - - int idx = 0; - while (current_process->desc[idx].type != DESC_EMPTY) { - idx++; - assert(idx < MAX_DESCRIPTORS); - } - - Descriptor *desc = ¤t_process->desc[idx]; - desc->type = DESC_DIRECTORY; - desc->real_d = dir; - - current_process->num_desc++; - CHECK_NON_EMPTY_DESC_INVARIANT; - return (DIR *)(intptr_t) idx; -} - -struct dirent *mock_readdir(DIR *dirp) -{ - if (dirp == NULL) { - errno = EBADF; // Bad file descriptor - return NULL; - } - int idx = (int)(intptr_t) dirp; - - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - errno = EBADF; - return NULL; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_DIRECTORY) { - errno = EBADF; - return NULL; - } - - // Forward to real readdir, errno is set by the real call - return readdir(desc->real_d); -} - -int mock_closedir(DIR *dirp) -{ - if (dirp == NULL) { - errno = EBADF; // Bad file descriptor - return -1; - } - int idx = (int)(intptr_t) dirp; - - if (idx < 0 || idx >= MAX_DESCRIPTORS) { - errno = EBADF; - return -1; - } - - Descriptor *desc = ¤t_process->desc[idx]; - if (desc->type != DESC_DIRECTORY) { - errno = EBADF; - return -1; - } - - close_desc(desc); - current_process->num_desc--; - CHECK_NON_EMPTY_DESC_INVARIANT; - return 0; -} - -#endif // !_WIN32 - -#endif // BUILD_TEST diff --git a/src/system.h b/src/system.h deleted file mode 100644 index a219ba1..0000000 --- a/src/system.h +++ /dev/null @@ -1,209 +0,0 @@ -#ifndef SYSTEM_INCLUDED -#define SYSTEM_INCLUDED - -#include -#include -#include - -#ifdef _WIN32 - -#define WIN32_LEAN_AND_MEAN - -#include // _mkdir -#include -#include -#include - -#else - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SOCKET int -#define INVALID_SOCKET ((SOCKET) -1) - -#endif - -#ifdef BUILD_TEST - -void startup_simulation(uint64_t seed_); -int spawn_simulated_process(char *args); -void update_simulation(void); -void cleanup_simulation(void); -uint64_t simulation_random_number(void); - -void* mock_malloc(size_t len); -void* mock_realloc(void *ptr, size_t len); -void mock_free(void *ptr); -int mock_remove(char *path); -int mock_rename(char *oldpath, char *newpath); -SOCKET mock_socket(int domain, int type, int protocol); -int mock_bind(SOCKET fd, void *addr, size_t addr_len); -int mock_listen(SOCKET fd, int backlog); -SOCKET mock_accept(SOCKET fd, void *addr, socklen_t *addr_len); -int mock_getsockopt(SOCKET fd, int level, int optname, void *optval, socklen_t *optlen); -int mock_recv(SOCKET fd, void *dst, int len, int flags); -int mock_send(SOCKET fd, void *src, int len, int flags); -int mock_connect(SOCKET fd, void *addr, size_t addr_len); - -#ifdef _WIN32 -int mock_closesocket(SOCKET fd); -int mock_ioctlsocket(SOCKET fd, long cmd, u_long *argp); -HANDLE mock_CreateFileW(WCHAR *lpFileName, DWORD dwDesiredAccess, DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, HANDLE hTemplateFile); -BOOL mock_CloseHandle(HANDLE handle); -BOOL mock_LockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh); -BOOL mock_UnlockFile(HANDLE hFile, DWORD dwFileOffsetLow, DWORD dwFileOffsetHigh, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh); -BOOL mock_FlushFileBuffers(HANDLE handle); -BOOL mock_ReadFile(HANDLE handle, char *dst, DWORD len, DWORD *num, OVERLAPPED *ov); -BOOL mock_WriteFile(HANDLE handle, char *src, DWORD len, DWORD *num, OVERLAPPED *ov); -DWORD mock_SetFilePointer(HANDLE hFile, LONG lDistanceToMove, PLONG lpDistanceToMoveHigh, DWORD dwMoveMethod); -BOOL mock_GetFileSizeEx(HANDLE handle, LARGE_INTEGER *buf); -BOOL mock_QueryPerformanceCounter(LARGE_INTEGER *lpPerformanceCount); -BOOL mock_QueryPerformanceFrequency(LARGE_INTEGER *lpFrequency); -char* mock__fullpath(char *path, char *dst, int cap); -int mock__mkdir(char *path); -HANDLE mock_FindFirstFileA(char *lpFileName, WIN32_FIND_DATAA *lpFindFileData); -BOOL mock_FindNextFileA(HANDLE hFindFile, WIN32_FIND_DATAA *lpFindFileData); -BOOL mock_FindClose(HANDLE hFindFile); -BOOL mock_MoveFileExW(WCHAR *lpExistingFileName, WCHAR *lpNewFileName, DWORD dwFlags); -#else -int mock_clock_gettime(clockid_t clockid, struct timespec *tp); -int mock_open(char *path, int flags, int mode); -int mock_close(int fd); -int mock_flock(int fd, int op); -int mock_fsync(int fd); -int mock_read(int fd, char *dst, int len); -int mock_write(int fd, char *src, int len); -off_t mock_lseek(int fd, off_t offset, int whence); -int mock_fstat(int fd, struct stat *buf); -int mock_mkstemp(char *path); -char* mock_realpath(char *path, char *dst); -int mock_mkdir(char *path, mode_t mode); -int mock_fcntl(int fd, int cmd, ...); -DIR* mock_opendir(char *name); -struct dirent* mock_readdir(DIR *dirp); -int mock_closedir(DIR *dirp); -#endif - -// Common -#define sys_malloc mock_malloc -#define sys_realloc mock_realloc -#define sys_free mock_free -#define sys_remove mock_remove -#define sys_rename mock_rename -#define sys_socket mock_socket -#define sys_bind mock_bind -#define sys_listen mock_listen -#define sys_accept mock_accept -#define sys_getsockopt mock_getsockopt -#define sys_recv mock_recv -#define sys_send mock_send -#define sys_connect mock_connect - -// Windows -#define sys__mkdir mock__mkdir -#define sys_closesocket mock_closesocket -#define sys_ioctlsocket mock_ioctlsocket -#define sys_CreateFileW mock_CreateFileW -#define sys_CloseHandle mock_CloseHandle -#define sys_LockFile mock_LockFile -#define sys_UnlockFile mock_UnlockFile -#define sys_FlushFileBuffers mock_FlushFileBuffers -#define sys_ReadFile mock_ReadFile -#define sys_WriteFile mock_WriteFile -#define sys_SetFilePointer mock_SetFilePointer -#define sys_GetFileSizeEx mock_GetFileSizeEx -#define sys__fullpath mock__fullpath -#define sys_QueryPerformanceCounter mock_QueryPerformanceCounter -#define sys_QueryPerformanceFrequency mock_QueryPerformanceFrequency -#define sys_FindFirstFileA mock_FindFirstFileA -#define sys_FindNextFileA mock_FindNextFileA -#define sys_FindClose mock_FindClose -#define sys_MoveFileExW mock_MoveFileExW - -// Linux -#define sys_mkdir mock_mkdir -#define sys_open mock_open -#define sys_close mock_close -#define sys_flock mock_flock -#define sys_fsync mock_fsync -#define sys_read mock_read -#define sys_write mock_write -#define sys_lseek mock_lseek -#define sys_fstat mock_fstat -#define sys_mkstemp mock_mkstemp -#define sys_realpath mock_realpath -#define sys_clock_gettime mock_clock_gettime -#define sys_fcntl mock_fcntl -#define sys_getrandom mock_getrandom -#define sys_opendir mock_opendir -#define sys_readdir mock_readdir -#define sys_closedir mock_closedir - -#else - -// Common -#define sys_malloc malloc -#define sys_realloc realloc -#define sys_free free -#define sys_remove remove -#define sys_rename rename -#define sys_socket socket -#define sys_bind bind -#define sys_listen listen -#define sys_accept accept -#define sys_getsockopt getsockopt -#define sys_recv recv -#define sys_send send -#define sys_connect connect - -// Windows -#define sys__mkdir _mkdir -#define sys_closesocket closesocket -#define sys_ioctlsocket ioctlsocket -#define sys_CreateFileW CreateFileW -#define sys_CloseHandle CloseHandle -#define sys_LockFile LockFile -#define sys_UnlockFile UnlockFile -#define sys_FlushFileBuffers FlushFileBuffers -#define sys_ReadFile ReadFile -#define sys_WriteFile WriteFile -#define sys_SetFilePointer SetFilePointer -#define sys_GetFileSizeEx GetFileSizeEx -#define sys__fullpath _fullpath -#define sys_QueryPerformanceCounter QueryPerformanceCounter -#define sys_QueryPerformanceFrequency QueryPerformanceFrequency -#define sys_FindFirstFileA FindFirstFileA -#define sys_FindNextFileA FindNextFileA -#define sys_FindClose FindClose -#define sys_MoveFileExW MoveFileExW - -// Linux -#define sys_mkdir mkdir -#define sys_open open -#define sys_close close -#define sys_flock flock -#define sys_fsync fsync -#define sys_read read -#define sys_write write -#define sys_lseek lseek -#define sys_fstat fstat -#define sys_mkstemp mkstemp -#define sys_realpath realpath -#define sys_clock_gettime clock_gettime -#define sys_fcntl fcntl -#define sys_opendir opendir -#define sys_readdir readdir -#define sys_closedir closedir - -#endif -#endif // SYSTEM_INCLUDED diff --git a/src/tcp.c b/src/tcp.c index a1f4565..1027345 100644 --- a/src/tcp.c +++ b/src/tcp.c @@ -1,8 +1,12 @@ + +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include #include -#include #include "tcp.h" -#include "system.h" #include "message.h" bool addr_eql(Address a, Address b) @@ -28,15 +32,15 @@ static int set_socket_blocking(SOCKET sock, bool value) { #ifdef _WIN32 u_long mode = !value; - if (sys_ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR) + if (ioctlsocket(sock, FIONBIO, &mode) == SOCKET_ERROR) return -1; #else - int flags = sys_fcntl(sock, F_GETFL, 0); + int flags = fcntl(sock, F_GETFL, 0); if (flags < 0) return -1; if (value) flags &= ~O_NONBLOCK; else flags |= O_NONBLOCK; - if (sys_fcntl(sock, F_SETFL, flags) < 0) + if (fcntl(sock, F_SETFL, flags) < 0) return -1; #endif @@ -45,7 +49,7 @@ static int set_socket_blocking(SOCKET sock, bool value) static SOCKET create_listen_socket(string addr, uint16_t port) { - SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0); + SOCKET fd = socket(AF_INET, SOCK_STREAM, 0); if (fd == INVALID_SOCKET) return INVALID_SOCKET; @@ -72,13 +76,13 @@ static SOCKET create_listen_socket(string addr, uint16_t port) return INVALID_SOCKET; } - if (sys_bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) { + if (bind(fd, (struct sockaddr*) &bind_buf, sizeof(bind_buf))) { CLOSE_SOCKET(fd); return INVALID_SOCKET; } int backlog = 32; - if (sys_listen(fd, backlog) < 0) { + if (listen(fd, backlog) < 0) { CLOSE_SOCKET(fd); return INVALID_SOCKET; } @@ -319,7 +323,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd assert(contexts[i] == NULL); if (polled[i].revents & POLLIN) { - SOCKET new_fd = sys_accept(tcp->listen_fd, NULL, NULL); + SOCKET new_fd = accept(tcp->listen_fd, NULL, NULL); if (new_fd != INVALID_SOCKET) { if (set_socket_blocking(new_fd, false) < 0) @@ -347,7 +351,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd int err = 0; socklen_t len = sizeof(err); - if (sys_getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0) + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, (void*) &err, &len) < 0 || err != 0) defer_close = true; else { conn->connecting = false; @@ -360,7 +364,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd if (polled[i].revents & POLLIN) { byte_queue_write_setmincap(&conn->input, 1<<9); ByteView buf = byte_queue_write_buf(&conn->input); - int num = sys_recv(conn->fd, (char*) buf.ptr, buf.len, 0); + int num = recv(conn->fd, (char*) buf.ptr, buf.len, 0); if (num == 0) defer_close = true; else if (num < 0) { @@ -388,7 +392,7 @@ int tcp_translate_events(TCP *tcp, Event *events, void **contexts, struct pollfd if (polled[i].revents & POLLOUT) { ByteView buf = byte_queue_read_buf(&conn->output); - int num = sys_send(conn->fd, (char*) buf.ptr, buf.len, 0); + int num = send(conn->fd, (char*) buf.ptr, buf.len, 0); if (num < 0) { if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) defer_close = true; @@ -432,7 +436,7 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output) return -1; int conn_idx = tcp->num_conns; - SOCKET fd = sys_socket(AF_INET, SOCK_STREAM, 0); + SOCKET fd = socket(AF_INET, SOCK_STREAM, 0); if (fd == INVALID_SOCKET) return -1; @@ -447,13 +451,13 @@ int tcp_connect(TCP *tcp, Address addr, int tag, ByteQueue **output) buf.sin_family = AF_INET; buf.sin_port = htons(addr.port); memcpy(&buf.sin_addr, &addr.ipv4, sizeof(IPv4)); - ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf)); + ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf)); } else { struct sockaddr_in6 buf; buf.sin6_family = AF_INET6; buf.sin6_port = htons(addr.port); memcpy(&buf.sin6_addr, &addr.ipv6, sizeof(IPv6)); - ret = sys_connect(fd, (struct sockaddr*) &buf, sizeof(buf)); + ret = connect(fd, (struct sockaddr*) &buf, sizeof(buf)); } bool connecting; diff --git a/src/tcp.h b/src/tcp.h index d2e1d70..02e6e8c 100644 --- a/src/tcp.h +++ b/src/tcp.h @@ -1,15 +1,23 @@ #ifndef TCP_INCLUDED #define TCP_INCLUDED -#include +#ifdef MAIN_SIMULATION +# define QUAKEY_ENABLE_MOCKS +# include +#else +# ifdef _WIN32 +# include +# endif +#endif -#include "system.h" #include "byte_queue.h" #ifdef _WIN32 -#define CLOSE_SOCKET sys_closesocket +#define CLOSE_SOCKET closesocket #else -#define CLOSE_SOCKET sys_close +#define SOCKET int +#define INVALID_SOCKET -1 +#define CLOSE_SOCKET close #endif #ifndef TCP_CONNECTION_LIMIT diff --git a/src/wal.c b/src/wal.c index bbaa5ba..a3f94a8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1,12 +1,14 @@ -#include -#include + +#ifdef MAIN_SIMULATION +#define QUAKEY_ENABLE_MOCKS +#endif + +#include #include -#include #include "wal.h" #include "file_system.h" #include "file_tree.h" -#include "system.h" #define WAL_MAGIC 0xcafebebe #define WAL_VERSION 1 @@ -190,7 +192,7 @@ static int swap_file(WAL *wal) } // MOVEFILE_REPLACE_EXISTING allows atomic overwrite - if (!sys_MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) { + if (!MoveFileExW(temp_path_w, old_path_w, MOVEFILE_REPLACE_EXISTING)) { file_unlock(temp_handle); file_close(temp_handle); remove_file_or_dir(temp_path); @@ -267,12 +269,12 @@ static int next_entry(Handle handle, WALEntry *entry) return -1; // Dynamically allocate path buffer - char *path_buffer = sys_malloc(path_len); + char *path_buffer = malloc(path_len); if (!path_buffer) return -1; if (read_exact(handle, path_buffer, path_len) <= 0) { - sys_free(path_buffer); + free(path_buffer); return -1; } @@ -315,13 +317,13 @@ static int next_entry(Handle handle, WALEntry *entry) goto cleanup_error; // Dynamically allocate hash buffers - SHA256 *hashes_buffer = sys_malloc(entry->num_chunks * sizeof(SHA256)); + SHA256 *hashes_buffer = malloc(entry->num_chunks * sizeof(SHA256)); if (!hashes_buffer) { goto cleanup_error; } if (read_exact(handle, (char*) hashes_buffer, entry->num_chunks * sizeof(SHA256)) <= 0) { - sys_free(hashes_buffer); + free(hashes_buffer); goto cleanup_error; } @@ -337,9 +339,9 @@ static int next_entry(Handle handle, WALEntry *entry) cleanup_error: if (entry->path.ptr) - sys_free((char*) entry->path.ptr); + free((char*) entry->path.ptr); if (entry->hashes) - sys_free(entry->hashes); + free(entry->hashes); return -1; } @@ -351,7 +353,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit) wal->file_path.ptr = NULL; // Copy file_path since the passed string may not have the same lifetime as WAL - char *path_copy = sys_malloc(file_path.len); + char *path_copy = malloc(file_path.len); if (!path_copy) return -1; memcpy(path_copy, file_path.ptr, file_path.len); @@ -482,9 +484,9 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit) // Free dynamically allocated fields from next_entry if (entry.path.ptr) - sys_free((char*) entry.path.ptr); + free((char*) entry.path.ptr); if (entry.hashes) - sys_free(entry.hashes); + free(entry.hashes); wal->entry_count++; } @@ -499,7 +501,7 @@ int wal_open(WAL *wal, FileTree *file_tree, string file_path, int entry_limit) return 0; error_cleanup_path: - sys_free(path_copy); + free(path_copy); return -1; } @@ -508,7 +510,7 @@ void wal_close(WAL *wal) file_unlock(wal->handle); file_close(wal->handle); if (wal->file_path.ptr) - sys_free((char*) wal->file_path.ptr); + free((char*) wal->file_path.ptr); } static int write_u8(Handle handle, uint8_t value) diff --git a/web/3p/chttp.c b/web/3p/chttp.c deleted file mode 100644 index 7c5465c..0000000 --- a/web/3p/chttp.c +++ /dev/null @@ -1,5077 +0,0 @@ -// cHTTP, an HTTP client and server library! -// -// This file was generated automatically. Do not modify directly. -// -// Refer to the end of this file for the license -#ifndef HTTP_DONT_INCLUDE -#include "chttp.h" -#endif - -//////////////////////////////////////////////////////////////////////////////////////// -// src/basic.c -//////////////////////////////////////////////////////////////////////////////////////// - -bool http_streq(HTTP_String s1, HTTP_String s2) -{ - if (s1.len != s2.len) - return false; - - for (int i = 0; i < s1.len; i++) - if (s1.ptr[i] != s2.ptr[i]) - return false; - - return true; -} - -static char to_lower(char c) -{ - if (c >= 'A' && c <= 'Z') - return c - 'A' + 'a'; - return c; -} - -bool http_streqcase(HTTP_String s1, HTTP_String s2) -{ - if (s1.len != s2.len) - return false; - - for (int i = 0; i < s1.len; i++) - if (to_lower(s1.ptr[i]) != to_lower(s2.ptr[i])) - return false; - - return true; -} - -HTTP_String http_trim(HTTP_String s) -{ - int i = 0; - while (i < s.len && (s.ptr[i] == ' ' || s.ptr[i] == '\t')) - i++; - - if (i == s.len) { - s.ptr = NULL; - s.len = 0; - } else { - s.ptr += i; - s.len -= i; - while (s.ptr[s.len-1] == ' ' || s.ptr[s.len-1] == '\t') - s.len--; - } - - return s; -} - -static bool is_printable(char c) -{ - return c >= ' ' && c <= '~'; -} - -void print_bytes(HTTP_String prefix, HTTP_String src) -{ - if (src.len == 0) - return; - - FILE *stream = stdout; - - bool new_line = true; - int cur = 0; - for (;;) { - int start = cur; - - while (cur < src.len && is_printable(src.ptr[cur])) - cur++; - - if (new_line) { - fwrite(prefix.ptr, 1, prefix.len, stream); - new_line = false; - } - - fwrite(src.ptr + start, 1, cur - start, stream); - - if (cur == src.len) - break; - - if (src.ptr[cur] == '\n') { - putc('\\', stream); - putc('n', stream); - putc('\n', stream); - new_line = true; - } else if (src.ptr[cur] == '\r') { - putc('\\', stream); - putc('r', stream); - } else { - putc('.', stream); - } - cur++; - } - putc('\n', stream); -} - -char *http_strerror(int code) -{ - switch (code) { - case HTTP_OK: return "No error"; - case HTTP_ERROR_UNSPECIFIED: return "Unspecified error"; - case HTTP_ERROR_OOM: return "Out of memory"; - case HTTP_ERROR_BADURL: return "Invalid URL"; - case HTTP_ERROR_REQLIMIT: return "Parallel request limit reached"; - case HTTP_ERROR_BADHANDLE: return "Invalid handle"; - case HTTP_ERROR_NOTLS: return "TLS support not built-in"; - } - return "???"; -} - -//////////////////////////////////////////////////////////////////////////////////////// -// src/parse.c -//////////////////////////////////////////////////////////////////////////////////////// -// From RFC 9112 -// request-target = origin-form -// / absolute-form -// / authority-form -// / asterisk-form -// origin-form = absolute-path [ "?" query ] -// absolute-form = absolute-URI -// authority-form = uri-host ":" port -// asterisk-form = "*" -// -// From RFC 9110 -// URI-reference = -// absolute-URI = -// relative-part = -// authority = -// uri-host = -// port = -// path-abempty = -// segment = -// query = -// -// absolute-path = 1*( "/" segment ) -// partial-URI = relative-part [ "?" query ] -// -// From RFC 3986: -// segment = *pchar -// pchar = unreserved / pct-encoded / sub-delims / ":" / "@" -// pct-encoded = "%" HEXDIG HEXDIG -// sub-delims = "!" / "$" / "&" / "'" / "(" / ")" -// / "*" / "+" / "," / ";" / "=" -// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" -// query = *( pchar / "/" / "?" ) -// absolute-URI = scheme ":" hier-part [ "?" query ] -// hier-part = "//" authority path-abempty -// / path-absolute -// / path-rootless -// / path-empty -// scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) - -typedef struct { - char *src; - int len; - int cur; -} Scanner; - -static int is_digit(char c) -{ - return c >= '0' && c <= '9'; -} - -static int is_alpha(char c) -{ - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); -} - -static int is_hex_digit(char c) -{ - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); -} - -// From RFC 3986: -// sub-delims = "!" / "$" / "&" / "'" / "(" / ")" -// / "*" / "+" / "," / ";" / "=" -static int is_sub_delim(char c) -{ - return c == '!' || c == '$' || c == '&' || c == '\'' - || c == '(' || c == ')' || c == '*' || c == '+' - || c == ',' || c == ';' || c == '='; -} - -// From RFC 3986: -// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" -static int is_unreserved(char c) -{ - return is_alpha(c) || is_digit(c) - || c == '-' || c == '.' - || c == '_' || c == '~'; -} - -// From RFC 3986: -// pchar = unreserved / pct-encoded / sub-delims / ":" / "@" -static int is_pchar(char c) -{ - return is_unreserved(c) || is_sub_delim(c) || c == ':' || c == '@'; -} - -static int is_tchar(char c) -{ - return is_digit(c) || is_alpha(c) - || c == '!' || c == '#' || c == '$' - || c == '%' || c == '&' || c == '\'' - || c == '*' || c == '+' || c == '-' - || c == '.' || c == '^' || c == '_' - || c == '~'; -} - -static int is_vchar(char c) -{ - return c >= ' ' && c <= '~'; -} - -#define CONSUME_OPTIONAL_SEQUENCE(scanner, func) \ - while ((scanner)->cur < (scanner)->len && (func)((scanner)->src[(scanner)->cur])) \ - (scanner)->cur++; - -static int -consume_absolute_path(Scanner *s) -{ - if (s->cur == s->len || s->src[s->cur] != '/') - return -1; // ERROR - s->cur++; - - for (;;) { - - CONSUME_OPTIONAL_SEQUENCE(s, is_pchar); - - if (s->cur == s->len || s->src[s->cur] != '/') - break; - s->cur++; - } - - return 0; -} - -// If abempty=1: -// path-abempty = *( "/" segment ) -// else: -// path-absolute = "/" [ segment-nz *( "/" segment ) ] -// path-rootless = segment-nz *( "/" segment ) -// path-empty = 0 -static int parse_path(Scanner *s, HTTP_String *path, int abempty) -{ - int start = s->cur; - - if (abempty) { - - // path-abempty - while (s->cur < s->len && s->src[s->cur] == '/') { - do - s->cur++; - while (s->cur < s->len && is_pchar(s->src[s->cur])); - } - - } else if (s->cur < s->len && (s->src[s->cur] == '/')) { - - // path-absolute - s->cur++; - if (s->cur < s->len && is_pchar(s->src[s->cur])) { - s->cur++; - for (;;) { - - CONSUME_OPTIONAL_SEQUENCE(s, is_pchar); - - if (s->cur == s->len || s->src[s->cur] != '/') - break; - s->cur++; - } - } - - } else if (s->cur < s->len && is_pchar(s->src[s->cur])) { - - // path-rootless - s->cur++; - for (;;) { - - CONSUME_OPTIONAL_SEQUENCE(s, is_pchar) - - if (s->cur == s->len || s->src[s->cur] != '/') - break; - s->cur++; - } - - } else { - // path->empty - // (do nothing) - } - - *path = (HTTP_String) { - s->src + start, - s->cur - start, - }; - if (path->len == 0) - path->ptr = NULL; - - return 0; -} - -// RFC 3986: -// query = *( pchar / "/" / "?" ) -static int is_query(char c) -{ - return is_pchar(c) || c == '/' || c == '?'; -} - -// RFC 3986: -// fragment = *( pchar / "/" / "?" ) -static int is_fragment(char c) -{ - return is_pchar(c) || c == '/' || c == '?'; -} - -static int little_endian(void) -{ - uint16_t x = 1; - return *((uint8_t*) &x); -} - -static void invert_bytes(void *p, int len) -{ - char *c = p; - for (int i = 0; i < len/2; i++) { - char tmp = c[i]; - c[i] = c[len-i-1]; - c[len-i-1] = tmp; - } -} - -static int parse_ipv4(Scanner *s, HTTP_IPv4 *ipv4) -{ - unsigned int out = 0; - int i = 0; - for (;;) { - - if (s->cur == s->len || !is_digit(s->src[s->cur])) - return -1; - - int b = 0; - do { - int x = s->src[s->cur++] - '0'; - if (b > (UINT8_MAX - x) / 10) - return -1; - b = b * 10 + x; - } while (s->cur < s->len && is_digit(s->src[s->cur])); - - out <<= 8; - out |= (unsigned char) b; - - i++; - if (i == 4) - break; - - if (s->cur == s->len || s->src[s->cur] != '.') - return -1; - s->cur++; - } - - if (little_endian()) - invert_bytes(&out, 4); - - ipv4->data = out; - return 0; -} - -static int hex_digit_to_int(char c) -{ - if (c >= 'a' && c <= 'f') return c - 'a' + 10; - if (c >= 'A' && c <= 'F') return c - 'A' + 10; - if (c >= '0' && c <= '9') return c - '0'; - return -1; -} - -static int parse_ipv6_comp(Scanner *s) -{ - unsigned short buf; - - if (s->cur == s->len || !is_hex_digit(s->src[s->cur])) - return -1; - buf = hex_digit_to_int(s->src[s->cur]); - s->cur++; - - if (s->cur == s->len || !is_hex_digit(s->src[s->cur])) - return buf; - buf <<= 4; - buf |= hex_digit_to_int(s->src[s->cur]); - s->cur++; - - if (s->cur == s->len || !is_hex_digit(s->src[s->cur])) - return buf; - buf <<= 4; - buf |= hex_digit_to_int(s->src[s->cur]); - s->cur++; - - if (s->cur == s->len || !is_hex_digit(s->src[s->cur])) - return buf; - buf <<= 4; - buf |= hex_digit_to_int(s->src[s->cur]); - s->cur++; - - return (int) buf; -} - -static int parse_ipv6(Scanner *s, HTTP_IPv6 *ipv6) -{ - unsigned short head[8]; - unsigned short tail[8]; - int head_len = 0; - int tail_len = 0; - - if (s->len - s->cur > 1 - && s->src[s->cur+0] == ':' - && s->src[s->cur+1] == ':') - s->cur += 2; - else { - - for (;;) { - - int ret = parse_ipv6_comp(s); - if (ret < 0) return ret; - - head[head_len++] = (unsigned short) ret; - if (head_len == 8) break; - - if (s->cur == s->len || s->src[s->cur] != ':') - return -1; - s->cur++; - - if (s->cur < s->len && s->src[s->cur] == ':') { - s->cur++; - break; - } - } - } - - if (head_len < 8) { - while (s->cur < s->len && is_hex_digit(s->src[s->cur])) { - - int ret = parse_ipv6_comp(s); - if (ret < 0) return ret; - - tail[tail_len++] = (unsigned short) ret; - if (head_len + tail_len == 8) break; - - if (s->cur == s->len || s->src[s->cur] != ':') - break; - s->cur++; - } - } - - for (int i = 0; i < head_len; i++) - ipv6->data[i] = head[i]; - - for (int i = 0; i < 8 - head_len - tail_len; i++) - ipv6->data[head_len + i] = 0; - - for (int i = 0; i < tail_len; i++) - ipv6->data[8 - tail_len + i] = tail[i]; - - if (little_endian()) - for (int i = 0; i < 8; i++) - invert_bytes(&ipv6->data[i], 2); - - return 0; -} - -// From RFC 3986: -// reg-name = *( unreserved / pct-encoded / sub-delims ) -static int is_regname(char c) -{ - return is_unreserved(c) || is_sub_delim(c); -} - -static int parse_regname(Scanner *s, HTTP_String *regname) -{ - if (s->cur == s->len || !is_regname(s->src[s->cur])) - return -1; - int start = s->cur; - do - s->cur++; - while (s->cur < s->len && is_regname(s->src[s->cur])); - regname->ptr = s->src + start; - regname->len = s->cur - start; - return 0; -} - -static int parse_host(Scanner *s, HTTP_Host *host) -{ - int ret; - if (s->cur < s->len && s->src[s->cur] == '[') { - - s->cur++; - - int start = s->cur; - HTTP_IPv6 ipv6; - ret = parse_ipv6(s, &ipv6); - if (ret < 0) return ret; - - host->mode = HTTP_HOST_MODE_IPV6; - host->ipv6 = ipv6; - host->text = (HTTP_String) { s->src + start, s->cur - start }; - - if (s->cur == s->len || s->src[s->cur] != ']') - return -1; - s->cur++; - - } else { - - int start = s->cur; - HTTP_IPv4 ipv4; - ret = parse_ipv4(s, &ipv4); - if (ret >= 0) { - host->mode = HTTP_HOST_MODE_IPV4; - host->ipv4 = ipv4; - } else { - s->cur = start; - - HTTP_String regname; - ret = parse_regname(s, ®name); - if (ret < 0) return ret; - - host->mode = HTTP_HOST_MODE_NAME; - host->name = regname; - } - host->text = (HTTP_String) { s->src + start, s->cur - start }; - } - - return 0; -} - -// scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) -static int is_scheme_head(char c) -{ - return is_alpha(c); -} - -static int is_scheme_body(char c) -{ - return is_alpha(c) - || is_digit(c) - || c == '+' - || c == '-' - || c == '.'; -} - -// userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) -// Note: percent-encoded characters (%XX) are not currently validated -static int is_userinfo(char c) -{ - return is_unreserved(c) || is_sub_delim(c) || c == ':'; -} - -// authority = [ userinfo "@" ] host [ ":" port ] -static int parse_authority(Scanner *s, HTTP_Authority *authority) -{ - HTTP_String userinfo; - { - int start = s->cur; - - CONSUME_OPTIONAL_SEQUENCE(s, is_userinfo); - - if (s->cur < s->len && s->src[s->cur] == '@') { - userinfo = (HTTP_String) { - s->src + start, - s->cur - start - }; - s->cur++; - } else { - // Rollback - s->cur = start; - userinfo = (HTTP_String) {NULL, 0}; - } - } - - HTTP_Host host; - { - int ret = parse_host(s, &host); - if (ret < 0) - return ret; - } - - int port = 0; - if (s->cur < s->len && s->src[s->cur] == ':') { - s->cur++; - if (s->cur < s->len && is_digit(s->src[s->cur])) { - port = s->src[s->cur++] - '0'; - while (s->cur < s->len && is_digit(s->src[s->cur])) { - int x = s->src[s->cur++] - '0'; - if (port > (UINT16_MAX - x) / 10) - return -1; // ERROR: Port too big - port = port * 10 + x; - } - } - } - - authority->userinfo = userinfo; - authority->host = host; - authority->port = port; - return 0; -} - -static int parse_uri(Scanner *s, HTTP_URL *url, int allow_fragment) -{ - HTTP_String scheme = {0}; - { - int start = s->cur; - if (s->cur == s->len || !is_scheme_head(s->src[s->cur])) - return -1; // ERROR: Missing scheme - do - s->cur++; - while (s->cur < s->len && is_scheme_body(s->src[s->cur])); - scheme = (HTTP_String) { - s->src + start, - s->cur - start, - }; - - if (s->cur == s->len || s->src[s->cur] != ':') - return -1; // ERROR: Missing ':' after scheme - s->cur++; - } - - int abempty = 0; - HTTP_Authority authority = {0}; - if (s->len - s->cur > 1 - && s->src[s->cur+0] == '/' - && s->src[s->cur+1] == '/') { - - s->cur += 2; - - int ret = parse_authority(s, &authority); - if (ret < 0) return ret; - - abempty = 1; - } - - HTTP_String path; - int ret = parse_path(s, &path, abempty); - if (ret < 0) return ret; - - HTTP_String query = {0}; - if (s->cur < s->len && s->src[s->cur] == '?') { - int start = s->cur; - do - s->cur++; - while (s->cur < s->len && is_query(s->src[s->cur])); - query = (HTTP_String) { - s->src + start, - s->cur - start, - }; - } - - HTTP_String fragment = {0}; - if (allow_fragment && s->cur < s->len && s->src[s->cur] == '#') { - int start = s->cur; - do - s->cur++; - while (s->cur < s->len && is_fragment(s->src[s->cur])); - fragment = (HTTP_String) { - s->src + start, - s->cur - start, - }; - } - - url->scheme = scheme; - url->authority = authority; - url->path = path; - url->query = query; - url->fragment = fragment; - - return 1; -} - -// authority-form = host ":" port -// host = IP-literal / IPv4address / reg-name -// IP-literal = "[" ( IPv6address / IPvFuture ) "]" -// reg-name = *( unreserved / pct-encoded / sub-delims ) -static int parse_authority_form(Scanner *s, HTTP_Host *host, int *port) -{ - int ret; - - ret = parse_host(s, host); - if (ret < 0) return ret; - - // Default port value - *port = 0; - - if (s->cur == s->len || s->src[s->cur] != ':') - return 0; // No port - s->cur++; - - if (s->cur == s->len || !is_digit(s->src[s->cur])) - return 0; // No port - - int buf = 0; - do { - int x = s->src[s->cur++] - '0'; - if (buf > (UINT16_MAX - x) / 10) - return -1; // ERROR - buf = buf * 10 + x; - } while (s->cur < s->len && is_digit(s->src[s->cur])); - - *port = buf; - return 0; -} - -static int parse_origin_form(Scanner *s, HTTP_String *path, HTTP_String *query) -{ - int ret, start; - - start = s->cur; - ret = consume_absolute_path(s); - if (ret < 0) return ret; - *path = (HTTP_String) { s->src + start, s->cur - start }; - - if (s->cur < s->len && s->src[s->cur] == '?') { - start = s->cur; - do - s->cur++; - while (s->cur < s->len && is_query(s->src[s->cur])); - *query = (HTTP_String) { s->src + start, s->cur - start }; - } else - *query = (HTTP_String) { NULL, 0 }; - - return 0; -} - -static int parse_asterisk_form(Scanner *s) -{ - if (s->len - s->cur < 2 - || s->src[s->cur+0] != '*' - || s->src[s->cur+1] != ' ') - return -1; - s->cur++; - return 0; -} - -static int parse_request_target(Scanner *s, HTTP_URL *url) -{ - int ret; - - memset(url, 0, sizeof(HTTP_URL)); - - // asterisk-form - ret = parse_asterisk_form(s); - if (ret >= 0) return ret; - - ret = parse_uri(s, url, 0); - if (ret >= 0) return ret; - - ret = parse_authority_form(s, &url->authority.host, &url->authority.port); - if (ret >= 0) return ret; - - ret = parse_origin_form(s, &url->path, &url->query); - if (ret >= 0) return ret; - - return -1; -} - -bool consume_str(Scanner *scan, HTTP_String token) -{ - assert(token.len > 0); - - if (token.len > scan->len - scan->cur) - return false; - - for (int i = 0; i < token.len; i++) - if (scan->src[scan->cur + i] != token.ptr[i]) - return false; - - scan->cur += token.len; - return true; -} - -static int is_header_body(char c) -{ - return is_vchar(c) || c == ' ' || c == '\t'; -} - -static int parse_headers(Scanner *s, HTTP_Header *headers, int max_headers) -{ - int num_headers = 0; - while (!consume_str(s, HTTP_STR("\r\n"))) { - - // RFC 9112: - // field-line = field-name ":" OWS field-value OWS - // - // RFC 9110: - // field-value = *field-content - // field-content = field-vchar - // [ 1*( SP / HTAB / field-vchar ) field-vchar ] - // field-vchar = VCHAR / obs-text - // obs-text = %x80-FF - - int start; - - if (s->cur == s->len || !is_tchar(s->src[s->cur])) - return -1; // ERROR - start = s->cur; - do - s->cur++; - while (s->cur < s->len && is_tchar(s->src[s->cur])); - HTTP_String name = { s->src + start, s->cur - start }; - - if (s->cur == s->len || s->src[s->cur] != ':') - return -1; // ERROR - s->cur++; - - start = s->cur; - CONSUME_OPTIONAL_SEQUENCE(s, is_header_body); - HTTP_String body = { s->src + start, s->cur - start }; - body = http_trim(body); - - if (num_headers < max_headers) - headers[num_headers++] = (HTTP_Header) { name, body }; - - if (!consume_str(s, HTTP_STR("\r\n"))) { - return -1; - } - } - - return num_headers; -} - -typedef enum { - TRANSFER_ENCODING_OPTION_CHUNKED, - TRANSFER_ENCODING_OPTION_COMPRESS, - TRANSFER_ENCODING_OPTION_DEFLATE, - TRANSFER_ENCODING_OPTION_GZIP, -} TransferEncodingOption; - -static bool is_space(char c) -{ - return c == ' ' || c == '\t'; -} - -static int -parse_transfer_encoding(HTTP_String src, TransferEncodingOption *dst, int max) -{ - Scanner s = { src.ptr, src.len, 0 }; - - int num = 0; - for (;;) { - - CONSUME_OPTIONAL_SEQUENCE(&s, is_space); - - TransferEncodingOption opt; - if (0) {} - else if (consume_str(&s, HTTP_STR("chunked"))) opt = TRANSFER_ENCODING_OPTION_CHUNKED; - else if (consume_str(&s, HTTP_STR("compress"))) opt = TRANSFER_ENCODING_OPTION_COMPRESS; - else if (consume_str(&s, HTTP_STR("deflate"))) opt = TRANSFER_ENCODING_OPTION_DEFLATE; - else if (consume_str(&s, HTTP_STR("gzip"))) opt = TRANSFER_ENCODING_OPTION_GZIP; - else return -1; // Invalid option - - if (num == max) - return -1; // Too many options - dst[num++] = opt; - - CONSUME_OPTIONAL_SEQUENCE(&s, is_space); - - if (s.cur == s.len) - break; - - if (s.src[s.cur] != ',') - return -1; // Missing comma separator - } - - return num; -} - -static int -parse_content_length(const char *src, int len, uint64_t *out) -{ - int cur = 0; - while (cur < len && (src[cur] == ' ' || src[cur] == '\t')) - cur++; - - if (cur == len || !is_digit(src[cur])) - return -1; - - uint64_t buf = 0; - do { - int d = src[cur++] - '0'; - if (buf > (UINT64_MAX - d) / 10) - return -1; - buf = buf * 10 + d; - } while (cur < len && is_digit(src[cur])); - - *out = buf; - return 0; -} - -static int parse_body(Scanner *s, - HTTP_Header *headers, int num_headers, - HTTP_String *body, bool body_expected) -{ - - // RFC 9112 section 6: - // The presence of a message body in a request is signaled by a Content-Length or - // Transfer-Encoding header field. Request message framing is independent of method - // semantics. - - int header_index = http_find_header(headers, num_headers, HTTP_STR("Transfer-Encoding")); - if (header_index != -1) { - - // RFC 9112 section 6.1: - // A server MAY reject a request that contains both Content-Length and Transfer-Encoding - // or process such a request in accordance with the Transfer-Encoding alone. Regardless, - // the server MUST close the connection after responding to such a request to avoid the - // potential attacks. - if (http_find_header(headers, num_headers, HTTP_STR("Content-Length")) != -1) - return -1; - - HTTP_String value = headers[header_index].value; - - // RFC 9112 section 6.1: - // If any transfer coding other than chunked is applied to a request's content, the - // sender MUST apply chunked as the final transfer coding to ensure that the message - // is properly framed. If any transfer coding other than chunked is applied to a - // response's content, the sender MUST either apply chunked as the final transfer - // coding or terminate the message by closing the connection. - - TransferEncodingOption opts[8]; - int num = parse_transfer_encoding(value, opts, HTTP_COUNT(opts)); - if (num != 1 || opts[0] != TRANSFER_ENCODING_OPTION_CHUNKED) - return -1; - - HTTP_String chunks_maybe[128]; - HTTP_String *chunks = chunks_maybe; - int num_chunks = 0; - int max_chunks = HTTP_COUNT(chunks_maybe); - - #define FREE_CHUNK_LIST \ - if (chunks != chunks_maybe) \ - free(chunks); - - char *content_start = s->src + s->cur; - - for (;;) { - - // RFC 9112 section 7.1: - // The chunked transfer coding wraps content in order to transfer it as a series of chunks, - // each with its own size indicator, followed by an OPTIONAL trailer section containing - // trailer fields. - - if (s->cur == s->len) { - FREE_CHUNK_LIST - return 0; // Incomplete request - } - - if (!is_hex_digit(s->src[s->cur])) { - FREE_CHUNK_LIST - return -1; - } - - int chunk_len = 0; - - do { - char c = s->src[s->cur++]; - int n = hex_digit_to_int(c); - if (chunk_len > (INT_MAX - n) / 16) { - FREE_CHUNK_LIST - return -1; // overflow - } - chunk_len = chunk_len * 16 + n; - } while (s->cur < s->len && is_hex_digit(s->src[s->cur])); - - if (s->cur == s->len) { - FREE_CHUNK_LIST - return 0; // Incomplete request - } - if (s->src[s->cur] != '\r') { - FREE_CHUNK_LIST - return -1; - } - s->cur++; - - if (s->cur == s->len) { - FREE_CHUNK_LIST - return 0; - } - if (s->src[s->cur] != '\n') { - FREE_CHUNK_LIST - return -1; - } - s->cur++; - - char *chunk_ptr = s->src + s->cur; - - if (chunk_len > s->len - s->cur) { - FREE_CHUNK_LIST - return 0; // Incomplete request - } - s->cur += chunk_len; - - if (s->cur == s->len) - return 0; // Incomplete request - if (s->src[s->cur] != '\r') { - FREE_CHUNK_LIST - return -1; - } - s->cur++; - - if (s->cur == s->len) { - FREE_CHUNK_LIST - return 0; // Incomplete request - } - if (s->src[s->cur] != '\n') { - FREE_CHUNK_LIST - return -1; - } - s->cur++; - - if (chunk_len == 0) - break; - - if (num_chunks == max_chunks) { - - max_chunks *= 2; - - HTTP_String *new_chunks = malloc(max_chunks * sizeof(HTTP_String)); - if (new_chunks == NULL) { - if (chunks != chunks_maybe) - free(chunks); - return -1; - } - - for (int i = 0; i < num_chunks; i++) - new_chunks[i] = chunks[i]; - - if (chunks != chunks_maybe) - free(chunks); - - chunks = new_chunks; - } - chunks[num_chunks++] = (HTTP_String) { chunk_ptr, chunk_len }; - } - - char *content_ptr = content_start; - for (int i = 0; i < num_chunks; i++) { - memmove(content_ptr, chunks[i].ptr, chunks[i].len); - content_ptr += chunks[i].len; - } - - *body = (HTTP_String) { - content_start, - content_ptr - content_start - }; - - if (chunks != chunks_maybe) - free(chunks); - - return 1; - } - - // RFC 9112 section 6.3: - // If a valid Content-Length header field is present without Transfer-Encoding, - // its decimal value defines the expected message body length in octets. - - header_index = http_find_header(headers, num_headers, HTTP_STR("Content-Length")); - if (header_index != -1) { - - // Have Content-Length - HTTP_String value = headers[header_index].value; - - uint64_t tmp; - if (parse_content_length(value.ptr, value.len, &tmp) < 0) - return -1; - if (tmp > INT_MAX) - return -1; - int len = (int) tmp; - - if (len > s->len - s->cur) - return 0; // Incomplete request - - *body = (HTTP_String) { s->src + s->cur, len }; - - s->cur += len; - return 1; - } - - // No Content-Length or Transfer-Encoding - if (body_expected) return -1; - - *body = (HTTP_String) { NULL, 0 }; - return 1; -} - -static int contains_head(char *src, int len) -{ - int cur = 0; - while (len - cur > 3) { - if (src[cur+0] == '\r' && - src[cur+1] == '\n' && - src[cur+2] == '\r' && - src[cur+3] == '\n') - return 1; - cur++; - } - return 0; -} - -static int parse_request(Scanner *s, HTTP_Request *req) -{ - if (!contains_head(s->src + s->cur, s->len - s->cur)) - return 0; - - req->secure = false; - - if (0) {} - else if (consume_str(s, HTTP_STR("GET "))) req->method = HTTP_METHOD_GET; - else if (consume_str(s, HTTP_STR("POST "))) req->method = HTTP_METHOD_POST; - else if (consume_str(s, HTTP_STR("PUT "))) req->method = HTTP_METHOD_PUT; - else if (consume_str(s, HTTP_STR("HEAD "))) req->method = HTTP_METHOD_HEAD; - else if (consume_str(s, HTTP_STR("DELETE "))) req->method = HTTP_METHOD_DELETE; - else if (consume_str(s, HTTP_STR("CONNECT "))) req->method = HTTP_METHOD_CONNECT; - else if (consume_str(s, HTTP_STR("OPTIONS "))) req->method = HTTP_METHOD_OPTIONS; - else if (consume_str(s, HTTP_STR("TRACE "))) req->method = HTTP_METHOD_TRACE; - else if (consume_str(s, HTTP_STR("PATCH "))) req->method = HTTP_METHOD_PATCH; - else return -1; - - { - Scanner s2 = *s; - int peek = s->cur; - while (peek < s->len && s->src[peek] != ' ') - peek++; - if (peek == s->len) - return -1; - s2.len = peek; - - int ret = parse_request_target(&s2, &req->url); - if (ret < 0) return ret; - - s->cur = s2.cur; - } - - if (consume_str(s, HTTP_STR(" HTTP/1.1\r\n"))) { - req->minor = 1; - } else if (consume_str(s, HTTP_STR(" HTTP/1.0\r\n")) || consume_str(s, HTTP_STR(" HTTP/1\r\n"))) { - req->minor = 0; - } else { - return -1; - } - - int num_headers = parse_headers(s, req->headers, HTTP_MAX_HEADERS); - if (num_headers < 0) - return num_headers; - req->num_headers = num_headers; - - // Request methods that typically don't have a body - bool body_expected = true; - if (req->method == HTTP_METHOD_GET || - req->method == HTTP_METHOD_HEAD || - req->method == HTTP_METHOD_DELETE || - req->method == HTTP_METHOD_OPTIONS || - req->method == HTTP_METHOD_TRACE) - body_expected = false; - - return parse_body(s, req->headers, req->num_headers, &req->body, body_expected); -} - -int http_find_header(HTTP_Header *headers, int num_headers, HTTP_String name) -{ - for (int i = 0; i < num_headers; i++) - if (http_streqcase(name, headers[i].name)) - return i; - return -1; -} - -static int parse_response(Scanner *s, HTTP_Response *res) -{ - if (!contains_head(s->src + s->cur, s->len - s->cur)) - return 0; - - if (consume_str(s, HTTP_STR("HTTP/1.1 "))) { - res->minor = 1; - } else if (consume_str(s, HTTP_STR("HTTP/1.0 ")) || consume_str(s, HTTP_STR("HTTP/1 "))) { - res->minor = 0; - } else { - return -1; - } - - if (s->len - s->cur < 4 - || !is_digit(s->src[s->cur+0]) - || !is_digit(s->src[s->cur+1]) - || !is_digit(s->src[s->cur+2]) - || s->src[s->cur+3] != ' ') - return -1; - res->status = - (s->src[s->cur+0] - '0') * 100 + - (s->src[s->cur+1] - '0') * 10 + - (s->src[s->cur+2] - '0') * 1; - s->cur += 4; - - // Parse reason phrase: HTAB / SP / VCHAR / obs-text - // Note: obs-text (obsolete text, octets 0x80-0xFF) is not validated - while (s->cur < s->len && ( - s->src[s->cur] == '\t' || - s->src[s->cur] == ' ' || - is_vchar(s->src[s->cur]))) - s->cur++; - - if (s->len - s->cur < 2 - || s->src[s->cur+0] != '\r' - || s->src[s->cur+1] != '\n') - return -1; - s->cur += 2; - - int num_headers = parse_headers(s, res->headers, HTTP_MAX_HEADERS); - if (num_headers < 0) - return num_headers; - res->num_headers = num_headers; - - // Responses with certain status codes don't have a body: - // - 1xx (Informational) - // - 204 (No Content) - // - 304 (Not Modified) - // Note: HEAD responses also don't have a body, but we can't determine - // that here without access to the request method - bool body_expected = true; - if ((res->status >= 100 && res->status < 200) || - res->status == 204 || - res->status == 304) - body_expected = false; - - return parse_body(s, res->headers, res->num_headers, &res->body, body_expected); -} - -int http_parse_ipv4(char *src, int len, HTTP_IPv4 *ipv4) -{ - Scanner s = {src, len, 0}; - int ret = parse_ipv4(&s, ipv4); - if (ret < 0) return ret; - return s.cur; -} - -int http_parse_ipv6(char *src, int len, HTTP_IPv6 *ipv6) -{ - Scanner s = {src, len, 0}; - int ret = parse_ipv6(&s, ipv6); - if (ret < 0) return ret; - return s.cur; -} - -int http_parse_url(char *src, int len, HTTP_URL *url) -{ - Scanner s = {src, len, 0}; - int ret = parse_uri(&s, url, 1); - if (ret == 1) - return s.cur; - return ret; -} - -int http_parse_request(char *src, int len, HTTP_Request *req) -{ - Scanner s = {src, len, 0}; - int ret = parse_request(&s, req); - if (ret == 1) - return s.cur; - return ret; -} - -int http_parse_response(char *src, int len, HTTP_Response *res) -{ - Scanner s = {src, len, 0}; - int ret = parse_response(&s, res); - if (ret == 1) - return s.cur; - return ret; -} - -HTTP_String http_get_cookie(HTTP_Request *req, HTTP_String name) -{ - // Simple cookie parsing - does not handle quoted values or special characters - // See RFC 6265 for full cookie specification - - for (int i = 0; i < req->num_headers; i++) { - - if (!http_streqcase(req->headers[i].name, HTTP_STR("Cookie"))) - continue; - - char *src = req->headers[i].value.ptr; - int len = req->headers[i].value.len; - int cur = 0; - - // Cookie: name1=value1; name2=value2; name3=value3 - - for (;;) { - - while (cur < len && src[cur] == ' ') - cur++; - - int off = cur; - while (cur < len && src[cur] != '=') - cur++; - - HTTP_String cookie_name = { src + off, cur - off }; - - if (cur == len) - break; - cur++; - - off = cur; - while (cur < len && src[cur] != ';') - cur++; - - HTTP_String cookie_value = { src + off, cur - off }; - - if (http_streq(name, cookie_name)) - return cookie_value; - - if (cur == len) - break; - cur++; - } - } - - return HTTP_STR(""); -} - -HTTP_String http_get_param(HTTP_String body, HTTP_String str, char *mem, int cap) -{ - // This is just a best-effort implementation - - char *src = body.ptr; - int len = body.len; - int cur = 0; - - if (cur < len && src[cur] == '?') - cur++; - - while (cur < len) { - - HTTP_String name; - { - int off = cur; - while (cur < len && src[cur] != '=' && src[cur] != '&') - cur++; - name = (HTTP_String) { src + off, cur - off }; - } - - HTTP_String body = HTTP_STR(""); - if (cur < len) { - cur++; - if (src[cur-1] == '=') { - int off = cur; - while (cur < len && src[cur] != '&') - cur++; - body = (HTTP_String) { src + off, cur - off }; - - if (cur < len) - cur++; - } - } - - if (http_streq(str, name)) { - - bool percent_encoded = false; - for (int i = 0; i < body.len; i++) - if (body.ptr[i] == '+' || body.ptr[i] == '%') { - percent_encoded = true; - break; - } - - if (!percent_encoded) - return body; - - if (body.len > cap) - return (HTTP_String) { NULL, 0 }; - - HTTP_String decoded = { mem, 0 }; - for (int i = 0; i < body.len; i++) { - - char c = body.ptr[i]; - if (c == '+') - c = ' '; - else { - if (body.ptr[i] == '%') { - if (body.len - i < 3 - || !is_hex_digit(body.ptr[i+1]) - || !is_hex_digit(body.ptr[i+2])) - return (HTTP_String) { NULL, 0 }; - - int h = hex_digit_to_int(body.ptr[i+1]); - int l = hex_digit_to_int(body.ptr[i+2]); - c = (h << 4) | l; - - i += 2; - } - } - - decoded.ptr[decoded.len++] = c; - } - - return decoded; - } - } - - return HTTP_STR(""); -} - -int http_get_param_i(HTTP_String body, HTTP_String str) -{ - char buf[128]; - HTTP_String out = http_get_param(body, str, buf, (int) sizeof(buf)); - if (out.len == 0 || !is_digit(out.ptr[0])) - return -1; - - int cur = 0; - int res = 0; - do { - int d = out.ptr[cur++] - '0'; - if (res > (INT_MAX - d) / 10) - return -1; - res = res * 10 + d; - } while (cur < out.len && is_digit(out.ptr[cur])); - - return res; -} - -bool http_match_host(HTTP_Request *req, HTTP_String domain, int port) -{ - int idx = http_find_header(req->headers, req->num_headers, HTTP_STR("Host")); - assert(idx != -1); // Requests without the host header are always rejected - - char tmp[1<<8]; - if (port > -1 && port != 80) { - int ret = snprintf(tmp, sizeof(tmp), "%.*s:%d", domain.len, domain.ptr, port); - assert(ret > 0); - domain = (HTTP_String) { tmp, ret }; - } - - HTTP_String host = req->headers[idx].value; - return http_streqcase(host, domain); -} - - -// , :: GMT -static int parse_date(Scanner *s, HTTP_Date *out) -{ - struct { HTTP_String str; HTTP_WeekDay val; } week_day_table[] = { - { HTTP_STR("Mon, "), HTTP_WEEKDAY_MON }, - { HTTP_STR("Tue, "), HTTP_WEEKDAY_TUE }, - { HTTP_STR("Wed, "), HTTP_WEEKDAY_WED }, - { HTTP_STR("Thu, "), HTTP_WEEKDAY_THU }, - { HTTP_STR("Fri, "), HTTP_WEEKDAY_FRI }, - { HTTP_STR("Sat, "), HTTP_WEEKDAY_SAT }, - { HTTP_STR("Sun, "), HTTP_WEEKDAY_SUN }, - }; - - bool found = false; - for (int i = 0; i < HTTP_COUNT(week_day_table); i++) - if (consume_str(s, week_day_table[i].str)) { - out->week_day = week_day_table[i].val; - found = true; - break; - } - if (!found) - return -1; - - if (1 >= s->len - s->cur - || !is_digit(s->src[s->cur+0]) - || !is_digit(s->src[s->cur+1])) - return -1; - out->day - = (s->src[s->cur+0] - '0') * 10 - + (s->src[s->cur+1] - '0') * 1; - s->cur += 2; - - struct { HTTP_String str; HTTP_Month val; } month_table[] = { - { HTTP_STR(" Jan "), HTTP_MONTH_JAN }, - { HTTP_STR(" Feb "), HTTP_MONTH_FEB }, - { HTTP_STR(" Mar "), HTTP_MONTH_MAR }, - { HTTP_STR(" Apr "), HTTP_MONTH_APR }, - { HTTP_STR(" May "), HTTP_MONTH_MAY }, - { HTTP_STR(" Jun "), HTTP_MONTH_JUN }, - { HTTP_STR(" Jul "), HTTP_MONTH_JUL }, - { HTTP_STR(" Aug "), HTTP_MONTH_AUG }, - { HTTP_STR(" Sep "), HTTP_MONTH_SEP }, - { HTTP_STR(" Oct "), HTTP_MONTH_OCT }, - { HTTP_STR(" Nov "), HTTP_MONTH_NOV }, - { HTTP_STR(" Dec "), HTTP_MONTH_DEC }, - }; - - found = false; - for (int i = 0; i < HTTP_COUNT(month_table); i++) - if (consume_str(s, month_table[i].str)) { - out->month = month_table[i].val; - found = true; - break; - } - if (!found) - return -1; - - if (3 >= s->len - s->cur - || !is_digit(s->src[s->cur+0]) - || !is_digit(s->src[s->cur+1]) - || !is_digit(s->src[s->cur+2]) - || !is_digit(s->src[s->cur+3])) - return -1; - out->year - = (s->src[s->cur+0] - '0') * 1000 - + (s->src[s->cur+1] - '0') * 100 - + (s->src[s->cur+2] - '0') * 10 - + (s->src[s->cur+3] - '0') * 1; - s->cur += 4; - - if (s->cur == s->len || s->src[s->cur] != ' ') - return -1; - s->cur++; - - if (7 >= s->len - s->cur - || !is_digit(s->src[s->cur+0]) - || !is_digit(s->src[s->cur+1]) - || s->src[s->cur+2] != ':' - || !is_digit(s->src[s->cur+3]) - || !is_digit(s->src[s->cur+4]) - || s->src[s->cur+5] != ':' - || !is_digit(s->src[s->cur+6]) - || !is_digit(s->src[s->cur+7]) - || s->src[s->cur+8] != ' ' - || s->src[s->cur+9] != 'G' - || s->src[s->cur+10] != 'M' - || s->src[s->cur+11] != 'T') - return -1; - out->hour - = (s->src[s->cur+0] - '0') * 10 - + (s->src[s->cur+1] - '0') * 1; - out->minute - = (s->src[s->cur+3] - '0') * 10 - + (s->src[s->cur+4] - '0') * 1; - out->second - = (s->src[s->cur+6] - '0') * 10 - + (s->src[s->cur+7] - '0') * 1; - s->cur += 12; - return 0; -} - -// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E -// ; US-ASCII characters excluding CTLs, -// ; whitespace, DQUOTE, comma, semicolon, -// ; and backslash -static bool is_cookie_octet(char c) -{ - return c == 0x21 || - (c >= 0x23 && c <= 0x2B) || - (c >= 0x2D && c <= 0x3A) || - (c >= 0x3C && c <= 0x5B) || - (c >= 0x5D && c <= 0x7E); -} - -int http_parse_set_cookie(HTTP_String str, HTTP_SetCookie *out) -{ - Scanner s = { str.ptr, str.len, 0 }; - - // cookie-name = token - if (s.cur == s.len || !is_tchar(s.src[s.cur])) - return -1; - int off = s.cur; - do - s.cur++; - while (s.cur < s.len && is_tchar(s.src[s.cur])); - out->name = (HTTP_String) { s.src + off, s.cur - off }; - - // cookie-pair = cookie-name "=" cookie-value - if (s.cur == s.len || s.src[s.cur] != '=') - return -1; - s.cur++; - - // cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) - if (s.cur < s.len && s.src[s.cur] == '"') { - s.cur++; // Consume opening double quote - int off = s.cur; - while (s.cur < s.len && is_cookie_octet(s.src[s.cur])) - s.cur++; - if (s.cur == s.len || s.src[s.cur] != '"') - return -1; // Missing closing double quote - out->value = (HTTP_String) { s.src + off, s.cur - off }; - s.cur++; // Consume closing double quote - } else { - int off = s.cur; - while (s.cur < s.len && is_cookie_octet(s.src[s.cur])) - s.cur++; - out->value = (HTTP_String) { s.src + off, s.cur - off }; - } - - // *( ";" SP cookie-av ) - // - // cookie-av = expires-av / max-age-av / domain-av / - // path-av / secure-av / httponly-av / - // extension-av - out->secure = false; - out->http_only = false; - out->have_date = false; - out->have_max_age = false; - out->have_domain = false; - out->have_path = false; - while (consume_str(&s, HTTP_STR("; "))) { - if (consume_str(&s, HTTP_STR("Expires="))) { - - // expires-av = "Expires=" sane-cookie-date - if (parse_date(&s, &out->date) < 0) - return -1; - out->have_date = true; - - } else if (consume_str(&s, HTTP_STR("Max-Age="))) { - - // max-age-av = "Max-Age=" non-zero-digit *DIGIT - - uint32_t value = 0; - if (s.cur == s.len || !is_digit(s.src[s.cur])) - return -1; - do { - int d = s.src[s.cur++] - '0'; - if (value > (UINT32_MAX - d) / 10) - return -1; - value = value * 10 + d; - } while (s.cur < s.len && is_digit(s.src[s.cur])); - - out->have_max_age = true; - out->max_age = value; - - } else if (consume_str(&s, HTTP_STR("Domain="))) { - - // domain-av = "Domain=" domain-value - // domain-value = - // ; defined in RFC 1034, Section 3.5 - // - // From RFC 1034: - // ::=