diff --git a/.gitignore b/.gitignore index ed2de37..18b098e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ coverage_html/ *.o *.a *.so -examples/example \ No newline at end of file +examples/example +.cluster/ diff --git a/cluster.sh b/cluster.sh index 97d5342..0d8800e 100755 --- a/cluster.sh +++ b/cluster.sh @@ -1,25 +1,24 @@ #!/usr/bin/env bash # -# cluster.sh - Spin up a ToastyFS cluster and optionally build & run -# an application that links against libtoastyfs. +# cluster.sh - Manage a local 3-node ToastyFS cluster. # # Usage: -# ./cluster.sh up Start the 3-node cluster -# ./cluster.sh down Stop the cluster -# ./cluster.sh status Show cluster status -# ./cluster.sh logs [node] Tail cluster logs (or a single node) +# ./cluster.sh up Build the server and start 3 nodes +# ./cluster.sh down Stop all nodes +# ./cluster.sh status Show which nodes are running +# ./cluster.sh logs [1|2|3] Tail logs (all nodes, or one) # ./cluster.sh run Build the library, compile # against it, start the cluster, run the # binary, then tear down the cluster. # -# The cluster exposes ports 8081-8083 on localhost, mapped to the three -# server nodes. Client applications should connect to: -# 127.0.0.1:8081 127.0.0.1:8082 127.0.0.1:8083 +# The three server nodes listen on: +# 127.0.0.1:8081 127.0.0.1:8082 127.0.0.1:8083 # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -cd "$SCRIPT_DIR" +CLUSTER_DIR="$SCRIPT_DIR/.cluster" +ADDRS=("127.0.0.1:8081" "127.0.0.1:8082" "127.0.0.1:8083") usage() { sed -n '3,14s/^# \?//p' "$0" @@ -27,26 +26,105 @@ usage() { } cluster_up() { - echo "==> Building server image and starting cluster..." - docker compose up --build -d + echo "==> Building server..." + make -C "$SCRIPT_DIR" toastyfs + + mkdir -p "$CLUSTER_DIR" + + for i in 1 2 3; do + pidfile="$CLUSTER_DIR/node${i}.pid" + if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + echo " Node $i already running (pid $(cat "$pidfile"))" + continue + fi + + node_dir="$CLUSTER_DIR/node${i}" + mkdir -p "$node_dir" + + # Build argument list: --addr for self, --peer for the others. + args=() + for j in 1 2 3; do + idx=$((j - 1)) + if [ "$j" -eq "$i" ]; then + args+=(--addr "${ADDRS[$idx]}") + else + args+=(--peer "${ADDRS[$idx]}") + fi + done + + ( + cd "$node_dir" + # ServerState is ~40 MB (MetaStore), which exceeds the + # default 8 MB stack. Raise the limit for the server. + ulimit -s unlimited + "$SCRIPT_DIR/toastyfs" "${args[@]}" \ + >> "$CLUSTER_DIR/node${i}.log" 2>&1 & + echo $! > "$pidfile" + ) + echo " Node $i started (pid $(cat "$pidfile")) on ${ADDRS[$((i-1))]}" + done echo "==> Cluster is running." - echo " Nodes: 127.0.0.1:8081 127.0.0.1:8082 127.0.0.1:8083" } cluster_down() { - echo "==> Stopping cluster..." - docker compose down + if [ ! -d "$CLUSTER_DIR" ]; then + echo "No cluster state found." + return + fi + + for i in 1 2 3; do + pidfile="$CLUSTER_DIR/node${i}.pid" + if [ -f "$pidfile" ]; then + pid="$(cat "$pidfile")" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + echo " Node $i stopped (pid $pid)" + else + echo " Node $i was not running" + fi + rm -f "$pidfile" + fi + done + + # Clean up node data directories so fresh starts don't trigger + # the crash-recovery path (vsr_boot_marker). + rm -rf "$CLUSTER_DIR" + echo "==> Cluster stopped." } cluster_status() { - docker compose ps + if [ ! -d "$CLUSTER_DIR" ]; then + echo "No cluster state found. Run './cluster.sh up' first." + return + fi + + printf "%-8s %-8s %-20s %s\n" "NODE" "PID" "ADDRESS" "STATUS" + for i in 1 2 3; do + pidfile="$CLUSTER_DIR/node${i}.pid" + addr="${ADDRS[$((i-1))]}" + if [ -f "$pidfile" ]; then + pid="$(cat "$pidfile")" + if kill -0 "$pid" 2>/dev/null; then + printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "running" + else + printf "%-8s %-8s %-20s %s\n" "node$i" "$pid" "$addr" "dead" + fi + else + printf "%-8s %-8s %-20s %s\n" "node$i" "-" "$addr" "not started" + fi + done } cluster_logs() { if [ $# -gt 0 ]; then - docker compose logs -f "$1" + logfile="$CLUSTER_DIR/node${1}.log" + if [ ! -f "$logfile" ]; then + echo "No log file for node $1." + return 1 + fi + tail -f "$logfile" else - docker compose logs -f + tail -f "$CLUSTER_DIR"/node*.log fi } diff --git a/examples/example.c b/examples/example.c index 4f79820..7f4cc21 100644 --- a/examples/example.c +++ b/examples/example.c @@ -36,12 +36,13 @@ static const char *error_name(ToastyFS_Error e) int main(void) { - /* Cluster addresses (mapped via docker-compose ports). */ - char *addrs[] = { - "127.0.0.1:8081", - "127.0.0.1:8082", - "127.0.0.1:8083", - }; + /* Cluster addresses (mapped via docker-compose ports). + * These must be writable char arrays because parse_addr_arg + * temporarily modifies the string in-place. */ + char addr1[] = "127.0.0.1:8081"; + char addr2[] = "127.0.0.1:8082"; + char addr3[] = "127.0.0.1:8083"; + char *addrs[] = { addr1, addr2, addr3 }; printf("Connecting to cluster...\n"); ToastyFS *tfs = toastyfs_init(1, addrs, 3); @@ -54,8 +55,8 @@ int main(void) int ret; /* ---- PUT ---- */ - char *key = "hello"; - char *data = "world"; + char key[] = "hello"; + char data[] = "world"; printf("PUT key=\"%s\" data=\"%s\" (%d bytes)\n", key, data, (int)strlen(data)); diff --git a/src/main.c b/src/main.c index 2721efb..b80d5cc 100644 --- a/src/main.c +++ b/src/main.c @@ -255,7 +255,14 @@ int main(int argc, char **argv) int main(int argc, char **argv) { int ret; - ServerState state; + + // ServerState is ~40 MB (MetaStore holds 4096 ObjectMeta entries), + // which exceeds the default stack size. Heap-allocate it. + ServerState *state = malloc(sizeof(ServerState)); + if (state == NULL) { + fprintf(stderr, "Failed to allocate ServerState\n"); + return -1; + } void* poll_ctxs[POLL_CAPACITY]; struct pollfd poll_array[POLL_CAPACITY]; @@ -263,7 +270,7 @@ int main(int argc, char **argv) int poll_timeout; ret = server_init( - &state, + state, argc, argv, poll_ctxs, @@ -284,7 +291,7 @@ int main(int argc, char **argv) #endif ret = server_tick( - &state, + state, poll_ctxs, poll_array, POLL_CAPACITY, @@ -295,7 +302,8 @@ int main(int argc, char **argv) return -1; } - server_free(&state); + server_free(state); + free(state); return 0; }