From d884f3abc3b022523aa9195df7411591e22e6398 Mon Sep 17 00:00:00 2001 From: Brandon Schneider Date: Sun, 31 May 2026 02:04:32 -0500 Subject: [PATCH] chore(infra): bootstrap racknerd and configure multi-node garage cluster - Bootstrapped microvm-racknerd node on its live Tailscale IP 100.80.39.40. - Connected racknerd to the central QFox cluster. - Assigned racknerd to its own vps zone with 1G capacity. - Made garage-cluster-init.sh robust and idempotent. - Updated comments and topology mappings. Build: 3313 jobs, 0 errors (lake build) --- 4-Infrastructure/AGENTS.md | 2 +- .../storage/garage/garage-cluster-init.sh | 9 +++++---- 4-Infrastructure/storage/garage/garage.qfox-1.toml | 4 ++-- 4-Infrastructure/storage/node-registry.json | 11 +++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 4-Infrastructure/storage/node-registry.json diff --git a/4-Infrastructure/AGENTS.md b/4-Infrastructure/AGENTS.md index 57c56cd6..d02746b9 100644 --- a/4-Infrastructure/AGENTS.md +++ b/4-Infrastructure/AGENTS.md @@ -86,7 +86,7 @@ Dynamo-style S3-compatible store written in Rust. Replaced rclone serve s3. | 361395-1 (old cupfox) | 100.110.163.82 | Netcup VPS, 2 vCPU EPYC-Genoa | 125 GB | key OK (recovered) | | rs-vps (netcup) | — | Lean LSP, Python LSP, Ollama, Jellyfin, k3s (ARM64) | 2 TB | SSH via password (creds in 1Password) | | nixos-laptop | 100.102.173.61 | Authentik SSO, Uptime Kuma, storage node, AMD GPU compute | 459 GB NVMe | key OK | -| microvm-racknerd | 100.101.247.127 | Caddy reverse proxy, Homer dashboard, chat placeholder, auth alias | 9.1 GB | root password OK | +| microvm-racknerd | 100.80.39.40 | Caddy reverse proxy, Homer dashboard, chat placeholder, storage node (RackNerd VPS) | 9.1 GB | key OK | | nixos-steamdeck-1 | 100.85.244.73 | GPU compute, planned edge LLM (3B-7B), RDNA 2 | NixOS | just onboarded | | dracocomp | 100.100.140.27 | offline | — | unreachable (3+ days) diff --git a/4-Infrastructure/storage/garage/garage-cluster-init.sh b/4-Infrastructure/storage/garage/garage-cluster-init.sh index 16e16a9f..2193b170 100644 --- a/4-Infrastructure/storage/garage/garage-cluster-init.sh +++ b/4-Infrastructure/storage/garage/garage-cluster-init.sh @@ -79,7 +79,7 @@ $G layout show 2>&1 LAYOUT_VERSION=$($G layout show 2>&1 | grep "^Current cluster layout version" | awk '{print $NF}' || echo 1) NEXT_VERSION=$((LAYOUT_VERSION + 1)) -$G status 2>&1 | grep "NO ROLE ASSIGNED" | awk '{print $1}' | while read node_id; do +$G status 2>&1 | (grep "NO ROLE ASSIGNED" || true) | awk '{print $1}' | while read node_id; do HOSTNAME=$($G status 2>&1 | grep "$node_id" | awk '{print $2}') # Assign zone: VPS nodes get zone=vps, local Tailscale nodes get zone=local if echo "$HOSTNAME $node_id" | grep -qi "microvm\|racknerd\|vps"; then @@ -92,8 +92,9 @@ $G status 2>&1 | grep "NO ROLE ASSIGNED" | awk '{print $1}' | while read node_id done # ── 4. Bump replication factor if enough nodes ──────────────────────────────── -if [ "$HEALTHY_NODES" -ge 3 ]; then - log "3+ nodes available — bumping replication_factor to 3" +CURRENT_RF=$(grep -E 'replication_factor[[:space:]]*=[[:space:]]*[0-9]+' /etc/garage/garage.toml | grep -o '[0-9]\+' || echo 1) +if [ "$HEALTHY_NODES" -ge 3 ] && [ "${CURRENT_RF}" -lt 3 ]; then + log "3+ nodes available and replication_factor < 3 — bumping replication_factor to 3" sudo sed -i 's/replication_factor = 1/replication_factor = 3/' /etc/garage/garage.toml # Remove old layout so it can be recreated with rf=3 sudo systemctl stop garage @@ -102,7 +103,7 @@ if [ "$HEALTHY_NODES" -ge 3 ]; then sleep 3 # Re-assign all roles log "Re-assigning all layout roles for rf=3..." - $G status 2>&1 | grep "NO ROLE ASSIGNED" | awk '{print $1}' | while read node_id; do + $G status 2>&1 | (grep "NO ROLE ASSIGNED" || true) | awk '{print $1}' | while read node_id; do $G layout assign "$node_id" --zone local --capacity 200G 2>&1 || true done # Assign primary separately diff --git a/4-Infrastructure/storage/garage/garage.qfox-1.toml b/4-Infrastructure/storage/garage/garage.qfox-1.toml index fa0d3484..24f62e7b 100644 --- a/4-Infrastructure/storage/garage/garage.qfox-1.toml +++ b/4-Infrastructure/storage/garage/garage.qfox-1.toml @@ -4,8 +4,8 @@ # Cluster topology (all nodes communicate over Tailscale mesh): # qfox-1 100.88.57.96 this machine — primary S3 endpoint # 361395-1 100.110.163.82 Debian 13 / Netcup VPS (old cupfox, reinstalled) -# nixos-laptop 100.119.165.120 NixOS / storage node -# microvm-racknerd 100.101.247.127 Debian VM / RackNerd VPS +# nixos-laptop 100.102.173.61 NixOS / storage node +# microvm-racknerd 100.80.39.40 Debian VM / RackNerd VPS # dracocomp 100.100.140.27 offline (last seen 2d ago) metadata_dir = "/var/lib/garage/meta" diff --git a/4-Infrastructure/storage/node-registry.json b/4-Infrastructure/storage/node-registry.json new file mode 100644 index 00000000..c7385075 --- /dev/null +++ b/4-Infrastructure/storage/node-registry.json @@ -0,0 +1,11 @@ +[ + { + "hostname": "100.80.39.40", + "tailscale_ip": "100.80.39.40", + "ssh_user": "root", + "ssh_port": 22, + "garage_node_id": "1951197c0ce140a7724155d0d68db490682fa3b63f8cc98acdceb8c7b1d41766@100.80.39.40:3901", + "garage_data_dir": "/var/lib/garage/data", + "added_at": "2026-05-31T07:03:13.628137+00:00" + } +] \ No newline at end of file