Research-Stack/4-Infrastructure/storage/garage/garage-cluster-init.sh

#!/usr/bin/env bash
# garage-cluster-init.sh
#
# Run after all nodes have been bootstrapped with garage-node-bootstrap.sh.
# This script:
#   1. Connects all nodes to the cluster (garage node connect)
#   2. Assigns layout roles (zone + capacity) to each node
#   3. Bumps replication_factor to 3 once 3+ nodes are available
#   4. Verifies bucket and key state
#
# Usage:
#   bash garage-cluster-init.sh
#
# Idempotent — safe to re-run.

set -euo pipefail

ADMIN_TOKEN=$(sudo grep GARAGE_ADMIN_TOKEN /etc/garage/garage.env | cut -d= -f2)
G="sudo garage -c /etc/garage/garage.toml --admin-token ${ADMIN_TOKEN}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REGISTRY="${SCRIPT_DIR}/../node-registry.json"

log() { echo "[cluster-init] $*" >&2; }

log "Current cluster status:"
$G status 2>&1
echo ""

# ── 1. Connect all registered nodes ───────────────────────────────────────────
if [ -f "$REGISTRY" ]; then
    log "Connecting nodes from registry..."
    python3 - << PYEOF
import json, subprocess, sys

with open("${REGISTRY}") as f:
    nodes = json.load(f)

for node in nodes:
    node_id = node.get("garage_node_id", "").strip()
    ip = node.get("tailscale_ip", "")
    if not node_id or "@" not in node_id:
        # node_id should be full_id@ip:port format
        if node_id:
            node_id = f"{node_id}@{ip}:3901"
        else:
            print(f"  SKIP {ip}: no garage_node_id recorded yet")
            continue
    print(f"  Connecting: {node_id[:16]}...@{ip}:3901")
    result = subprocess.run(
        ["sudo", "garage", "-c", "/etc/garage/garage.toml",
         "--admin-token", "${ADMIN_TOKEN}",
         "node", "connect", node_id],
        capture_output=True, text=True
    )
    if result.returncode == 0 or "already" in result.stdout.lower():
        print(f"    OK")
    else:
        print(f"    WARN: {result.stderr.strip()[:80]}")
PYEOF
fi

sleep 2
log "Cluster after connect:"
$G status 2>&1
echo ""

# ── 2. Count healthy nodes ─────────────────────────────────────────────────────
HEALTHY_NODES=$($G status 2>&1 | grep -c "v2\." || echo 0)
log "Healthy nodes: ${HEALTHY_NODES}"

# ── 3. Assign layout roles ─────────────────────────────────────────────────────
log "Assigning layout roles..."
$G layout show 2>&1

# Assign each unroled node a zone based on its tag
# Zone strategy: group local nodes together, VPS (microvm-racknerd) in its own zone
# This ensures Garage spreads replicas across failure domains.
LAYOUT_VERSION=$($G layout show 2>&1 | grep "^Current cluster layout version" | awk '{print $NF}' || echo 1)
NEXT_VERSION=$((LAYOUT_VERSION + 1))

$G status 2>&1 | (grep "NO ROLE ASSIGNED" || true) | awk '{print $1}' | while read node_id; do
    HOSTNAME=$($G status 2>&1 | grep "$node_id" | awk '{print $2}')
    # Assign zone: VPS nodes get zone=vps, local Tailscale nodes get zone=local
    if echo "$HOSTNAME $node_id" | grep -qi "microvm\|racknerd\|vps"; then
        ZONE="vps"
    else
        ZONE="local"
    fi
    log "  Assigning $node_id ($HOSTNAME) → zone=$ZONE, 200G"
    $G layout assign "$node_id" --zone "$ZONE" --capacity 200G 2>&1 || true
done

# ── 4. Bump replication factor if enough nodes ────────────────────────────────
CURRENT_RF=$(grep -E 'replication_factor[[:space:]]*=[[:space:]]*[0-9]+' /etc/garage/garage.toml | grep -o '[0-9]\+' || echo 1)
if [ "$HEALTHY_NODES" -ge 3 ] && [ "${CURRENT_RF}" -lt 3 ]; then
    log "3+ nodes available and replication_factor < 3 — bumping replication_factor to 3"
    sudo sed -i 's/replication_factor = 1/replication_factor = 3/' /etc/garage/garage.toml
    # Remove old layout so it can be recreated with rf=3
    sudo systemctl stop garage
    sudo rm -f /var/lib/garage/meta/cluster_layout
    sudo systemctl start garage
    sleep 3
    # Re-assign all roles
    log "Re-assigning all layout roles for rf=3..."
    $G status 2>&1 | (grep "NO ROLE ASSIGNED" || true) | awk '{print $1}' | while read node_id; do
        $G layout assign "$node_id" --zone local --capacity 200G 2>&1 || true
    done
    # Assign primary separately
    $G layout assign 3e08a71b73fa2b10 --zone local --capacity 900G --tag primary 2>&1 || true
    NEXT_VERSION=1
fi

# ── 5. Apply layout ────────────────────────────────────────────────────────────
log "Applying layout (version ${NEXT_VERSION})..."
$G layout apply --version "${NEXT_VERSION}" 2>&1

sleep 2

# ── 6. Verify buckets and keys ─────────────────────────────────────────────────
log "Verifying buckets..."
EXISTING=$($G bucket list 2>&1)
for bucket in research-stack db-scratch rds-overflow snap-zone gdrive-mirror; do
    if echo "$EXISTING" | grep -q "$bucket"; then
        echo "  OK  $bucket"
    else
        $G bucket create "$bucket" 2>&1
        echo "  CREATED  $bucket"
    fi
done

KEY_ID="GK55105d55675994caea2c2d3d"
for bucket in research-stack db-scratch rds-overflow snap-zone gdrive-mirror; do
    $G bucket allow --read --write --owner "$bucket" --key "$KEY_ID" 2>&1 | grep -v "^==" || true
done

log ""
log "Final cluster status:"
$G status 2>&1
$G bucket list 2>&1