#!/usr/bin/env bash # verify-internal.sh — internal routing smoke test # # Run on nixos-laptop (100.102.173.61) after nixos-rebuild switch, # BEFORE deploying the edge Caddy. Exercises the host-Caddy → Traefik # → Ingress path directly so you know the internal leg is clean. # # Usage: # bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh # bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh --remote # (runs via ssh root@100.102.173.61 if not already on the node) # # Exit 0 = all checks passed. # Exit 1 = one or more checks failed; details printed to stderr. set -euo pipefail NODE_IP="100.102.173.61" PASS=0 FAIL=0 # ── helpers ────────────────────────────────────────────────────────────────── green() { printf '\033[32m✓ %s\033[0m\n' "$*"; } red() { printf '\033[31m✗ %s\033[0m\n' "$*" >&2; } check() { local label="$1"; shift if "$@" 2>/dev/null; then green "$label" (( PASS++ )) || true else red "$label" (( FAIL++ )) || true fi } # Run from remote if requested and not already on the node. if [[ "${1:-}" == "--remote" ]]; then echo "[verify] running remotely on $NODE_IP..." exec ssh "root@$NODE_IP" bash -s < "$0" fi echo echo "═══════════════════════════════════════════════════════" echo " Internal routing smoke test — $(date -u '+%Y-%m-%dT%H:%M:%SZ')" echo " Target node: $NODE_IP" echo "═══════════════════════════════════════════════════════" echo # ── 0. Listener sanity ─────────────────────────────────────────────────────── echo "── 0. Port listeners ──" check "host Caddy owns :80" \ bash -c "ss -tlnp | grep -q ':80 '" check "Traefik NodePort :30080 is accessible" \ nc -z -w 2 "$NODE_IP" 30080 echo # ── 1. Root path: Homer or Authentik redirect ──────────────────────────────── echo "── 1. GET / (expect 200 Homer or 302 → Authentik) ──" root_status=$(curl -so /dev/null -w '%{http_code}' \ -H "Host: researchstack.info" \ -H "X-Forwarded-Proto: https" \ --max-time 5 \ "http://$NODE_IP/") check "/ returns 200, 302 or 404 (got $root_status)" \ bash -c '[[ '"$root_status"' == 200 || '"$root_status"' == 302 || '"$root_status"' == 404 ]]' if [[ "$root_status" == "302" ]]; then root_loc=$(curl -so /dev/null -w '%{redirect_url}' \ -H "Host: researchstack.info" \ -H "X-Forwarded-Proto: https" \ --max-time 5 \ "http://$NODE_IP/") check "/ redirect points to auth.researchstack.info (not an http://10x... internal URL)" \ bash -c '[[ '"\"$root_loc\""' == *"auth.researchstack.info"* ]]' fi echo # ── 2. API bypass: /api/* must NOT 302 to Authentik ───────────────────────── echo "── 2. /api/* paths must not redirect to Authentik ──" for path in /api/registry/health /api/jobs/health /api/blobs/health; do status=$(curl -so /dev/null -w '%{http_code}' \ -H "Host: researchstack.info" \ --max-time 5 \ "http://$NODE_IP$path") check "$path → $status (not 302)" \ bash -c '[[ '"$status"' != 302 ]]' if [[ "$status" == "302" ]]; then loc=$(curl -so /dev/null -w '%{redirect_url}' \ -H "Host: researchstack.info" \ --max-time 5 \ "http://$NODE_IP$path") red " redirect target: $loc" fi done echo # ── 3. auth.researchstack.info → Authentik (no forward_auth loop) ─────────── echo "── 3. auth.researchstack.info serves Authentik (200 or 302, no loop) ──" auth_status=$(curl -so /dev/null -w '%{http_code}' \ -H "Host: auth.researchstack.info" \ -H "X-Forwarded-Proto: https" \ --max-time 5 \ "http://$NODE_IP/") check "auth.researchstack.info returns 200 or 302 (got $auth_status)" \ bash -c '[[ '"$auth_status"' == 200 || '"$auth_status"' == 302 ]]' # If it's a redirect, the location must not point back to an internal IP if [[ "$auth_status" == "302" ]]; then auth_loc=$(curl -sI \ -H "Host: auth.researchstack.info" \ -H "X-Forwarded-Proto: https" \ --max-time 5 \ "http://$NODE_IP/" | grep -Fi 'Location:' | tr -d '\r\n') check "auth redirect is relative or does not point to internal IP ($auth_loc)" \ bash -c '[[ ! '"\"$auth_loc\""' =~ (100\.|127\.|10\.|192\.) ]]' fi # Response body should look like Authentik (may be empty if redirecting to flow) auth_body=$(curl -s \ -H "Host: auth.researchstack.info" \ -H "X-Forwarded-Proto: https" \ -L --max-redirs 3 --max-time 10 \ "http://$NODE_IP/" 2>/dev/null || true) check "auth response contains Authentik markers" \ grep -q -E "authentik|ak-flow|Sign in" <<< "$auth_body" echo # ── 4. Host header passthrough ──────────────────────────────────────────────── echo "── 4. Host header passthrough (server Caddy → Traefik) ──" # The Traefik dashboard is disabled, but /ping is available by default ping_status=$(curl -so /dev/null -w '%{http_code}' \ --max-time 5 \ "http://$NODE_IP:30080/ping" 2>/dev/null || echo "000") check "Traefik /ping on :30080 responds (got $ping_status — 200/404 both ok, 000 = not reachable)" \ bash -c '[[ '"$ping_status"' != 000 ]]' # Confirm Host is forwarded by checking a real Ingress path jobs_status=$(curl -so /dev/null -w '%{http_code}' \ -H "Host: researchstack.info" \ --max-time 5 \ "http://$NODE_IP/api/jobs/health") check "/api/jobs/health via host Caddy → Traefik → service (got $jobs_status, not 000)" \ bash -c '[[ '"$jobs_status"' != 000 ]]' echo # ── 5. X-Forwarded-Proto passthrough ───────────────────────────────────────── echo "── 5. X-Forwarded-Proto preserved through server Caddy ──" # Hermes placeholder echoes back X-Forwarded headers in its /health response # if deployed. If not, check that Traefik doesn't return 000 on the path. proto_status=$(curl -so /dev/null -w '%{http_code}' \ -H "Host: researchstack.info" \ -H "X-Forwarded-Proto: https" \ --max-time 5 \ "http://$NODE_IP/api/registry/health") check "X-Forwarded-Proto: https request routes without error (got $proto_status)" \ bash -c '[[ '"$proto_status"' != 000 ]]' echo echo "═══════════════════════════════════════════════════════" printf " Results: %d passed, %d failed\n" "$PASS" "$FAIL" echo "═══════════════════════════════════════════════════════" echo if (( FAIL > 0 )); then echo "[verify] FAIL — fix the red checks before deploying the edge." >&2 exit 1 fi echo "[verify] PASS — internal leg is clean. Safe to deploy k3s-edge." echo " Next: nixos-rebuild switch on microvm-racknerd, then run:" echo " cd 4-Infrastructure/k3s-flake/tests && npm test"