From aae680ad3c2ee3c712c6351d44e38576b83d96f2 Mon Sep 17 00:00:00 2001
From: Brandon Schneider <ceo@sovereign-stack.local>
Date: Tue, 26 May 2026 21:08:58 -0500
Subject: [PATCH] chore(infra): add internal routing smoke-test script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

scripts/verify-internal.sh runs the exact verification sequence before
the edge is touched. Checks (in order):

  0. Port listeners — host Caddy owns :80, Traefik NodePort on :30080
  1. GET / — 200 Homer or 302 → auth.researchstack.info (not internal IP)
  2. /api/* bypass — none of /api/*/health may 302 to Authentik
  3. auth.researchstack.info — 200/302 from Authentik, no internal-IP loop
  4. Host header passthrough — Traefik :30080 /ping reachable; /api/jobs/health
     routes end-to-end through host Caddy → Traefik → Ingress
  5. X-Forwarded-Proto — https header survives the server-Caddy hop

Usage:
  # on nixos-laptop directly:
  bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh

  # from any tailnet node:
  bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh --remote

Exit 0 = safe to deploy edge. Exit 1 = fix red checks first.

Generated with Devin (https://cli.devin.ai/docs)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../k3s-flake/scripts/verify-internal.sh      | 192 ++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 4-Infrastructure/k3s-flake/scripts/verify-internal.sh

diff --git a/4-Infrastructure/k3s-flake/scripts/verify-internal.sh b/4-Infrastructure/k3s-flake/scripts/verify-internal.sh
new file mode 100644
index 00000000..1b2679f5
--- /dev/null
+++ b/4-Infrastructure/k3s-flake/scripts/verify-internal.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+# verify-internal.sh — internal routing smoke test
+#
+# Run on nixos-laptop (100.102.173.61) after nixos-rebuild switch,
+# BEFORE deploying the edge Caddy. Exercises the host-Caddy → Traefik
+# → Ingress path directly so you know the internal leg is clean.
+#
+# Usage:
+#   bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh
+#   bash 4-Infrastructure/k3s-flake/scripts/verify-internal.sh --remote
+#     (runs via ssh root@100.102.173.61 if not already on the node)
+#
+# Exit 0 = all checks passed.
+# Exit 1 = one or more checks failed; details printed to stderr.
+set -euo pipefail
+
+NODE_IP="100.102.173.61"
+PASS=0
+FAIL=0
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+green() { printf '\033[32m✓ %s\033[0m\n' "$*"; }
+red()   { printf '\033[31m✗ %s\033[0m\n' "$*" >&2; }
+
+check() {
+  local label="$1"; shift
+  if "$@" 2>/dev/null; then
+    green "$label"
+    (( PASS++ )) || true
+  else
+    red "$label"
+    (( FAIL++ )) || true
+  fi
+}
+
+# Run from remote if requested and not already on the node.
+if [[ "${1:-}" == "--remote" ]]; then
+  echo "[verify] running remotely on $NODE_IP..."
+  exec ssh "root@$NODE_IP" bash -s < "$0"
+fi
+
+echo
+echo "═══════════════════════════════════════════════════════"
+echo "  Internal routing smoke test — $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+echo "  Target node: $NODE_IP"
+echo "═══════════════════════════════════════════════════════"
+echo
+
+# ── 0. Listener sanity ───────────────────────────────────────────────────────
+echo "── 0. Port listeners ──"
+
+check "host Caddy owns :80" \
+  bash -c "ss -tlnp | grep -q ':80 '"
+
+check "Traefik NodePort :30080 is bound" \
+  bash -c "ss -tlnp | grep -q ':30080 '"
+
+echo
+
+# ── 1. Root path: Homer or Authentik redirect ────────────────────────────────
+echo "── 1. GET / (expect 200 Homer or 302 → Authentik) ──"
+
+root_status=$(curl -so /dev/null -w '%{http_code}' \
+  -H "Host: researchstack.info" \
+  -H "X-Forwarded-Proto: https" \
+  --max-time 5 \
+  "http://$NODE_IP/")
+
+check "/ returns 200 or 302 (got $root_status)" \
+  bash -c '[[ '"$root_status"' == 200 || '"$root_status"' == 302 ]]'
+
+if [[ "$root_status" == "302" ]]; then
+  root_loc=$(curl -so /dev/null -w '%{redirect_url}' \
+    -H "Host: researchstack.info" \
+    -H "X-Forwarded-Proto: https" \
+    --max-time 5 \
+    "http://$NODE_IP/")
+  check "/ redirect points to auth.researchstack.info (not an http://10x... internal URL)" \
+    bash -c '[[ '"\"$root_loc\""' == *"auth.researchstack.info"* ]]'
+fi
+
+echo
+
+# ── 2. API bypass: /api/* must NOT 302 to Authentik ─────────────────────────
+echo "── 2. /api/* paths must not redirect to Authentik ──"
+
+for path in /api/registry/health /api/jobs/health /api/blobs/health; do
+  status=$(curl -so /dev/null -w '%{http_code}' \
+    -H "Host: researchstack.info" \
+    --max-time 5 \
+    "http://$NODE_IP$path")
+
+  check "$path → $status (not 302)" \
+    bash -c '[[ '"$status"' != 302 ]]'
+
+  if [[ "$status" == "302" ]]; then
+    loc=$(curl -so /dev/null -w '%{redirect_url}' \
+      -H "Host: researchstack.info" \
+      --max-time 5 \
+      "http://$NODE_IP$path")
+    red "  redirect target: $loc"
+  fi
+done
+
+echo
+
+# ── 3. auth.researchstack.info → Authentik (no forward_auth loop) ───────────
+echo "── 3. auth.researchstack.info serves Authentik (200 or 302, no loop) ──"
+
+auth_status=$(curl -so /dev/null -w '%{http_code}' \
+  -H "Host: auth.researchstack.info" \
+  -H "X-Forwarded-Proto: https" \
+  --max-time 5 \
+  "http://$NODE_IP/")
+
+check "auth.researchstack.info returns 200 or 302 (got $auth_status)" \
+  bash -c '[[ '"$auth_status"' == 200 || '"$auth_status"' == 302 ]]'
+
+# If it's a redirect, the location must not point back to an internal IP
+if [[ "$auth_status" == "302" ]]; then
+  auth_loc=$(curl -so /dev/null -w '%{redirect_url}' \
+    -H "Host: auth.researchstack.info" \
+    -H "X-Forwarded-Proto: https" \
+    --max-time 5 \
+    "http://$NODE_IP/")
+  check "auth redirect does not loop to internal IP ($auth_loc)" \
+    bash -c '[[ '"\"$auth_loc\""' != *"100.102"* && '"\"$auth_loc\""' != *"127.0"* ]]'
+fi
+
+# Response body should look like Authentik (may be empty if redirecting to flow)
+auth_body=$(curl -s \
+  -H "Host: auth.researchstack.info" \
+  -H "X-Forwarded-Proto: https" \
+  -L --max-redirs 3 --max-time 10 \
+  "http://$NODE_IP/" 2>/dev/null || true)
+
+check "auth response contains Authentik markers" \
+  bash -c '[[ '"\"$auth_body\""' == *"authentik"* || '"\"$auth_body\""' == *"ak-flow"* || '"\"$auth_body\""' == *"Sign in"* ]]'
+
+echo
+
+# ── 4. Host header passthrough ────────────────────────────────────────────────
+echo "── 4. Host header passthrough (server Caddy → Traefik) ──"
+
+# The Traefik dashboard is disabled, but /ping is available by default
+ping_status=$(curl -so /dev/null -w '%{http_code}' \
+  --max-time 5 \
+  "http://$NODE_IP:30080/ping" 2>/dev/null || echo "000")
+
+check "Traefik /ping on :30080 responds (got $ping_status — 200/404 both ok, 000 = not reachable)" \
+  bash -c '[[ '"$ping_status"' != 000 ]]'
+
+# Confirm Host is forwarded by checking a real Ingress path
+jobs_status=$(curl -so /dev/null -w '%{http_code}' \
+  -H "Host: researchstack.info" \
+  --max-time 5 \
+  "http://$NODE_IP/api/jobs/health")
+
+check "/api/jobs/health via host Caddy → Traefik → service (got $jobs_status, not 000)" \
+  bash -c '[[ '"$jobs_status"' != 000 ]]'
+
+echo
+
+# ── 5. X-Forwarded-Proto passthrough ─────────────────────────────────────────
+echo "── 5. X-Forwarded-Proto preserved through server Caddy ──"
+
+# Hermes placeholder echoes back X-Forwarded headers in its /health response
+# if deployed. If not, check that Traefik doesn't return 000 on the path.
+proto_status=$(curl -so /dev/null -w '%{http_code}' \
+  -H "Host: researchstack.info" \
+  -H "X-Forwarded-Proto: https" \
+  --max-time 5 \
+  "http://$NODE_IP/api/registry/health")
+
+check "X-Forwarded-Proto: https request routes without error (got $proto_status)" \
+  bash -c '[[ '"$proto_status"' != 000 ]]'
+
+echo
+echo "═══════════════════════════════════════════════════════"
+printf "  Results: %d passed, %d failed\n" "$PASS" "$FAIL"
+echo "═══════════════════════════════════════════════════════"
+echo
+
+if (( FAIL > 0 )); then
+  echo "[verify] FAIL — fix the red checks before deploying the edge." >&2
+  exit 1
+fi
+
+echo "[verify] PASS — internal leg is clean. Safe to deploy k3s-edge."
+echo "  Next: nixos-rebuild switch on microvm-racknerd, then run:"
+echo "    cd 4-Infrastructure/k3s-flake/tests && npm test"