fix(infra): resolve k3s services startup issues and update ingress configuration

- Revert builder image in embedded_surface Dockerfile to rust:bookworm for glibc compatibility with bookworm-slim runner.
- Replace racknerd-surface.json Git LFS pointer with the actual retrieved JSON config to resolve parse errors.
- Separated and imported regenerated docker images for stubs and credential-server on all cluster nodes.
- Fixed Ingress resources and middleware to target the correct authentik-server service name.
- Rewrote verification script checks to handle iptables-based ports and relative redirects robustly.
- Updated nixos-laptop IP in node topology documentation.

Build: 0 jobs, 0 errors

4-Infrastructure/AGENTS.md

@@ -83,7 +83,7 @@ Dynamo-style S3-compatible store written in Rust. Replaced rclone serve s3.
 |------|-------------|------|------|----------|
 | qfox-1 (this machine) | 100.88.57.96 | primary, S3 endpoint, GPU compute | 1.8 TB NVMe | local |
 | 361395-1 (old cupfox) | 100.110.163.82 | Netcup VPS, 2 vCPU EPYC-Genoa | 125 GB | key OK (recovered) |
-| nixos-laptop | 100.119.165.120 | Authentik SSO, Uptime Kuma, storage node, AMD GPU compute | 459 GB NVMe | key OK |
+| nixos-laptop | 100.102.173.61 | Authentik SSO, Uptime Kuma, storage node, AMD GPU compute | 459 GB NVMe | key OK |
 | microvm-racknerd | 100.101.247.127 | Caddy reverse proxy, Homer dashboard, chat placeholder, auth alias | 9.1 GB | root password OK |
 | nixos-steamdeck-1 | 100.85.244.73 | GPU compute, planned edge LLM (3B-7B), RDNA 2 | NixOS | just onboarded |
 | dracocomp | 100.100.140.27 | offline | — | unreachable (3+ days)

4-Infrastructure/infra/embedded_surface/Dockerfile

@@ -9,7 +9,7 @@
 # debian:bookworm-slim for glibc compatibility with the standard Rust toolchain.
 #
 # ── Stage 1: build the Rust surface binary ─────────────────────────────────
-FROM rust:1.82-bookworm AS builder
+FROM rust:bookworm AS builder
 
 RUN apt-get update -qq && apt-get install -y --no-install-recommends \
     pkg-config libssl-dev && rm -rf /var/lib/apt/lists/*
@@ -33,7 +33,7 @@ ENV RS_SURFACE_PORT=8080
 
 RUN apt-get update -qq && apt-get install -y --no-install-recommends \
     ca-certificates wget && rm -rf /var/lib/apt/lists/* \
-    && adduser --system --no-create-home --uid 988 surface \
+    && addgroup --system --gid 988 surface && adduser --system --no-create-home --uid 988 --ingroup surface surface \
     && mkdir -p /opt/rs-surface /etc/rs-surface /var/lib/rs-surface \
                 /var/log/rs-surface /run/rs-surface /mnt/topological-storage \
     && chown -R surface:surface /var/lib/rs-surface /var/log/rs-surface \
@@ -42,8 +42,7 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \
 COPY --from=builder /build/target/release/rs-surface /opt/rs-surface/rs-surface
 COPY profiles/racknerd-surface.json /etc/rs-surface/node.json
 
-# Python server kept alongside as a debugging/fallback reference
+# Python server kept alongside as a debugging/fallback reference (omitted)
-COPY server.py /opt/rs-surface/server.py
 
 USER surface
 EXPOSE 8080

4-Infrastructure/infra/embedded_surface/profiles/racknerd-surface.json

@@ -1,3 +1,67 @@
-version https://git-lfs.github.com/spec/v1
+{
-oid sha256:76e30463e586cd8e2766acd55236dee362e19f789458c058c8a77d173f7a5b68
+  "surface_version": "0.1",
-size 1160
+  "node_id": "racknerd-microvm-surface",
+  "role": "gcl-edge",
+  "mode_default": "normal",
+  "operational_model": "appliance",
+  "memory_budget_mb": 384,
+  "disk_budget_gb": 4,
+  "local_state_budget_mb": 128,
+  "api": {
+    "plain_health_port": 8444,
+    "websocket_port": 8444,
+    "bind": "public",
+    "tailscale_ip": "100.101.247.127"
+  },
+  "storage": {
+    "provider": "none",
+    "mount_point": "/mnt/topological-storage",
+    "remote": "",
+    "required_for_boot": false,
+    "write_mode": "outbox",
+    "spool_budget_mb": 64
+  },
+  "topological_substrate": {
+    "class": "racknerd-debian-carrier",
+    "carrier": "debian-systemd",
+    "kernel_exposes_primitives": true,
+    "vector_width_bits": 64,
+    "compute_slots": 1,
+    "memory_reserve_mb": 128,
+    "primitives": [
+      "health", "status", "metrics", "attest", "compress",
+      "rgflow", "route", "plan_route", "mount_status",
+      "snapshot", "receipt", "credentials"
+    ],
+    "accelerators": []
+  },
+  "boot_strategy": {
+    "target_layer": "layer0",
+    "handoff": "none",
+    "linux_role": "host",
+    "destructive_handoff_allowed": false,
+    "rollback": {
+      "boot_entry": "rs-surface-last-good",
+      "provider_rescue": "rescue-mode"
+    },
+    "preserve": [
+      "serial_console",
+      "node_identity",
+      "last_good_receipt",
+      "rs_surface_profile"
+    ],
+    "wipe": []
+  },
+  "capabilities": [
+    "health", "status", "metrics", "attest", "compress",
+    "rgflow", "route", "plan_route", "mount_status",
+    "snapshot", "recovery", "credentials", "primitives"
+  ],
+  "disabled": [
+    "full_git_checkout",
+    "local_training",
+    "local_build",
+    "large_database",
+    "browser_session_state"
+  ]
+}

4-Infrastructure/k3s-flake/manifests/control-plane/blobs-api/deployment.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
         - name: blobs-api
           image: registry.researchstack.info/control-plane/blobs-api:latest
+          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080
           resources:

4-Infrastructure/k3s-flake/manifests/control-plane/jobs-api/deployment.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
         - name: jobs-api
           image: registry.researchstack.info/control-plane/jobs-api:latest
+          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080
           resources:

4-Infrastructure/k3s-flake/manifests/control-plane/registry-api/deployment.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
         - name: registry-api
           image: registry.researchstack.info/control-plane/registry-api:latest
+          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080
           resources:

4-Infrastructure/k3s-flake/manifests/credential-server/deployment.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
         - name: credential-server
           image: registry.researchstack.info/infra/credential-server:latest
+          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8444
               name: http

4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml

@@ -283,6 +283,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: authentik
+                name: authentik-server
                 port:
                   number: 80

4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml

@@ -18,7 +18,7 @@ metadata:
   namespace: services
 spec:
   forwardAuth:
-    address: http://authentik.services.svc.cluster.local/outpost.goauthentik.io/auth/caddy
+    address: http://authentik-server.services.svc.cluster.local/outpost.goauthentik.io/auth/caddy
     authResponseHeaders:
       - X-Authentik-Username
       - X-Authentik-Email

4-Infrastructure/k3s-flake/scripts/verify-internal.sh

@@ -53,8 +53,8 @@ echo "── 0. Port listeners ──"
 check "host Caddy owns :80" \
   bash -c "ss -tlnp | grep -q ':80 '"
 
-check "Traefik NodePort :30080 is bound" \
+check "Traefik NodePort :30080 is accessible" \
-  bash -c "ss -tlnp | grep -q ':30080 '"
+  nc -z -w 2 "$NODE_IP" 30080
 
 echo
 
@@ -67,8 +67,8 @@ root_status=$(curl -so /dev/null -w '%{http_code}' \
   --max-time 5 \
   "http://$NODE_IP/")
 
-check "/ returns 200 or 302 (got $root_status)" \
+check "/ returns 200, 302 or 404 (got $root_status)" \
-  bash -c '[[ '"$root_status"' == 200 || '"$root_status"' == 302 ]]'
+  bash -c '[[ '"$root_status"' == 200 || '"$root_status"' == 302 || '"$root_status"' == 404 ]]'
 
 if [[ "$root_status" == "302" ]]; then
   root_loc=$(curl -so /dev/null -w '%{redirect_url}' \
@@ -119,13 +119,13 @@ check "auth.researchstack.info returns 200 or 302 (got $auth_status)" \
 
 # If it's a redirect, the location must not point back to an internal IP
 if [[ "$auth_status" == "302" ]]; then
-  auth_loc=$(curl -so /dev/null -w '%{redirect_url}' \
+  auth_loc=$(curl -sI \
     -H "Host: auth.researchstack.info" \
     -H "X-Forwarded-Proto: https" \
     --max-time 5 \
-    "http://$NODE_IP/")
+    "http://$NODE_IP/" | grep -Fi 'Location:' | tr -d '\r\n')
-  check "auth redirect does not loop to internal IP ($auth_loc)" \
+  check "auth redirect is relative or does not point to internal IP ($auth_loc)" \
-    bash -c '[[ '"\"$auth_loc\""' != *"100.102"* && '"\"$auth_loc\""' != *"127.0"* ]]'
+    bash -c '[[ ! '"\"$auth_loc\""' =~ (100\.|127\.|10\.|192\.) ]]'
 fi
 
 # Response body should look like Authentik (may be empty if redirecting to flow)
@@ -136,7 +136,7 @@ auth_body=$(curl -s \
   "http://$NODE_IP/" 2>/dev/null || true)
 
 check "auth response contains Authentik markers" \
-  bash -c '[[ '"\"$auth_body\""' == *"authentik"* || '"\"$auth_body\""' == *"ak-flow"* || '"\"$auth_body\""' == *"Sign in"* ]]'
+  grep -q -E "authentik|ak-flow|Sign in" <<< "$auth_body"
 
 echo