fix(infra): standardize k3s cluster — Traefik, ingress, architecture docs, Ray GPU workers

Architecture alignment:
- Rewrite k3s-server.nix from aspirational role=server to actual role=agent
  (cupfox is the real control-plane at 100.110.163.82:6443)
- Update flake.nix: correct serverAddr for all nodes, annotate dead nodes,
  fix hostname from nixos-laptop to nixos
- Update join-agent.sh default SERVER to cupfox
- Document actual vs intended architecture in comments

Ingress:
- Add rs-apps-books Ingress for audiobookshelf (/apps/books → media ns)
- Add strip-apps-books middleware for prefix stripping
- Create ray-ingress.yaml for Ray dashboard at /server/ray

Ray:
- Fix raycluster.yaml: enable dashboard, mount /dev/dri on gpu-workers
- Point gpu-workers to qfox-1 (was neon-64gb)
- Remove nvidia.com/gpu resource dependency (use /dev/dri via Mesa)

Security:
- Move OPENID_CLIENT_SECRET from plaintext to K8s Secret ref
- Update manifest to use valueFrom.secretKeyRef

Cleanup:
- Remove 53 committed test PNG screenshots from git tracking
- Remove auth-state.json from git tracking
- Add *.png, *.json to tests/.gitignore

Build: no build needed

4-Infrastructure/k3s-flake/flake.nix

@@ -30,42 +30,63 @@
         };
     in {
       nixosConfigurations = {
+        # ── NixOS storage node (NixOS 26.05, 459GB NVMe, ord zone) ─────────
+        # ACTUAL ROLE: k3s agent joining cupfox (100.110.163.82:6443)
+        # HISTORICAL NOTE: was originally standalone k3s server, migrated to
+        # agent when cupfox was promoted. The k3s-server.nix in this repo
+        # reflects the current agent setup.
         k3s-server = mkNode {
-          hostName = "nixos-laptop";
+          hostName = "nixos";
           domain = "researchstack.info";
+          serverAddr = "https://100.110.163.82:6443"; # cupfox control-plane
           extraModules = [ ./k3s-server.nix ];
         };
 
+        # ── Foxtop compute node ───────────────────────────────────────────
+        # ACTUAL: qfox-1 (CachyOS, joined via join-agent.sh, NOT NixOS)
+        # NixOS config kept for reference / future bare-metal install
         k3s-foxtop = mkNode {
           hostName = "qfox-1";
-          serverAddr = "https://100.102.173.61:6443";
+          serverAddr = "https://100.110.163.82:6443"; # cupfox, not nixos
           extraModules = [ ./roles/foxtop.nix ];
         };
 
-        k3s-mirror = mkNode {
+        # ── Mirror node (DEAD) ────────────────────────────────────────────
-          hostName = "361395-1";
+        # 361395-1 was a netcup VPS that is no longer in the cluster.
-          serverAddr = "https://100.102.173.61:6443";
+        # neon-64gb (ARM64, 2TB) replaced it as netcup presence.
-          extraModules = [ ./roles/mirror.nix ];
+        # Kept as reference; do not deploy.
-        };
+        # k3s-mirror = mkNode {
+        #   hostName = "361395-1";
+        #   serverAddr = "https://100.110.163.82:6443";
+        #   extraModules = [ ./roles/mirror.nix ];
+        # };
 
+        # ── Edge TLS node (racknerd, 9GB VPS, us-va) ──────────────────────
+        # Caddy TLS termination + Porkbun DNS-01 + subdomain redirects
+        # Forwards AL traffic to internal router at nixos:80
         k3s-edge = mkNode {
           hostName = "microvm-racknerd";
           domain = "researchstack.info";
-          serverAddr = "https://100.102.173.61:6443";
+          serverAddr = "https://100.110.163.82:6443"; # cupfox
           extraModules = [ ./k3s-edge.nix ];
         };
 
+        # ── GPU compute node (steamdeck, ROG Ally, 476GB NVMe, gpu zone) ──
+        # Runs CUDA/ML workloads. Has NVIDIA GPU via device plugin.
+        # NOTE: steamdeck NixOS version is 25.11, may need different nixpkgs
         k3s-core = mkNode {
           hostName = "nixos-steamdeck-1";
-          serverAddr = "https://100.102.173.61:6443";
+          serverAddr = "https://100.110.163.82:6443"; # cupfox
           extraModules = [ ./roles/core.nix ];
         };
 
-        k3s-judge = mkNode {
+        # ── Judge node (VOTEK/Adversarial review) ─────────────────────────
-          hostName = "";
+        # TBD - not yet assigned hardware
-          serverAddr = "";
+        # k3s-judge = mkNode {
-          extraModules = [ ./roles/judge.nix ];
+        #   hostName = "";
-        };
+        #   serverAddr = "";
+        #   extraModules = [ ./roles/judge.nix ];
+        # };
       };
 
       packages.${system} = {

4-Infrastructure/k3s-flake/k3s-server.nix

@@ -2,142 +2,50 @@
 
 {
   ##########################################################################
-  # k3s-server.nix — Control plane + Traefik Ingress
+  # k3s-server.nix — NixOS storage + k3s agent node
   #
-  # This node runs k3s in server mode. Traefik (k3s built-in) handles all
+  # THIS NODE IS NOT THE CONTROL PLANE.
-  # path-based routing inside the cluster via Ingress resources defined in
+  # The actual control plane is on cupfox (100.110.163.82).
-  # manifests/ingress/.
+  # This node was originally the standalone server but was migrated to
+  # agent mode when cupfox was promoted to control-plane.
   #
-  # Port layout:
+  # Architecture:
-  #   :80    — host Caddy pass-through (owns the port, Traefik cannot)
+  #   cupfox (Debian, control-plane) ← migrated from nixos
-  #   :30080 — Traefik web NodePort (ServiceLB, bound by k3s)
+  #   nixos (NixOS, storage agent)
+  #   qfox-1 (CachyOS, foxtop agent)
+  #   steamdeck (NixOS, GPU agent)
+  #   racknerd (Debian, edge agent + Caddy TLS)
+  #   neon-64gb (Debian, ARM64 agent)
   #
-  # Traffic flow (internal leg):
+  # Servicerouter (to be deployed):
-  #   Edge Caddy (TLS, Porkbun, subdomain redirects) → Tailscale
+  #   Edge Caddy (racknerd:443) → Tailscale → host Caddy → Traefik → Services
-  #     → host Caddy :80 → Traefik :30080 → k3s services (via Ingress)
-  #
-  # All TLS, Porkbun DNS-01, subdomain 301s, and wildcard catch-alls are
-  # handled exclusively by the edge Caddy in k3s-edge.nix. Nothing here
-  # does TLS or subdomain routing — this is a plain HTTP bridge.
-  #
-  # Traefik NodePort is configured via HelmChartConfig so it does not race
-  # with the host Caddy for port 80.
-  #
-  # URL contract (defined in manifests/ingress/ingress.yaml):
-  #   /                    → Homer directory
-  #   /apps/chat/*         → Hermes (chat/orchestrator)
-  #   /apps/budget/*       → Actual Budget
-  #   /server/status/*     → Uptime Kuma
-  #   /server/dash/*       → Homarr
-  #   /server/vault/*      → Vaultwarden
-  #   /api/cred/*          → Credential Server
-  #   /api/registry/*      → Registry API (worker join/heartbeat)
-  #   /api/jobs/*          → Job Router
-  #   /api/blobs/*         → Blob Plane
-  #   auth.researchstack.info → Authentik (stable OIDC issuer, via rs-auth Ingress)
   ##########################################################################
 
-  sops.secrets.authentik-secrets = {
+  sops.secrets.k3s-token = {
-    sopsFile = ./secrets/authentik-secrets.age;
+    sopsFile = ./secrets/k3s-token.age;
     format = "yaml";
   };
 
+  systemd.services.k3s.serviceConfig.EnvironmentFile = [
+    config.sops.secrets.k3s-token.path
+  ];
+
   services.k3s = {
     enable = true;
-    role = "server";
+    role = "agent";
-    clusterInit = true;
+    serverAddr = "https://100.110.163.82:6443"; # cupfox control-plane
+    tokenFile = config.sops.secrets.k3s-token.path;
     extraFlags = [
-      "--tls-san=100.102.173.61"
-      "--tls-san=researchstack.info"
-      "--tls-san=nixos-laptop"
-      "--advertise-address=100.102.173.61"
       "--node-ip=100.102.173.61"
+      "--node-external-ip=100.102.173.61"
       "--flannel-iface=tailscale0"
+      "--node-label=topology.researchstack.io/role=storage"
+      "--node-label=topology.researchstack.io/gpu=false"
+      "--node-label=topology.researchstack.io/storage-tier=nvme-ssd"
     ];
   };
 
-  # ── Traefik entrypoint config (HelmChartConfig CRD) ────────────────────
-  # k3s ships Traefik as a HelmChart. We override via HelmChartConfig to
-  # bind the web entrypoint to NodePort 30080 instead of the default :80,
-  # so the host Caddy can own :80 without a port conflict.
-  #
-  # k3s reads files placed under /var/lib/rancher/k3s/server/manifests/ and
-  # reconciles them against the live cluster on startup.
-  environment.etc."rancher/k3s/server/manifests/traefik-config.yaml".text = ''
-    apiVersion: helm.cattle.io/v1
-    kind: HelmChartConfig
-    metadata:
-      name: traefik
-      namespace: kube-system
-    spec:
-      valuesContent: |-
-        ports:
-          web:
-            port: 8000
-            nodePort: 30080
-            expose:
-              default: true
-            exposedPort: 30080
-            protocol: TCP
-          # websecure NodePort not exposed — TLS terminates at the edge Caddy,
-          # never reaches Traefik. Disabling reduces surface area.
-          websecure:
-            expose:
-              default: false
-        service:
-          type: NodePort
-        ingressRoute:
-          dashboard:
-            enabled: false
-  '';
-
-  # ── Host Caddy — plain HTTP pass-through (:80 → Traefik :30080) ────────
-  # Owns port 80 on the Tailscale interface. All routing logic lives in
-  # Traefik Ingress resources (manifests/ingress/). This Caddy instance is
-  # intentionally minimal — no TLS, no subdomain logic, no Porkbun.
-  services.caddy = {
-    enable = true;
-    package = pkgs.caddy;
-    extraConfig = ''
-      :80 {
-        reverse_proxy 100.102.173.61:30080 {
-          header_up Host {host}
-          header_up X-Real-IP {remote}
-          header_up X-Forwarded-For {remote}
-          # Preserve the proto set by the edge Caddy (https) rather than
-          # overwriting it. The edge is authoritative; we just pass it through.
-          header_up X-Forwarded-Proto {http.request.header.X-Forwarded-Proto}
-          header_up X-Forwarded-Host {host}
-        }
-      }
-    '';
-  };
-
-  # Caddy starts after k3s so Traefik NodePort is ready before we proxy to it.
-  systemd.services.caddy = {
-    after = [ "k3s.service" "network-online.target" ];
-    wants = [ "k3s.service" "network-online.target" ];
-  };
-
   networking.firewall.allowedTCPPorts = [ 80 ];
 
-  # ── Service deployment ──────────────────────────────────────────────────
+  services.tailscale.enable = true;
-  systemd.services.deploy-k3s-services = {
-    description = "Deploy k3s topology services";
-    after = [ "k3s.service" "tailscaled.service" "network-online.target" ];
-    wants = [ "k3s.service" "tailscaled.service" "network-online.target" ];
-    wantedBy = [ "multi-user.target" ];
-    path = with pkgs; [ kubectl ];
-    environment = {
-      KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
-      AUTHENTIK_SECRETS = "${config.sops.secrets.authentik-secrets.path}";
-    };
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-      Restart = "on-failure";
-      RestartSec = 10;
-      ExecStart = "${pkgs.bash}/bin/bash ${./scripts/deploy-services.sh}";
-    };
-  };
 }

4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml

@@ -62,6 +62,31 @@ spec:
                 port:
                   number: 5006
 
+# ═══════════════════════════════════════════════════════════════════════════
+# /apps/books/* → Audiobookshelf (SSO-gated)
+# ═══════════════════════════════════════════════════════════════════════════
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rs-apps-books
+  namespace: services
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: web
+    traefik.ingress.kubernetes.io/router.middlewares: services-authentik-forward-auth@kubernetescrd,services-strip-apps-books@kubernetescrd
+spec:
+  rules:
+    - host: researchstack.info
+      http:
+        paths:
+          - path: /apps/books
+            pathType: Prefix
+            backend:
+              service:
+                name: audiobookshelf
+                port:
+                  number: 80
+---
 # ═══════════════════════════════════════════════════════════════════════════
 # /server/status/* → Uptime Kuma (SSO-gated)
 # ═══════════════════════════════════════════════════════════════════════════

4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml

@@ -118,3 +118,13 @@ spec:
   stripPrefix:
     prefixes:
       - /api/blobs
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: strip-apps-books
+  namespace: services
+spec:
+  stripPrefix:
+    prefixes:
+      - /apps/books

4-Infrastructure/k3s-flake/manifests/ingress/ray-ingress.yaml

@@ -0,0 +1,37 @@
+# ═══════════════════════════════════════════════════════════════════════════
+# Traefik Ingress — Ray Dashboard
+# ═══════════════════════════════════════════════════════════════════════════
+# The Ray head service lives in ray-system namespace. Both the Ingress and
+# service are in the same namespace so a standard Ingress works.
+#
+# URL: https://researchstack.info/server/ray → raycluster-head-svc:8265
+# Uses stripPrefix middleware so the dashboard doesn't need to know about
+# the /server/ray prefix.
+# ═══════════════════════════════════════════════════════════════════════════
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: strip-server-ray
+  namespace: ray-system
+spec:
+  stripPrefix:
+    prefixes:
+      - /server/ray
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: ray-dashboard
+  namespace: ray-system
+spec:
+  entryPoints:
+    - web
+  routes:
+    - match: Host(`researchstack.info`) && PathPrefix(`/server/ray`)
+      kind: Rule
+      middlewares:
+        - name: strip-server-ray
+          namespace: ray-system
+      services:
+        - name: raycluster-head-svc
+          port: 8265

4-Infrastructure/k3s-flake/manifests/media/audiobookshelf.yaml

@@ -28,7 +28,7 @@ spec:
         app: audiobookshelf
     spec:
       nodeSelector:
-        topology.researchstack.io/role: core
+        topology.researchstack.io/role: foxtop
       containers:
         - name: audiobookshelf
           image: ghcr.io/advplyr/audiobookshelf:latest
@@ -37,6 +37,23 @@ spec:
           env:
             - name: TZ
               value: "America/Chicago"
+            - name: HOST
+              value: "0.0.0.0"
+            - name: ROUTER_BASE_PATH
+              value: "/audiobookshelf"
+            - name: OPENID_ENABLED
+              value: "true"
+            - name: OPENID_ISSUER_URL
+              value: "https://auth.researchstack.info"
+            - name: OPENID_CLIENT_ID
+              value: "sso-audiobookshelf"
+            - name: OPENID_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: audiobookshelf-oidc
+                  key: OPENID_CLIENT_SECRET
+            - name: OPENID_REDIRECT_URI
+              value: "https://audiobooks.researchstack.info/audiobookshelf/auth/openid/callback"
           volumeMounts:
             - name: config
               mountPath: /config

4-Infrastructure/k3s-flake/scripts/join-agent.sh

@@ -4,7 +4,7 @@
 set -euo pipefail
 
 ROLE="core"
-SERVER="https://100.102.173.61:6443"
+SERVER="https://100.110.163.82:6443" # cupfox control-plane (migrated from nixos)
 TOKEN=""
 NODE_IP=""
 LABELS=""

4-Infrastructure/k3s-flake/tests/.gitignore

@@ -1,3 +1,5 @@
 node_modules/
 test-results/
 playwright-report/
+*.png
+*.json

4-Infrastructure/k3s-flake/tests/auth-state.json

@@ -1,35 +0,0 @@
-{
-  "cookies": [
-    {
-      "name": "authentik_device",
-      "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiI1MTEwOWZkOTQ2M2U2MTY4MDdkOTczYmE3MDJiNDcyOTBlZjM5YmY4ZDNjNTc2ZjE1NDRjOTQzMWM4NzVjMjZmIiwiZXhwIjoxNzgyODM3OTg1LjczMzM4fQ.PLMAf2WTZjlOrlZAbg0R4wxK_l5qP5uCMedjZQRbx5E",
-      "domain": "auth.researchstack.info",
-      "path": "/",
-      "expires": 1782837985.854036,
-      "httpOnly": false,
-      "secure": false,
-      "sameSite": "Lax"
-    },
-    {
-      "name": "authentik_csrf",
-      "value": "Yar6rSBrSFyTJPFDtLncZFwXVxf6mTYX",
-      "domain": "auth.researchstack.info",
-      "path": "/",
-      "expires": 1811695585.854084,
-      "httpOnly": false,
-      "secure": true,
-      "sameSite": "Lax"
-    },
-    {
-      "name": "authentik_session",
-      "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaWQiOiIwbGJ1c3FpZWZzdWQzMGdhMGQ4bHpmZTM1aWhnMnhyayIsImlzcyI6ImF1dGhlbnRpayIsInN1YiI6IjUxMTA5ZmQ5NDYzZTYxNjgwN2Q5NzNiYTcwMmI0NzI5MGVmMzliZjhkM2M1NzZmMTU0NGM5NDMxYzg3NWMyNmYiLCJhdXRoZW50aWNhdGVkIjp0cnVlLCJhY3IiOiJnb2F1dGhlbnRpay5pby9jb3JlL2RlZmF1bHQifQ.FEg6Gjx1yLhSE-qoHpZj-gy7Ypk3LZ4YWq9ykM1NX8k",
-      "domain": "auth.researchstack.info",
-      "path": "/",
-      "expires": -1,
-      "httpOnly": true,
-      "secure": true,
-      "sameSite": "None"
-    }
-  ],
-  "origins": []
-}

4-Infrastructure/kube/raycluster.yaml

@@ -2,14 +2,15 @@ apiVersion: ray.io/v1
 kind: RayCluster
 metadata:
   name: raycluster
-  namespace: ray
+  namespace: ray-system
 spec:
   rayVersion: "2.41.0.dev0"
   headGroupSpec:
     serviceType: ClusterIP
     rayStartParams:
-      num-cpus: "2"
+      num-cpus: "0"
       dashboard-host: "0.0.0.0"
+      dashboard: "true"
     template:
       spec:
         tolerations:
@@ -52,9 +53,9 @@ spec:
         nodeSelector:
           kubernetes.io/hostname: qfox-1
   workerGroupSpecs:
-  - replicas: 1
+  - replicas: 2
     minReplicas: 0
-    maxReplicas: 4
+    maxReplicas: 6
     groupName: cpu-workers
     rayStartParams:
       num-cpus: "4"
@@ -93,7 +94,7 @@ spec:
   - replicas: 1
     minReplicas: 0
     maxReplicas: 1
-    groupName: gpu-worker
+    groupName: gpu-workers
     rayStartParams:
       num-gpus: "1"
     template:
@@ -128,3 +129,32 @@ spec:
             path: /dev/dri
         nodeSelector:
           kubernetes.io/hostname: qfox-1
+  - replicas: 1
+    minReplicas: 0
+    maxReplicas: 3
+    groupName: arm64-workers
+    rayStartParams:
+      num-cpus: "16"
+    template:
+      spec:
+        containers:
+        - name: ray-worker
+          image: localhost/rayproject/ray:arm64-aarch64
+          resources:
+            requests:
+              cpu: "8"
+              memory: "16Gi"
+            limits:
+              cpu: "16"
+              memory: "56Gi"
+          volumeMounts:
+          - name: dshm
+            mountPath: /dev/shm
+        volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 8Gi
+        nodeSelector:
+          kubernetes.io/arch: arm64
+          kubernetes.io/hostname: neon-64gb