diff --git a/4-Infrastructure/k3s-flake/flake.nix b/4-Infrastructure/k3s-flake/flake.nix index 0ff985ad..96f842f4 100644 --- a/4-Infrastructure/k3s-flake/flake.nix +++ b/4-Infrastructure/k3s-flake/flake.nix @@ -30,42 +30,63 @@ }; in { nixosConfigurations = { + # ── NixOS storage node (NixOS 26.05, 459GB NVMe, ord zone) ───────── + # ACTUAL ROLE: k3s agent joining cupfox (100.110.163.82:6443) + # HISTORICAL NOTE: was originally standalone k3s server, migrated to + # agent when cupfox was promoted. The k3s-server.nix in this repo + # reflects the current agent setup. k3s-server = mkNode { - hostName = "nixos-laptop"; + hostName = "nixos"; domain = "researchstack.info"; + serverAddr = "https://100.110.163.82:6443"; # cupfox control-plane extraModules = [ ./k3s-server.nix ]; }; + # ── Foxtop compute node ─────────────────────────────────────────── + # ACTUAL: qfox-1 (CachyOS, joined via join-agent.sh, NOT NixOS) + # NixOS config kept for reference / future bare-metal install k3s-foxtop = mkNode { hostName = "qfox-1"; - serverAddr = "https://100.102.173.61:6443"; + serverAddr = "https://100.110.163.82:6443"; # cupfox, not nixos extraModules = [ ./roles/foxtop.nix ]; }; - k3s-mirror = mkNode { - hostName = "361395-1"; - serverAddr = "https://100.102.173.61:6443"; - extraModules = [ ./roles/mirror.nix ]; - }; + # ── Mirror node (DEAD) ──────────────────────────────────────────── + # 361395-1 was a netcup VPS that is no longer in the cluster. + # neon-64gb (ARM64, 2TB) replaced it as netcup presence. + # Kept as reference; do not deploy. + # k3s-mirror = mkNode { + # hostName = "361395-1"; + # serverAddr = "https://100.110.163.82:6443"; + # extraModules = [ ./roles/mirror.nix ]; + # }; + # ── Edge TLS node (racknerd, 9GB VPS, us-va) ────────────────────── + # Caddy TLS termination + Porkbun DNS-01 + subdomain redirects + # Forwards AL traffic to internal router at nixos:80 k3s-edge = mkNode { hostName = "microvm-racknerd"; domain = "researchstack.info"; - serverAddr = "https://100.102.173.61:6443"; + serverAddr = "https://100.110.163.82:6443"; # cupfox extraModules = [ ./k3s-edge.nix ]; }; + # ── GPU compute node (steamdeck, ROG Ally, 476GB NVMe, gpu zone) ── + # Runs CUDA/ML workloads. Has NVIDIA GPU via device plugin. + # NOTE: steamdeck NixOS version is 25.11, may need different nixpkgs k3s-core = mkNode { hostName = "nixos-steamdeck-1"; - serverAddr = "https://100.102.173.61:6443"; + serverAddr = "https://100.110.163.82:6443"; # cupfox extraModules = [ ./roles/core.nix ]; }; - k3s-judge = mkNode { - hostName = ""; - serverAddr = ""; - extraModules = [ ./roles/judge.nix ]; - }; + # ── Judge node (VOTEK/Adversarial review) ───────────────────────── + # TBD - not yet assigned hardware + # k3s-judge = mkNode { + # hostName = ""; + # serverAddr = ""; + # extraModules = [ ./roles/judge.nix ]; + # }; }; packages.${system} = { diff --git a/4-Infrastructure/k3s-flake/k3s-server.nix b/4-Infrastructure/k3s-flake/k3s-server.nix index 76846e9d..3038b5cd 100644 --- a/4-Infrastructure/k3s-flake/k3s-server.nix +++ b/4-Infrastructure/k3s-flake/k3s-server.nix @@ -2,142 +2,50 @@ { ########################################################################## - # k3s-server.nix — Control plane + Traefik Ingress + # k3s-server.nix — NixOS storage + k3s agent node # - # This node runs k3s in server mode. Traefik (k3s built-in) handles all - # path-based routing inside the cluster via Ingress resources defined in - # manifests/ingress/. + # THIS NODE IS NOT THE CONTROL PLANE. + # The actual control plane is on cupfox (100.110.163.82). + # This node was originally the standalone server but was migrated to + # agent mode when cupfox was promoted to control-plane. # - # Port layout: - # :80 — host Caddy pass-through (owns the port, Traefik cannot) - # :30080 — Traefik web NodePort (ServiceLB, bound by k3s) + # Architecture: + # cupfox (Debian, control-plane) ← migrated from nixos + # nixos (NixOS, storage agent) + # qfox-1 (CachyOS, foxtop agent) + # steamdeck (NixOS, GPU agent) + # racknerd (Debian, edge agent + Caddy TLS) + # neon-64gb (Debian, ARM64 agent) # - # Traffic flow (internal leg): - # Edge Caddy (TLS, Porkbun, subdomain redirects) → Tailscale - # → host Caddy :80 → Traefik :30080 → k3s services (via Ingress) - # - # All TLS, Porkbun DNS-01, subdomain 301s, and wildcard catch-alls are - # handled exclusively by the edge Caddy in k3s-edge.nix. Nothing here - # does TLS or subdomain routing — this is a plain HTTP bridge. - # - # Traefik NodePort is configured via HelmChartConfig so it does not race - # with the host Caddy for port 80. - # - # URL contract (defined in manifests/ingress/ingress.yaml): - # / → Homer directory - # /apps/chat/* → Hermes (chat/orchestrator) - # /apps/budget/* → Actual Budget - # /server/status/* → Uptime Kuma - # /server/dash/* → Homarr - # /server/vault/* → Vaultwarden - # /api/cred/* → Credential Server - # /api/registry/* → Registry API (worker join/heartbeat) - # /api/jobs/* → Job Router - # /api/blobs/* → Blob Plane - # auth.researchstack.info → Authentik (stable OIDC issuer, via rs-auth Ingress) + # Servicerouter (to be deployed): + # Edge Caddy (racknerd:443) → Tailscale → host Caddy → Traefik → Services ########################################################################## - sops.secrets.authentik-secrets = { - sopsFile = ./secrets/authentik-secrets.age; + sops.secrets.k3s-token = { + sopsFile = ./secrets/k3s-token.age; format = "yaml"; }; + systemd.services.k3s.serviceConfig.EnvironmentFile = [ + config.sops.secrets.k3s-token.path + ]; + services.k3s = { enable = true; - role = "server"; - clusterInit = true; + role = "agent"; + serverAddr = "https://100.110.163.82:6443"; # cupfox control-plane + tokenFile = config.sops.secrets.k3s-token.path; extraFlags = [ - "--tls-san=100.102.173.61" - "--tls-san=researchstack.info" - "--tls-san=nixos-laptop" - "--advertise-address=100.102.173.61" "--node-ip=100.102.173.61" + "--node-external-ip=100.102.173.61" "--flannel-iface=tailscale0" + "--node-label=topology.researchstack.io/role=storage" + "--node-label=topology.researchstack.io/gpu=false" + "--node-label=topology.researchstack.io/storage-tier=nvme-ssd" ]; }; - # ── Traefik entrypoint config (HelmChartConfig CRD) ──────────────────── - # k3s ships Traefik as a HelmChart. We override via HelmChartConfig to - # bind the web entrypoint to NodePort 30080 instead of the default :80, - # so the host Caddy can own :80 without a port conflict. - # - # k3s reads files placed under /var/lib/rancher/k3s/server/manifests/ and - # reconciles them against the live cluster on startup. - environment.etc."rancher/k3s/server/manifests/traefik-config.yaml".text = '' - apiVersion: helm.cattle.io/v1 - kind: HelmChartConfig - metadata: - name: traefik - namespace: kube-system - spec: - valuesContent: |- - ports: - web: - port: 8000 - nodePort: 30080 - expose: - default: true - exposedPort: 30080 - protocol: TCP - # websecure NodePort not exposed — TLS terminates at the edge Caddy, - # never reaches Traefik. Disabling reduces surface area. - websecure: - expose: - default: false - service: - type: NodePort - ingressRoute: - dashboard: - enabled: false - ''; - - # ── Host Caddy — plain HTTP pass-through (:80 → Traefik :30080) ──────── - # Owns port 80 on the Tailscale interface. All routing logic lives in - # Traefik Ingress resources (manifests/ingress/). This Caddy instance is - # intentionally minimal — no TLS, no subdomain logic, no Porkbun. - services.caddy = { - enable = true; - package = pkgs.caddy; - extraConfig = '' - :80 { - reverse_proxy 100.102.173.61:30080 { - header_up Host {host} - header_up X-Real-IP {remote} - header_up X-Forwarded-For {remote} - # Preserve the proto set by the edge Caddy (https) rather than - # overwriting it. The edge is authoritative; we just pass it through. - header_up X-Forwarded-Proto {http.request.header.X-Forwarded-Proto} - header_up X-Forwarded-Host {host} - } - } - ''; - }; - - # Caddy starts after k3s so Traefik NodePort is ready before we proxy to it. - systemd.services.caddy = { - after = [ "k3s.service" "network-online.target" ]; - wants = [ "k3s.service" "network-online.target" ]; - }; - networking.firewall.allowedTCPPorts = [ 80 ]; - # ── Service deployment ────────────────────────────────────────────────── - systemd.services.deploy-k3s-services = { - description = "Deploy k3s topology services"; - after = [ "k3s.service" "tailscaled.service" "network-online.target" ]; - wants = [ "k3s.service" "tailscaled.service" "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - path = with pkgs; [ kubectl ]; - environment = { - KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; - AUTHENTIK_SECRETS = "${config.sops.secrets.authentik-secrets.path}"; - }; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - Restart = "on-failure"; - RestartSec = 10; - ExecStart = "${pkgs.bash}/bin/bash ${./scripts/deploy-services.sh}"; - }; - }; + services.tailscale.enable = true; } diff --git a/4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml b/4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml index 9d171282..d0fe0071 100644 --- a/4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml +++ b/4-Infrastructure/k3s-flake/manifests/ingress/ingress.yaml @@ -62,6 +62,31 @@ spec: port: number: 5006 +# ═══════════════════════════════════════════════════════════════════════════ +# /apps/books/* → Audiobookshelf (SSO-gated) +# ═══════════════════════════════════════════════════════════════════════════ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rs-apps-books + namespace: services + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web + traefik.ingress.kubernetes.io/router.middlewares: services-authentik-forward-auth@kubernetescrd,services-strip-apps-books@kubernetescrd +spec: + rules: + - host: researchstack.info + http: + paths: + - path: /apps/books + pathType: Prefix + backend: + service: + name: audiobookshelf + port: + number: 80 +--- # ═══════════════════════════════════════════════════════════════════════════ # /server/status/* → Uptime Kuma (SSO-gated) # ═══════════════════════════════════════════════════════════════════════════ diff --git a/4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml b/4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml index 575f4b22..25ecf90c 100644 --- a/4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml +++ b/4-Infrastructure/k3s-flake/manifests/ingress/middleware.yaml @@ -118,3 +118,13 @@ spec: stripPrefix: prefixes: - /api/blobs +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: strip-apps-books + namespace: services +spec: + stripPrefix: + prefixes: + - /apps/books diff --git a/4-Infrastructure/k3s-flake/manifests/ingress/ray-ingress.yaml b/4-Infrastructure/k3s-flake/manifests/ingress/ray-ingress.yaml new file mode 100644 index 00000000..d1da7b24 --- /dev/null +++ b/4-Infrastructure/k3s-flake/manifests/ingress/ray-ingress.yaml @@ -0,0 +1,37 @@ +# ═══════════════════════════════════════════════════════════════════════════ +# Traefik Ingress — Ray Dashboard +# ═══════════════════════════════════════════════════════════════════════════ +# The Ray head service lives in ray-system namespace. Both the Ingress and +# service are in the same namespace so a standard Ingress works. +# +# URL: https://researchstack.info/server/ray → raycluster-head-svc:8265 +# Uses stripPrefix middleware so the dashboard doesn't need to know about +# the /server/ray prefix. +# ═══════════════════════════════════════════════════════════════════════════ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: strip-server-ray + namespace: ray-system +spec: + stripPrefix: + prefixes: + - /server/ray +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: ray-dashboard + namespace: ray-system +spec: + entryPoints: + - web + routes: + - match: Host(`researchstack.info`) && PathPrefix(`/server/ray`) + kind: Rule + middlewares: + - name: strip-server-ray + namespace: ray-system + services: + - name: raycluster-head-svc + port: 8265 diff --git a/4-Infrastructure/k3s-flake/manifests/media/audiobookshelf.yaml b/4-Infrastructure/k3s-flake/manifests/media/audiobookshelf.yaml index 9fb320cc..0d7c0cfe 100644 --- a/4-Infrastructure/k3s-flake/manifests/media/audiobookshelf.yaml +++ b/4-Infrastructure/k3s-flake/manifests/media/audiobookshelf.yaml @@ -28,7 +28,7 @@ spec: app: audiobookshelf spec: nodeSelector: - topology.researchstack.io/role: core + topology.researchstack.io/role: foxtop containers: - name: audiobookshelf image: ghcr.io/advplyr/audiobookshelf:latest @@ -37,6 +37,23 @@ spec: env: - name: TZ value: "America/Chicago" + - name: HOST + value: "0.0.0.0" + - name: ROUTER_BASE_PATH + value: "/audiobookshelf" + - name: OPENID_ENABLED + value: "true" + - name: OPENID_ISSUER_URL + value: "https://auth.researchstack.info" + - name: OPENID_CLIENT_ID + value: "sso-audiobookshelf" + - name: OPENID_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: audiobookshelf-oidc + key: OPENID_CLIENT_SECRET + - name: OPENID_REDIRECT_URI + value: "https://audiobooks.researchstack.info/audiobookshelf/auth/openid/callback" volumeMounts: - name: config mountPath: /config diff --git a/4-Infrastructure/k3s-flake/scripts/join-agent.sh b/4-Infrastructure/k3s-flake/scripts/join-agent.sh index e7b77dc4..6fb590f7 100644 --- a/4-Infrastructure/k3s-flake/scripts/join-agent.sh +++ b/4-Infrastructure/k3s-flake/scripts/join-agent.sh @@ -4,7 +4,7 @@ set -euo pipefail ROLE="core" -SERVER="https://100.102.173.61:6443" +SERVER="https://100.110.163.82:6443" # cupfox control-plane (migrated from nixos) TOKEN="" NODE_IP="" LABELS="" diff --git a/4-Infrastructure/k3s-flake/tests/.gitignore b/4-Infrastructure/k3s-flake/tests/.gitignore index dbd64df8..3fce1f06 100644 --- a/4-Infrastructure/k3s-flake/tests/.gitignore +++ b/4-Infrastructure/k3s-flake/tests/.gitignore @@ -1,3 +1,5 @@ node_modules/ test-results/ playwright-report/ +*.png +*.json diff --git a/4-Infrastructure/k3s-flake/tests/auth-state.json b/4-Infrastructure/k3s-flake/tests/auth-state.json deleted file mode 100644 index 05a83878..00000000 --- a/4-Infrastructure/k3s-flake/tests/auth-state.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cookies": [ - { - "name": "authentik_device", - "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiI1MTEwOWZkOTQ2M2U2MTY4MDdkOTczYmE3MDJiNDcyOTBlZjM5YmY4ZDNjNTc2ZjE1NDRjOTQzMWM4NzVjMjZmIiwiZXhwIjoxNzgyODM3OTg1LjczMzM4fQ.PLMAf2WTZjlOrlZAbg0R4wxK_l5qP5uCMedjZQRbx5E", - "domain": "auth.researchstack.info", - "path": "/", - "expires": 1782837985.854036, - "httpOnly": false, - "secure": false, - "sameSite": "Lax" - }, - { - "name": "authentik_csrf", - "value": "Yar6rSBrSFyTJPFDtLncZFwXVxf6mTYX", - "domain": "auth.researchstack.info", - "path": "/", - "expires": 1811695585.854084, - "httpOnly": false, - "secure": true, - "sameSite": "Lax" - }, - { - "name": "authentik_session", - "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzaWQiOiIwbGJ1c3FpZWZzdWQzMGdhMGQ4bHpmZTM1aWhnMnhyayIsImlzcyI6ImF1dGhlbnRpayIsInN1YiI6IjUxMTA5ZmQ5NDYzZTYxNjgwN2Q5NzNiYTcwMmI0NzI5MGVmMzliZjhkM2M1NzZmMTU0NGM5NDMxYzg3NWMyNmYiLCJhdXRoZW50aWNhdGVkIjp0cnVlLCJhY3IiOiJnb2F1dGhlbnRpay5pby9jb3JlL2RlZmF1bHQifQ.FEg6Gjx1yLhSE-qoHpZj-gy7Ypk3LZ4YWq9ykM1NX8k", - "domain": "auth.researchstack.info", - "path": "/", - "expires": -1, - "httpOnly": true, - "secure": true, - "sameSite": "None" - } - ], - "origins": [] -} \ No newline at end of file diff --git a/4-Infrastructure/k3s-flake/tests/authentik-admin-home.png b/4-Infrastructure/k3s-flake/tests/authentik-admin-home.png deleted file mode 100644 index 3039c101..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-admin-home.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-login-step1.png b/4-Infrastructure/k3s-flake/tests/authentik-login-step1.png deleted file mode 100644 index 6a562c68..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-login-step1.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-login-step2.png b/4-Infrastructure/k3s-flake/tests/authentik-login-step2.png deleted file mode 100644 index 5c71c9a9..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-login-step2.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-outpost-edit-clicked.png b/4-Infrastructure/k3s-flake/tests/authentik-outpost-edit-clicked.png deleted file mode 100644 index c8f65f88..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-outpost-edit-clicked.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-outpost-updated.png b/4-Infrastructure/k3s-flake/tests/authentik-outpost-updated.png deleted file mode 100644 index 22e0c6cb..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-outpost-updated.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-outposts-page.png b/4-Infrastructure/k3s-flake/tests/authentik-outposts-page.png deleted file mode 100644 index 9c8851ce..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-outposts-page.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/authentik-sidebar-expanded.png b/4-Infrastructure/k3s-flake/tests/authentik-sidebar-expanded.png deleted file mode 100644 index 3039c101..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/authentik-sidebar-expanded.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/homarr-after-login.png b/4-Infrastructure/k3s-flake/tests/homarr-after-login.png deleted file mode 100644 index 485b4be3..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/homarr-after-login.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/homarr-login-page.png b/4-Infrastructure/k3s-flake/tests/homarr-login-page.png deleted file mode 100644 index 485b4be3..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/homarr-login-page.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/homarr-manage-boards.png b/4-Infrastructure/k3s-flake/tests/homarr-manage-boards.png deleted file mode 100644 index 8db0db5d..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/homarr-manage-boards.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/homarr-status.png b/4-Infrastructure/k3s-flake/tests/homarr-status.png deleted file mode 100644 index 0396e79b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/homarr-status.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-check-initial.png b/4-Infrastructure/k3s-flake/tests/jellyfin-check-initial.png deleted file mode 100644 index d0630ac2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-check-initial.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-direct.png b/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-direct.png deleted file mode 100644 index 0e59e453..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-direct.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-plugins-clicked.png b/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-plugins-clicked.png deleted file mode 100644 index 01204229..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard-plugins-clicked.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard.png b/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard.png deleted file mode 100644 index f83a5b5f..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-dashboard.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-home.png b/4-Infrastructure/k3s-flake/tests/jellyfin-home.png deleted file mode 100644 index 6124090b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-home.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-logged-in-state.png b/4-Infrastructure/k3s-flake/tests/jellyfin-logged-in-state.png deleted file mode 100644 index 6124090b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-logged-in-state.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-login-filled.png b/4-Infrastructure/k3s-flake/tests/jellyfin-login-filled.png deleted file mode 100644 index c1aed0c2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-login-filled.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-login-initial.png b/4-Infrastructure/k3s-flake/tests/jellyfin-login-initial.png deleted file mode 100644 index d0630ac2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-login-initial.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-login-step-0.png b/4-Infrastructure/k3s-flake/tests/jellyfin-login-step-0.png deleted file mode 100644 index 42508eaa..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-login-step-0.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-plugins-loaded.png b/4-Infrastructure/k3s-flake/tests/jellyfin-plugins-loaded.png deleted file mode 100644 index dc356aa3..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-plugins-loaded.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-plugins.png b/4-Infrastructure/k3s-flake/tests/jellyfin-plugins.png deleted file mode 100644 index 2a0bc4b0..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-plugins.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-status-loaded.png b/4-Infrastructure/k3s-flake/tests/jellyfin-status-loaded.png deleted file mode 100644 index d0630ac2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-status-loaded.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-status.png b/4-Infrastructure/k3s-flake/tests/jellyfin-status.png deleted file mode 100644 index c558c1e2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-status.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step1.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step1.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step1.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2-filled.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2-filled.png deleted file mode 100644 index 0d30876f..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2-filled.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2.png deleted file mode 100644 index 30ea2a4b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step2.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3-debug.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3-debug.png deleted file mode 100644 index 2c7e8403..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3-debug.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3.png deleted file mode 100644 index 0375625f..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step3.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step4.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step4.png deleted file mode 100644 index 2dcfbbfd..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step4.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5-debug.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5-debug.png deleted file mode 100644 index e5ad70be..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5-debug.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5.png deleted file mode 100644 index e5ad70be..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-step5.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-start.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-start.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-start.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step1.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step1.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step1.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2-filled.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2-filled.png deleted file mode 100644 index 0d30876f..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2-filled.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2.png deleted file mode 100644 index 30ea2a4b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step2.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step3.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step3.png deleted file mode 100644 index 2a08bc2c..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step3.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step4.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step4.png deleted file mode 100644 index 2dcfbbfd..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step4.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step5.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step5.png deleted file mode 100644 index e5ad70be..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v2-step5.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-finished.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-finished.png deleted file mode 100644 index d0630ac2..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-finished.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-home.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-home.png deleted file mode 100644 index 6124090b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-home.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-start.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-start.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-start.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step1.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step1.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step1.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2-filled.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2-filled.png deleted file mode 100644 index 0d30876f..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2-filled.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2.png deleted file mode 100644 index 30ea2a4b..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step2.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step3.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step3.png deleted file mode 100644 index 6fe1cd41..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step3.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step4.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step4.png deleted file mode 100644 index 2dcfbbfd..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step4.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step5.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step5.png deleted file mode 100644 index e5ad70be..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step5.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step6.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step6.png deleted file mode 100644 index 675e6fd7..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard-v3-step6.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard.png b/4-Infrastructure/k3s-flake/tests/jellyfin-wizard.png deleted file mode 100644 index 1338c398..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/jellyfin-wizard.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/uptime-kuma-dashboard.png b/4-Infrastructure/k3s-flake/tests/uptime-kuma-dashboard.png deleted file mode 100644 index 54ab5aa6..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/uptime-kuma-dashboard.png and /dev/null differ diff --git a/4-Infrastructure/k3s-flake/tests/uptime-kuma-logged-in.png b/4-Infrastructure/k3s-flake/tests/uptime-kuma-logged-in.png deleted file mode 100644 index f36893f0..00000000 Binary files a/4-Infrastructure/k3s-flake/tests/uptime-kuma-logged-in.png and /dev/null differ diff --git a/4-Infrastructure/kube/raycluster.yaml b/4-Infrastructure/kube/raycluster.yaml index 0151fd3b..e2e9cdf8 100644 --- a/4-Infrastructure/kube/raycluster.yaml +++ b/4-Infrastructure/kube/raycluster.yaml @@ -2,14 +2,15 @@ apiVersion: ray.io/v1 kind: RayCluster metadata: name: raycluster - namespace: ray + namespace: ray-system spec: rayVersion: "2.41.0.dev0" headGroupSpec: serviceType: ClusterIP rayStartParams: - num-cpus: "2" + num-cpus: "0" dashboard-host: "0.0.0.0" + dashboard: "true" template: spec: tolerations: @@ -52,9 +53,9 @@ spec: nodeSelector: kubernetes.io/hostname: qfox-1 workerGroupSpecs: - - replicas: 1 + - replicas: 2 minReplicas: 0 - maxReplicas: 4 + maxReplicas: 6 groupName: cpu-workers rayStartParams: num-cpus: "4" @@ -93,7 +94,7 @@ spec: - replicas: 1 minReplicas: 0 maxReplicas: 1 - groupName: gpu-worker + groupName: gpu-workers rayStartParams: num-gpus: "1" template: @@ -128,3 +129,32 @@ spec: path: /dev/dri nodeSelector: kubernetes.io/hostname: qfox-1 + - replicas: 1 + minReplicas: 0 + maxReplicas: 3 + groupName: arm64-workers + rayStartParams: + num-cpus: "16" + template: + spec: + containers: + - name: ray-worker + image: localhost/rayproject/ray:arm64-aarch64 + resources: + requests: + cpu: "8" + memory: "16Gi" + limits: + cpu: "16" + memory: "56Gi" + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi + nodeSelector: + kubernetes.io/arch: arm64 + kubernetes.io/hostname: neon-64gb