Research-Stack/4-Infrastructure/k3s-flake/flake.nix
Brandon Schneider 6d9c4e23de fix(infra): standardize k3s cluster — Traefik, ingress, architecture docs, Ray GPU workers
Architecture alignment:
- Rewrite k3s-server.nix from aspirational role=server to actual role=agent
  (cupfox is the real control-plane at 100.110.163.82:6443)
- Update flake.nix: correct serverAddr for all nodes, annotate dead nodes,
  fix hostname from nixos-laptop to nixos
- Update join-agent.sh default SERVER to cupfox
- Document actual vs intended architecture in comments

Ingress:
- Add rs-apps-books Ingress for audiobookshelf (/apps/books → media ns)
- Add strip-apps-books middleware for prefix stripping
- Create ray-ingress.yaml for Ray dashboard at /server/ray

Ray:
- Fix raycluster.yaml: enable dashboard, mount /dev/dri on gpu-workers
- Point gpu-workers to qfox-1 (was neon-64gb)
- Remove nvidia.com/gpu resource dependency (use /dev/dri via Mesa)

Security:
- Move OPENID_CLIENT_SECRET from plaintext to K8s Secret ref
- Update manifest to use valueFrom.secretKeyRef

Cleanup:
- Remove 53 committed test PNG screenshots from git tracking
- Remove auth-state.json from git tracking
- Add *.png, *.json to tests/.gitignore

Build: no build needed
2026-05-31 23:13:04 -05:00

111 lines
4.3 KiB
Nix

{
description = "Unified NixOS k3s topology Research Stack. Zero embedded IPs or secrets.";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
sops-nix = {
url = "github:Mic92/sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { self, nixpkgs, sops-nix }:
let
lib = nixpkgs.lib;
system = "x86_64-linux";
caddyPorkbun = import ./pkgs/caddy-porkbun.nix {
pkgs = nixpkgs.legacyPackages.${system};
};
mkNode = { hostName, extraModules ? [ ], serverAddr ? null, domain ? null }:
assert hostName != "";
nixpkgs.lib.nixosSystem {
inherit system;
specialArgs = { inherit hostName serverAddr domain caddyPorkbun; };
modules = [
sops-nix.nixosModules.sops
./k3s-configuration.nix
] ++ extraModules;
};
in {
nixosConfigurations = {
# ── NixOS storage node (NixOS 26.05, 459GB NVMe, ord zone) ─────────
# ACTUAL ROLE: k3s agent joining cupfox (100.110.163.82:6443)
# HISTORICAL NOTE: was originally standalone k3s server, migrated to
# agent when cupfox was promoted. The k3s-server.nix in this repo
# reflects the current agent setup.
k3s-server = mkNode {
hostName = "nixos";
domain = "researchstack.info";
serverAddr = "https://100.110.163.82:6443"; # cupfox control-plane
extraModules = [ ./k3s-server.nix ];
};
# ── Foxtop compute node ───────────────────────────────────────────
# ACTUAL: qfox-1 (CachyOS, joined via join-agent.sh, NOT NixOS)
# NixOS config kept for reference / future bare-metal install
k3s-foxtop = mkNode {
hostName = "qfox-1";
serverAddr = "https://100.110.163.82:6443"; # cupfox, not nixos
extraModules = [ ./roles/foxtop.nix ];
};
# ── Mirror node (DEAD) ────────────────────────────────────────────
# 361395-1 was a netcup VPS that is no longer in the cluster.
# neon-64gb (ARM64, 2TB) replaced it as netcup presence.
# Kept as reference; do not deploy.
# k3s-mirror = mkNode {
# hostName = "361395-1";
# serverAddr = "https://100.110.163.82:6443";
# extraModules = [ ./roles/mirror.nix ];
# };
# ── Edge TLS node (racknerd, 9GB VPS, us-va) ──────────────────────
# Caddy TLS termination + Porkbun DNS-01 + subdomain redirects
# Forwards AL traffic to internal router at nixos:80
k3s-edge = mkNode {
hostName = "microvm-racknerd";
domain = "researchstack.info";
serverAddr = "https://100.110.163.82:6443"; # cupfox
extraModules = [ ./k3s-edge.nix ];
};
# ── GPU compute node (steamdeck, ROG Ally, 476GB NVMe, gpu zone) ──
# Runs CUDA/ML workloads. Has NVIDIA GPU via device plugin.
# NOTE: steamdeck NixOS version is 25.11, may need different nixpkgs
k3s-core = mkNode {
hostName = "nixos-steamdeck-1";
serverAddr = "https://100.110.163.82:6443"; # cupfox
extraModules = [ ./roles/core.nix ];
};
# ── Judge node (VOTEK/Adversarial review) ─────────────────────────
# TBD - not yet assigned hardware
# k3s-judge = mkNode {
# hostName = "";
# serverAddr = "";
# extraModules = [ ./roles/judge.nix ];
# };
};
packages.${system} = {
nixos-anywhere = nixpkgs.legacyPackages.${system}.nixos-anywhere;
caddy-porkbun = caddyPorkbun;
};
apps.${system}.install-edge = {
type = "app";
program = "${nixpkgs.legacyPackages.${system}.writeShellScript "install-edge" ''
set -euo pipefail
TARGET="''${1:-}"
if [ -z "$TARGET" ]; then
echo "Usage: nix run .#install-edge -- root@172.245.19.182"
exit 1
fi
exec nix run github:nix-community/nixos-anywhere -- \
--flake .#k3s-edge "$TARGET"
''}";
};
};
}