mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-30 18:56:16 +00:00
e5fb0a5f4d
Lean: update Semantics modules, add new numerics/physics data files Hardware: update FPGA bitstreams (tangnano9k_uart_loopback) Infra: k3s-flake tests, netcup-vps configuration, VCN compute substrate Docs: ARCHITECTURE, specs, citation updates
797 lines
31 KiB
Nix
797 lines
31 KiB
Nix
{ config, pkgs, lib, ... }:
|
|
|
|
{
|
|
imports = [
|
|
# Hardware configuration (if needed)
|
|
];
|
|
|
|
# ── System identity ───────────────────────────────────────────────────────
|
|
networking.hostName = "rs-vps";
|
|
time.timeZone = "Europe/Berlin";
|
|
|
|
# ── Boot ──────────────────────────────────────────────────────────────────
|
|
boot.loader.grub.enable = lib.mkDefault true;
|
|
boot.loader.grub.device = "/dev/vda";
|
|
boot.loader.grub.efiSupport = true;
|
|
boot.loader.grub.efiInstallAsRemovable = true;
|
|
|
|
# ── BTRFS support ───────────────────────────────────────────────────────────
|
|
boot.supportedFilesystems = lib.mkAfter [ "btrfs" ];
|
|
|
|
# ── Kernel modules (mirrors Debian lsmod) ───────────────────────────────────
|
|
# VirtIO (KVM guest) — must be loaded early
|
|
boot.kernelModules = [
|
|
# Storage
|
|
"virtio_blk"
|
|
"virtio_scsi"
|
|
# Network
|
|
"virtio_net"
|
|
# Memory ballooning (SCP control panel needs this)
|
|
"virtio_balloon"
|
|
# Serial console (SCP VNC/console access)
|
|
"virtio_serial"
|
|
# GPU (SPICE display + 3D acceleration via virglrenderer)
|
|
"virtio_gpu"
|
|
"virtio_dma_buf"
|
|
"virtio-gpu" # virtio-gpu 3D with virglrenderer
|
|
"drm"
|
|
"drm_kms_helper"
|
|
"drm_shmem_helper"
|
|
"ttm" # GPU memory manager (needed by DRM)
|
|
# USB (keyboard/mouse in SPICE console)
|
|
"xhci_pci"
|
|
"usbcore"
|
|
"usb_common"
|
|
"usbhid"
|
|
# SCSI / CDROM
|
|
"scsi_mod"
|
|
"scsi_common"
|
|
"sr_mod"
|
|
"cdrom"
|
|
# Filesystem
|
|
"ext4"
|
|
"mbcache"
|
|
"jbd2"
|
|
"crc16"
|
|
"crc32c_generic"
|
|
# EFI
|
|
"efi_pstore"
|
|
"efivarfs"
|
|
# QEMU firmware config (SCP reads guest info via this)
|
|
"qemu_fw_cfg"
|
|
# IPTables (if firewall is active)
|
|
"ip_tables"
|
|
"x_tables"
|
|
];
|
|
boot.extraModulePackages = with pkgs; [ btrfs-progs ];
|
|
|
|
# ── X11 / SPICE configuration ───────────────────────────────────────────────
|
|
services.xserver = {
|
|
enable = true;
|
|
videoDrivers = [ "virtiogpu" "fbdev" ];
|
|
};
|
|
|
|
# ── Kernel command line ───────────────────────────────────────────────────────
|
|
# net.ifnames=0 = eth0 naming (matches Debian convention)
|
|
boot.kernelParams = [
|
|
"net.ifnames=0"
|
|
"console=tty0"
|
|
"quiet"
|
|
];
|
|
|
|
# ── Filesystems ────────────────────────────────────────────────────────────
|
|
# Real partition layout from netcup CCP:
|
|
# vda1: EFI System Partition (vfat, /boot/efi)
|
|
# Partition layout (GPT):
|
|
# vda1: /boot (ext4, 1GB)
|
|
# vda2: /nix (btrfs @ subvol, 64GB)
|
|
# vda3: / (btrfs @ subvol) + /home (@home) + /var (@var) + /var/log (@log)
|
|
# Swap: file on btrfs (no separate partition needed)
|
|
fileSystems = {
|
|
"/boot" = {
|
|
device = "/dev/disk/by-uuid/62bea288-8fcf-45c3-aeec-d7b4cc8aa681";
|
|
fsType = "ext4";
|
|
};
|
|
|
|
"/nix" = {
|
|
device = "/dev/disk/by-uuid/f9741995-1a17-4ac3-ac03-478c76e41998";
|
|
fsType = "btrfs";
|
|
options = [ "compress=zstd:3" "ssd" "noatime" "subvol=@" ];
|
|
};
|
|
|
|
"/" = {
|
|
device = "/dev/disk/by-uuid/c505c4a6-9133-4d10-a461-f050721d4fb2";
|
|
fsType = "btrfs";
|
|
options = [ "compress=zstd:3" "ssd" "noatime" "subvol=@" ];
|
|
};
|
|
|
|
"/home" = {
|
|
device = "/dev/disk/by-uuid/c505c4a6-9133-4d10-a461-f050721d4fb2";
|
|
fsType = "btrfs";
|
|
options = [ "compress=zstd:3" "ssd" "noatime" "subvol=@home" ];
|
|
};
|
|
|
|
"/var" = {
|
|
device = "/dev/disk/by-uuid/c505c4a6-9133-4d10-a461-f050721d4fb2";
|
|
fsType = "btrfs";
|
|
options = [ "compress=zstd:3" "ssd" "noatime" "subvol=@var" ];
|
|
};
|
|
|
|
"/var/log" = {
|
|
device = "/dev/disk/by-uuid/c505c4a6-9133-4d10-a461-f050721d4fb2";
|
|
fsType = "btrfs";
|
|
options = [ "compress=zstd:3" "ssd" "noatime" "subvol=@log" ];
|
|
};
|
|
};
|
|
|
|
swapDevices = [
|
|
{ device = "/swapfile"; size = 65536; }
|
|
];
|
|
|
|
# ── Nix caching ────────────────────────────────────────────────────────────
|
|
nix.settings = {
|
|
# Use the global NixOS cache
|
|
substituters = [ "https://cache.nixos.org" ];
|
|
trusted-substituters = [ "https://cache.nixos.org" ];
|
|
|
|
# Lean / mathlib community cache (accelerates lake builds)
|
|
extra-substituters = [ "https://leanprover-community.github.io" ];
|
|
extra-trusted-public-keys = [
|
|
"leanprover-community.github.io-1:a8UP+R2uLj3/r6nGCoDSo1R4+/tJ1BLC5W3gNiV/Es="
|
|
];
|
|
|
|
# Use 4 cores to avoid crashing the VPS
|
|
max-jobs = 4;
|
|
cores = 4;
|
|
|
|
# Keep 50 generations per user profile
|
|
keep-derivations = true;
|
|
keep-outputs = true;
|
|
};
|
|
|
|
# ── Tailscale mesh networking ──────────────────────────────────────────────
|
|
services.tailscale = {
|
|
enable = true;
|
|
# "server" makes this a relay node (exit node for other machines)
|
|
useRoutingFeatures = "server";
|
|
# Enable tailscale SSH (auth via tailnet)
|
|
extraUpFlags = [
|
|
"--accept-dns=false"
|
|
"--operator=root"
|
|
];
|
|
};
|
|
|
|
# ── Users ─────────────────────────────────────────────────────────────────
|
|
users.users.researcher = {
|
|
isNormalUser = true;
|
|
description = "Research Stack developer";
|
|
extraGroups = [
|
|
"wheel"
|
|
"docker" # podman docker compat socket
|
|
"k3s" # k3s API access
|
|
"keys"
|
|
"tailscale"
|
|
];
|
|
openssh.authorizedKeys.keys = [
|
|
# Add SSH keys here
|
|
];
|
|
};
|
|
|
|
# ── OpenSSH ────────────────────────────────────────────────────────────────
|
|
services.openssh = {
|
|
enable = true;
|
|
settings = {
|
|
PermitRootLogin = "yes";
|
|
PasswordAuthentication = false;
|
|
};
|
|
};
|
|
|
|
# ── nix-ld (run x86_64 binaries on ARM64) ────────────────────────────────
|
|
programs.nix-ld.enable = true;
|
|
|
|
# ── Build parallelism tunables ──────────────────────────────────────────────
|
|
# Set by systemd service env in individual services, but also available globally
|
|
# ── Lean LSP services ──────────────────────────────────────────────────────
|
|
systemd.services.lean-lsp-mcp = {
|
|
description = "Lean LSP MCP server (v4.19.0)";
|
|
after = [ "network.target" "tailscaled.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = let
|
|
lean-lsp = pkgs.writeShellScriptBin "lean-lsp-run" ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
export ELAN_TOOLCHAIN=4.19.0
|
|
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
|
|
export PATH="${pkgs.uv}/bin:$HOME/.elan/bin:$PATH"
|
|
exec uv tool run --from lean-lsp-mcp lean-lsp-mcp --port 8765 --stdio
|
|
'';
|
|
in ''
|
|
exec ${lean-lsp}/bin/lean-lsp-run
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
User = "researcher";
|
|
Group = "researcher";
|
|
WorkingDirectory = "/home/researcher";
|
|
Environment = [
|
|
"ELAN_TOOLCHAIN=4.19.0"
|
|
"HOME=/home/researcher"
|
|
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
|
"LAKE_JOBS=16"
|
|
"XDG_CACHE_HOME=/home/researcher/.cache"
|
|
];
|
|
};
|
|
};
|
|
|
|
systemd.services.lean-lsp-mathlib = {
|
|
description = "Lean LSP MCP server (v4.30.0-rc2, mathlib)";
|
|
after = [ "network.target" "tailscaled.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = let
|
|
lean-lsp-mathlib = pkgs.writeShellScriptBin "lean-lsp-mathlib-run" ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
export ELAN_TOOLCHAIN=4.30.0-rc2
|
|
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
|
|
export PATH="${pkgs.uv}/bin:$HOME/.elan/bin:$PATH"
|
|
exec uv tool run --from lean-lsp-mcp lean-lsp-mcp --port 8766 --stdio
|
|
'';
|
|
in ''
|
|
exec ${lean-lsp-mathlib}/bin/lean-lsp-mathlib-run
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
User = "researcher";
|
|
Group = "researcher";
|
|
WorkingDirectory = "/home/researcher";
|
|
Environment = [
|
|
"ELAN_TOOLCHAIN=4.30.0-rc2"
|
|
"HOME=/home/researcher"
|
|
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
|
"LAKE_JOBS=16"
|
|
"XDG_CACHE_HOME=/home/researcher/.cache"
|
|
];
|
|
};
|
|
};
|
|
|
|
# ── Python LSP TCP on port 8767 ──────────────────────────────────────────
|
|
systemd.services.pylsp = {
|
|
description = "Python LSP server (TCP)";
|
|
after = [ "network.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = let
|
|
pythonEnv = pkgs.python311.withPackages (ps: with ps; [
|
|
numpy scipy sympy pandas matplotlib seaborn
|
|
uncertainties pint
|
|
beautifulsoup4 lxml
|
|
requests httpx
|
|
fastapi uvicorn pydantic
|
|
python-dateutil
|
|
]);
|
|
pylsp-script = pkgs.writeShellScriptBin "pylsp-run" ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
exec ${pythonEnv}/bin/python -m pylsp --host 0.0.0.0 --port 8767
|
|
'';
|
|
in ''
|
|
exec ${pylsp-script}/bin/pylsp-run
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
User = "researcher";
|
|
Group = "researcher";
|
|
WorkingDirectory = "/home/researcher";
|
|
Environment = [
|
|
"HOME=/home/researcher"
|
|
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
|
"PYTHONUNBUFFERED=1"
|
|
];
|
|
};
|
|
};
|
|
|
|
# ── Ollama inference server on port 11434 ─────────────────────────────────
|
|
systemd.services.ollama = {
|
|
description = "Ollama LLM inference server";
|
|
after = [ "network.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
export OLLAMA_HOST=0.0.0.0:11434
|
|
export OLLAMA_MODELS=/home/researcher/.ollama/models
|
|
mkdir -p $OLLAMA_MODELS
|
|
exec ${pkgs.ollama}/bin/ollama serve
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
User = "researcher";
|
|
Group = "researcher";
|
|
WorkingDirectory = "/home/researcher";
|
|
Environment = [
|
|
"HOME=/home/researcher"
|
|
"PATH=${pkgs.ollama}/bin:${pkgs.curl}/bin"
|
|
"OLLAMA_MODELS=/home/researcher/.ollama/models"
|
|
];
|
|
};
|
|
};
|
|
|
|
# ── Ollama model puller (oneshot) ────────────────────────────────────────
|
|
# Usage: sudo -u researcher systemctl start ollama-model-pull@"deepseek-r1:7b"
|
|
systemd.services.ollama-model-pull = {
|
|
description = "Ollama model puller";
|
|
after = [ "ollama.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
script = ''
|
|
#!${pkgs.bash}/bin/bash
|
|
MODEL="''${1:-deepseek-r1:7b}"
|
|
exec ${pkgs.ollama}/bin/ollama pull "$MODEL"
|
|
'';
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = "researcher";
|
|
RemainAfterExit = true;
|
|
};
|
|
};
|
|
|
|
# ── PostgreSQL (ENE database) ───────────────────────────────────────────────
|
|
services.postgresql = {
|
|
enable = true;
|
|
package = pkgs.postgresql_16;
|
|
# Enable JIT for analytical queries on ENE data
|
|
settings.jit = "on";
|
|
# Memory tuning for 64GB RAM
|
|
settings.shared_buffers = "16GB";
|
|
settings.effective_cache_size = "48GB";
|
|
settings.work_mem = "256MB";
|
|
settings.maintenance_work_mem = "2GB";
|
|
settings.effective_io_concurrency = 200;
|
|
settings.max_worker_processes = "16";
|
|
# Enable parallel query execution
|
|
settings.max_parallel_workers_per_gather = "8";
|
|
settings.max_parallel_workers = "16";
|
|
# Logging
|
|
settings.log_destination = "stderr";
|
|
settings.log_line_prefix = "ene %p %u@%d ";
|
|
};
|
|
|
|
# ── ENE database restoration (oneshot) ───────────────────────────────────
|
|
# Copies backed-up ENE data from external disk to the VPS
|
|
# Run manually after mounting the backup drive:
|
|
# sudo systemctl start ene-restore
|
|
systemd.services.ene-restore = {
|
|
description = "Restore ENE from backup";
|
|
after = [ "postgresql.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
script = let
|
|
backupDir = "/mnt/aws-backup-20260529"; # mount backup disk here first
|
|
in ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
export PGHOST=/run/postgresql
|
|
export PGDATA=/var/lib/postgresql/16/data
|
|
export PGUSER=postgres
|
|
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
|
|
|
|
echo "Creating ENE database..."
|
|
sudo -u postgres psql -c "CREATE DATABASE ene;" 2>/dev/null || true
|
|
|
|
echo "Restoring ENE schema..."
|
|
sudo -u postgres psql -d ene -f "${backupDir}/rds_dump/ene_schema.sql" 2>/dev/null || true
|
|
|
|
echo "Restoring ENE data..."
|
|
sudo -u postgres pg_restore -d ene "${backupDir}/rds_dump/ene_full.dump" 2>/dev/null || true
|
|
|
|
echo "Importing CSV tables..."
|
|
for tbl in $(ls "${backupDir}/rds_tables/"*.csv 2>/dev/null | xargs -I{} basename {} .csv); do
|
|
echo " Importing $tbl..."
|
|
sudo -u postgres psql -d ene -c "\\COPY $tbl FROM '${backupDir}/rds_tables/$tbl.csv' WITH (FORMAT csv, HEADER true)" 2>/dev/null || true
|
|
done
|
|
|
|
echo "ENE restore complete."
|
|
'';
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
};
|
|
|
|
# ── Caddy reverse proxy (HTTPS for LSP endpoints) ──────────────────────────
|
|
# Certificates via Let's Encrypt / Caddy DNS challenge (configure domain first)
|
|
services.caddy = {
|
|
enable = true;
|
|
virtualHosts = {
|
|
# TLS will work once a domain points to this VPS
|
|
# "rs-vps.example.com" = {
|
|
# extraConfig = ''
|
|
# reverse_proxy /lean* localhost:8765
|
|
# reverse_proxy /python* localhost:8767
|
|
# reverse_proxy /ollama* localhost:11434
|
|
# reverse_proxy localhost:8096
|
|
# '';
|
|
# };
|
|
};
|
|
globalConfig = ''
|
|
admin off
|
|
auto_https off
|
|
'';
|
|
};
|
|
|
|
# ── Jellyfin media server (port 8096) ────────────────────────────────────
|
|
# Stream video, audio, and images to any device.
|
|
# Uses system ffmpeg (software encode on ARM64).
|
|
systemd.services.jellyfin = {
|
|
description = "Jellyfin media server";
|
|
after = [ "network.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
export JELLYFIN_DATA_DIR=/home/researcher/.local/share/jellyfin
|
|
export JELLYFIN_CONFIG_DIR=/home/researcher/.config/jellyfin
|
|
export JELLYFIN_LOG_DIR=/home/researcher/.local/log/jellyfin
|
|
export JELLYFIN_CACHE_DIR=/home/researcher/.cache/jellyfin
|
|
mkdir -p $JELLYFIN_DATA_DIR $JELLYFIN_CONFIG_DIR $JELLYFIN_LOG_DIR $JELLYFIN_CACHE_DIR
|
|
exec ${pkgs.jellyfin}/bin/jellyfin \
|
|
--datadir "$JELLYFIN_DATA_DIR" \
|
|
--configdir "$JELLYFIN_CONFIG_DIR" \
|
|
--logdir "$JELLYFIN_LOG_DIR" \
|
|
--cachedir "$JELLYFIN_CACHE_DIR" \
|
|
--ffmpeg ${pkgs.ffmpeg}/bin/ffmpeg \
|
|
--webdir ${pkgs.jellyfin-web}/share/jellyfin-web
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "always";
|
|
RestartSec = "10s";
|
|
User = "researcher";
|
|
Group = "researcher";
|
|
Environment = [
|
|
"HOME=/home/researcher"
|
|
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
|
];
|
|
};
|
|
};
|
|
|
|
# ── Prometheus node exporter (port 9100) ──────────────────────────────────
|
|
# Exposes hardware/OS metrics for monitoring (Uptime Kuma, etc.)
|
|
services.prometheus.exporters = {
|
|
node = {
|
|
enable = true;
|
|
port = 9100;
|
|
enabledCollectors = [ "systemd" "logind" ];
|
|
# Don't firewall this — only expose on internal interfaces
|
|
openFirewall = false;
|
|
};
|
|
};
|
|
|
|
# ── Automatic NixOS upgrades (weekly) ───────────────────────────────────
|
|
systemd.timers.nix-upgrade = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig.OnCalendar = "weekly";
|
|
};
|
|
systemd.services.nix-upgrade = {
|
|
description = "NixOS channel upgrade";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${pkgs.nix}/bin/nix-channel --update nixos";
|
|
User = "root";
|
|
};
|
|
};
|
|
|
|
# ── Health check watchdog ────────────────────────────────────────────────
|
|
# Restarts LSP/Ollama if they become unresponsive
|
|
systemd.services.health-check = {
|
|
description = "LSP and Ollama health watchdog";
|
|
after = [
|
|
"lean-lsp-mcp.service"
|
|
"lean-lsp-mathlib.service"
|
|
"pylsp.service"
|
|
"ollama.service"
|
|
"vcn-lupine-daemon.service"
|
|
];
|
|
wantedBy = [ "multi-user.target" ];
|
|
script = ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -euo pipefail
|
|
|
|
check_service() {
|
|
local name="$1"; shift
|
|
local url="$1"; shift
|
|
if ! curl -sf --max-time 5 "$url" > /dev/null 2>&1; then
|
|
echo "[health] $name unhealthy at $url — restarting..."
|
|
systemctl restart "$name"
|
|
fi
|
|
}
|
|
|
|
# Lean LSP (streamable-http health check via --timeout flag)
|
|
# check_service lean-lsp-mcp http://localhost:8765/health 2>/dev/null || true
|
|
# Python LSP has no built-in health endpoint — check port
|
|
if ! nc -z localhost 8767 2>/dev/null; then
|
|
echo "[health] pylsp port 8767 not listening — restarting..."
|
|
systemctl restart pylsp
|
|
fi
|
|
# Ollama API
|
|
if ! curl -sf --max-time 5 http://localhost:11434/api/tags > /dev/null 2>&1; then
|
|
echo "[health] ollama unhealthy — restarting..."
|
|
systemctl restart ollama
|
|
fi
|
|
'';
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
# Run every 5 minutes
|
|
};
|
|
};
|
|
systemd.timers.health-check = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig.OnCalendar = "*-*-* *:0/5:00";
|
|
};
|
|
|
|
# ── VCN-LUPINE unified compute bridge daemon ─────────────────────────────────
|
|
# Bridges IPC (libcuda preload + braid encoders) to GPU node (qfox-1) over MKV.
|
|
systemd.services.vcn-lupine-daemon = {
|
|
description = "VCN-LUPINE unified compute transport daemon";
|
|
after = [ "network.target" "tailscaled.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
|
|
script = ''
|
|
mkdir -p /run/vcn-lupine
|
|
export LUPINE_GPU_NODE="100.88.57.96"
|
|
export LUPINE_GPU_PORT="14834"
|
|
export PYTHONPATH="/home/researcher/repo/4-Infrastructure/shim:$PYTHONPATH"
|
|
exec ${pkgs.python3}/bin/python3 \
|
|
/home/researcher/repo/4-Infrastructure/shim/vcn_lupine_daemon.py \
|
|
--gpu-node "$LUPINE_GPU_NODE" \
|
|
--gpu-port "$LUPINE_GPU_PORT" \
|
|
>> /var/log/vcn-lupine-daemon.log 2>&1
|
|
'';
|
|
|
|
serviceConfig = {
|
|
Type = "simple";
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
RuntimeDirectory = "vcn-lupine";
|
|
RuntimeDirectoryMode = "0755";
|
|
LogsDirectory = "vcn-lupine";
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
ReadOnlyPaths = [ "/" ];
|
|
ReadWritePaths = [ "/run/vcn-lupine" "/var/log" ];
|
|
PrivateTmp = true;
|
|
NoNewPrivileges = true;
|
|
CapabilityBoundingSet = [ "" ];
|
|
DeviceAllow = [ "/dev/null" "r" ];
|
|
};
|
|
};
|
|
|
|
# Replaces Docker; manages containers without a daemon
|
|
virtualisation.podman = {
|
|
enable = true;
|
|
# Docker-compatible socket for tools that expect Docker
|
|
dockerCompat = true;
|
|
};
|
|
|
|
# ── k3s (lightweight Kubernetes) ─────────────────────────────────────────
|
|
# Single-node k3s for orchestrating long-running compute workloads
|
|
services.k3s = {
|
|
enable = true;
|
|
# Token and certs for node authentication
|
|
# token = "changeme"; # Set a secure token
|
|
# Master role only (no agents)
|
|
role = "server";
|
|
# Disable traefik (we have Caddy for ingress)
|
|
disableAgent = true;
|
|
# Extra args for the server
|
|
extraFlags = [
|
|
"--disable traefik"
|
|
"--disable servicelb"
|
|
"--disable metrics-server"
|
|
"--write-kubeconfig-mode 0644"
|
|
];
|
|
};
|
|
|
|
# ── Networking ─────────────────────────────────────────────────────────────
|
|
networking.firewall = {
|
|
enable = true;
|
|
allowedTCPPorts = [
|
|
22 # SSH
|
|
80 # HTTP (Caddy)
|
|
443 # HTTPS (Caddy)
|
|
6443 # k3s Kubernetes API
|
|
2379 # etcd client
|
|
2380 # etcd peer
|
|
8096 # Jellyfin media server
|
|
8765 # Lean LSP (v4.19.0)
|
|
8766 # Lean LSP (v4.30.0-rc2)
|
|
8767 # Python LSP
|
|
8920 # Jellyfin-alt HTTP
|
|
9100 # Prometheus node exporter
|
|
11434 # Ollama
|
|
];
|
|
allowedUDPPorts = [
|
|
1900 # Jellyfin discovery (SSDP)
|
|
];
|
|
};
|
|
|
|
# ── Packages ────────────────────────────────────────────────────────────────
|
|
environment.systemPackages = with pkgs; [
|
|
git git-lfs
|
|
bash bashInteractive coreutils findutils gnugrep gnused gawk
|
|
util-linux
|
|
curl wget rsync zstd xz gnutar gzip
|
|
ripgrep jq
|
|
gnumake
|
|
nodejs_22
|
|
uv
|
|
elan
|
|
texliveFull
|
|
gnuplot maxima octave
|
|
typst
|
|
openssh cacert
|
|
less which file procps htop
|
|
glibc binutils
|
|
openssl pkg-config
|
|
graphviz
|
|
postgresql_16
|
|
podman
|
|
tailscale # mesh networking
|
|
netcat-openbsd # for health check port probing
|
|
|
|
# ── ARM64-optimized HPC / math packages ───────────────────────────
|
|
openblas # Multi-threaded BLAS, ARM64 Neoverse-optimized
|
|
lapack
|
|
petsc # Scientific computing (sparse/direct solvers)
|
|
slepc # Eigenvalue solver (PETSc-based)
|
|
pari # PARI/GP (number theory)
|
|
gap # Groups, Algorithms, Programming
|
|
singular # Polynomial algebra, Gröbner bases
|
|
symengine # Fast C++ symbolic (SymPy backend)
|
|
fftw # FFT (single/double precision)
|
|
suitesparse # CHOLMOD, UMFPACK, SPQR
|
|
z3 # SMT solver
|
|
julia # Julia (latest, ARM64 native)
|
|
|
|
# ── Audio / video processing (DSP volunteer computing) ─────────────────
|
|
ffmpeg # Core: transcoding, streaming, filtering
|
|
flac # Free Lossless Audio Codec
|
|
libvpx # VP8/VP9 video codec
|
|
libaom # AV1 codec
|
|
dav1d # AV1 decoder (fast, ASM-optimized)
|
|
mediainfo # Inspect audio/video metadata
|
|
sox # Sound exchange — audio processing CLI
|
|
portaudio # Cross-platform audio I/O
|
|
libsndfile # Audio file I/O
|
|
alsa-lib # ALSA audio library
|
|
|
|
# ── Jellyfin media server ───────────────────────────────────────────────
|
|
jellyfin # Media server (TV, movies, music)
|
|
jellyfin-web # Web UI
|
|
|
|
# ── BTRFS tools ────────────────────────────────────────────────────────
|
|
btrfs-progs # mkfs, subvolume, send/receive, scrub, balance
|
|
btrfs-heatmap # Visualize BTRFS space usage
|
|
|
|
# ── NUMA / ARM64 performance ───────────────────────────────────────────
|
|
numactl # NUMA policy control for BLAS/PETSc/Julia threading
|
|
|
|
# ── Framebuffer / DRM / GPU acceleration ─────────────────────────────────
|
|
mesa # OpenGL, VA-API, video acceleration
|
|
libGL # OpenGL runtime
|
|
libGLU # GLU (OpenGL utility library)
|
|
virglrenderer # VirtIO-gpu 3D acceleration (VRAM share with host)
|
|
swiftshader # Software Vulkan (for containers without GPU)
|
|
libva # VA-API (video acceleration)
|
|
# ── SPICE remote display ───────────────────────────────────────────────
|
|
xorg.libX11 # X11 display server library
|
|
xorg.libXext # X11 extensions
|
|
xorg.libXrender # X Render extension
|
|
xf86-video-fbdev # Linux framebuffer X11 driver
|
|
];
|
|
|
|
# ── Performance tuning for ARM64 (Ampere/Altra Neoverse) ─────────────────────
|
|
# 64GB RAM, 18 cores, NUMA-aware (if Ampere Altra)
|
|
|
|
# ── CPU governor: performance mode ────────────────────────────────────────
|
|
# Prefer consistent clocks over power saving for compute workloads
|
|
|
|
|
|
boot.kernel.sysctl = {
|
|
# ── General memory ─────────────────────────────────────────────────
|
|
# Swappiness: aggressive RAM use, minimal swap (we have 64GB)
|
|
"vm.swappiness" = 10;
|
|
# Dirty page cache: prefer page cache over swapping
|
|
"vm.dirty_ratio" = 60;
|
|
"vm.dirty_background_ratio" = 15;
|
|
"vm.dirty_expire_centisecs" = 6000;
|
|
|
|
# ── Transparent hugepages ───────────────────────────────────────────
|
|
"vm.nr_hugepages" = 1024;
|
|
# Always use hugepages (not "madvise") for predictable allocation
|
|
"vm.nr_overcommit_hugepages" = 1024;
|
|
|
|
# ── ARM64-specific memory / cache tuning ────────────────────────────
|
|
# VM max for ARM64 (default is only 16TB; bump for large workloads)
|
|
"vm.max_map_count" = 1048576;
|
|
# Shared memory: raise to 32GB for Julia, PETSc, BLAS
|
|
"kernel.shmmax" = 34359738368; # 32GB
|
|
"kernel.shmall" = 8388608; # 32GB worth of pages
|
|
"kernel.shmmin" = 4096;
|
|
"kernel.shmni" = 8192;
|
|
|
|
# ── ARM64 cache / TLB tuning ───────────────────────────────────────
|
|
# Enable the ARM64 hardware DBM (Dirty Bit Map) feature for faster
|
|
# page dirtying tracking — benefits in-memory workloads
|
|
"vm.zone_reclaim_mode" = 0; # Disable zone reclaim on NUMA (let remote access)
|
|
|
|
# ── File descriptor limits ──────────────────────────────────────────
|
|
"fs.file-max" = 524288;
|
|
"fs.nr_open" = 524288;
|
|
"fs.inotify.max_user_watches" = 524288;
|
|
|
|
# ── Network buffers (LSP/TCP services) ────────────────────────────
|
|
"net.core.rmem_max" = 16777216;
|
|
"net.core.wmem_max" = 16777216;
|
|
"net.core.rmem_default" = 16777216;
|
|
"net.core.wmem_default" = 16777216;
|
|
"net.core.netdev_max_backlog" = 5000;
|
|
"net.ipv4.tcp_rmem" = "4096 87380 16777216";
|
|
"net.ipv4.tcp_wmem" = "4096 65536 16777216";
|
|
"net.ipv4.tcp_fastopen" = 3; # Fast Open for LSP connections
|
|
|
|
# ── POSIX message queues ───────────────────────────────────────────
|
|
"kernel.msgmax" = 65536;
|
|
"kernel.msgmni" = 4096;
|
|
};
|
|
|
|
# ── Raise default file descriptor limit for all users ──────────────────
|
|
security.pam.loginLimits = [
|
|
{ domain = "*"; type = "soft"; item = "nofile"; value = "524288"; }
|
|
{ domain = "*"; type = "hard"; item = "nofile"; value = "524288"; }
|
|
{ domain = "*"; type = "soft"; item = "memlock"; value = "unlimited"; }
|
|
{ domain = "*"; type = "hard"; item = "memlock"; value = "unlimited"; }
|
|
];
|
|
|
|
# ── ARM64 Numactl for Julia / PETSc / OpenBLAS threading ──────────────
|
|
# Pin BLAS/PETSc threads to specific NUMA nodes on Ampere Altra
|
|
environment.variables = {
|
|
# Build parallelism
|
|
LAKE_JOBS = "16";
|
|
MAKEFLAGS = "-j16";
|
|
NIX_BUILD_CORES = "16";
|
|
# OpenBLAS: use all 18 cores, prefer threads over OpenMP
|
|
OPENBLAS_NUM_THREADS = "16";
|
|
OPENBLAS_VERBOSE = "0";
|
|
OPENBLAS_AFFINITY = "1"; # Compact affinity (cores 0-15 on node 0)
|
|
# PETSc: thread pinning
|
|
PETSC_OPTIONS = "-matpthread"; # Use pthread threading, not OpenMP
|
|
# Julia: use all cores for BLAS/LAPACK
|
|
JULIA_NUM_THREADS = "16";
|
|
# FFTW: plan with wisdom for reuse
|
|
OMP_NUM_THREADS = "16";
|
|
# tmpfs for /tmp means scratch goes to RAM, no disk I/O bottleneck
|
|
XDG_CACHE_HOME = "/home/researcher/.cache";
|
|
};
|
|
|
|
# ── Security ────────────────────────────────────────────────────────────────
|
|
security.sudo.wheelNeedsPassword = false;
|
|
|
|
# ── System state ────────────────────────────────────────────────────────────
|
|
system.stateVersion = "25.11";
|
|
}
|