Add EC2 recovery backup: NixOS config, AppFlowy compose/env template, credential server bootstrap, recovery guide

- ec2-configuration.nix: full NixOS config for aws-nixos-node-1
- docker-compose.minimal.yml: AppFlowy Cloud compose with search_path fix
- .env.example: sanitized AppFlowy env template
- nixos-setup-cred-server.sh: credential server bootstrap
- RECOVERY.md: step-by-step rebuild instructions
- .gitignore: secrets dir excluded
- credential_provider.py reverted to repo HEAD (EC2 had hardcoded AWS creds)
- racknerd_root.txt removed from working tree

.gitignore

@@ -217,3 +217,7 @@ data
 
 # Rust build artifacts
 **/target/
+
+# Recovery secrets backup - DO NOT COMMIT
+4-Infrastructure/infra/secrets/
+

4-Infrastructure/infra/RECOVERY.md

@@ -0,0 +1,25 @@
+# Recovery Guide
+
+If EC2 server aws-nixos-node-1 (100.69.1.43) dies, rebuild from these files.
+
+## Critical Files (committed to repo)
+- `ec2-configuration.nix` — Full NixOS config (Caddy routes, AppFlowy, Forgejo, credential server)
+- `docker-compose.minimal.yml` — Custom AppFlowy Cloud compose (in 5-Applications/AppFlowy-Cloud/)
+- `nixos-setup-cred-server.sh` — Credential server bootstrap
+- `credential_provider.py` — Credential resolution chain
+- `credential_server.py` — Webhook handler
+- `.env.example` — Sanitized AppFlowy Cloud env template
+
+## Critical Files (NOT committed — in `secrets/` dir, gitignored)
+- `credentials.json` — All 8 API provider keys
+- `appflowy.env` — AppFlowy Cloud env (RDS host, JWT, encryption key)
+- `tailscale-auth.key` — Tailscale auth key
+
+## Recovery Flow
+1. Launch new NixOS EC2 → copy `ec2-configuration.nix` to `/etc/nixos/`
+2. Restore secrets from gitignored backup
+3. `nixos-rebuild switch` — brings up Caddy, credential server, Forgejo, Heimdall
+4. Deploy AppFlowy: copy compose + `.env` to `/var/lib/AppFlowy-Cloud/`, start stack
+5. RDS (2,685 records) survives independently — reconnect AppFlowy to it
+6. Forgejo repos lost unless backed up separately
+7. Heimdall tiles lost unless backed up separately

4-Infrastructure/infra/ec2-configuration.nix

@@ -0,0 +1,124 @@
+{ config, pkgs, ... }:
+{
+  imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
+
+  networking.hostName = "aws-nixos-node-1";
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+  documentation.enable = false;
+  documentation.nixos.enable = false;
+  documentation.man.enable = false;
+  documentation.info.enable = false;
+  documentation.doc.enable = false;
+
+  nix.settings.auto-optimise-store = true;
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 7d";
+  };
+
+  environment.systemPackages = with pkgs; [ tailscale python3 ];
+
+  services.tailscale.enable = true;
+  services.tailscale.authKeyFile = "/etc/tailscale-auth.key";
+  services.tailscale.extraUpFlags = [ "--ssh" ];
+
+  virtualisation.podman.enable = true;
+
+  virtualisation.oci-containers = {
+    backend = "podman";
+    containers = {
+      heimdall = {
+        image = "lscr.io/linuxserver/heimdall:latest";
+        autoStart = true;
+        ports = [ "127.0.0.1:8090:80" ];
+        volumes = [ "/var/lib/heimdall:/config" ];
+        environment = {
+          PUID = "1000";
+          PGID = "1000";
+          TZ = "America/Chicago";
+        };
+      };
+      forgejo = {
+        image = "codeberg.org/forgejo/forgejo:1.21";
+        autoStart = true;
+        ports = [
+          "0.0.0.0:3000:3000"
+        ];
+        volumes = [
+          "/var/lib/forgejo/data:/data"
+        ];
+        environment = {
+          USER_UID = "1000";
+          USER_GID = "1000";
+          FORGEJO__server__DOMAIN = "researchstack.info";
+          FORGEJO__server__ROOT_URL = "https://researchstack.info/git/";
+          FORGEJO__server__HTTP_PORT = "3000";
+          FORGEJO__server__SSH_PORT = "22";
+        };
+      };
+    };
+  };
+
+  services.caddy = {
+    enable = true;
+    extraConfig = ''
+      researchstack.info {
+        handle_path /api/* {
+          reverse_proxy localhost:8444
+        }
+        handle_path /git/* {
+          reverse_proxy localhost:3000
+        }
+        handle_path /appflowy/* {
+          reverse_proxy localhost:8000
+        }
+        handle {
+          basic_auth {
+            admin $2b$12$yUaZ1sxezq84rRDQ3Fa48OdTx52vU7bldIfmav4cwsRn227pFJQDq
+          }
+          reverse_proxy localhost:8090
+        }
+      }
+
+# git.researchstack.info subdomain removed - use /git/* path instead
+    '';
+  };
+
+  systemd.services.rs-credential-server = {
+    description = "Research Stack Credential Server";
+    after = [ "network-online.target" "tailscaled.service" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "simple";
+      User = "root";
+      WorkingDirectory = "/opt/rs-surface";
+      ExecStart = "${pkgs.python3}/bin/python3 /opt/rs-surface/credential_server.py --port 8444 --bind 127.0.0.1";
+      Restart = "always";
+      RestartSec = "5";
+      Environment = [
+        "RS_CREDENTIAL_CONFIG=/etc/rs-surface/credentials.json"
+        "RS_CREDENTIAL_SERVER=http://100.101.247.127:8444"
+        "RS_SURFACE_NODE_ID=aws-nixos-node-1"
+      ];
+    };
+  };
+
+
+  systemd.services.appflowy-cloud = {
+    description = "AppFlowy Cloud podman-compose stack";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    path = with pkgs; [ podman podman-compose ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      WorkingDirectory = "/var/lib/AppFlowy-Cloud";
+      ExecStart = "${pkgs.podman-compose}/bin/podman-compose -f docker-compose.minimal.yml up -d";
+      ExecStop = "${pkgs.podman-compose}/bin/podman-compose -f docker-compose.minimal.yml down";
+    };
+  };
+}

4-Infrastructure/infra/nixos-setup-cred-server.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+
+# Create unit file
+cat > /etc/systemd/system/rs-credential-server.service << 'UNITEOF'
+[Unit]
+Description=Research Stack Credential Server
+After=network-online.target tailscaled.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/rs-surface
+ExecStart=/run/current-system/sw/bin/python3 /opt/rs-surface/credential_server.py --port 8444 --bind 0.0.0.0
+Restart=always
+RestartSec=5
+Environment=RS_CREDENTIAL_CONFIG=/etc/rs-surface/credentials.json
+Environment=RS_CREDENTIAL_SERVER=http://100.101.247.127:8444
+Environment=RS_SURFACE_NODE_ID=aws-nixos-node-1
+
+[Install]
+WantedBy=multi-user.target
+UNITEOF
+
+systemctl daemon-reload
+systemctl enable rs-credential-server
+systemctl restart rs-credential-server
+
+echo "Service status:"
+systemctl status rs-credential-server --no-pager --lines=5
+echo ""
+echo "Testing health endpoint..."
+curl -sf --connect-timeout 5 http://localhost:8444/health && echo "OK" || echo "FAIL"
+
+# Test remote credential resolution
+echo ""
+echo "Testing remote credential resolution..."
+curl -sf --connect-timeout 10 http://localhost:8444/credentials/deepseek | python3 -m json.tool

5-Applications/AppFlowy-Cloud/.env.example

@@ -0,0 +1,41 @@
+# AppFlowy Cloud - Minimal Configuration
+APPFLOWY_BASE_URL=http://researchstack.info/appflowy
+APPFLOWY_WS_BASE_URL=ws://researchstack.info/appflowy/ws/v1
+
+# External RDS
+RDS_HOST=database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com
+RDS_PORT=5432
+RDS_DB=postgres
+RDS_USER=postgres
+
+# Redis
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+# Storage (local filesystem via minio)
+APPFLOWY_S3_ACCESS_KEY=CHANGEME
+APPFLOWY_S3_SECRET_KEY=CHANGEME
+APPFLOWY_S3_BUCKET=appflowy
+
+# Auth
+GOTRUE_ADMIN_EMAIL=admin@researchstack.info
+GOTRUE_ADMIN_PASSWORD=CHANGEME
+GOTRUE_BASE_URL=http://researchstack.info/appflowy/gotrue
+GOTRUE_SITE_URL=http://researchstack.info
+GOTRUE_DISABLE_SIGNUP=false
+
+# Encryption
+APPFLOWY_ENCRYPTION_KEY=CHANGE_THIS
+APPFLOWY_JWT_SECRET=CHANGE_THIS
+
+# External URLs (for Caddy proxying)
+API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue
+APPFLOWY_WEB_URL=https://researchstack.info/appflowy
+APPFLOWY_DATABASE_URL=postgres://postgres@database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com:5432/postgres?sslmode=require
+GOTRUE_JWT_SECRET=${APPFLOWY_JWT_SECRET}
+GOTRUE_DATABASE_URL=${APPFLOWY_DATABASE_URL}
+APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999
+APPFLOWY_GOTRUE_JWT_SECRET=${APPFLOWY_JWT_SECRET}
+APPFLOWY_S3_CREATE_BUCKET=false
+APPFLOWY_S3_USE_MINIO=false
+GOTRUE_MAILER_AUTOCONFIRM=true

5-Applications/AppFlowy-Cloud/docker-compose.minimal.yml

@@ -0,0 +1,85 @@
+version: "3.9"
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: appflowy_local
+      POSTGRES_DB: appflowy
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  gotrue:
+    image: docker.io/appflowyinc/gotrue:latest
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      - GOTRUE_ADMIN_EMAIL=admin@researchstack.info
+      - GOTRUE_ADMIN_PASSWORD=changeme789
+      - GOTRUE_DISABLE_SIGNUP=false
+      - GOTRUE_SITE_URL=https://researchstack.info/appflowy
+      - GOTRUE_URI_ALLOW_LIST=**
+      - GOTRUE_JWT_SECRET=${APPFLOWY_JWT_SECRET:-changeme_jwt_secret_at_least_32_chars_long}
+      - GOTRUE_JWT_EXP=3600
+      - GOTRUE_DB_DRIVER=postgres
+      - API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue
+      - DATABASE_URL=postgres://postgres:appflowy_local@postgres:5432/appflowy?sslmode=disable&options=-c%20search_path%3Dauth%2Cpublic
+      - PORT=9999
+      - GOTRUE_MAILER_AUTOCONFIRM=true
+      - GOTRUE_RATE_LIMIT_EMAIL_SENT=100
+      - GOTRUE_LOG_LEVEL=info
+      - GOTRUE_JWT_ADMIN_GROUP_NAME=supabase_admin
+      - REDIS_ENABLED=true
+      - REDIS_URL=redis://redis:6379
+
+  appflowy_cloud:
+    image: docker.io/appflowyinc/appflowy_cloud:latest
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      gotrue:
+        condition: service_started
+    environment:
+      - APPFLOWY_ENVIRONMENT=production
+      - APPFLOWY_DATABASE_URL=postgres://postgres:appflowy_local@postgres:5432/appflowy?sslmode=disable
+      - APPFLOWY_REDIS_URI=redis://redis:6379
+      - APPFLOWY_GOTRUE_JWT_SECRET=${APPFLOWY_JWT_SECRET:-changeme_jwt_secret_at_least_32_chars_long}
+      - APPFLOWY_GOTRUE_JWT_EXP=3600
+      - APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999
+      - APPFLOWY_BASE_URL=https://researchstack.info/appflowy
+      - APPFLOWY_WS_BASE_URL=wss://researchstack.info/appflowy/ws/v1
+      - APPFLOWY_WEB_URL=https://researchstack.info/appflowy
+      - APPFLOWY_ENCRYPTION_KEY=${APPFLOWY_ENCRYPTION_KEY:-changeme_encryption_key_at_least_32_chars_long}
+      - APPFLOWY_JWT_SECRET=${APPFLOWY_JWT_SECRET:-changeme_jwt_secret_at_least_32_chars_long}
+      - APPFLOWY_S3_CREATE_BUCKET=false
+      - APPFLOWY_S3_USE_MINIO=false
+      - APPFLOWY_ACCESS_CONTROL=false
+
+      - APPFLOWY_S3_ENABLED=false
+      - APPFLOWY_SEARCH_ENABLED=false
+      - RUST_LOG=info
+    ports:
+      - "127.0.0.1:8000:8000"
+
+volumes:
+  pgdata: