diff --git a/scripts/neon-startup.py b/scripts/neon-startup.py index 59c53fd0..7909fe26 100644 --- a/scripts/neon-startup.py +++ b/scripts/neon-startup.py @@ -1,150 +1,179 @@ #!/usr/bin/env python3 -"""neon-startup.py — Restart all neon-64gb services in dependency order. -Run after a reboot: python3 neon-startup.py - -Handles the GoTrue migration ordering bug by pre-running migrations -from the Docker image with proper template expansion. +"""neon-startup.py — Staged neon-64gb service startup with health checks. + +Each stage waits for its dependencies before proceeding. +A stage failure logs an error but doesn't cascade-kill other stages. """ -import subprocess, time, sys +import subprocess, time, sys, os HOST = "allaun@100.92.88.64" -PSQL = "PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres /home/allaun/.nix-profile/bin/psql -h localhost -U postgres -d appflowy" +SSH_OPTS = "-o ConnectTimeout=15 -o ServerAliveInterval=10" +SSH = f"ssh {SSH_OPTS} {HOST}" -def ssh(cmd: str, timeout: int = 30): - r = subprocess.run(["ssh", "-o", "ConnectTimeout=10", HOST, cmd], - capture_output=True, text=True, timeout=timeout, env={**ENV, "PATH": "/usr/bin:/bin"}) - return r.returncode, r.stdout, r.stderr +def ssh(cmd: str, timeout: int = 30) -> tuple[int, str, str]: + r = subprocess.run(f"{SSH} '{cmd}'", capture_output=True, text=True, + timeout=timeout, shell=True, + env={**os.environ, "PATH": "/usr/bin:/bin"}) + return r.returncode, r.stdout.strip(), r.stderr.strip() -def ssh_simple(cmd: str, timeout: int = 30): - return subprocess.run(["ssh", "-o", "ConnectTimeout=10", HOST, cmd], - capture_output=True, text=True, timeout=timeout).returncode +def run(cmd: str, timeout: int = 30) -> int: + rc, _, _ = ssh(cmd, timeout) + return rc -def log(msg): - print(f" {msg}") +def health(url: str) -> str: + rc, out, _ = ssh(f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{url}", timeout=10) + return out if out else "down" -def pg(cmd: str): - rc = ssh_simple(f'{PSQL} -c "{cmd}"', timeout=15) - return rc == 0 +def wait_for(url: str, label: str, timeout: int = 60): + deadline = time.time() + timeout + while time.time() < deadline: + h = health(url) + if h in ("200", "302", "307"): + print(f" ✅ {label} ({h})") + return True + time.sleep(3) + h = health(url) + print(f" ⚠️ {label} not healthy after {timeout}s (last: {h})") + return False -def main(): - print("=== 1. Postgres ===") - ssh_simple("podman restart arxiv-pg", timeout=15) - time.sleep(5) - rc = ssh_simple("pg_isready -h localhost", timeout=10) - if rc != 0: - print(" Postgres start failed") - sys.exit(1) - log("Postgres ready") +def log(s): print(s) - print("=== 2. Redis ===") - ssh_simple("podman rm -f redis 2>/dev/null; podman run -d --name redis --network host docker.io/library/redis:7-alpine", timeout=20) - time.sleep(3) - log("Redis started") +# ── Stage 1: Infrastructure ───────────────────────────────────────── - print("=== 3. Database setup ===") - ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='"'"'appflowy'"'"'" | grep -q 1 || ' - 'PGPASSWORD=postgres psql -h localhost -U postgres -c "CREATE DATABASE appflowy"', timeout=10) - ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy ' - '-c "ALTER DATABASE appflowy SET search_path TO auth, public"', timeout=10) - ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy ' - '-c "DROP SCHEMA IF EXISTS auth CASCADE; CREATE SCHEMA auth"', timeout=10) - log("Database ready") +log("=== Stage 1: Postgres ===") +run("podman start arxiv-pg 2>/dev/null", timeout=10) +time.sleep(5) +if run("pg_isready -h localhost", timeout=10) == 0: + log(" ✅ Postgres ready") +else: + log(" ❌ Postgres failed") - print("=== 4. GoTrue migrations ===") - # Run all migrations with template expansion via Python - result = subprocess.run( - ["podman", "run", "--rm", "docker.io/appflowyinc/gotrue:0.15.1", "ls", "/migrations/"], +log("=== Stage 2: Redis ===") +run("podman rm -f redis 2>/dev/null; podman run -d --name redis --network host docker.io/library/redis:7-alpine", timeout=15) +wait_for("6379", "Redis", timeout=15) + +# ── Stage 3: Database setup ───────────────────────────────────────── + +log("=== Stage 3: Database ===") +PSQL = "PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres /home/allaun/.nix-profile/bin/psql -h localhost -U postgres" +run(f'{PSQL} -tc "SELECT 1 FROM pg_database WHERE datname='"'appflowy'"'" | grep -q 1 || ' + f'{PSQL} -c "CREATE DATABASE appflowy"', timeout=10) +run(f'{PSQL} -d appflowy -c "ALTER DATABASE appflowy SET search_path TO auth, public"', timeout=10) +run(f'{PSQL} -d appflowy -c "DROP SCHEMA IF EXISTS auth CASCADE; CREATE SCHEMA auth"', timeout=10) +log(" ✅ Database ready") + +# ── Stage 4: GoTrue migrations ────────────────────────────────────── + +log("=== Stage 4: GoTrue migrations ===") +MIG_IMAGE = "docker.io/appflowyinc/gotrue:0.15.1" +result = subprocess.run( + ["podman", "run", "--rm", MIG_IMAGE, "ls", "/migrations/"], + capture_output=True, text=True, timeout=15 +) +files = sorted([f for f in result.stdout.strip().split("\n") if f.endswith(".up.sql")]) + +# Batch all SQL into ONE connection to avoid overwhelming postgres +all_sql = "" +for fname in files: + r = subprocess.run( + ["podman", "run", "--rm", MIG_IMAGE, "cat", f"/migrations/{fname}"], capture_output=True, text=True, timeout=15 ) - files = sorted([f for f in result.stdout.strip().split("\n") if f.endswith(".up.sql")]) - log(f"Running {len(files)} migrations") - ok = 0 - for fname in files: - r = subprocess.run( - ["podman", "run", "--rm", "docker.io/appflowyinc/gotrue:0.15.1", "cat", f"/migrations/{fname}"], - capture_output=True, text=True, timeout=15 - ) - sql = r.stdout.replace("{{ index .Options \"Namespace\" }}", "auth") - # Pipe SQL through SSH to psql on neon - r2 = subprocess.run( - ["ssh", HOST, f"PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy"], - input=sql, capture_output=True, text=True, timeout=30 - ) - if r2.returncode == 0: - ok += 1 - else: - ver = fname.split("_")[0] - subprocess.run(["ssh", HOST, f'{PSQL} -c "INSERT INTO auth.schema_migrations (version) VALUES (\\'{ver}\\') ON CONFLICT DO NOTHING"'], - capture_output=True, timeout=10) - log(f"{ok}/{len(files)} direct, remaining tracked in schema_migrations") - - # Copy schema_migrations to public schema - subprocess.run(["ssh", HOST, f'{PSQL} -c "DROP TABLE IF EXISTS public.schema_migrations; CREATE TABLE public.schema_migrations AS SELECT * FROM auth.schema_migrations"'], - capture_output=True, timeout=10) - log("Migration tracking synced") + all_sql += r.stdout.replace("{{ index .Options \"Namespace\" }}", "auth") + "\n" - print("=== 5. GoTrue ===") - ssh_simple("podman rm -f appflowy_gotrue_1 2>/dev/null", timeout=5) - ssh_simple( - 'podman run -d --name appflowy_gotrue_1 --network host ' - '-e GOTRUE_ADMIN_EMAIL=admin@researchstack.info ' - '-e GOTRUE_ADMIN_PASSWORD=admin123 ' - '-e GOTRUE_DISABLE_SIGNUP=false ' - '-e GOTRUE_SITE_URL=https://researchstack.info/appflowy ' - '-e GOTRUE_URI_ALLOW_LIST=** ' - '-e GOTRUE_JWT_SECRET=jwt_ ' - '-e GOTRUE_JWT_EXP=3600 ' - '-e GOTRUE_DB_DRIVER=postgres ' - '-e API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue ' - '-e DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy" ' - '-e PORT=9999 ' - '-e GOTRUE_MAILER_AUTOCONFIRM=true ' - '-e GOTRUE_LOG_LEVEL=info ' - '-e REDIS_ENABLED=true ' - '-e REDIS_URL=redis://localhost:6379 ' - 'docker.io/appflowyinc/gotrue:0.15.1', timeout=20) - time.sleep(10) - rc, out, _ = ssh("curl -s -o /dev/null -w '%{http_code}' http://localhost:9999/health", timeout=10) - if "200" in out: - log("GoTrue: running") +# Run all migrations in one batch via SSH +proc = subprocess.Popen( + f"{SSH} '{PSQL} -d appflowy -v ON_ERROR_STOP=0'", + shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE +) +stdout, stderr = proc.communicate(input=all_sql.encode(), timeout=120) +log(f" ✅ {len(files)} migrations attempted (batched)") + +# Mark all versions done +all_versions = "','".join(f.split("_")[0] for f in files) +run(f'{PSQL} -d appflowy -c "INSERT INTO auth.schema_migrations (version) SELECT unnest(array[\'{all_versions}\']) ON CONFLICT DO NOTHING"', timeout=30) + +# Sync to public schema for GoTrue binary +run(f'{PSQL} -d appflowy -c "DROP TABLE IF EXISTS public.schema_migrations; CREATE TABLE public.schema_migrations AS SELECT * FROM auth.schema_migrations"', timeout=10) +log(" Migration tracking synced") + +# ── Stage 5: GoTrue service ───────────────────────────────────────── + +log("=== Stage 5: GoTrue ===") +run("podman rm -f appflowy_gotrue_1 2>/dev/null", timeout=5) +run( + 'podman run -d --name appflowy_gotrue_1 --network host ' + '-e GOTRUE_ADMIN_EMAIL=admin@researchstack.info ' + '-e GOTRUE_ADMIN_PASSWORD=admin123 ' + '-e GOTRUE_DISABLE_SIGNUP=false ' + '-e GOTRUE_SITE_URL=https://researchstack.info/appflowy ' + '-e GOTRUE_URI_ALLOW_LIST=** ' + '-e GOTRUE_JWT_SECRET=jwt_ ' + '-e GOTRUE_JWT_EXP=3600 ' + '-e GOTRUE_DB_DRIVER=postgres ' + '-e API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue ' + '-e DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy" ' + '-e PORT=9999 ' + '-e GOTRUE_MAILER_AUTOCONFIRM=true ' + '-e GOTRUE_LOG_LEVEL=info ' + '-e REDIS_ENABLED=true ' + '-e REDIS_URL=redis://localhost:6379 ' + 'docker.io/appflowyinc/gotrue:0.15.1', timeout=20) +wait_for("9999/health", "GoTrue", timeout=30) + +# ── Stage 6: AppFloyo Cloud ───────────────────────────────────────── + +log("=== Stage 6: AppFloyo Cloud ===") +run("podman rm -f appflowy_cloud 2>/dev/null", timeout=5) +run( + 'podman run -d --name appflowy_cloud --network host ' + '-e APPFLOWY_DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy?sslmode=disable&options=-c%20search_path=public" ' + '-e APPFLOWY_ENVIRONMENT=production ' + '-e APPFLOWY_REDIS_URI=redis://localhost:6379 ' + '-e APPFLOWY_GOTRUE_JWT_SECRET=jwt_ ' + '-e APPFLOWY_GOTRUE_JWT_EXP=3600 ' + '-e APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999 ' + '-e APPFLOWY_BASE_URL=https://researchstack.info/appflowy ' + '-e APPFLOWY_WEB_URL=https://researchstack.info/appflowy ' + '-e APPFLOWY_ENCRYPTION_KEY=sk_fix_me_32_chars_abcdef1234567890 ' + '-e APPFLOWY_JWT_SECRET=jwt_ ' + '-e APPFLOWY_CREATE_BUCKET=false ' + '-e APPFLOWY_S3_ENABLED=false ' + '-e APPFLOWY_SEARCH_ENABLED=false ' + '-e RUST_LOG=info ' + 'docker.io/appflowyinc/appflowy_cloud:latest', timeout=20) +wait_for("8000/api/health", "AppFloyo", timeout=45) + +# ── Stage 7: Authentik SSO ────────────────────────────────────────── + +log("=== Stage 7: Authentik ===") +run("podman start authentik-server 2>/dev/null", timeout=10) +run("podman start authentik-worker 2>/dev/null", timeout=10) +wait_for("30001/", "Authentik", timeout=45) + +# ── Stage 8: Supporting services ──────────────────────────────────── + +log("=== Stage 8: Other services ===") +for svc, port, label in [("couchdb", "5984/", "CouchDB"), + ("homarr-web", "7575/auth/login", "Homarr"), + ("www-static", "8082/", "www-static")]: + run(f"podman start {svc} 2>/dev/null", timeout=10) + wait_for(port, label, timeout=30) + +# ── Final report ──────────────────────────────────────────────────── + +log("\n=== Service Summary ===") +for name, url in [("Postgres (pg)", "5432 (pg_isready)"), + ("Redis", "6379 (redis-cli ping)"), + ("GoTrue", "9999/health"), + ("AppFloyo", "8000/api/health"), + ("Authentik", "30001/"), + ("CouchDB", "5984/"), + ("Homarr", "7575/auth/login")]: + if "pg" in name.lower(): + rc = run("pg_isready -h localhost", timeout=5) + log(f" {'✅' if rc==0 else '❌'} {name}") else: - log(f"GoTrue health: {out.strip()}") - - print("=== 6. AppFloyo Cloud ===") - ssh_simple("podman rm -f appflowy_cloud 2>/dev/null", timeout=5) - ssh_simple( - 'podman run -d --name appflowy_cloud --network host ' - '-e APPFLOWY_DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy?sslmode=disable&options=-c%20search_path=public" ' - '-e APPFLOWY_ENVIRONMENT=production ' - '-e APPFLOWY_REDIS_URI=redis://localhost:6379 ' - '-e APPFLOWY_GOTRUE_JWT_SECRET=jwt_ ' - '-e APPFLOWY_GOTRUE_JWT_EXP=3600 ' - '-e APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999 ' - '-e APPFLOWY_BASE_URL=https://researchstack.info/appflowy ' - '-e APPFLOWY_WEB_URL=https://researchstack.info/appflowy ' - '-e APPFLOWY_ENCRYPTION_KEY=sk_fix_me_32_chars_abcdef1234567890 ' - '-e APPFLOWY_JWT_SECRET=jwt_ ' - '-e APPFLOWY_CREATE_BUCKET=false ' - '-e APPFLOWY_S3_ENABLED=false ' - '-e APPFLOWY_SEARCH_ENABLED=false ' - '-e RUST_LOG=info ' - 'docker.io/appflowyinc/appflowy_cloud:latest', timeout=20) - - print("=== 7. Other services ===") - for svc in ["couchdb", "homarr-web", "www-static", "hermes-dashboard"]: - ssh_simple(f"podman start {svc} 2>/dev/null", timeout=10) - - time.sleep(20) - - print("=== 8. Health check ===") - for name, url in [("GoTrue", "9999/health"), ("AppFloyo", "8000/api/health"), - ("Homarr", "7575/auth/login"), ("CouchDB", "5984/")]: - rc, out, _ = ssh(f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{url}", timeout=10) - tag = "ok" if "200" in out or "302" in out or "307" in out else "?" - log(f" {name}: {out.strip()} {tag}") - - print("=== Done ===") - -if __name__ == "__main__": - main() + h = health(url.split("/")[0] + "/" + "/".join(url.split("/")[1:])) + log(f" {'✅' if h in ('200','302','307') else '❌'} {name} ({h})")