mirror of
https://github.com/allaunthefox/SilverSight.git
synced 2026-07-31 01:25:21 +00:00
fix: staged neon startup with batched GoTrue migrations, Podman lock workaround
This commit is contained in:
parent
2d48f6a327
commit
29b0582b12
1 changed files with 162 additions and 133 deletions
|
|
@ -1,150 +1,179 @@
|
|||
#!/usr/bin/env python3
|
||||
"""neon-startup.py — Restart all neon-64gb services in dependency order.
|
||||
Run after a reboot: python3 neon-startup.py
|
||||
|
||||
Handles the GoTrue migration ordering bug by pre-running migrations
|
||||
from the Docker image with proper template expansion.
|
||||
"""neon-startup.py — Staged neon-64gb service startup with health checks.
|
||||
|
||||
Each stage waits for its dependencies before proceeding.
|
||||
A stage failure logs an error but doesn't cascade-kill other stages.
|
||||
"""
|
||||
|
||||
import subprocess, time, sys
|
||||
import subprocess, time, sys, os
|
||||
|
||||
HOST = "allaun@100.92.88.64"
|
||||
PSQL = "PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres /home/allaun/.nix-profile/bin/psql -h localhost -U postgres -d appflowy"
|
||||
SSH_OPTS = "-o ConnectTimeout=15 -o ServerAliveInterval=10"
|
||||
SSH = f"ssh {SSH_OPTS} {HOST}"
|
||||
|
||||
def ssh(cmd: str, timeout: int = 30):
|
||||
r = subprocess.run(["ssh", "-o", "ConnectTimeout=10", HOST, cmd],
|
||||
capture_output=True, text=True, timeout=timeout, env={**ENV, "PATH": "/usr/bin:/bin"})
|
||||
return r.returncode, r.stdout, r.stderr
|
||||
def ssh(cmd: str, timeout: int = 30) -> tuple[int, str, str]:
|
||||
r = subprocess.run(f"{SSH} '{cmd}'", capture_output=True, text=True,
|
||||
timeout=timeout, shell=True,
|
||||
env={**os.environ, "PATH": "/usr/bin:/bin"})
|
||||
return r.returncode, r.stdout.strip(), r.stderr.strip()
|
||||
|
||||
def ssh_simple(cmd: str, timeout: int = 30):
|
||||
return subprocess.run(["ssh", "-o", "ConnectTimeout=10", HOST, cmd],
|
||||
capture_output=True, text=True, timeout=timeout).returncode
|
||||
def run(cmd: str, timeout: int = 30) -> int:
|
||||
rc, _, _ = ssh(cmd, timeout)
|
||||
return rc
|
||||
|
||||
def log(msg):
|
||||
print(f" {msg}")
|
||||
def health(url: str) -> str:
|
||||
rc, out, _ = ssh(f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{url}", timeout=10)
|
||||
return out if out else "down"
|
||||
|
||||
def pg(cmd: str):
|
||||
rc = ssh_simple(f'{PSQL} -c "{cmd}"', timeout=15)
|
||||
return rc == 0
|
||||
def wait_for(url: str, label: str, timeout: int = 60):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
h = health(url)
|
||||
if h in ("200", "302", "307"):
|
||||
print(f" ✅ {label} ({h})")
|
||||
return True
|
||||
time.sleep(3)
|
||||
h = health(url)
|
||||
print(f" ⚠️ {label} not healthy after {timeout}s (last: {h})")
|
||||
return False
|
||||
|
||||
def main():
|
||||
print("=== 1. Postgres ===")
|
||||
ssh_simple("podman restart arxiv-pg", timeout=15)
|
||||
time.sleep(5)
|
||||
rc = ssh_simple("pg_isready -h localhost", timeout=10)
|
||||
if rc != 0:
|
||||
print(" Postgres start failed")
|
||||
sys.exit(1)
|
||||
log("Postgres ready")
|
||||
def log(s): print(s)
|
||||
|
||||
print("=== 2. Redis ===")
|
||||
ssh_simple("podman rm -f redis 2>/dev/null; podman run -d --name redis --network host docker.io/library/redis:7-alpine", timeout=20)
|
||||
time.sleep(3)
|
||||
log("Redis started")
|
||||
# ── Stage 1: Infrastructure ─────────────────────────────────────────
|
||||
|
||||
print("=== 3. Database setup ===")
|
||||
ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='"'"'appflowy'"'"'" | grep -q 1 || '
|
||||
'PGPASSWORD=postgres psql -h localhost -U postgres -c "CREATE DATABASE appflowy"', timeout=10)
|
||||
ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy '
|
||||
'-c "ALTER DATABASE appflowy SET search_path TO auth, public"', timeout=10)
|
||||
ssh('PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy '
|
||||
'-c "DROP SCHEMA IF EXISTS auth CASCADE; CREATE SCHEMA auth"', timeout=10)
|
||||
log("Database ready")
|
||||
log("=== Stage 1: Postgres ===")
|
||||
run("podman start arxiv-pg 2>/dev/null", timeout=10)
|
||||
time.sleep(5)
|
||||
if run("pg_isready -h localhost", timeout=10) == 0:
|
||||
log(" ✅ Postgres ready")
|
||||
else:
|
||||
log(" ❌ Postgres failed")
|
||||
|
||||
print("=== 4. GoTrue migrations ===")
|
||||
# Run all migrations with template expansion via Python
|
||||
result = subprocess.run(
|
||||
["podman", "run", "--rm", "docker.io/appflowyinc/gotrue:0.15.1", "ls", "/migrations/"],
|
||||
log("=== Stage 2: Redis ===")
|
||||
run("podman rm -f redis 2>/dev/null; podman run -d --name redis --network host docker.io/library/redis:7-alpine", timeout=15)
|
||||
wait_for("6379", "Redis", timeout=15)
|
||||
|
||||
# ── Stage 3: Database setup ─────────────────────────────────────────
|
||||
|
||||
log("=== Stage 3: Database ===")
|
||||
PSQL = "PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres /home/allaun/.nix-profile/bin/psql -h localhost -U postgres"
|
||||
run(f'{PSQL} -tc "SELECT 1 FROM pg_database WHERE datname='"'appflowy'"'" | grep -q 1 || '
|
||||
f'{PSQL} -c "CREATE DATABASE appflowy"', timeout=10)
|
||||
run(f'{PSQL} -d appflowy -c "ALTER DATABASE appflowy SET search_path TO auth, public"', timeout=10)
|
||||
run(f'{PSQL} -d appflowy -c "DROP SCHEMA IF EXISTS auth CASCADE; CREATE SCHEMA auth"', timeout=10)
|
||||
log(" ✅ Database ready")
|
||||
|
||||
# ── Stage 4: GoTrue migrations ──────────────────────────────────────
|
||||
|
||||
log("=== Stage 4: GoTrue migrations ===")
|
||||
MIG_IMAGE = "docker.io/appflowyinc/gotrue:0.15.1"
|
||||
result = subprocess.run(
|
||||
["podman", "run", "--rm", MIG_IMAGE, "ls", "/migrations/"],
|
||||
capture_output=True, text=True, timeout=15
|
||||
)
|
||||
files = sorted([f for f in result.stdout.strip().split("\n") if f.endswith(".up.sql")])
|
||||
|
||||
# Batch all SQL into ONE connection to avoid overwhelming postgres
|
||||
all_sql = ""
|
||||
for fname in files:
|
||||
r = subprocess.run(
|
||||
["podman", "run", "--rm", MIG_IMAGE, "cat", f"/migrations/{fname}"],
|
||||
capture_output=True, text=True, timeout=15
|
||||
)
|
||||
files = sorted([f for f in result.stdout.strip().split("\n") if f.endswith(".up.sql")])
|
||||
log(f"Running {len(files)} migrations")
|
||||
ok = 0
|
||||
for fname in files:
|
||||
r = subprocess.run(
|
||||
["podman", "run", "--rm", "docker.io/appflowyinc/gotrue:0.15.1", "cat", f"/migrations/{fname}"],
|
||||
capture_output=True, text=True, timeout=15
|
||||
)
|
||||
sql = r.stdout.replace("{{ index .Options \"Namespace\" }}", "auth")
|
||||
# Pipe SQL through SSH to psql on neon
|
||||
r2 = subprocess.run(
|
||||
["ssh", HOST, f"PATH=$HOME/.nix-profile/bin:$PATH PGPASSWORD=postgres psql -h localhost -U postgres -d appflowy"],
|
||||
input=sql, capture_output=True, text=True, timeout=30
|
||||
)
|
||||
if r2.returncode == 0:
|
||||
ok += 1
|
||||
else:
|
||||
ver = fname.split("_")[0]
|
||||
subprocess.run(["ssh", HOST, f'{PSQL} -c "INSERT INTO auth.schema_migrations (version) VALUES (\\'{ver}\\') ON CONFLICT DO NOTHING"'],
|
||||
capture_output=True, timeout=10)
|
||||
log(f"{ok}/{len(files)} direct, remaining tracked in schema_migrations")
|
||||
|
||||
# Copy schema_migrations to public schema
|
||||
subprocess.run(["ssh", HOST, f'{PSQL} -c "DROP TABLE IF EXISTS public.schema_migrations; CREATE TABLE public.schema_migrations AS SELECT * FROM auth.schema_migrations"'],
|
||||
capture_output=True, timeout=10)
|
||||
log("Migration tracking synced")
|
||||
all_sql += r.stdout.replace("{{ index .Options \"Namespace\" }}", "auth") + "\n"
|
||||
|
||||
print("=== 5. GoTrue ===")
|
||||
ssh_simple("podman rm -f appflowy_gotrue_1 2>/dev/null", timeout=5)
|
||||
ssh_simple(
|
||||
'podman run -d --name appflowy_gotrue_1 --network host '
|
||||
'-e GOTRUE_ADMIN_EMAIL=admin@researchstack.info '
|
||||
'-e GOTRUE_ADMIN_PASSWORD=admin123 '
|
||||
'-e GOTRUE_DISABLE_SIGNUP=false '
|
||||
'-e GOTRUE_SITE_URL=https://researchstack.info/appflowy '
|
||||
'-e GOTRUE_URI_ALLOW_LIST=** '
|
||||
'-e GOTRUE_JWT_SECRET=jwt_ '
|
||||
'-e GOTRUE_JWT_EXP=3600 '
|
||||
'-e GOTRUE_DB_DRIVER=postgres '
|
||||
'-e API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue '
|
||||
'-e DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy" '
|
||||
'-e PORT=9999 '
|
||||
'-e GOTRUE_MAILER_AUTOCONFIRM=true '
|
||||
'-e GOTRUE_LOG_LEVEL=info '
|
||||
'-e REDIS_ENABLED=true '
|
||||
'-e REDIS_URL=redis://localhost:6379 '
|
||||
'docker.io/appflowyinc/gotrue:0.15.1', timeout=20)
|
||||
time.sleep(10)
|
||||
rc, out, _ = ssh("curl -s -o /dev/null -w '%{http_code}' http://localhost:9999/health", timeout=10)
|
||||
if "200" in out:
|
||||
log("GoTrue: running")
|
||||
# Run all migrations in one batch via SSH
|
||||
proc = subprocess.Popen(
|
||||
f"{SSH} '{PSQL} -d appflowy -v ON_ERROR_STOP=0'",
|
||||
shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = proc.communicate(input=all_sql.encode(), timeout=120)
|
||||
log(f" ✅ {len(files)} migrations attempted (batched)")
|
||||
|
||||
# Mark all versions done
|
||||
all_versions = "','".join(f.split("_")[0] for f in files)
|
||||
run(f'{PSQL} -d appflowy -c "INSERT INTO auth.schema_migrations (version) SELECT unnest(array[\'{all_versions}\']) ON CONFLICT DO NOTHING"', timeout=30)
|
||||
|
||||
# Sync to public schema for GoTrue binary
|
||||
run(f'{PSQL} -d appflowy -c "DROP TABLE IF EXISTS public.schema_migrations; CREATE TABLE public.schema_migrations AS SELECT * FROM auth.schema_migrations"', timeout=10)
|
||||
log(" Migration tracking synced")
|
||||
|
||||
# ── Stage 5: GoTrue service ─────────────────────────────────────────
|
||||
|
||||
log("=== Stage 5: GoTrue ===")
|
||||
run("podman rm -f appflowy_gotrue_1 2>/dev/null", timeout=5)
|
||||
run(
|
||||
'podman run -d --name appflowy_gotrue_1 --network host '
|
||||
'-e GOTRUE_ADMIN_EMAIL=admin@researchstack.info '
|
||||
'-e GOTRUE_ADMIN_PASSWORD=admin123 '
|
||||
'-e GOTRUE_DISABLE_SIGNUP=false '
|
||||
'-e GOTRUE_SITE_URL=https://researchstack.info/appflowy '
|
||||
'-e GOTRUE_URI_ALLOW_LIST=** '
|
||||
'-e GOTRUE_JWT_SECRET=jwt_ '
|
||||
'-e GOTRUE_JWT_EXP=3600 '
|
||||
'-e GOTRUE_DB_DRIVER=postgres '
|
||||
'-e API_EXTERNAL_URL=https://researchstack.info/appflowy/gotrue '
|
||||
'-e DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy" '
|
||||
'-e PORT=9999 '
|
||||
'-e GOTRUE_MAILER_AUTOCONFIRM=true '
|
||||
'-e GOTRUE_LOG_LEVEL=info '
|
||||
'-e REDIS_ENABLED=true '
|
||||
'-e REDIS_URL=redis://localhost:6379 '
|
||||
'docker.io/appflowyinc/gotrue:0.15.1', timeout=20)
|
||||
wait_for("9999/health", "GoTrue", timeout=30)
|
||||
|
||||
# ── Stage 6: AppFloyo Cloud ─────────────────────────────────────────
|
||||
|
||||
log("=== Stage 6: AppFloyo Cloud ===")
|
||||
run("podman rm -f appflowy_cloud 2>/dev/null", timeout=5)
|
||||
run(
|
||||
'podman run -d --name appflowy_cloud --network host '
|
||||
'-e APPFLOWY_DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy?sslmode=disable&options=-c%20search_path=public" '
|
||||
'-e APPFLOWY_ENVIRONMENT=production '
|
||||
'-e APPFLOWY_REDIS_URI=redis://localhost:6379 '
|
||||
'-e APPFLOWY_GOTRUE_JWT_SECRET=jwt_ '
|
||||
'-e APPFLOWY_GOTRUE_JWT_EXP=3600 '
|
||||
'-e APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999 '
|
||||
'-e APPFLOWY_BASE_URL=https://researchstack.info/appflowy '
|
||||
'-e APPFLOWY_WEB_URL=https://researchstack.info/appflowy '
|
||||
'-e APPFLOWY_ENCRYPTION_KEY=sk_fix_me_32_chars_abcdef1234567890 '
|
||||
'-e APPFLOWY_JWT_SECRET=jwt_ '
|
||||
'-e APPFLOWY_CREATE_BUCKET=false '
|
||||
'-e APPFLOWY_S3_ENABLED=false '
|
||||
'-e APPFLOWY_SEARCH_ENABLED=false '
|
||||
'-e RUST_LOG=info '
|
||||
'docker.io/appflowyinc/appflowy_cloud:latest', timeout=20)
|
||||
wait_for("8000/api/health", "AppFloyo", timeout=45)
|
||||
|
||||
# ── Stage 7: Authentik SSO ──────────────────────────────────────────
|
||||
|
||||
log("=== Stage 7: Authentik ===")
|
||||
run("podman start authentik-server 2>/dev/null", timeout=10)
|
||||
run("podman start authentik-worker 2>/dev/null", timeout=10)
|
||||
wait_for("30001/", "Authentik", timeout=45)
|
||||
|
||||
# ── Stage 8: Supporting services ────────────────────────────────────
|
||||
|
||||
log("=== Stage 8: Other services ===")
|
||||
for svc, port, label in [("couchdb", "5984/", "CouchDB"),
|
||||
("homarr-web", "7575/auth/login", "Homarr"),
|
||||
("www-static", "8082/", "www-static")]:
|
||||
run(f"podman start {svc} 2>/dev/null", timeout=10)
|
||||
wait_for(port, label, timeout=30)
|
||||
|
||||
# ── Final report ────────────────────────────────────────────────────
|
||||
|
||||
log("\n=== Service Summary ===")
|
||||
for name, url in [("Postgres (pg)", "5432 (pg_isready)"),
|
||||
("Redis", "6379 (redis-cli ping)"),
|
||||
("GoTrue", "9999/health"),
|
||||
("AppFloyo", "8000/api/health"),
|
||||
("Authentik", "30001/"),
|
||||
("CouchDB", "5984/"),
|
||||
("Homarr", "7575/auth/login")]:
|
||||
if "pg" in name.lower():
|
||||
rc = run("pg_isready -h localhost", timeout=5)
|
||||
log(f" {'✅' if rc==0 else '❌'} {name}")
|
||||
else:
|
||||
log(f"GoTrue health: {out.strip()}")
|
||||
|
||||
print("=== 6. AppFloyo Cloud ===")
|
||||
ssh_simple("podman rm -f appflowy_cloud 2>/dev/null", timeout=5)
|
||||
ssh_simple(
|
||||
'podman run -d --name appflowy_cloud --network host '
|
||||
'-e APPFLOWY_DATABASE_URL="postgres://postgres:postgres@localhost:5432/appflowy?sslmode=disable&options=-c%20search_path=public" '
|
||||
'-e APPFLOWY_ENVIRONMENT=production '
|
||||
'-e APPFLOWY_REDIS_URI=redis://localhost:6379 '
|
||||
'-e APPFLOWY_GOTRUE_JWT_SECRET=jwt_ '
|
||||
'-e APPFLOWY_GOTRUE_JWT_EXP=3600 '
|
||||
'-e APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999 '
|
||||
'-e APPFLOWY_BASE_URL=https://researchstack.info/appflowy '
|
||||
'-e APPFLOWY_WEB_URL=https://researchstack.info/appflowy '
|
||||
'-e APPFLOWY_ENCRYPTION_KEY=sk_fix_me_32_chars_abcdef1234567890 '
|
||||
'-e APPFLOWY_JWT_SECRET=jwt_ '
|
||||
'-e APPFLOWY_CREATE_BUCKET=false '
|
||||
'-e APPFLOWY_S3_ENABLED=false '
|
||||
'-e APPFLOWY_SEARCH_ENABLED=false '
|
||||
'-e RUST_LOG=info '
|
||||
'docker.io/appflowyinc/appflowy_cloud:latest', timeout=20)
|
||||
|
||||
print("=== 7. Other services ===")
|
||||
for svc in ["couchdb", "homarr-web", "www-static", "hermes-dashboard"]:
|
||||
ssh_simple(f"podman start {svc} 2>/dev/null", timeout=10)
|
||||
|
||||
time.sleep(20)
|
||||
|
||||
print("=== 8. Health check ===")
|
||||
for name, url in [("GoTrue", "9999/health"), ("AppFloyo", "8000/api/health"),
|
||||
("Homarr", "7575/auth/login"), ("CouchDB", "5984/")]:
|
||||
rc, out, _ = ssh(f"curl -s -o /dev/null -w '%{{http_code}}' http://localhost:{url}", timeout=10)
|
||||
tag = "ok" if "200" in out or "302" in out or "307" in out else "?"
|
||||
log(f" {name}: {out.strip()} {tag}")
|
||||
|
||||
print("=== Done ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
h = health(url.split("/")[0] + "/" + "/".join(url.split("/")[1:]))
|
||||
log(f" {'✅' if h in ('200','302','307') else '❌'} {name} ({h})")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue