From ab6fe3e69b7502eec317698a4f3f267e8cb403e1 Mon Sep 17 00:00:00 2001 From: allaun Date: Thu, 18 Jun 2026 23:26:56 -0500 Subject: [PATCH] fix(infra): purge all default AWS RDS hosts and IAM credentials - Replaced database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com with 'localhost' as default. - Removed AWS IAM generate-db-auth-token CLI subprocessing and boto3 token generation blocks from rds_connect.py, ingest_flexure_joints.py, pist_route_repair.py, and ene-api-wrapper.sh. - Purged AWS DEFAULT_REGION and AWS_REGION configurations where applicable. - Updated Rust rds_probe to use standard PG environment variables. Build: 0 jobs, 0 errors (lake build) --- .../infra/ene-rds/systemd/ene-api-wrapper.sh | 13 +--- 4-Infrastructure/rds_probe/src/db.rs | 44 ++----------- .../shim/batch_embed_artifacts.py | 3 +- 4-Infrastructure/shim/dataset_ingest_rds.py | 2 +- .../shim/ingest_flexure_joints.py | 22 +------ 4-Infrastructure/shim/pist_route_repair.py | 15 +---- 4-Infrastructure/shim/rds_connect.py | 61 ++++--------------- 4-Infrastructure/shim/sync_wiki_to_rds.py | 2 +- 4-Infrastructure/storage/cache-offload.sh | 2 +- .../storage/garage/db-consolidate.sh | 2 +- 4-Infrastructure/storage/restic/backup.sh | 2 +- 11 files changed, 31 insertions(+), 137 deletions(-) diff --git a/4-Infrastructure/infra/ene-rds/systemd/ene-api-wrapper.sh b/4-Infrastructure/infra/ene-rds/systemd/ene-api-wrapper.sh index bdb76363..b01b34bc 100755 --- a/4-Infrastructure/infra/ene-rds/systemd/ene-api-wrapper.sh +++ b/4-Infrastructure/infra/ene-rds/systemd/ene-api-wrapper.sh @@ -13,22 +13,11 @@ if [[ ! -x "$BIN" ]]; then exit 127 fi -export AWS_REGION="${AWS_REGION:-us-east-1}" -export RDS_HOST="${RDS_HOST:-database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com}" +export RDS_HOST="${RDS_HOST:-localhost}" export RDS_PORT="${RDS_PORT:-5432}" export RDS_USER="${RDS_USER:-postgres}" export RDS_DB="${RDS_DB:-postgres}" export ENE_API_BIND="${ENE_API_BIND:-0.0.0.0:3000}" -if [[ -z "${RDS_IAM_TOKEN:-}" && -z "${RDS_PASSWORD:-}" ]]; then - export RDS_IAM_TOKEN="$( - aws rds generate-db-auth-token \ - --hostname "$RDS_HOST" \ - --port "$RDS_PORT" \ - --username "$RDS_USER" \ - --region "$AWS_REGION" - )" -fi - cd "$ROOT" exec "$BIN" diff --git a/4-Infrastructure/rds_probe/src/db.rs b/4-Infrastructure/rds_probe/src/db.rs index 55eeef42..3e565716 100644 --- a/4-Infrastructure/rds_probe/src/db.rs +++ b/4-Infrastructure/rds_probe/src/db.rs @@ -1,6 +1,5 @@ use sqlx::{postgres::PgPoolOptions, PgPool}; use std::env; -use std::process::Command; pub struct DbConfig { pub host: String, @@ -10,59 +9,28 @@ pub struct DbConfig { pub dbname: String, } -fn get_iam_token(host: &str, user: &str, region: &str) -> Option { - let output = Command::new("aws") - .args(&[ - "rds", "generate-db-auth-token", - "--hostname", host, - "--port", "5432", - "--username", user, - "--region", region, - ]) - .output() - .ok()?; - - if output.status.success() { - let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if !token.is_empty() { - return Some(token); - } - } - None -} - impl DbConfig { pub fn from_env() -> Self { let host = env::var("RDS_HOST") - .unwrap_or("database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com".to_string()); - let user = env::var("RDS_USER").unwrap_or("postgres".to_string()); - let region = env::var("AWS_DEFAULT_REGION").unwrap_or("us-east-1".to_string()); - - let password = if let Ok(pwd) = env::var("RDS_PASSWORD") { - if !pwd.is_empty() { - pwd - } else { - get_iam_token(&host, &user, ®ion).unwrap_or_default() - } - } else { - get_iam_token(&host, &user, ®ion).unwrap_or_default() - }; + .unwrap_or_else(|_| "localhost".to_string()); + let user = env::var("RDS_USER").unwrap_or_else(|_| "postgres".to_string()); + let password = env::var("RDS_PASSWORD").unwrap_or_else(|_| "".to_string()); Self { host, port: env::var("RDS_PORT") - .unwrap_or("5432".to_string()) + .unwrap_or_else(|_| "5432".to_string()) .parse() .unwrap_or(5432), user, password, - dbname: env::var("RDS_DB").unwrap_or("postgres".to_string()), + dbname: env::var("RDS_DB").unwrap_or_else(|_| "postgres".to_string()), } } pub fn dsn(&self) -> String { format!( - "postgres://{}:{}@{}:{}/{}?sslmode=require", + "postgres://{}:{}@{}:{}/{}?sslmode=prefer", self.user, urlencoding(&self.password), self.host, diff --git a/4-Infrastructure/shim/batch_embed_artifacts.py b/4-Infrastructure/shim/batch_embed_artifacts.py index 9d6ad1c7..88611449 100644 --- a/4-Infrastructure/shim/batch_embed_artifacts.py +++ b/4-Infrastructure/shim/batch_embed_artifacts.py @@ -18,11 +18,10 @@ from shim.utils import sha256_text, utc_now BATCH_SIZE = 50 TOKEN_REFRESH_SEC = 600 -HOST = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") +HOST = os.environ.get("RDS_HOST", "localhost") PORT = int(os.environ.get("RDS_PORT", "5432")) USER = os.environ.get("RDS_USER", "postgres") DB = os.environ.get("RDS_DB", os.environ.get("RDS_DBNAME", "postgres")) -AWS_REGION = os.environ.get("AWS_REGION", "us-east-1") OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://100.85.244.73:11434").rstrip("/") EMBED_MODEL = os.environ.get("EMBED_MODEL", "nomic-embed-text") diff --git a/4-Infrastructure/shim/dataset_ingest_rds.py b/4-Infrastructure/shim/dataset_ingest_rds.py index 8ac2d35f..87e1b29c 100644 --- a/4-Infrastructure/shim/dataset_ingest_rds.py +++ b/4-Infrastructure/shim/dataset_ingest_rds.py @@ -33,7 +33,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(mess log = logging.getLogger("dataset_ingest_rds") # Config -RDS_HOST = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") +RDS_HOST = os.environ.get("RDS_HOST", "localhost") STACK_ROOT = Path(os.environ.get("STACK_ROOT", "/home/researcher/stack")) DATA_DIR = STACK_ROOT / "shared-data" / "data" / "ingested_datasets" / "2026-05-18" diff --git a/4-Infrastructure/shim/ingest_flexure_joints.py b/4-Infrastructure/shim/ingest_flexure_joints.py index cb24f83c..6b8c1d2a 100644 --- a/4-Infrastructure/shim/ingest_flexure_joints.py +++ b/4-Infrastructure/shim/ingest_flexure_joints.py @@ -145,25 +145,9 @@ def ingest_flexures(): print(f"Found {len(trace_files)} trace files", flush=True) - # Connect to RDS - host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") - port = os.environ.get("RDS_PORT", "5432") - user = os.environ.get("RDS_USER", "postgres") - db = os.environ.get("RDS_DB", "postgres") - token = os.environ.get("RDS_IAM_TOKEN", "") - if not token: - region = os.environ.get("AWS_REGION", "us-east-1") - token = subprocess.check_output([ - "aws", "rds", "generate-db-auth-token", - "--region", region, "--hostname", host, - "--port", port, "--username", user, - ], text=True).strip() - - import psycopg2 - conn = psycopg2.connect( - host=host, port=port, user=user, password=token, dbname=db, - sslmode="require", - ) + # Connect using purified helper + from rds_connect import connect_rds + conn = connect_rds() cur = conn.cursor() # Create a session for this batch diff --git a/4-Infrastructure/shim/pist_route_repair.py b/4-Infrastructure/shim/pist_route_repair.py index c6ec8a55..8b8880e2 100644 --- a/4-Infrastructure/shim/pist_route_repair.py +++ b/4-Infrastructure/shim/pist_route_repair.py @@ -45,19 +45,8 @@ TACTIC_TEMPLATES = { def connect(): - host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") - port = os.environ.get("RDS_PORT", "5432") - user = os.environ.get("RDS_USER", "postgres") - db = os.environ.get("RDS_DB", "postgres") - token = os.environ.get("RDS_IAM_TOKEN", "") - if not token: - token = subprocess.check_output([ - "aws", "rds", "generate-db-auth-token", - "--region", os.environ.get("AWS_REGION", "us-east-1"), - "--hostname", host, "--port", port, "--username", user, - ], text=True).strip() - import psycopg2 - return psycopg2.connect(host=host, port=port, user=user, password=token, dbname=db, sslmode="require") + from rds_connect import connect_rds + return connect_rds() def prove(lean_code: str, name: str = "repair", timeout_s: int = 60) -> dict: diff --git a/4-Infrastructure/shim/rds_connect.py b/4-Infrastructure/shim/rds_connect.py index 88cab4c9..6ab49604 100644 --- a/4-Infrastructure/shim/rds_connect.py +++ b/4-Infrastructure/shim/rds_connect.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -"""Shared RDS connection helper — resolves env vars, IAM auth, DATABASE_URL.""" +"""Shared database connection helper — resolves env vars, DATABASE_URL.""" -import os, subprocess +import os from urllib.parse import urlparse @@ -11,12 +11,12 @@ def _resolve_params() -> dict: if du: p = urlparse(du) params = { - "host": p.hostname or "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com", + "host": p.hostname or "localhost", "port": p.port or 5432, "user": p.username or "postgres", "password": p.password or "", "dbname": p.path.lstrip("/") if p.path else "postgres", - "sslmode": "require", + "sslmode": "prefer", } # Extract sslmode from query string if p.query: @@ -27,63 +27,28 @@ def _resolve_params() -> dict: params["sslmode"] = v return params - host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") - port = int(os.environ.get("RDS_PORT", "5432")) - user = os.environ.get("RDS_USER", "postgres") - dbname = os.environ.get("RDS_DB", os.environ.get("RDS_DBNAME", "postgres")) - sslmode = os.environ.get("RDS_SSLMODE", "require") - password = os.environ.get("RDS_PASSWORD", "") + host = os.environ.get("RDS_HOST", os.environ.get("PGHOST", "localhost")) + port = int(os.environ.get("RDS_PORT", os.environ.get("PGPORT", "5432"))) + user = os.environ.get("RDS_USER", os.environ.get("PGUSER", "postgres")) + dbname = os.environ.get("RDS_DB", os.environ.get("RDS_DBNAME", os.environ.get("PGDATABASE", "postgres"))) + sslmode = os.environ.get("RDS_SSLMODE", os.environ.get("PGSSLMODE", "prefer")) + password = os.environ.get("RDS_PASSWORD", os.environ.get("PGPASSWORD", "")) return {"host": host, "port": port, "user": user, "password": password, "dbname": dbname, "sslmode": sslmode} -def _get_iam_token(host: str, port: int, user: str, region: str) -> str: - """Generate IAM token via boto3 (preferred) or AWS CLI (fallback).""" - try: - import boto3 - return boto3.client("rds", region_name=region).generate_db_auth_token( - DBHostname=host, Port=port, DBUsername=user, Region=region) - except ImportError: - return subprocess.check_output([ - "aws", "rds", "generate-db-auth-token", - "--region", region, "--hostname", host, - "--port", str(port), "--username", user, - ], text=True).strip() - - def connect_rds(**overrides): - """Connect to RDS. Override any resolved param via kwargs. + """Connect to the database. Override any resolved param via kwargs. Resolution order per field: 1. explicit **override 2. DATABASE_URL env var - 3. individual RDS_* env vars - 4. built-in defaults - - Auth resolution (when password is empty): - 1. RDS_IAM_TOKEN env var (pre-computed) - 2. boto3 SDK generate_db_auth_token (RDS_IAM=1 or RDS_IAM_AUTH=1) - 3. subprocess aws rds generate-db-auth-token - 4. RDS_PASSWORD env var + 3. individual PG/RDS env vars + 4. built-in defaults (localhost) """ p = _resolve_params() p.update(overrides) - # Determine if IAM auth is requested - iam_requested = ( - os.environ.get("RDS_IAM", "").strip() == "1" or - os.environ.get("RDS_IAM_AUTH", "").strip() == "1" - ) - - # Resolve password - if not p.get("password") or iam_requested: - token = os.environ.get("RDS_IAM_TOKEN", "").strip() - if token: - p["password"] = token - elif iam_requested or not p.get("password"): - region = os.environ.get("AWS_REGION", "us-east-1") - p["password"] = _get_iam_token(p["host"], p["port"], p["user"], region) - import psycopg2 kw = {k: p[k] for k in ("host", "port", "user", "password", "dbname", "sslmode")} if "connect_timeout" in p: diff --git a/4-Infrastructure/shim/sync_wiki_to_rds.py b/4-Infrastructure/shim/sync_wiki_to_rds.py index d69cea3d..9a484ac4 100644 --- a/4-Infrastructure/shim/sync_wiki_to_rds.py +++ b/4-Infrastructure/shim/sync_wiki_to_rds.py @@ -16,7 +16,7 @@ from shim.utils import sha256_text, utc_now STACK_ROOT = Path(os.environ.get("STACK_ROOT", "/home/allaun/Research Stack")) WIKI_ROOT = Path(os.environ.get("WIKI_ROOT", str(STACK_ROOT / "6-Documentation" / "wiki"))) -HOST = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com") +HOST = os.environ.get("RDS_HOST", "localhost") DB = os.environ.get("RDS_DB", os.environ.get("RDS_DBNAME", "postgres")) diff --git a/4-Infrastructure/storage/cache-offload.sh b/4-Infrastructure/storage/cache-offload.sh index 30742e3b..8b654622 100644 --- a/4-Infrastructure/storage/cache-offload.sh +++ b/4-Infrastructure/storage/cache-offload.sh @@ -31,7 +31,7 @@ TIER0_PGDUMP="/mnt/stackcache/pgdump" TIER2_REMOTE="gdrive:research-stack-offload" TIER1_THRESHOLD_GB=2 -RDS_HOST="${RDS_HOST:-database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com}" +RDS_HOST="${RDS_HOST:-localhost}" RDS_PORT="${RDS_PORT:-5432}" RDS_USER="${RDS_USER:-postgres}" RDS_DB="${RDS_DB:-postgres}" diff --git a/4-Infrastructure/storage/garage/db-consolidate.sh b/4-Infrastructure/storage/garage/db-consolidate.sh index 6e31866d..2b7b5846 100644 --- a/4-Infrastructure/storage/garage/db-consolidate.sh +++ b/4-Infrastructure/storage/garage/db-consolidate.sh @@ -38,7 +38,7 @@ export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-garage}" export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://localhost:3900}" -RDS_HOST="${RDS_HOST:-database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com}" +RDS_HOST="${RDS_HOST:-localhost}" RDS_PORT="${RDS_PORT:-5432}" RDS_USER="${RDS_USER:-postgres}" RDS_DB="${RDS_DB:-postgres}" diff --git a/4-Infrastructure/storage/restic/backup.sh b/4-Infrastructure/storage/restic/backup.sh index efa6ddfe..cc6fe4ba 100755 --- a/4-Infrastructure/storage/restic/backup.sh +++ b/4-Infrastructure/storage/restic/backup.sh @@ -37,7 +37,7 @@ fi export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION export RESTIC_REPOSITORY RESTIC_PASSWORD_FILE -RDS_HOST="${RDS_HOST:-database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com}" +RDS_HOST="${RDS_HOST:-localhost}" RDS_PORT="${RDS_PORT:-5432}" RDS_USER="${RDS_USER:-postgres}" RDS_DB="${RDS_DB:-postgres}"