#!/usr/bin/env python3 """ Auto-pipeline: Lean build -> extract metadata -> populate DB -> RRC classify. Runs after every push to Semantics Lean sources. Usage: python3 auto_pipeline.py # full pipeline python3 auto_pipeline.py --db-only # recreate DB schema only python3 auto_pipeline.py --ci # CI mode (no build, just extract+predict) python3 auto_pipeline.py --spark # also run Spark guide-path analysis """ import subprocess, json, os, sys, argparse, time, hashlib from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent NEON_PG = os.environ.get("NEON_PG", "postgres://postgres:postgres@100.92.88.64:5432/research_stack") try: import psycopg2 import psycopg2.extras HAS_DB = True except ImportError: HAS_DB = False def db(): if not HAS_DB: return None return psycopg2.connect(NEON_PG, connect_timeout=5) def sh(cmd, **kw): return subprocess.run(cmd, shell=True, capture_output=True, text=True, **kw) def sql(sql_str, params=None): if not HAS_DB: return try: conn = db() if conn is None: return with conn.cursor() as cur: cur.execute(sql_str, params or ()) conn.commit() conn.close() except Exception as e: print(f" [db] {e}") try: conn.close() except: pass # --- 1. Extract Lean metadata --- SCAN_DIRS = [ ROOT / "formal", ] def extract_theorems(): theorems = [] for d in SCAN_DIRS: if not d.exists(): continue for f in sorted(d.rglob("*.lean")): if ".lake" in str(f): continue rel = f.relative_to(ROOT) text = f.read_text() for line_no, line in enumerate(text.split("\n"), 1): for kw in ("theorem ", "lemma ", "def "): if kw in line: name = line.split(kw)[1].split()[0].split(":")[0].split(" ")[0] theorems.append({ "name": name, "kind": kw.strip(), "file": str(rel), "line": line_no, "has_sorry": "sorry" in text, }) return theorems # --- 2. Populate ENE DB --- def _bulk_insert(cur, table, columns, rows, conflict_col="pkg"): """Bulk insert rows using a single VALUES clause (avoids psycopg2.executemany slowness).""" if not rows: return cols = ", ".join(columns) placeholders = ", ".join(f"({', '.join(['%s'] * len(columns))})" for _ in rows) flat = [v for row in rows for v in row] cur.execute( f"INSERT INTO {table} ({cols}) VALUES {placeholders} ON CONFLICT ({conflict_col}) DO NOTHING", flat ) def populate_ene(theorems): if not HAS_DB: return try: conn = psycopg2.connect(NEON_PG, connect_timeout=5) with conn.cursor() as cur: batch = [] for t in theorems: pkg = f"lean:{t['file']}:{t['name']}" batch.append((pkg, 'lean_theorem', t['name'], t['file'], 'lean')) if len(batch) >= 500: _bulk_insert(cur, "ene.packages", ["pkg", "package_type", "title", "source", "domain"], batch) batch = [] if batch: _bulk_insert(cur, "ene.packages", ["pkg", "package_type", "title", "source", "domain"], batch) # Seed spectral regions region_rows = [(f"region:{r}", 'spectral_region', r) for r in ["CANONICAL_pair0", "CANONICAL_pair1", "CANONICAL_pair2", "CANONICAL_pair3", "ROSSBY_all"]] _bulk_insert(cur, "ene.packages", ["pkg", "package_type", "title"], region_rows) # Seed default RRC classifications for i, shape in enumerate(["logogramProjection", "cognitiveLoadField", "signalShapedRouteCompiler", "angrySphinxGate", "rossbyDrift"]): eq_id = f"builtin:shape:{shape}" cur.execute( "INSERT INTO ene.packages (pkg, package_type, title) VALUES (%s, 'rrc_shape', %s) ON CONFLICT (pkg) DO NOTHING", (eq_id, shape) ) cur.execute( "INSERT INTO ene.rrc_classifications (id, equation_id, shape, pist_label, spectral_radius, weak_axes, score) " "VALUES (gen_random_uuid(), %s, %s, 'auto_seeded', 0.5, 4, %s) ON CONFLICT DO NOTHING", (eq_id, shape, 1.0 - i * 0.1) ) conn.commit() conn.close() except Exception as e: print(f" DB error: {e}") try: conn.close() except: pass # --- 3. Run RRC Classification --- def run_rrc(): semantics_path = ROOT / "formal" / "CoreFormalism" if not (semantics_path / "lakefile.lean").exists(): semantics_path = ROOT / "formal" / "RRCLib" if not (semantics_path / "lakefile.lean").exists(): semantics_path = ROOT print(f" Build path: {semantics_path}") if not (semantics_path / "lakefile.lean").exists(): print(" No Lean workspace found; skipping build") return True r = sh(f"cd {semantics_path} && lake build 2>&1", timeout=600) if r.returncode != 0: print(f" Lean build FAILED: {r.stderr[-300:]}") return False return True def run_rrc_classification(theorems): if not HAS_DB: return [] classifications = [] try: conn = psycopg2.connect(NEON_PG, connect_timeout=5) with conn.cursor() as cur: for t in theorems[:100]: eq_id = f"lean:{t['file']}:{t['name']}" name_lower = t['name'].lower() if 'sidon' in name_lower or 'levelset' in name_lower: shape, pist, radius, axes = 'logogramProjection', 'SidonLabelClassifier', 0.85, 4 elif 'cartan' in name_lower or 'hachimoji' in name_lower or 'encode' in name_lower: shape, pist, radius, axes = 'cognitiveLoadField', 'CartanEnergyGate', 0.72, 2 elif 'eigensolid' in name_lower or 'convergence' in name_lower: shape, pist, radius, axes = 'signalShapedRouteCompiler', 'EigensolidConvergence', 0.65, 4 elif 'angry' in name_lower or 'gate' in name_lower or 'collision' in name_lower: shape, pist, radius, axes = 'angrySphinxGate', 'AngrySphinxGate', 0.58, 1 elif 'rossby' in name_lower or 'scar' in name_lower or 'famm' in name_lower: shape, pist, radius, axes = 'rossbyDrift', 'RossbyDriftClassifier', 0.45, 2 else: shape, pist, radius, axes = 'logogramProjection', 'GenericClassifier', 0.3, 4 cur.execute( "INSERT INTO ene.rrc_classifications (equation_id, shape, pist_label, spectral_radius, weak_axes, score) " "VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING", (eq_id, shape, pist, radius, axes, radius * 0.9) ) cur.execute( "INSERT INTO ene.shape_predictions (equation_id, shape, model_version, confidence, evidence) " "VALUES (%s, %s, 'auto_pipeline_v1', %s, ARRAY['auto_classified']) ON CONFLICT DO NOTHING", (eq_id, shape, 0.85) ) classifications.append({"equation_id": eq_id, "shape": shape}) conn.commit() conn.close() except Exception as e: print(f" [db] Classification error: {e}") try: conn.close() except: pass return classifications # --- 4. Run Spark guide-path analysis (optional) --- def run_spark_analysis(): spark_master = os.environ.get("SPARK_MASTER", "spark://100.92.88.64:7077") spark_script = """ import sys sys.path.insert(0, "/opt/spark/work-dir") from pyspark.sql import SparkSession spark = SparkSession.builder \\ .appName("ENE-guide-path-analysis") \\ .master("{master}") \\ .config("spark.jars", "/opt/spark/jars/postgresql-42.7.5.jar") \\ .getOrCreate() # Load scars and routes from PostgreSQL scars_df = spark.read \\ .format("jdbc") \\ .option("url", "{pg_url}") \\ .option("dbtable", "ene.scars") \\ .option("user", "{pg_user}") \\ .option("password", "{pg_pass}") \\ .load() routes_df = spark.read \\ .format("jdbc") \\ .option("url", "{pg_url}") \\ .option("dbtable", "ene.routes") \\ .option("user", "{pg_user}") \\ .option("password", "{pg_pass}") \\ .load() # Guide-path analysis: which scarred regions have highest pressure? high_pressure = scars_df.filter("scar_pressure > 100").orderBy("scar_pressure", ascending=False) high_pressure.show() # Route cost analysis: cheapest routes by route_type route_costs = routes_df.groupBy("route_type").avg("cost").orderBy("avg(cost)") route_costs.show() spark.stop() """.format( master=spark_master, pg_url="jdbc:postgresql://localhost:5432/research_stack", pg_user="postgres", pg_pass="postgres" ) spark_script_path = Path("/tmp/spark_guide_path.py") spark_script_path.write_text(spark_script) r = sh( f"cd {ROOT} && " f"ssh allaun@100.92.88.64 '" f"podman exec spark-worker mkdir -p /opt/spark/work-dir && " f"podman cp /tmp/spark_guide_path.py spark-worker:/opt/spark/work-dir/spark_guide_path.py 2>&1 && " f"podman exec spark-worker /opt/spark/bin/spark-submit " f"--master {spark_master} " f"/opt/spark/work-dir/spark_guide_path.py 2>&1'", timeout=120 ) return r.returncode == 0 # --- Main --- def main(): parser = argparse.ArgumentParser() parser.add_argument("--db-only", action="store_true") parser.add_argument("--ci", action="store_true") parser.add_argument("--spark", action="store_true") args = parser.parse_args() if args.db_only: if not HAS_DB: print("psycopg2 not installed; run: pip install psycopg2-binary") sys.exit(1) sql_path = Path(__file__).with_name("ene_schema.sql") schema_sql = sql_path.read_text() conn = db() with conn.cursor() as cur: cur.execute(schema_sql) conn.commit() conn.close() print("Schema applied") return if not HAS_DB: print("[pipeline] WARNING: psycopg2 not installed; DB operations skipped") if not args.ci: print("[pipeline] Building Lean...") ok = run_rrc() print(f"[pipeline] Build {'OK' if ok else 'FAILED'}") print("[pipeline] Extracting theorems...") theorems = extract_theorems() print(f" Found {len(theorems)} theorem/lemma/def sites") print("[pipeline] Populating ENE...") populate_ene(theorems) print(" Done") print("[pipeline] RRC classification...") classifications = run_rrc_classification(theorems) print(f" Classified {len(classifications)} theorems into spectral shapes") if args.spark: print("[pipeline] Running Spark guide-path analysis...") ok = run_spark_analysis() print(f" Spark analysis {'OK' if ok else 'FAILED'}") print("[pipeline] Complete") print(f"[pipeline] DB: {NEON_PG}") if __name__ == "__main__": main()