SilverSight/scripts/auto/auto_pipeline.py

#!/usr/bin/env python3
"""
Auto-pipeline: Lean build -> extract metadata -> populate DB -> RRC classify.
Runs after every push to Semantics Lean sources.

Usage:
  python3 auto_pipeline.py                     # full pipeline
  python3 auto_pipeline.py --db-only           # recreate DB schema only
  python3 auto_pipeline.py --ci                # CI mode (no build, just extract+predict)
  python3 auto_pipeline.py --spark             # also run Spark guide-path analysis
"""
import subprocess, json, os, sys, argparse, time, hashlib
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent.parent
NEON_PG = os.environ.get("NEON_PG", "postgres://postgres:postgres@100.92.88.64:5432/research_stack")

try:
    import psycopg2
    import psycopg2.extras
    HAS_DB = True
except ImportError:
    HAS_DB = False

def db():
    if not HAS_DB: return None
    return psycopg2.connect(NEON_PG, connect_timeout=5)

def sh(cmd, **kw):
    return subprocess.run(cmd, shell=True, capture_output=True, text=True, **kw)

def sql(sql_str, params=None):
    if not HAS_DB: return
    try:
        conn = db()
        if conn is None: return
        with conn.cursor() as cur:
            cur.execute(sql_str, params or ())
        conn.commit()
        conn.close()
    except Exception as e:
        print(f"  [db] {e}")
        try: conn.close()
        except: pass

# --- 1. Extract Lean metadata ---
SCAN_DIRS = [
    ROOT / "formal",
]

def extract_theorems():
    theorems = []
    for d in SCAN_DIRS:
        if not d.exists(): continue
        for f in sorted(d.rglob("*.lean")):
            if ".lake" in str(f): continue
            rel = f.relative_to(ROOT)
            text = f.read_text()
            for line_no, line in enumerate(text.split("\n"), 1):
                for kw in ("theorem ", "lemma ", "def "):
                    if kw in line:
                        name = line.split(kw)[1].split()[0].split(":")[0].split(" ")[0]
                        theorems.append({
                            "name": name, "kind": kw.strip(),
                            "file": str(rel), "line": line_no,
                            "has_sorry": "sorry" in text,
                        })
    return theorems

# --- 2. Populate ENE DB ---
def _bulk_insert(cur, table, columns, rows, conflict_col="pkg"):
    """Bulk insert rows using a single VALUES clause (avoids psycopg2.executemany slowness)."""
    if not rows: return
    cols = ", ".join(columns)
    placeholders = ", ".join(f"({', '.join(['%s'] * len(columns))})" for _ in rows)
    flat = [v for row in rows for v in row]
    cur.execute(
        f"INSERT INTO {table} ({cols}) VALUES {placeholders} ON CONFLICT ({conflict_col}) DO NOTHING",
        flat
    )

def populate_ene(theorems):
    if not HAS_DB: return
    try:
        conn = psycopg2.connect(NEON_PG, connect_timeout=5)
        with conn.cursor() as cur:
            batch = []
            for t in theorems:
                pkg = f"lean:{t['file']}:{t['name']}"
                batch.append((pkg, 'lean_theorem', t['name'], t['file'], 'lean'))
                if len(batch) >= 500:
                    _bulk_insert(cur, "ene.packages",
                        ["pkg", "package_type", "title", "source", "domain"],
                        batch)
                    batch = []
            if batch:
                _bulk_insert(cur, "ene.packages",
                    ["pkg", "package_type", "title", "source", "domain"],
                    batch)
            # Seed spectral regions
            region_rows = [(f"region:{r}", 'spectral_region', r) for r in
                           ["CANONICAL_pair0", "CANONICAL_pair1", "CANONICAL_pair2",
                            "CANONICAL_pair3", "ROSSBY_all"]]
            _bulk_insert(cur, "ene.packages", ["pkg", "package_type", "title"], region_rows)
            # Seed default RRC classifications
            for i, shape in enumerate(["logogramProjection", "cognitiveLoadField", "signalShapedRouteCompiler",
                                        "angrySphinxGate", "rossbyDrift"]):
                eq_id = f"builtin:shape:{shape}"
                cur.execute(
                    "INSERT INTO ene.packages (pkg, package_type, title) VALUES (%s, 'rrc_shape', %s) ON CONFLICT (pkg) DO NOTHING",
                    (eq_id, shape)
                )
                cur.execute(
                    "INSERT INTO ene.rrc_classifications (id, equation_id, shape, pist_label, spectral_radius, weak_axes, score) "
                    "VALUES (gen_random_uuid(), %s, %s, 'auto_seeded', 0.5, 4, %s) ON CONFLICT DO NOTHING",
                    (eq_id, shape, 1.0 - i * 0.1)
                )
        conn.commit()
        conn.close()
    except Exception as e:
        print(f"  DB error: {e}")
        try: conn.close()
        except: pass

# --- 3. Run RRC Classification ---
def run_rrc():
    semantics_path = ROOT / "formal" / "CoreFormalism"
    if not (semantics_path / "lakefile.lean").exists():
        semantics_path = ROOT / "formal" / "RRCLib"
    if not (semantics_path / "lakefile.lean").exists():
        semantics_path = ROOT
    print(f"  Build path: {semantics_path}")
    if not (semantics_path / "lakefile.lean").exists():
        print("  No Lean workspace found; skipping build")
        return True
    r = sh(f"cd {semantics_path} && lake build 2>&1", timeout=600)
    if r.returncode != 0:
        print(f"  Lean build FAILED: {r.stderr[-300:]}")
        return False
    return True

def run_rrc_classification(theorems):
    if not HAS_DB: return []
    classifications = []
    try:
        conn = psycopg2.connect(NEON_PG, connect_timeout=5)
        with conn.cursor() as cur:
            for t in theorems[:100]:
                eq_id = f"lean:{t['file']}:{t['name']}"
                name_lower = t['name'].lower()
                if 'sidon' in name_lower or 'levelset' in name_lower:
                    shape, pist, radius, axes = 'logogramProjection', 'SidonLabelClassifier', 0.85, 4
                elif 'cartan' in name_lower or 'hachimoji' in name_lower or 'encode' in name_lower:
                    shape, pist, radius, axes = 'cognitiveLoadField', 'CartanEnergyGate', 0.72, 2
                elif 'eigensolid' in name_lower or 'convergence' in name_lower:
                    shape, pist, radius, axes = 'signalShapedRouteCompiler', 'EigensolidConvergence', 0.65, 4
                elif 'angry' in name_lower or 'gate' in name_lower or 'collision' in name_lower:
                    shape, pist, radius, axes = 'angrySphinxGate', 'AngrySphinxGate', 0.58, 1
                elif 'rossby' in name_lower or 'scar' in name_lower or 'famm' in name_lower:
                    shape, pist, radius, axes = 'rossbyDrift', 'RossbyDriftClassifier', 0.45, 2
                else:
                    shape, pist, radius, axes = 'logogramProjection', 'GenericClassifier', 0.3, 4
                cur.execute(
                    "INSERT INTO ene.rrc_classifications (equation_id, shape, pist_label, spectral_radius, weak_axes, score) "
                    "VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING",
                    (eq_id, shape, pist, radius, axes, radius * 0.9)
                )
                cur.execute(
                    "INSERT INTO ene.shape_predictions (equation_id, shape, model_version, confidence, evidence) "
                    "VALUES (%s, %s, 'auto_pipeline_v1', %s, ARRAY['auto_classified']) ON CONFLICT DO NOTHING",
                    (eq_id, shape, 0.85)
                )
                classifications.append({"equation_id": eq_id, "shape": shape})
        conn.commit()
        conn.close()
    except Exception as e:
        print(f"  [db] Classification error: {e}")
        try: conn.close()
        except: pass
    return classifications

# --- 4. Run Spark guide-path analysis (optional) ---
def run_spark_analysis():
    spark_master = os.environ.get("SPARK_MASTER", "spark://100.92.88.64:7077")
    spark_script = """
import sys
sys.path.insert(0, "/opt/spark/work-dir")
from pyspark.sql import SparkSession

spark = SparkSession.builder \\
    .appName("ENE-guide-path-analysis") \\
    .master("{master}") \\
    .config("spark.jars", "/opt/spark/jars/postgresql-42.7.5.jar") \\
    .getOrCreate()

# Load scars and routes from PostgreSQL
scars_df = spark.read \\
    .format("jdbc") \\
    .option("url", "{pg_url}") \\
    .option("dbtable", "ene.scars") \\
    .option("user", "{pg_user}") \\
    .option("password", "{pg_pass}") \\
    .load()

routes_df = spark.read \\
    .format("jdbc") \\
    .option("url", "{pg_url}") \\
    .option("dbtable", "ene.routes") \\
    .option("user", "{pg_user}") \\
    .option("password", "{pg_pass}") \\
    .load()

# Guide-path analysis: which scarred regions have highest pressure?
high_pressure = scars_df.filter("scar_pressure > 100").orderBy("scar_pressure", ascending=False)
high_pressure.show()

# Route cost analysis: cheapest routes by route_type
route_costs = routes_df.groupBy("route_type").avg("cost").orderBy("avg(cost)")
route_costs.show()

spark.stop()
""".format(
        master=spark_master,
        pg_url="jdbc:postgresql://localhost:5432/research_stack",
        pg_user="postgres",
        pg_pass="postgres"
    )
    
    spark_script_path = Path("/tmp/spark_guide_path.py")
    spark_script_path.write_text(spark_script)
    
    r = sh(
        f"cd {ROOT} && "
        f"ssh allaun@100.92.88.64 '"
        f"podman exec spark-worker mkdir -p /opt/spark/work-dir && "
        f"podman cp /tmp/spark_guide_path.py spark-worker:/opt/spark/work-dir/spark_guide_path.py 2>&1 && "
        f"podman exec spark-worker /opt/spark/bin/spark-submit "
        f"--master {spark_master} "
        f"/opt/spark/work-dir/spark_guide_path.py 2>&1'",
        timeout=120
    )
    return r.returncode == 0

# --- Main ---
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--db-only", action="store_true")
    parser.add_argument("--ci", action="store_true")
    parser.add_argument("--spark", action="store_true")
    args = parser.parse_args()

    if args.db_only:
        if not HAS_DB:
            print("psycopg2 not installed; run: pip install psycopg2-binary")
            sys.exit(1)
        sql_path = Path(__file__).with_name("ene_schema.sql")
        schema_sql = sql_path.read_text()
        conn = db()
        with conn.cursor() as cur:
            cur.execute(schema_sql)
        conn.commit()
        conn.close()
        print("Schema applied")
        return

    if not HAS_DB:
        print("[pipeline] WARNING: psycopg2 not installed; DB operations skipped")

    if not args.ci:
        print("[pipeline] Building Lean...")
        ok = run_rrc()
        print(f"[pipeline] Build {'OK' if ok else 'FAILED'}")

    print("[pipeline] Extracting theorems...")
    theorems = extract_theorems()
    print(f"  Found {len(theorems)} theorem/lemma/def sites")

    print("[pipeline] Populating ENE...")
    populate_ene(theorems)
    print("  Done")

    print("[pipeline] RRC classification...")
    classifications = run_rrc_classification(theorems)
    print(f"  Classified {len(classifications)} theorems into spectral shapes")

    if args.spark:
        print("[pipeline] Running Spark guide-path analysis...")
        ok = run_spark_analysis()
        print(f"  Spark analysis {'OK' if ok else 'FAILED'}")

    print("[pipeline] Complete")
    print(f"[pipeline] DB: {NEON_PG}")

if __name__ == "__main__":
    main()