Research-Stack/4-Infrastructure/shim/ingest_57_flexures.py

#!/usr/bin/env python3
"""Ingest 57 theorem vectors into ene.flexures with full v2 spectral features."""

import hashlib
import json
import os
import subprocess
import sys
import uuid
from collections import Counter, defaultdict
from pathlib import Path

VECTORS_PATH = os.path.join(os.path.dirname(__file__), "../..", "shared-data/pist_trace_scaled_vectors.jsonl")
REPORT_PATH = os.path.join(os.path.dirname(__file__), "../..", "shared-data/pist_flexure_v2_report.json")


def classify_tactic(name: str) -> str:
    name_lower = name.lower()
    if "rw" in name_lower: return "rewrite"
    if "simp" in name_lower: return "normalization"
    if "omega" in name_lower: return "arithmetic"
    if "induct" in name_lower: return "induction"
    if "ring" in name_lower or "calc" in name_lower: return "algebraic"
    if "cases" in name_lower or "constructor" in name_lower: return "case_analysis"
    if "apply" in name_lower or "intro" in name_lower or "have" in name_lower: return "discharge"
    if "rfl" in name_lower: return "reflexivity"
    if "logic" in name_lower or "order" in name_lower: return "discharge"
    if "with_import" in name_lower or "algebra" in name_lower: return "normalization"
    if "expect_fail" in name_lower or "fail" in name_lower: return "unknown"
    if "complex" in name_lower: return "induction"
    return "unknown"


def main():
    with open(VECTORS_PATH) as f:
        records = [json.loads(line) for line in f]
    print(f"Vectors: {len(records)}", flush=True)

    host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
    port = os.environ.get("RDS_PORT", "5432")
    user = os.environ.get("RDS_USER", "postgres")
    db = os.environ.get("RDS_DB", "postgres")
    token = os.environ.get("RDS_IAM_TOKEN", "")
    if not token:
        token = subprocess.check_output([
            "aws", "rds", "generate-db-auth-token",
            "--region", os.environ.get("AWS_REGION", "us-east-1"),
            "--hostname", host, "--port", port, "--username", user,
        ], text=True).strip()

    import psycopg2
    conn = psycopg2.connect(host=host, port=port, user=user, password=token, dbname=db, sslmode="require")
    cur = conn.cursor()

    # Create new session (keep old data)
    session_id = str(uuid.uuid4())
    cur.execute(
        "INSERT INTO ene.sessions (id, title, event_type, content, metadata) "
        "VALUES (%s, %s, 'flexure_ingest', '57-theorem v2 spectral batch', %s::jsonb)",
        (session_id, "Tier 2B 57-theorem v2", json.dumps({"source": "57_batch", "count": len(records)})),
    )

    joint_labels = Counter()
    tactic_families = Counter()
    total_joints = 0

    for r in records:
        name = r.get("name", "?")
        status = r.get("status", "failed")
        tf = classify_tactic(name)
        
        # Create a single flexure per theorem (the matrix represents the whole proof path)
        flex_id = str(uuid.uuid4())
        spectral = {k: r.get(k, 0) for k in ["matrix_size", "rank", "spectral_gap", "laplacian_zero_count",
                                               "density", "adjacency_eigenvalue_max", "adjacency_eigenvalue_second",
                                               "laplacian_eigenvalue_max", "laplacian_eigenvalue_min",
                                               "singular_value_max", "trace", "frobenius_norm"]}
        signals = json.dumps({
            "tactic": name, "tactic_family": tf, "delta_score": abs(r.get("gap", 0)) * 10,
            "joint_label": f"{tf}_{status}", "domain": "mixed", "proof_method": tf,
            "rrc_shape": "?", "obstruction": None, "matrix_rank": r.get("rank", 0),
            "n_unique_states": r.get("n", 0), "spectral": spectral,
            "feature_version": "flexure-spectrum-v2",
        })
        chosen = json.dumps({"theorem": name, "status": status})
        available = json.dumps([{"step": 0, "tactic_family": tf}])
        sidon = int(hashlib.sha256(name.encode()).hexdigest()[:4], 16) % 255
        
        cur.execute(
            "INSERT INTO ene.flexures (id, session_id, step_index, pre_sidon_label, pre_residual, "
            "available_crossings, chosen_crossing, decision_signals, post_sidon_label, post_residual, converged) "
            "VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s, %s, %s)",
            (flex_id, session_id, 0, sidon, 0.5, available, chosen, signals,
             sidon % 255, 0.5, status == "verified"),
        )
        tactic_families[tf] += 1
        joint_labels[f"{tf}_{status}"] += 1
        total_joints += 1

    conn.commit()
    print(f"Inserted: {total_joints} flexures", flush=True)

    # Motif discovery
    cur.execute(
        "SELECT decision_signals->>'tactic_family', decision_signals->>'joint_label', converged, count(*) "
        "FROM ene.flexures WHERE session_id = %s GROUP BY 1, 2, 3 HAVING count(*) >= 2",
        (session_id,),
    )
    motif_count = 0
    for row in cur.fetchall():
        fam, label, converged, freq = row
        sig = hashlib.sha256(f"{fam}_{label}".encode()).hexdigest()[:16]
        cur.execute(
            "INSERT INTO ene.flexure_patterns (id, pattern_signature, pre_conditions, decision_rules, outcome_stats, frequency) "
            "VALUES (%s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s) "
            "ON CONFLICT (pattern_signature) DO UPDATE SET frequency = ene.flexure_patterns.frequency + %s",
            (str(uuid.uuid4()), sig, json.dumps({"tactic_family": fam}),
             json.dumps({"joint_type": label}), json.dumps({"status": "verified" if converged else "failed", "count": freq}),
             freq, freq),
        )
        motif_count += 1

    conn.commit()
    print(f"Motifs: {motif_count}", flush=True)
    print(f"Tactic families: {dict(tactic_families.most_common(5))}", flush=True)
    print(f"Joint labels: {dict(joint_labels.most_common(5))}", flush=True)
    print(f"Session: {session_id}", flush=True)

    with open(REPORT_PATH, "w") as f:
        json.dump({"session_id": session_id, "total_flexures": total_joints, "motifs": motif_count,
                    "tactic_families": dict(tactic_families.most_common(5)),
                    "joint_labels": dict(joint_labels.most_common(5))}, f, indent=2)
    print(f"Report: {REPORT_PATH}", flush=True)
    conn.close()


if __name__ == "__main__":
    main()