diff --git a/docs/SPECTRAL_CODEBOOK_GENERATOR.md b/docs/SPECTRAL_CODEBOOK_GENERATOR.md index 2e137670..df47b3f1 100644 --- a/docs/SPECTRAL_CODEBOOK_GENERATOR.md +++ b/docs/SPECTRAL_CODEBOOK_GENERATOR.md @@ -138,6 +138,88 @@ clusters, all extra rows fall inside existing clusters.** neither silently — it emits the gap-derived boundaries above and flags the discrepancy here for resolution. +## Neon data layer (neon-64gb) + +`python/spectral_codebook_db.py` syncs the codebook into the ENE schema on +the neon-64gb Postgres (`$NEON_PG`, default +`postgres://postgres:postgres@100.92.88.64:5432/research_stack` — same +convention as `scripts/auto/auto_pipeline.py`). + +```sh +python3 python/spectral_codebook_db.py # DRY RUN (default): summary + sample SQL, writes nothing +python3 python/spectral_codebook.py --sync-db # same dry run from the generator CLI +python3 python/spectral_codebook_db.py --verify-schema # + READ-ONLY column check against the live DB +python3 python/spectral_codebook_db.py --emit-sql # write data/spectral_codebook_sync.sql (no DB contact) +python3 python/spectral_codebook_db.py --apply # actually upsert (psycopg2, or falls back to --emit-sql) +``` + +The default is always a dry run; only an explicit `--apply` writes to the +database. `--apply` has **not** been run — `ene.rrc_predictions` is +untouched as of this writing. Live-DB access (even read-only +`--verify-schema`) requires the user's go-ahead per the standing "ask +before any DB work" rule; schema compatibility was instead verified +offline against `scripts/auto/ene_schema.sql` (tested). + +### Landing table: `ene.rrc_predictions` (empty — natural target) + +One flat, SQL-typed row per equation (deterministic uuid5 ids, so reruns +upsert idempotently): + +| Column | Value | +|---|---| +| `id` | uuid5(namespace, equation_id) | +| `equation_id` | e.g. `rrc_eq_01ab6e9c32652d06` | +| `proxy_pred` | gap-aware cluster codeword `C0`–`C8` | +| `exact_pred` | shape from exact λ under the **current** ClassifyN thresholds (1.5/4.0 Q16.16, integer semantics mirrored) | +| `matrix_hash` | `charpoly=;pos10=` — exact char-poly (similarity key) + base-10 positional hash (identity key; injective, entries ≤ 9) | +| `confidence` | 1.0 for unique char-poly fingerprints; 1/k in a k-way collision class | + +Dry-run counts: 250 rows — per cluster C0=35, C1=20, C2=13, C3=79, C4=29, +C5=15, C6=22, C7=19, C8=18; per shape LogogramProjection=69, +SignalShapedRouteCompiler=131, CognitiveLoadField=50; confidence 1.0 for +183 rows, <1.0 for 67. + +### Spark path + +The row shape is deliberately flat so the Spark cluster on neon-64gb +(master `spark://100.92.88.64:7077`, podman spark-worker, JDBC +`postgresql-42.7.5`) can read it directly, the same JDBC pattern the +auto-pipeline Spark analysis uses for `ene.scars` (11 rows) and +`ene.routes` (5 rows): + +```python +df = (spark.read.format("jdbc") + .option("url", "jdbc:postgresql://100.92.88.64:5432/research_stack") + .option("dbtable", "ene.rrc_predictions") + .option("user", "postgres").option("password", "postgres") + .option("driver", "org.postgresql.Driver") + .load()) +df.groupBy("proxy_pred", "exact_pred").count().orderBy("proxy_pred").show() +``` + +### Table map & staleness findings + +| Table | Rows | Status | +|---|---|---| +| `ene.rrc_classifications` | 120 | **STALE** — all `spectral_radius` values are 0.3–0.85, i.e. old flat-regime power-iteration artifacts, classified 2026-07-01 05:11 *before* the exact-eigenvalue correction (this codebook shows real ρ ∈ {0} ∪ {1} ∪ (1, 17]; the corpus has nothing in (0, 1)). Equation ids there are `lean:formal/...` paths, not `rrc_eq_*` hashes. **Recommend re-classification via this codebook** and reconciling the id conventions. | +| `ene.rrc_predictions` | 0 | Empty — landing table for this sync (`matrix_hash` column already fits the fingerprint). | +| `ene.shape_predictions` | 100 | Existing shape model output; `exact_pred` here can serve as cross-check. | +| `ene.shape_ground_truth` | 0 | Empty. | +| `ene.eigensolid_snapshots` | 0 | Empty (root_hash, crossing_matrix, sidon_slack, residual_series, …) — future home for full spectral profiles. | +| `ene.braid_strands`, `ene.receipts`, `ene.sidon_labels` | 0 | Empty. | +| `ene.scars` / `ene.routes` | 11 / 5 | Read by the Spark cluster via JDBC (auto-pipeline `--spark`). | + +(Row counts and staleness ranges as enumerated by the coordinating agent +on 2026-07-01; re-check with `--verify-schema` + read-only SELECTs before +acting on them.) + +### arxiv-pg citation layer + +A separate Postgres runs in the `arxiv-pg` container on neon-64gb (arxiv +DB with pgvector, `concept_citations`). It has **no published port** — +access is podman-exec only. It is the citation/grounding layer for +equation provenance; this sync does not (and cannot) connect to it. + ## Output schema (`data/spectral_codebook.json`) - header: `schema` (`spectral_codebook_v2`), `quantization` (factor, diff --git a/python/spectral_codebook.py b/python/spectral_codebook.py index 71388fb2..a6916280 100644 --- a/python/spectral_codebook.py +++ b/python/spectral_codebook.py @@ -586,6 +586,11 @@ def main(argv: Optional[List[str]] = None) -> int: help="Also run over the 278-row RRC/Q16_16Manifold corpus") ap.add_argument("--no-write", action="store_true", help="Report only; do not write the JSON") + ap.add_argument("--sync-db", action="store_true", + help="DRY-RUN sync preview to ene.rrc_predictions on the " + "neon-64gb Postgres ($NEON_PG). Always a dry run from " + "this entry point; use python/spectral_codebook_db.py " + "--apply to actually insert.") args = ap.parse_args(argv) cb = build_codebook(args.matrices, args.gap_factor, args.min_support) @@ -633,6 +638,11 @@ def main(argv: Optional[List[str]] = None) -> int: print(f" boundaries match 250-matrix codebook: {mc['boundaries_match_250']} " f"({mc['clusters']} clusters)") + if args.sync_db: + import spectral_codebook_db + print("\n DB sync (dry run):") + spectral_codebook_db.sync_db(codebook=cb) # never applies from here + if not args.no_write: args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(doc, indent=2) + "\n") diff --git a/python/spectral_codebook_db.py b/python/spectral_codebook_db.py new file mode 100644 index 00000000..60947f65 --- /dev/null +++ b/python/spectral_codebook_db.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +""" +spectral_codebook_db.py — Sync the spectral codebook into the Neon ENE layer. + +Writes one row per equation into `ene.rrc_predictions` on the neon-64gb +Postgres (schema in scripts/auto/ene_schema.sql): + + id uuid — deterministic uuid5 of the equation_id (idempotent reruns) + equation_id text + proxy_pred text — gap-aware cluster codeword ("C0".."C8") + exact_pred text — exact-λ shape under the CURRENT ClassifyN.lean + thresholds (signalThreshold 1.5 / oberthHigh 4.0, + Q16.16 integer semantics mirrored exactly) + matrix_hash text — canonical fingerprint string: + "charpoly=;pos10=" + charpoly = exact integer char-poly coefficients + (similarity key); pos10 = base-10 positional hash + (identity key — injective since entries ≤ 9, + unlike ClassifyN.hashMatrix's base 5) + confidence float — 1.0 for a unique charpoly fingerprint, + 1/k inside a k-way charpoly collision class + +SAFETY: the default is a DRY RUN — it prints the row summary and sample SQL +and writes NOTHING. `--apply` performs the insert via psycopg2; if psycopg2 +is not importable, `--apply` (or `--emit-sql`) writes +data/spectral_codebook_sync.sql for `psql $NEON_PG/research_stack -f ...`. + +The row shape is flat/SQL-typed so the Spark cluster on neon-64gb +(spark://100.92.88.64:7077, postgresql-42.7.5 JDBC) can read +ene.rrc_predictions directly — see docs/SPECTRAL_CODEBOOK_GENERATOR.md. + +Usage: + python3 python/spectral_codebook_db.py # dry run + python3 python/spectral_codebook_db.py --verify-schema # + read-only DB check + python3 python/spectral_codebook_db.py --emit-sql # write .sql, no DB + python3 python/spectral_codebook_db.py --apply # actually insert +""" +from __future__ import annotations + +import argparse +import os +import sys +import uuid +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import spectral_codebook as sc + +NEON_PG_DEFAULT = "postgres://postgres:postgres@100.92.88.64:5432/research_stack" +DEFAULT_SQL_OUT = sc.REPO_ROOT / "data" / "spectral_codebook_sync.sql" + +# Q16.16 thresholds from formal/SilverSight/PIST/ClassifyN.lean +Q16_SCALE = 65536 +SIGNAL_THRESHOLD_RAW = 98304 # 1.5 +OBERTH_HIGH_RAW = 262144 # 4.0 + +# uuid5 namespace for deterministic row ids (rerun-idempotent) +ROW_NS = uuid.uuid5(uuid.NAMESPACE_URL, "silversight/ene/rrc_predictions") + + +def neon_dsn() -> str: + """NEON_PG env convention (scripts/auto/auto_pipeline.py); default db + research_stack is appended when the env var carries only host:port.""" + dsn = os.environ.get("NEON_PG", NEON_PG_DEFAULT) + if dsn.rstrip("/").count("/") < 3: # postgres://user:pw@host:port (no db) + dsn = dsn.rstrip("/") + "/research_stack" + return dsn + + +def classify_exact_shape(lam: float) -> str: + """Mirror ClassifyN.lean's spectralRadiusToColor → colorToShapeName on + the exact spectral radius, using the CURRENT Lean thresholds (1.5/4.0 + Q16.16) with integer Q16.16 semantics.""" + raw = int(lam * Q16_SCALE) # truncation, as in Q16.16 fixed point + if raw >= OBERTH_HIGH_RAW: + return "CognitiveLoadField" # red > 0 + if raw >= SIGNAL_THRESHOLD_RAW: + green = (raw - SIGNAL_THRESHOLD_RAW) * Q16_SCALE // (OBERTH_HIGH_RAW - SIGNAL_THRESHOLD_RAW) + if green > 0: + return "SignalShapedRouteCompiler" + return "LogogramProjection" # blue-or-zero branch + + +def positional_hash(mat: Sequence[Sequence[int]], base: int = 10) -> int: + """Positional hash in the loop order of ClassifyN.hashMatrix, but with a + base ≥ max_entry + 1 (corpus entries reach 9), which makes it injective + on entry patterns — ClassifyN's base 5 is not.""" + acc, pw = 0, 1 + for row in mat: + for val in row: + acc += val * pw + pw *= base + return acc + + +def matrix_hash(charpoly: Sequence[int], mat: Sequence[Sequence[int]]) -> str: + cp = ",".join(str(c) for c in charpoly) + return f"charpoly={cp};pos10={positional_hash(mat)}" + + +def build_rows(codebook: sc.Codebook) -> List[dict]: + """Flat SQL-typed rows for ene.rrc_predictions, one per equation.""" + # k-way charpoly collision classes → confidence 1/k + cp_size: Dict[tuple, int] = {} + for p in codebook.profiles.values(): + key = tuple(p["charpoly"]) + cp_size[key] = cp_size.get(key, 0) + 1 + + rows = [] + for eid in sorted(codebook.profiles): + p = codebook.profiles[eid] + cw, _idx = codebook.codeword_of(eid) + k = cp_size[tuple(p["charpoly"])] + rows.append({ + "id": str(uuid.uuid5(ROW_NS, eid)), + "equation_id": eid, + "proxy_pred": cw, + "exact_pred": classify_exact_shape(p["spectral_radius"]), + "matrix_hash": matrix_hash(p["charpoly"], codebook.matrices[eid]), + "confidence": round(1.0 / k, 6), + }) + return rows + + +INSERT_SQL = ( + "INSERT INTO ene.rrc_predictions " + "(id, equation_id, proxy_pred, exact_pred, matrix_hash, confidence) " + "VALUES (%(id)s, %(equation_id)s, %(proxy_pred)s, %(exact_pred)s, " + "%(matrix_hash)s, %(confidence)s) " + "ON CONFLICT (id) DO UPDATE SET " + "proxy_pred = EXCLUDED.proxy_pred, exact_pred = EXCLUDED.exact_pred, " + "matrix_hash = EXCLUDED.matrix_hash, confidence = EXCLUDED.confidence, " + "predicted_at = NOW()" +) + + +def _sql_literal(v) -> str: + if isinstance(v, str): + return "'" + v.replace("'", "''") + "'" + return str(v) + + +def render_sql(rows: List[dict]) -> str: + """Standalone .sql rendering of the same upsert (for psql, no driver).""" + lines = [ + "-- spectral_codebook → ene.rrc_predictions sync", + "-- generated by python/spectral_codebook_db.py (schema: scripts/auto/ene_schema.sql)", + "BEGIN;", + ] + for r in rows: + vals = ", ".join(_sql_literal(r[c]) for c in + ("id", "equation_id", "proxy_pred", "exact_pred", + "matrix_hash", "confidence")) + lines.append( + "INSERT INTO ene.rrc_predictions " + "(id, equation_id, proxy_pred, exact_pred, matrix_hash, confidence) " + f"VALUES ({vals}) " + "ON CONFLICT (id) DO UPDATE SET " + "proxy_pred = EXCLUDED.proxy_pred, exact_pred = EXCLUDED.exact_pred, " + "matrix_hash = EXCLUDED.matrix_hash, confidence = EXCLUDED.confidence, " + "predicted_at = NOW();" + ) + lines.append("COMMIT;") + return "\n".join(lines) + "\n" + + +def summarize(rows: List[dict]) -> str: + by_cluster: Dict[str, int] = {} + by_shape: Dict[str, int] = {} + for r in rows: + by_cluster[r["proxy_pred"]] = by_cluster.get(r["proxy_pred"], 0) + 1 + by_shape[r["exact_pred"]] = by_shape.get(r["exact_pred"], 0) + 1 + unique = sum(1 for r in rows if r["confidence"] == 1.0) + out = [f"{len(rows)} rows → ene.rrc_predictions"] + out.append(" per cluster (proxy_pred): " + ", ".join( + f"{k}={by_cluster[k]}" for k in sorted(by_cluster, key=lambda c: int(c[1:])))) + out.append(" per shape (exact_pred): " + ", ".join( + f"{k}={v}" for k, v in sorted(by_shape.items()))) + out.append(f" confidence: {unique} unique fingerprints at 1.0, " + f"{len(rows) - unique} in collision classes (<1.0)") + return "\n".join(out) + + +def verify_schema(dsn: str) -> List[str]: + """READ-ONLY check that ene.rrc_predictions has the expected columns.""" + import psycopg2 # only needed for this optional check + expected = {"id", "equation_id", "proxy_pred", "exact_pred", + "matrix_hash", "confidence", "predicted_at"} + with psycopg2.connect(dsn) as conn: + conn.set_session(readonly=True) + with conn.cursor() as cur: + cur.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'ene' AND table_name = 'rrc_predictions'" + ) + cols = {r[0] for r in cur.fetchall()} + missing = sorted(expected - cols) + if missing: + raise RuntimeError(f"ene.rrc_predictions missing columns: {missing}") + return sorted(cols) + + +def apply_rows(dsn: str, rows: List[dict]) -> int: + import psycopg2 + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.executemany(INSERT_SQL, rows) + conn.commit() + return len(rows) + + +def sync_db(codebook: Optional[sc.Codebook] = None, + apply: bool = False, + emit_sql: Optional[Path] = None, + verify: bool = False, + dsn: Optional[str] = None) -> List[dict]: + """Build rows and dry-run/emit/apply. Returns the rows. + + Dry run (apply=False, emit_sql=None) touches neither network nor disk. + """ + cb = codebook or sc.build_codebook() + rows = build_rows(cb) + print(summarize(rows)) + + if verify: + cols = verify_schema(dsn or neon_dsn()) + print(f" schema check (read-only): ene.rrc_predictions columns OK: {cols}") + + if apply: + try: + import psycopg2 # noqa: F401 + except ImportError: + emit_sql = emit_sql or DEFAULT_SQL_OUT + print(" psycopg2 not importable — falling back to SQL emission") + else: + n = apply_rows(dsn or neon_dsn(), rows) + print(f" APPLIED: upserted {n} rows into ene.rrc_predictions") + return rows + + if emit_sql is not None: + emit_sql.parent.mkdir(parents=True, exist_ok=True) + emit_sql.write_text(render_sql(rows)) + print(f" wrote {emit_sql} " + f"(psql $NEON_PG/research_stack -f {emit_sql.name})") + elif not apply: + print(" DRY RUN — nothing written. Sample statement:") + print(" " + render_sql(rows[:1]).splitlines()[3]) + print(" Use --apply to insert, --emit-sql to write " + f"{DEFAULT_SQL_OUT.relative_to(sc.REPO_ROOT)}") + return rows + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser( + description="Sync spectral codebook → ene.rrc_predictions (dry-run by default)") + ap.add_argument("--apply", action="store_true", + help="Actually upsert rows (default: dry run, writes nothing)") + ap.add_argument("--emit-sql", nargs="?", type=Path, const=DEFAULT_SQL_OUT, + default=None, metavar="PATH", + help=f"Write upsert SQL to PATH (default {DEFAULT_SQL_OUT}) " + "instead of connecting") + ap.add_argument("--verify-schema", action="store_true", + help="Read-only check of ene.rrc_predictions columns") + ap.add_argument("--dsn", default=None, + help="Postgres DSN (default: $NEON_PG or the neon-64gb URL)") + args = ap.parse_args(argv) + sync_db(apply=args.apply, emit_sql=args.emit_sql, + verify=args.verify_schema, dsn=args.dsn) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_spectral_codebook.py b/tests/test_spectral_codebook.py index 31fbf5ea..67fb1b67 100644 --- a/tests/test_spectral_codebook.py +++ b/tests/test_spectral_codebook.py @@ -199,5 +199,126 @@ class TestSpectralCodebook(unittest.TestCase): self.assertTrue(mc["boundaries_match_250"]) +class TestSpectralCodebookDbSync(unittest.TestCase): + """DB sync layer (spectral_codebook_db): dry-run only, no network.""" + + @classmethod + def setUpClass(cls): + import spectral_codebook_db as db + cls.db = db + cls.codebook = sc.Codebook(sc.parse_matrices_lean()) + cls.rows = db.build_rows(cls.codebook) + + def test_rows_shape(self): + self.assertEqual(len(self.rows), 250) + cols = {"id", "equation_id", "proxy_pred", "exact_pred", + "matrix_hash", "confidence"} + for r in self.rows: + self.assertEqual(set(r), cols) + self.assertTrue(r["proxy_pred"].startswith("C")) + self.assertIn(r["exact_pred"], { + "LogogramProjection", "SignalShapedRouteCompiler", + "CognitiveLoadField", + }) + self.assertTrue(r["matrix_hash"].startswith("charpoly=")) + self.assertIn(";pos10=", r["matrix_hash"]) + # deterministic uuid5 ids, unique per equation + self.assertEqual(len({r["id"] for r in self.rows}), 250) + + def test_confidence_matches_collision_classes(self): + col = self.codebook.collision_report() + unique = [r for r in self.rows if r["confidence"] == 1.0] + self.assertEqual(len(unique), col["identified_by_charpoly"]) + by_eid = {r["equation_id"]: r for r in self.rows} + for cls_ids in col["charpoly_collision_classes"].values(): + k = len(cls_ids) + for eid in cls_ids: + self.assertAlmostEqual(by_eid[eid]["confidence"], 1.0 / k, places=6) + + def test_exact_pred_mirrors_classifyn_thresholds(self): + # ClassifyN.lean: 1.5/4.0 Q16.16 thresholds + self.assertEqual(self.db.classify_exact_shape(0.0), "LogogramProjection") + self.assertEqual(self.db.classify_exact_shape(1.0), "LogogramProjection") + self.assertEqual(self.db.classify_exact_shape(1.5), "LogogramProjection") # green==0 edge + self.assertEqual(self.db.classify_exact_shape(2.0), "SignalShapedRouteCompiler") + self.assertEqual(self.db.classify_exact_shape(4.0), "CognitiveLoadField") + self.assertEqual(self.db.classify_exact_shape(16.99), "CognitiveLoadField") + + def test_positional_hash_injective_over_corpus(self): + """Base-10 positional hash separates all 238 distinct matrices + (ClassifyN's base-5 hash offers no such guarantee: entries reach 9).""" + max_entry = max(x for m in self.codebook.matrices.values() + for row in m for x in row) + self.assertLessEqual(max_entry, 9) + hashes = {self.db.positional_hash(m) for m in self.codebook.matrix_groups} + self.assertEqual(len(hashes), len(self.codebook.matrix_groups)) + + def test_dry_run_writes_nothing(self): + import contextlib + import io + + out = io.StringIO() + with contextlib.redirect_stdout(out): + rows = self.db.sync_db(codebook=self.codebook) # pure dry run + text = out.getvalue() + self.assertEqual(len(rows), 250) + self.assertIn("250 rows", text) + self.assertIn("DRY RUN", text) + self.assertFalse( + (sc.REPO_ROOT / "data" / "spectral_codebook_sync.sql").exists(), + "dry run must not write the sync SQL file", + ) + + def test_emit_sql_renders_all_rows(self): + import contextlib + import io + import tempfile + + with tempfile.TemporaryDirectory() as td: + target = Path(td) / "sync.sql" + with contextlib.redirect_stdout(io.StringIO()): + self.db.sync_db(codebook=self.codebook, emit_sql=target) + text = target.read_text() + self.assertEqual(text.count("INSERT INTO ene.rrc_predictions"), 250) + self.assertTrue(text.startswith("--")) + self.assertIn("BEGIN;", text) + self.assertIn("COMMIT;", text) + self.assertIn("ON CONFLICT (id) DO UPDATE", text) + + def test_schema_matches_repo_ddl(self): + """Offline check: emitted columns ⊆ ene.rrc_predictions DDL in + scripts/auto/ene_schema.sql (live-DB check is --verify-schema).""" + import re + + ddl = (sc.REPO_ROOT / "scripts" / "auto" / "ene_schema.sql").read_text() + m = re.search( + r"CREATE TABLE IF NOT EXISTS ene\.rrc_predictions \((.*?)\);", + ddl, re.DOTALL, + ) + self.assertIsNotNone(m) + ddl_cols = set(re.findall(r"(\w+)\s+(?:UUID|TEXT|FLOAT|INT|TIMESTAMPTZ)", m.group(1))) + emitted = {"id", "equation_id", "proxy_pred", "exact_pred", + "matrix_hash", "confidence"} + self.assertTrue(emitted <= ddl_cols, ddl_cols) + + def test_neon_dsn_convention(self): + import os + + old = os.environ.pop("NEON_PG", None) + try: + self.assertEqual( + self.db.neon_dsn(), + "postgres://postgres:postgres@100.92.88.64:5432/research_stack", + ) + # auto_pipeline.py convention: host:port only → db appended + os.environ["NEON_PG"] = "postgres://postgres:postgres@100.92.88.64:5432" + self.assertTrue(self.db.neon_dsn().endswith("/research_stack")) + finally: + if old is None: + os.environ.pop("NEON_PG", None) + else: + os.environ["NEON_PG"] = old + + if __name__ == "__main__": unittest.main()