SilverSight/tests/test_spectral_codebook.py

"""test_spectral_codebook.py — Gap-aware spectral codebook round-trip tests.

Validates the spectral_codebook module against the 250-matrix PIST corpus:
exact char-poly fingerprints (Cayley–Hamilton in integer arithmetic),
pure-stdlib vs NumPy spectral-radius agreement, gap-aware quantization
invariants, and the encode/decode round trip over all 250 equations.
"""

from __future__ import annotations

import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "python"))

import spectral_codebook as sc


def _mat_mul(a, b):
    n = len(a)
    return [
        [sum(a[i][t] * b[t][j] for t in range(n)) for j in range(n)]
        for i in range(n)
    ]


class TestSpectralCodebook(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.matrices = sc.parse_matrices_lean()
        cls.codebook = sc.Codebook(cls.matrices)

    # ── parsing ─────────────────────────────────────────────────────

    def test_parse_250_8x8(self):
        self.assertEqual(len(self.matrices), 250)
        for eid, mat in self.matrices.items():
            self.assertEqual(len(mat), 8, eid)
            self.assertTrue(all(len(r) == 8 for r in mat), eid)
            self.assertTrue(all(x >= 0 for r in mat for x in r), eid)

    # ── exact char poly ─────────────────────────────────────────────

    def test_charpoly_identity(self):
        # det(λI − I) = (λ − 1)^8 → coefficients are signed binomials
        ident = tuple(tuple(1 if i == j else 0 for j in range(8)) for i in range(8))
        expect = (-8, 28, -56, 70, -56, 28, -8, 1)
        self.assertEqual(sc.charpoly_coeffs(ident), expect)

    def test_cayley_hamilton_integer(self):
        """p(M) = 0 exactly in integer arithmetic (spot check)."""
        for eid in sorted(self.matrices)[::50]:  # 5 spread-out matrices
            m = [list(r) for r in self.matrices[eid]]
            coeffs = sc.charpoly_coeffs(m)
            n = len(m)
            acc = [[1 if i == j else 0 for j in range(n)] for i in range(n)]  # M^0
            total = [[coeffs[-1] if i == j else 0 for j in range(n)] for i in range(n)]
            for k in range(1, n + 1):
                acc = _mat_mul(acc, m)  # M^k
                c = 1 if k == n else coeffs[n - 1 - k]
                for i in range(n):
                    for j in range(n):
                        total[i][j] += c * acc[i][j]
            self.assertTrue(
                all(x == 0 for row in total for x in row),
                f"Cayley–Hamilton failed for {eid}",
            )

    def test_charpoly_trace_relation(self):
        for eid, mat in self.matrices.items():
            coeffs = self.codebook.profiles[eid]["charpoly"]
            self.assertEqual(coeffs[0], -sum(mat[i][i] for i in range(8)), eid)

    # ── spectral radius ─────────────────────────────────────────────

    def test_pure_root_finder_matches_numpy(self):
        """Durand–Kerner on the exact char poly agrees with the fast path."""
        for eid in sorted(self.matrices)[::10]:  # 25 matrices
            p = self.codebook.profiles[eid]
            pure = sc._durand_kerner_max_modulus(p["charpoly"])
            self.assertAlmostEqual(pure, p["spectral_radius"], places=4, msg=eid)

    def test_peripheral_spectra_are_unit(self):
        """The 9 'mid band' λ in the committed raw JSON (0.5 ≤ λ < 1) are
        power-iteration non-convergence artifacts: exact ρ = 1 for all 9."""
        import json

        raw_path = Path(__file__).resolve().parent.parent / "data" / "spectral_codebook_raw.json"
        raw = json.loads(raw_path.read_text())
        artifacts = [
            e["equation_id"] for e in raw["entries"]
            if 0.5 <= e["spectral_radius"] < 1.0
        ]
        self.assertEqual(len(artifacts), 9)
        for eid in artifacts:
            self.assertAlmostEqual(
                self.codebook.profiles[eid]["spectral_radius"], 1.0, places=6, msg=eid
            )

    def test_nilpotent_class(self):
        """Every ρ = 0 matrix has char poly exactly x^8 (nilpotent), and the
        pure path detects it with no iteration."""
        zeros = [
            eid for eid, p in self.codebook.profiles.items()
            if p["spectral_radius"] == 0.0
        ]
        self.assertEqual(len(zeros), 35)
        for eid in zeros:
            coeffs = self.codebook.profiles[eid]["charpoly"]
            self.assertEqual(list(coeffs), [0] * 8, eid)
            self.assertEqual(sc._durand_kerner_max_modulus(coeffs), 0.0)

    # ── fingerprints & collisions ───────────────────────────────────

    def test_charpoly_strictly_finer_than_lambda(self):
        col = self.codebook.collision_report()
        self.assertEqual(col["corpus_size"], 250)
        self.assertEqual(col["distinct_matrices"], 238)
        self.assertEqual(col["duplicate_matrix_groups"], 11)
        self.assertEqual(col["duplicate_matrix_members"], 23)
        self.assertGreater(col["unique_charpoly"], col["unique_lambda_4dp"])
        self.assertGreater(
            col["identified_by_charpoly"], col["identified_by_lambda_4dp"]
        )
        # char poly is NOT injective: collision classes must be explicit
        self.assertGreater(col["charpoly_collision_groups"], 0)
        self.assertEqual(
            sum(len(v) for v in col["charpoly_collision_classes"].values()),
            col["charpoly_collision_members"],
        )

    # ── quantization invariants ─────────────────────────────────────

    def test_boundaries_sorted_and_gapped(self):
        b = self.codebook.boundaries
        self.assertEqual(b, sorted(b))
        self.assertGreater(len(b), 0)
        # no λ value sits on a boundary
        for p in self.codebook.profiles.values():
            self.assertNotIn(p["spectral_radius"], b)

    def test_min_support_guard(self):
        """Every cluster holds ≥ MIN_SUPPORT distinct matrices."""
        for row in self.codebook.cluster_table():
            self.assertGreaterEqual(
                row["distinct_matrices"], sc.MIN_SUPPORT, row["codeword"]
            )

    def test_cluster_partition_covers_corpus(self):
        table = self.codebook.cluster_table()
        self.assertEqual(sum(r["count"] for r in table), 250)
        self.assertEqual(sum(r["distinct_matrices"] for r in table), 238)

    # ── round trip ──────────────────────────────────────────────────

    def test_decode_is_bijective_over_250(self):
        seen = set()
        for eid in self.matrices:
            cw, idx = self.codebook.codeword_of(eid)
            self.assertNotIn((cw, idx), seen)
            seen.add((cw, idx))
            self.assertEqual(self.codebook.decode(cw, idx), eid)
        self.assertEqual(len(seen), 250)

    def test_encode_decode_round_trip(self):
        """encode(matrix) → decode must return the same equation for every
        unique matrix, and a byte-identical matrix for exact duplicates
        (identical inputs are information-theoretically indistinguishable)."""
        dup_members = {
            eid
            for ids in self.codebook.matrix_groups.values()
            if len(ids) > 1
            for eid in ids
        }
        for eid, mat in self.matrices.items():
            cw, idx = self.codebook.encode([list(r) for r in mat])
            self.assertIsNotNone(idx, eid)
            decoded = self.codebook.decode(cw, idx)
            if eid in dup_members:
                self.assertEqual(self.matrices[decoded], mat, eid)
            else:
                self.assertEqual(decoded, eid)

    def test_encode_novel_matrix(self):
        novel = [[3 if i == j else 1 for j in range(8)] for i in range(8)]
        cw, idx = self.codebook.encode(novel)
        self.assertIsNone(idx)
        self.assertTrue(cw.startswith("C"))

    # ── 278-row manifold ────────────────────────────────────────────

    def test_manifold_278_no_new_gaps(self):
        mc = sc.manifold_check(self.codebook)
        self.assertEqual(mc["rows"], 278)
        self.assertEqual(mc["unique_ids"], 250)
        self.assertEqual(mc["extra_rows"], 28)
        self.assertEqual(mc["ids_without_matrix"], [])
        self.assertTrue(mc["boundaries_match_250"])


class TestSpectralCodebookDbSync(unittest.TestCase):
    """DB sync layer (spectral_codebook_db): dry-run only, no network."""

    @classmethod
    def setUpClass(cls):
        import spectral_codebook_db as db
        cls.db = db
        cls.codebook = sc.Codebook(sc.parse_matrices_lean())
        cls.rows = db.build_rows(cls.codebook)

    def test_rows_shape(self):
        self.assertEqual(len(self.rows), 250)
        cols = {"id", "equation_id", "proxy_pred", "exact_pred",
                "matrix_hash", "confidence"}
        for r in self.rows:
            self.assertEqual(set(r), cols)
            self.assertTrue(r["proxy_pred"].startswith("C"))
            self.assertIn(r["exact_pred"], {
                "LogogramProjection", "SignalShapedRouteCompiler",
                "CognitiveLoadField",
            })
            self.assertTrue(r["matrix_hash"].startswith("charpoly="))
            self.assertIn(";pos10=", r["matrix_hash"])
        # deterministic uuid5 ids, unique per equation
        self.assertEqual(len({r["id"] for r in self.rows}), 250)

    def test_confidence_matches_collision_classes(self):
        col = self.codebook.collision_report()
        unique = [r for r in self.rows if r["confidence"] == 1.0]
        self.assertEqual(len(unique), col["identified_by_charpoly"])
        by_eid = {r["equation_id"]: r for r in self.rows}
        for cls_ids in col["charpoly_collision_classes"].values():
            k = len(cls_ids)
            for eid in cls_ids:
                self.assertAlmostEqual(by_eid[eid]["confidence"], 1.0 / k, places=6)

    def test_exact_pred_mirrors_classifyn_thresholds(self):
        # ClassifyN.lean: 1.5/4.0 Q16.16 thresholds
        self.assertEqual(self.db.classify_exact_shape(0.0), "LogogramProjection")
        self.assertEqual(self.db.classify_exact_shape(1.0), "LogogramProjection")
        self.assertEqual(self.db.classify_exact_shape(1.5), "LogogramProjection")  # green==0 edge
        self.assertEqual(self.db.classify_exact_shape(2.0), "SignalShapedRouteCompiler")
        self.assertEqual(self.db.classify_exact_shape(4.0), "CognitiveLoadField")
        self.assertEqual(self.db.classify_exact_shape(16.99), "CognitiveLoadField")

    def test_positional_hash_injective_over_corpus(self):
        """Base-10 positional hash separates all 238 distinct matrices
        (ClassifyN's base-5 hash offers no such guarantee: entries reach 9)."""
        max_entry = max(x for m in self.codebook.matrices.values()
                        for row in m for x in row)
        self.assertLessEqual(max_entry, 9)
        hashes = {self.db.positional_hash(m) for m in self.codebook.matrix_groups}
        self.assertEqual(len(hashes), len(self.codebook.matrix_groups))

    def test_dry_run_writes_nothing(self):
        import contextlib
        import io

        out = io.StringIO()
        with contextlib.redirect_stdout(out):
            rows = self.db.sync_db(codebook=self.codebook)  # pure dry run
        text = out.getvalue()
        self.assertEqual(len(rows), 250)
        self.assertIn("250 rows", text)
        self.assertIn("DRY RUN", text)
        self.assertFalse(
            (sc.REPO_ROOT / "data" / "spectral_codebook_sync.sql").exists(),
            "dry run must not write the sync SQL file",
        )

    def test_emit_sql_renders_all_rows(self):
        import contextlib
        import io
        import tempfile

        with tempfile.TemporaryDirectory() as td:
            target = Path(td) / "sync.sql"
            with contextlib.redirect_stdout(io.StringIO()):
                self.db.sync_db(codebook=self.codebook, emit_sql=target)
            text = target.read_text()
            self.assertEqual(text.count("INSERT INTO ene.rrc_predictions"), 250)
            self.assertTrue(text.startswith("--"))
            self.assertIn("BEGIN;", text)
            self.assertIn("COMMIT;", text)
            self.assertIn("ON CONFLICT (id) DO UPDATE", text)

    def test_schema_matches_repo_ddl(self):
        """Offline check: emitted columns ⊆ ene.rrc_predictions DDL in
        scripts/auto/ene_schema.sql (live-DB check is --verify-schema)."""
        import re

        ddl = (sc.REPO_ROOT / "scripts" / "auto" / "ene_schema.sql").read_text()
        m = re.search(
            r"CREATE TABLE IF NOT EXISTS ene\.rrc_predictions \((.*?)\);",
            ddl, re.DOTALL,
        )
        self.assertIsNotNone(m)
        ddl_cols = set(re.findall(r"(\w+)\s+(?:UUID|TEXT|FLOAT|INT|TIMESTAMPTZ)", m.group(1)))
        emitted = {"id", "equation_id", "proxy_pred", "exact_pred",
                   "matrix_hash", "confidence"}
        self.assertTrue(emitted <= ddl_cols, ddl_cols)

    def test_neon_dsn_convention(self):
        import os

        old = os.environ.pop("NEON_PG", None)
        try:
            self.assertEqual(
                self.db.neon_dsn(),
                "postgres://postgres:postgres@100.92.88.64:5432/research_stack",
            )
            # auto_pipeline.py convention: host:port only → db appended
            os.environ["NEON_PG"] = "postgres://postgres:postgres@100.92.88.64:5432"
            self.assertTrue(self.db.neon_dsn().endswith("/research_stack"))
        finally:
            if old is None:
                os.environ.pop("NEON_PG", None)
            else:
                os.environ["NEON_PG"] = old


if __name__ == "__main__":
    unittest.main()