Research-Stack/4-Infrastructure/shim/rrc_genre_decompose.py

#!/usr/bin/env python3
"""
rrc_genre_decompose.py — Decompose the operator grammar into named irreducible sectors.

Uses the role-pure genres (conditional, dataflow, balanced-algebra) as probes
to identify the irreducible components of the affine A₂ grammar.

Genres:
  - CONDITIONAL: all-relation (overflow gates, bounds, if-then-else)
  - DATAFLOW: all-arrow (pipeline stages, JTAG→SUBLEQ→GCL→LUT)
  - BALANCED_ALGEBRA: mixed relation+binary_op+greek (Sidon, algebraic)
  - ANALYSIS: greek_letter + nary_operator (PDE, integration, summation)
  - GATE: binary_op + accent (logic gates, circuits)

Each genre defines a projection operator onto a sector of the grammar graph.
The decomposition factors the full 7D role space into ~4-5 irreducible sectors.
"""
from __future__ import annotations

import json
import math
import re
import sys
from collections import Counter, defaultdict
from pathlib import Path

RECEIPT_PATH = Path("archive/experimental-shim-probes/rrc_equation_classifier_receipt.json")
OUT_PATH = Path("shared-data/data/rrc_genre_decomposition_v1.json")

GENRES = {
    "conditional": {
        "label": "Conditional / Constraint",
        "dominant_roles": ["relation"],
        "min_role_fraction": 0.5,
        "description": "Relational constraints, bounds, overflow gates, if-then-else",
    },
    "dataflow": {
        "label": "Dataflow / Pipeline",
        "dominant_roles": ["arrow"],
        "min_role_fraction": 0.3,
        "description": "Pipeline stages, data transformation chains, JTAG→SUBLEQ→GCL",
    },
    "balanced_algebra": {
        "label": "Balanced Algebra",
        "dominant_roles": ["relation", "binary_op", "greek_letter"],
        "min_role_fraction": 0.6,
        "description": "Sidon sets, additive combinatorics, algebraic equations",
    },
    "analysis": {
        "label": "Analysis / PDE",
        "dominant_roles": ["greek_letter", "nary_operator"],
        "min_role_fraction": 0.4,
        "description": "Partial derivatives, integrals, summations, kinetic equations",
    },
    "gate": {
        "label": "Gate / Circuit",
        "dominant_roles": ["binary_op", "accent"],
        "min_role_fraction": 0.3,
        "description": "Logic gates, bitwise operations, circuit components",
    },
    "reconstruction": {
        "label": "Graph Reconstruction",
        "dominant_roles": ["binary_op", "relation", "arrow"],
        "min_role_fraction": 0.4,
        "description": "Graph reconstruction conjecture: deck→graph isomorphism, Kelly's lemma, Tutte polynomial recognizability",
        "arxiv_papers": ["2604.16567", "2601.00620", "2402.14986", "2504.02353", "1606.02926", "1301.4121"],
        "lean_proofs": ["bipartite_reconstruction.lean", "graph_conjecture2.lean"],
    },
}


def role_histogram(text: str) -> dict[str, int]:
    if not text:
        return {}
    LATEX_ROLES = {
        'sum': 'nary_operator', 'prod': 'nary_operator', 'int': 'nary_operator',
        'to': 'arrow', 'mapsto': 'arrow', 'rightarrow': 'arrow', 'leftarrow': 'arrow',
        'longrightarrow': 'arrow', 'longleftarrow': 'arrow',
        'alpha': 'greek_letter', 'beta': 'greek_letter', 'gamma': 'greek_letter',
        'delta': 'greek_letter', 'theta': 'greek_letter', 'lambda': 'greek_letter',
        'mu': 'greek_letter', 'pi': 'greek_letter', 'rho': 'greek_letter',
        'sigma': 'greek_letter', 'tau': 'greek_letter', 'phi': 'greek_letter',
        'omega': 'greek_letter', 'Gamma': 'greek_letter', 'Delta': 'greek_letter',
        'leq': 'relation', 'geq': 'relation', 'neq': 'relation', 'equiv': 'relation',
        'approx': 'relation', 'subset': 'relation', 'subseteq': 'relation',
        'in': 'relation', 'forall': 'relation', 'exists': 'relation',
        'times': 'binary_op', 'cdot': 'binary_op', 'circ': 'binary_op',
        'oplus': 'binary_op', 'otimes': 'binary_op', 'wedge': 'binary_op', 'vee': 'binary_op',
        'cup': 'binary_op', 'cap': 'binary_op',
        'partial': 'operator', 'nabla': 'operator', 'infty': 'operator',
    }
    hist = Counter()
    for cmd in re.findall(r'\\([a-zA-Z]+)', text):
        if cmd in LATEX_ROLES:
            hist[LATEX_ROLES[cmd]] += 1
    for ch in text:
        if ord(ch) > 127:
            if ch in 'αβγδεζηθικλμνξπρστυφχψω':
                hist['greek_letter'] += 1
            elif ch in '≤≥≠≡≈≅∈⊂⊆∀∃':
                hist['relation'] += 1
            elif ch in '→↦⇒⇐↔⟶⟵':
                hist['arrow'] += 1
            elif ch in '+−×⋅∘⊕⊗∩∪':
                hist['binary_op'] += 1
            elif ch in '∂∇∞∅':
                hist['operator'] += 1
            elif ch in '∫∑∏':
                hist['nary_operator'] += 1
    return dict(hist)


def classify_genre(name: str, eq_text: str, route_hint: str = "") -> str:
    combined = name + " " + str(eq_text) + " " + route_hint
    rh = role_histogram(combined)
    total = sum(rh.values())
    if total == 0:
        return "unclassified"
    scores = {}
    for gname, gdef in GENRES.items():
        dominant = sum(rh.get(r, 0) for r in gdef["dominant_roles"])
        scores[gname] = dominant / total
    # Prefer pure genres (high fraction) but fall back to best match
    winners = [g for g, s in scores.items() if s >= GENRES[g]["min_role_fraction"]]
    if winners:
        return max(winners, key=lambda g: scores[g])
    return max(scores, key=scores.get) if scores else "unclassified"


def entropy(vec: list[float]) -> float:
    """Shannon entropy of a probability distribution."""
    total = sum(vec) or 1
    return -sum((v / total) * math.log2(v / total) for v in vec if v > 0)


def kl_divergence(p: dict[str, float], q: dict[str, float]) -> float:
    """KL-divergence between two role distributions."""
    all_keys = set(p) | set(q)
    total_p = sum(p.values()) or 1
    total_q = sum(q.values()) or 1
    d = 0.0
    for k in all_keys:
        pk = p.get(k, 0) / total_p
        qk = q.get(k, 0) / total_q
        if pk > 0 and qk > 0:
            d += pk * math.log2(pk / qk)
    return d


def main():
    d = json.loads(RECEIPT_PATH.read_text())
    eqs = d["compiled_equations"]

    genre_counts = Counter()
    genre_routes = defaultdict(Counter)
    genre_examples = defaultdict(list)
    genre_role_dist = defaultdict(lambda: Counter())

    for e in eqs:
        rec = e["equation_record"]
        name = rec.get("name", "?")
        eq_text = str(rec.get("equation", ""))
        route = rec.get("manifold_route", "unclassified")
        genre = classify_genre(name, eq_text, route)
        genre_counts[genre] += 1
        genre_routes[genre][route] += 1
        if len(genre_examples[genre]) < 5:
            genre_examples[genre].append(name)
        rh = role_histogram(name + " " + eq_text)
        for r, c in rh.items():
            genre_role_dist[genre][r] += c

    print("=== RRC Genre Decomposition ===")
    print(f"  Total equations: {len(eqs)}")
    print()
    for genre, count in genre_counts.most_common():
        pct = 100 * count / len(eqs)
        routes = dict(genre_routes[genre].most_common(3))
        routes_str = ", ".join(f"{r}({c})" for r, c in routes.items())
        role_dist = dict(genre_role_dist[genre].most_common(6))
        role_str = ", ".join(f"{r}={c}" for r, c in role_dist.items())
        print(f"  {genre:25s} {count:4d} ({pct:5.1f}%) | roles: {role_str}")
        print(f"    examples: {', '.join(genre_examples[genre][:3])}")
        print(f"    routes: {routes_str}")
        print()

    # Cross-genre KL divergence
    print("=== Cross-genre KL divergence (bits) ===")
    genres_ordered = [g for g, _ in genre_counts.most_common() if genre_role_dist[g]]
    for i, g1 in enumerate(genres_ordered):
        for g2 in genres_ordered[i + 1:]:
            d = kl_divergence(genre_role_dist[g1], genre_role_dist[g2])
            print(f"  D_KL({g1:20s} || {g2:20s}) = {d:.3f} bits")

    # Genre entropy (purity measure)
    print()
    print("=== Genre purity (avg entropy per equation) ===")
    for genre, count in genre_counts.most_common():
        entropies = []
        for e in eqs:
            rec = e["equation_record"]
            name = rec.get("name", "?")
            eq_text = str(rec.get("equation", ""))
            if classify_genre(name, eq_text) == genre:
                rh = role_histogram(name + " " + eq_text)
                vec = list(rh.values())
                if vec:
                    entropies.append(entropy(vec))
        avg_h = sum(entropies) / max(len(entropies), 1)
        print(f"  {genre:25s} avg_entropy = {avg_h:.3f} bits (lower = purer)")

    # Write JSON output
    decomposition = {
        "schema": "rrc_genre_decomposition_v1",
        "description": "Irreducible sector decomposition of the operator grammar",
        "genres": {},
        "cross_genre_kl": [],
    }
    for genre, count in genre_counts.most_common():
        routes = dict(genre_routes[genre].most_common(3))
        roles = dict(genre_role_dist[genre].most_common(6))
        decomposition["genres"][genre] = {
            "count": count,
            "percentage": round(100 * count / len(eqs), 1),
            "definition": GENRES.get(genre, {}).get("description", ""),
            "examples": genre_examples[genre],
            "top_routes": routes,
            "top_roles": roles,
        }
    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
    OUT_PATH.write_text(json.dumps(decomposition, indent=2, ensure_ascii=False))
    print(f"\nSaved to {OUT_PATH}")


if __name__ == "__main__":
    main()