Research-Stack/5-Applications/scripts/ingest_unified_compression_synthesis.py

#!/usr/bin/env python3
"""
Ingest: Unified Compression Architecture Synthesis
==================================================
Synthesis of Density Field Encoding + GCCL-GEC + OAC + Hypercube-Rhomboid
+ S3C Shells + FAMM + PIST + erans + Radius-Ratio into a single coherent
compression pipeline for the Research Stack.
"""

import json, time
from pathlib import Path

RESEARCH_STACK = Path("/home/allaun/Documents/Research Stack")

SYNTHESIS = {
    "id": "unified-compression-architecture-synthesis-v1",
    "source": "Synthesis of 5 ingested theories: density-field, gccl-gec, oac, hypercube-rhomboid, hutter-rhomboid",
    "title": "Unified Compression Architecture: Semantic Density Fields with Glyph Eigen Codec and Sheared Manifold Geometry",
    "date": "2026-05-07",

    "core_synthesis": (
        "The unified architecture treats text as an n-dimensional semantic density field, "
        "encodes its topological skeleton via GCCL-GEC glyph packets, applies hypercube-rhomboid "
        "shear for correlation modeling, uses OAC for speculative lazy manifestation, "
        "coordinates via S3C shells, temporally warps via FAMM, encodes perturbations via PIST, "
        "entropy-codes residuals via erans, and quantizes local admissibility via radius-ratio."
    ),

    "architectural_layers": {
        "Layer_0_Raw_Representation": {
            "input": "UTF-8 byte sequence C ∈ Byteⁿ",
            "limitation": "1D orthogonal hypercube — independent byte positions, no semantic structure",
            "transformation": "Parse into semantic density field ρ: M → ℝ⁺ where M is n-D semantic manifold"
        },
        "Layer_1_Density_Field_Extraction": {
            "operation": "Compute semantic density field ρ(x⃗) from corpus structure",
            "topological_features": {
                "peaks": "Named entities, article centers, key concepts (local maxima)",
                "ridges": "Hyperlinks, citations, semantic connections (1D maxima)",
                "saddles": "Topic transitions, paragraph boundaries (mixed Hessian)",
                "vortices": "Cyclic references, template instantiations (rotational flow)",
                "voids": "Template structures, expected absence (local minima)",
                "level_sets": "Paragraphs, sections, articles (iso-density surfaces)"
            },
            "morse_smale_complex": "Critical points + separatrices = topological skeleton of meaning",
            "output": "Topological skeleton S (peaks, ridges, saddles, vortices, voids) + perturbation field δ"
        },
        "Layer_2_Hypercube_Rhomboid_Shear": {
            "operation": "Apply shear matrix A to orthogonal UTF-8 hypercube → correlated semantic rhomboid",
            "shear_matrix": "A_{ij} = δ_{ij} + α_{ij} where α encodes mutual information between dimensions i, j",
            "gram_matrix": "G = A^T A is the compression dictionary — eigenvectors = principal correlation directions, eigenvalues = compression gains",
            "volume_preservation": "det(A) = 1 — information volume preserved, only geometry changed",
            "information_gravity": "g_{μν} = δ_{μν} + κ·I_{μν} where I_{μν} is mutual information, κ is gravitational coupling",
            "output": "Sheared manifold S̃ = A·S where correlated axes lean into each other"
        },
        "Layer_3_S3C_Shell_Coordinate_Encoding": {
            "operation": "Encode positions within sheared manifold via S3C shell coordinates",
            "s3c_split": "n = k² + a with mirror complement b⁰, next-shell tension b⁺, mass = a·b⁰, mirror_delta = a - b⁰",
            "coordinate_mapping": {
                "k": "Shell index = structural depth / semantic distance from peak",
                "a": "Angular offset = intra-cluster position within shell",
                "b⁰": "Mirror complement = remaining path to next peak",
                "throat": "a ≈ b⁰ = maximum compression (maximum predictive confidence)",
                "mass": "Connection strength between topological features"
            },
            "output": "Shell-encoded coordinates (k, a, b⁰, b⁺, mass, throat_class) for each packet"
        },
        "Layer_4_GCCL_GEC_Packet_Encoding": {
            "operation": "Encode each topological feature as glyph packet Γᵢ = γᵢ ⊗ χᵢ ⊗ κᵢ ⊗ τᵢ ⊗ UᵢΛᵢaᵢ ⊗ θᵢ ⊗ εᵢ",
            "packet_fields": {
                "γᵢ": "Visible glyph / emoji / math symbol / PUA codepoint — invokes callable reconstruction kernel",
                "χᵢ": "Chirality vector ⟨G_geo, G_comp, G_load, G_spec, G_topo, G_arith⟩ — law-axis selection",
                "κᵢ": "Local context = S3C shell coordinate (k, a) from Layer 3",
                "τᵢ": "Type witness from TypeBook (WikiArticle<Mathematics>, Infobox<Person>, CitationGraph...)",
                "UᵢΛᵢaᵢ": "Eigen descriptor from EigenBook (basis, spectrum, sparse coefficients)",
                "θᵢ": "Parameters from side stream (mode selectors, eigenbook indices)",
                "εᵢ": "Residual bytes = exact repair to generated prediction"
            },
            "books": {
                "GlyphBook": "Maps codepoints to callable kernels",
                "ChiralityBook": "Maps chirality vectors to law-axes",
                "TypeBook": "Maps datatypes to structural laws",
                "EigenBook": "Stores reusable geometric descriptors"
            },
            "gain_test": "ΔGCL(Γᵢ) = literal_cost - encoded_cost(γᵢ, χᵢ, κᵢ, τᵢ, UΛa, θᵢ, εᵢ). Accept iff ΔGCL > 0.",
            "output": "Packet stream Γ = {Γ₁, Γ₂, ..., Γₙ} + parameter stream Θ + residual stream Ε"
        },
        "Layer_5_OAC_Speculative_Manifestation": {
            "operation": "Test compression motifs as Observer-Admissible Cavities before committing",
            "oac_definition": "OAC = (Sₙ, A_O, T, V, R, ε) — latent cavity that only manifests under lawful observer touch",
            "touch_operator": "touch(O, OACᵢ, q) → (Sᵢ, rᵢ, εᵢ, ρᵢ) if admissible, (Vᵢ, scarᵢ, ρᵢ) otherwise",
            "admissibility_gate": "Accept iff L(Sₙ) + L(route) + L(ε) < L(x)",
            "spherion_shaping": {
                "positive_pyramids": "Protrusions = confident predictions (high-Q narrow)",
                "negative_pyramids": "Voids = expected-but-missing features (cheaper to encode absence)",
                "shape_parameters": "Height = amplitude, base = duration, slope = transition, asymmetry = skew, apex = precision"
            },
            "sn_nn_grammar": "Sₙ(nⁿ) recursive shell grammar — nth shell contains n choices of previous shell state",
            "address_space_rule": "OAC ⊄ A_substrate; only receipt, accepted route, and residual may commit",
            "output": "Accepted packets + void scars (FAMM failure memory) + receipts"
        },
        "Layer_6_Radius_Ratio_Local_Quantization": {
            "operation": "Quantize local scale ratio ρ into admissible motif class via radius-ratio rule",
            "radius_ratio_rule": "Local scale ratio → smallest stable coordination geometry",
            "coordination_analogy": {
                "CN3 (triangular)": "ρ ∈ [0.155, 0.225) — minimal coordination",
                "CN4 (tetrahedral)": "ρ ∈ [0.225, 0.414) — 4-way coordination",
                "CN6 (octahedral)": "ρ ∈ [0.414, 0.732) — 6-way coordination",
                "CN8 (cubic)": "ρ ∈ [0.732, 1.0] — 8-way coordination"
            },
            "compression_mapping": "ρᵢ = s_center(i) / median(s(N(i))) → admissible kernel class + residual",
            "motif_classes": [
                "WikiArticle", "Infobox", "CitationGraph", "SectionTree",
                "TableMatrix", "ListRegion", "FormulaRegion", "MarkupRegion"
            ],
            "quantizer": "Continuous local metric witness → finite motif alphabet → decode rule + nibble residual",
            "output": "Motif_id + orientation + scale + law_id + residual_nibble_stream"
        },
        "Layer_7_FAMM_Temporal_Sequencing": {
            "operation": "Preshape delay profile for temporal path through packet stream",
            "uniform_delay": "Orthogonal time hypercube — fixed context window",
            "preshaped_delay": "Sheared time rhomboid — context stretches for high-entropy regions, compresses for low-entropy template regions",
            "delay_profile": "Delay = path integral through field gradient. Fast regions = high density (predictable). Slow regions = low density (needs context).",
            "q16_16_fixed_point": "Delays encoded in Q16.16 fixed-point for hardware-native determinism",
            "eigenvalue_derivation": "Preshaped delays based on eigenvalue spectra from waveprobe manifold generation",
            "output": "Delay profile D(t) for packet stream sequencing"
        },
        "Layer_8_PIST_Perturbation_Encoding": {
            "operation": "Encode perturbation field δ via PIST n-D bundle encoding",
            "pist_modes": {
                "cartesian": "Orthogonal n-D encoding (baseline)",
                "bundle": "Fiber dimensions encode correlated features",
                "radial": "Fully collapsed angular coordinates"
            },
            "fiber_mapping": {
                "fiber₀": "Topological feature type (peak, ridge, saddle, vortex, void)",
                "fiber₁": "Shell index k (semantic distance)",
                "fiber₂": "Throat class (compression quality)",
                "fiber₃": "Mass (connection strength)",
                "fiber₄": "Local curvature / distortion"
            },
            "n_dims": "Typically 3-4D: topic, time, authority, style",
            "compression_level": "PIST level 3 for offload pipeline (balanced speed/ratio)",
            "output": "PIST-encoded perturbation bundle B"
        },
        "Layer_9_erans_Residual_Entropy_Coding": {
            "operation": "Entropy-code residual stream Ε and parameter stream Θ via enumerative rANS",
            "erans_principle": "Enumerative rANS is optimal for exact histogram coding",
            "histogram_exact": "Residual values are discretized to exact histogram bins",
            "enumerative_coding": "Rank-based coding of symbols within exact histogram",
            "isa_agnostic": "SIMD opportunistic, scalar fallback required — no instruction set assumed",
            "licensing": "erans reference only — algorithmic ideas ingested, no code incorporated",
            "output": "Entropy-coded residual stream E_erc and parameter stream P_erc"
        },
        "Layer_10_Archive_Assembly": {
            "magic": "UCA1 (Unified Compression Architecture v1)",
            "structure": [
                "DECOMPRESSOR_PROFILE D",
                "GLYPHBOOK 𝔊",
                "CHIRALITYBOOK Χ",
                "TYPEBOOK Τ",
                "EIGENBOOK 𝕌",
                "SHEAR_MATRIX A (Gram matrix G = A^T A)",
                "REGION_INDEX I",
                "PACKET_STREAM Γ",
                "PARAMETER_STREAM Θ",
                "RESIDUAL_STREAM Ε",
                "DELAY_PROFILE D_FAMM",
                "S3C_SHELL_COORDINATES",
                "RECEIPT_SECTION R",
                "CHECKSUM_SECTION (SHA256)"
            ],
            "decode_invariant": "sha256(decode(archive)) == sha256(original_corpus)",
            "output": "Compressed archive A = D ⊕ 𝔊 ⊕ Χ ⊕ Τ ⊕ 𝕌 ⊕ A ⊕ I ⊕ Γ ⊕ Θ ⊕ Ε ⊕ D_FAMM ⊕ S3C ⊕ R ⊕ SHA256"
        }
    },

    "data_flow_summary": {
        "encode_pipeline": [
            "Raw bytes C",
            "↓ Parse to semantic density field ρ",
            "↓ Extract Morse-Smale topological skeleton S",
            "↓ Apply shear matrix A → sheared manifold S̃",
            "↓ Encode positions via S3C shells (k, a, b⁰, b⁺)",
            "↓ Quantize via radius-ratio → motif classes",
            "↓ Encode each feature as GCCL-GEC packet Γᵢ",
            "↓ Test via OAC gate → accept/reject",
            "↓ Preshape FAMM delay profile D_FAMM",
            "↓ Encode perturbations via PIST bundle B",
            "↓ Entropy-code residuals/params via erans",
            "↓ Assemble archive with books, shear matrix, receipts"
        ],
        "decode_pipeline": [
            "Load archive A",
            "↓ Load decompressor profile D",
            "↓ Load books (𝔊, Χ, Τ, 𝕌)",
            "↓ Load shear matrix A (Gram matrix G)",
            "↓ Load S3C shell coordinates",
            "↓ Load FAMM delay profile D_FAMM",
            "↓ Load packet stream Γ, params Θ, residuals Ε",
            "↓ For each Γᵢ: resolve glyph, chirality, type, eigen descriptor",
            "↓ Apply shear inverse A⁻¹ to reconstruct expected structure",
            "↓ Generate predicted byte span ŝᵢ",
            "↓ Apply residual εᵢ",
            "↓ Emit exact span sᵢ",
            "↓ Concatenate spans via FAMM sequencing",
            "↓ Verify SHA256 checksum",
            "↓ Output original corpus C"
        ]
    },

    "component_interdependencies": {
        "density_field_to_gccl": "Topological features (peaks, ridges, saddles, vortices, voids) ARE the packet types in GCCL-GEC",
        "gccl_to_oac": "Each glyph packet Γᵢ is tested as an OAC before committing to stream",
        "hypercube_rhomboid_to_s3c": "Shear matrix A defines the manifold geometry that S3C shells navigate",
        "s3c_to_famm": "Shell index k determines delay class in FAMM preshaping",
        "radius_ratio_to_gccl": "Local scale ratio quantizes to motif class → selects glyph kernel γᵢ",
        "famm_to_pist": "Delay profile determines temporal bundling of perturbation fibers",
        "pist_to_erans": "PIST-encoded perturbation bundle is entropy-coded via erans",
        "oac_to_receipts": "Accepted OAC routes become receipts; rejected become FAMM scars",
        "shear_matrix_to_eigenbook": "Gram matrix G = A^T A eigenvectors ARE the eigenbasis U in EigenBook",
        "all_to_gain_test": "Every component must pass ΔGCL > 0 before inclusion in archive"
    },

    "compression_gain_sources": {
        "geometric_shear": "15-30% on structured regions (infoboxes, citations, templates) by collapsing empty angles between correlated axes",
        "topological_skeleton": "50-150MB skeleton vs 1GB raw for enwik9 via Morse-Smale complex",
        "glyph_kernels": "Reusable callable kernels avoid storing repeated structures explicitly",
        "s3c_shells": "5-10% on positional encoding overhead via structural coordinate encoding",
        "oac_speculation": "Avoids 2-5% bloat from bad motif choices via lazy manifestation",
        "famm_context": "10-20% context efficiency via preshaped information-geometric windows",
        "pist_bundle": "5-8% on token encoding via fiber-dimensional correlation capture",
        "erans_entropy": "Optimal exact histogram coding for residuals",
        "radius_ratio_quantization": "Continuous witness → finite motif alphabet reduces entropy",
        "gram_dictionary": "MB → KB dictionary overhead via shear matrix as unified dictionary"
    },

    "estimated_aggregate_gain": {
        "structured_regions": "15-30% gain on ~40% of enwik (infoboxes, citations, templates, lists, headings, markup)",
        "free_text_regions": "5-10% gain on ~60% of enwik (natural language paragraphs)",
        "dictionary_overhead": "MB → KB (Gram matrix replaces LZ dictionary + transformer weights)",
        "context_efficiency": "10-20% more predictive power per context byte",
        "skeleton_compression": "O(N_peaks + N_ridges) ≪ O(N_bytes) — ~100MB vs 1GB for enwik9 skeleton",
        "overall_compressed_size": "Estimated 12-22% reduction vs current best Hutter compressors, plus navigable manifold capability",
        "novel_capability": "Produces navigable structure — query 'show me all articles 2 links from France' without full decompression"
    },

    "implementation_priority": {
        "Phase_0_Foundation": {
            "tasks": [
                "Implement S3C shell coordinate codec (s3c_shell_codec.py)",
                "Implement radius-ratio local quantizer (radius_ratio_quantizer.py)",
                "Implement basic FAMM delay line with Q16.16 (famm_q16_16.py)"
            ],
            "rationale": "These are the coordinate and temporal foundations for all higher layers"
        },
        "Phase_1_Density_Field": {
            "tasks": [
                "Implement semantic density field extraction (density_field_extractor.py)",
                "Implement Morse-Smale complex computation (morse_smale_complex.py)",
                "Implement topological feature classifier (topological_classifier.py)"
            ],
            "rationale": "Density field is the representation that all other layers operate on"
        },
        "Phase_2_GCCL_GEC_Core": {
            "tasks": [
                "Implement GlyphBook (glyphbook.py)",
                "Implement TypeBook with WikiArticle, Infobox, CitationGraph kernels (typebook.py)",
                "Implement basic packet encoder/decoder (gccl_packet.py)",
                "Implement gain test ΔGCL (gccl_gain_test.py)"
            ],
            "rationale": "GCCL-GEC is the packet-level codec that carries all compression"
        },
        "Phase_3_Shear_and_Eigen": {
            "tasks": [
                "Implement shear matrix learning (shear_matrix_learner.py)",
                "Implement Gram matrix extraction (gram_matrix.py)",
                "Implement EigenBook with UΛa descriptors (eigenbook.py)"
            ],
            "rationale": "Shear matrix and eigen descriptors capture correlation structure"
        },
        "Phase_4_OAC_and_Speculation": {
            "tasks": [
                "Implement OAC touch operator (oac_touch_gate.py)",
                "Implement spherion shape quantizer (spherion_quantizer.py)",
                "Implement Sₙ(nⁿ) recursive shell grammar (sn_nn_grammar.py)"
            ],
            "rationale": "OAC enables safe speculative compression without output pollution"
        },
        "Phase_5_PIST_and_erans": {
            "tasks": [
                "Integrate existing PIST n-D bundle encoding for perturbations",
                "Implement erans enumerative rANS wrapper (erans_entropy.py)",
                "Ensure ISA-agnostic scalar fallback"
            ],
            "rationale": "PIST encodes perturbations; erans entropy-codes residuals"
        },
        "Phase_6_Integration": {
            "tasks": [
                "Implement full encode pipeline (uca_encode.py)",
                "Implement full decode pipeline (uca_decode.py)",
                "Implement archive format (uca_archive.py)",
                "Add SHA256 verification"
            ],
            "rationale": "Integrate all layers into end-to-end codec"
        },
        "Phase_7_Benchmark": {
            "tasks": [
                "Benchmark on enwik8/enwik9",
                "Compare vs current Hutter best",
                "Measure navigable query performance",
                "Profile each layer's contribution"
            ],
            "rationale": "Validate gains and identify bottlenecks"
        }
    },

    "keeper_phrases": [
        "The density field is the manifold; the glyph packets are the navigators; the shear matrix is the map.",
        "Don't store the text. Store the cheapest lawful generator of its topological skeleton.",
        "A citation is not 200 bytes. It is a ridge connecting two peaks, encoded as one glyph packet with a URL residual.",
        "The Gram matrix of the shear IS the dictionary, the context model, the token encoding, and the structure detector — all at once.",
        "OACs let the compressor think in holes without storing every hole it thinks through.",
        "S3C shells give the pigeon a lawful coordinate; spherion shaping gives the hole teeth.",
        "Stop predicting the next token. Shear the manifold until the next token is obvious.",
        "The Morse-Smale complex is the topological skeleton of meaning. GCCL-GEC is the lawful engine that navigates it.",
        "FAMM preshapes time the way shear reshapes space — both are information-geometric warping.",
        "Every '{{cite web' is the same OAC cavity. Pay for the cavity once, pay for the URL residual each time.",
        "The Hutter Prize is manifold learning disguised as sequence modeling.",
        "ρ = |ε| / |raw_span|. This is the only number that matters.",
        "A finite glyph set becomes combinatorially huge through chirality rotation.",
        "The datatype is the engine. UTF-8 is only the exhaust."
    ],

    "metadata": {
        "ingested_at": time.time(),
        "tags": [
            "unified-compression-architecture",
            "density-field-encoding",
            "gccl-gec",
            "observer-admissible-cavities",
            "hypercube-rhomboid",
            "s3c-shells",
            "famm",
            "pist",
            "erans",
            "radius-ratio",
            "morse-smale-complex",
            "gram-matrix",
            "shear-matrix",
            "topological-compression",
            "navigable-compression",
            "hutter-prize",
            "information-geometry",
            "semantic-manifold"
        ]
    }
}


def ingest():
    germane_dir = RESEARCH_STACK / "shared-data/data/germane/research"
    germane_dir.mkdir(parents=True, exist_ok=True)

    out_path = germane_dir / "unified_compression_architecture_synthesis_v1.json"
    with open(out_path, 'w') as f:
        json.dump(SYNTHESIS, f, indent=2)

    print(f"✓ Ingested: {out_path}")

    index_path = germane_dir / "research_ingestion_index.json"
    index = []
    if index_path.exists():
        with open(index_path) as f:
            index = json.load(f)

    index.append({
        "id": SYNTHESIS["id"],
        "title": SYNTHESIS["title"],
        "date": SYNTHESIS["date"],
        "source": SYNTHESIS["source"],
        "ingested_at": SYNTHESIS["metadata"]["ingested_at"],
        "tags": SYNTHESIS["metadata"]["tags"],
    })

    with open(index_path, 'w') as f:
        json.dump(index, f, indent=2)

    print(f"✓ Index: {len(index)} entries")

    print(f"\nArchitectural layers (10):")
    for layer, data in SYNTHESIS["architectural_layers"].items():
        if "operation" in data:
            print(f"  {layer}: {data['operation'][:70]}...")

    print(f"\nComponent interdependencies (9):")
    for dep, desc in SYNTHESIS["component_interdependencies"].items():
        print(f"  {dep} → {desc[:60]}...")

    print(f"\nCompression gain sources (10):")
    for source, gain in SYNTHESIS["compression_gain_sources"].items():
        print(f"  {source}: {gain[:60]}...")

    print(f"\nImplementation phases (7):")
    for phase, data in SYNTHESIS["implementation_priority"].items():
        print(f"  {phase}: {len(data['tasks'])} tasks — {data['rationale'][:50]}...")

    print(f"\nKeeper phrases ({len(SYNTHESIS['keeper_phrases'])}):")
    for p in SYNTHESIS["keeper_phrases"]:
        print(f"  → {p}")


if __name__ == "__main__":
    ingest()