Research-Stack/4-Infrastructure/shim/semantic_compression_theoretical_limits_prior.py

#!/usr/bin/env python3
"""Semantic compression theoretical-limits prior.

This records the user's pasted Consensus thread as a route/evaluator prior.
It is not a semantic-compression proof and it does not relax byte-exact
promotion.  The practical extraction is a set of limit coordinates that can
shape DD pruning, receipts, and evaluator fields.
"""

from __future__ import annotations

import argparse
import hashlib
import json
from pathlib import Path
from typing import Any


DEFAULT_RECEIPT = Path(
    "4-Infrastructure/shim/semantic_compression_theoretical_limits_prior_receipt.json"
)
DEFAULT_CURRICULUM = Path(
    "4-Infrastructure/shim/semantic_compression_theoretical_limits_prior_curriculum.jsonl"
)


CONSENSUS_SOURCE_SUMMARY = {
    "thread_title": "Semantic Compression Theoretical Limits",
    "prompt": "unified math models of theoretical limits in semantic compression",
    "mode": "Deep",
    "search_count": 21,
    "citation_graph_uses": 1,
    "retrieved_count_reported": 2_777_471,
    "eligible_count_reported": 1_500,
    "included_count_reported": 50,
    "consensus_meter": {
        "question": "Are there unified mathematical models that define the theoretical limits of semantic compression?",
        "n": 9,
        "yes_percent": 100,
    },
    "claim_boundary": (
        "Consensus thread is a source-bundle prior. It supplies theoretical-limit "
        "coordinates and citations, not local compression evidence."
    ),
}


LIMIT_FAMILIES = [
    {
        "id": "semantic_information_bounds",
        "source_examples": [
            "Information-theoretic limits on compression of semantic information",
            "A Mathematical Theory of Semantic Communication",
            "Semantic Information Theory and Applications",
        ],
        "useful_shape": (
            "model a semantic source with conditional independence, Bayesian or "
            "probabilistic structure, and derive lower/upper rate bounds"
        ),
        "dd_use": "bound semantic-sidecar claims and require explicit semantic source model IDs",
        "receipt_fields": [
            "semantic_source_model_id",
            "conditional_independence_receipt",
            "semantic_entropy_bound_bits",
            "side_information_policy",
        ],
        "failure_mode": "semantic bound claimed without an explicit source model or local byte receipt",
    },
    {
        "id": "semantic_rate_distortion",
        "source_examples": [
            "Semantic Rate-Distortion Theory with Applications",
            "A Rate-Distortion Framework for Characterizing Semantic Information",
            "Semantic Compression with Side Information: A Rate-Distortion Perspective",
            "Fundamental Limitation of Semantic Communications: Neural Estimation for Rate-Distortion",
        ],
        "useful_shape": (
            "define a semantic distortion variable, estimate or bound the semantic "
            "rate-distortion function, and compare rate against task outcome"
        ),
        "dd_use": "treat semantic distortion as a diagnostic constraint, never as byte promotion",
        "receipt_fields": [
            "semantic_distortion_metric_id",
            "rate_distortion_estimator_id",
            "side_information_bits",
            "task_success_metric",
        ],
        "failure_mode": "semantic distortion score improves while decoded bytes do not match",
    },
    {
        "id": "rate_distortion_perception_bottleneck",
        "source_examples": [
            "Rate-Distortion-Perception Trade-Off in Information Theory, Generative Models, and Intelligent Communications",
            "Semantic Communication via Rate Distortion Perception Bottleneck",
            "Rate-Distortion-Perception Theory for Semantic Communication",
        ],
        "useful_shape": (
            "perceptual constraints may require extra rate; bottleneck objective "
            "balances task/perception/bit distortion"
        ),
        "dd_use": "charge perceptual or semantic witness bytes explicitly in the route budget",
        "receipt_fields": [
            "perception_constraint_id",
            "bottleneck_lambda",
            "extra_rate_for_perception_bits",
            "diagnostic_quality_score",
        ],
        "failure_mode": "perception quality hides sidecar or witness debt",
    },
    {
        "id": "information_bottleneck_and_ordered_latents",
        "source_examples": [
            "Efficient compression in color naming and its evolution",
            "Information-Ordered Bottlenecks for Adaptive Semantic Compression",
            "Ordered embeddings and intrinsic dimensionalities with information-ordered bottlenecks",
            "Adversarial Information Bottleneck",
        ],
        "useful_shape": (
            "compress observations while preserving task-relevant variables; order "
            "latent coordinates by marginal information or robustness"
        ),
        "dd_use": "rank route features and prune low-information sidecar lanes",
        "receipt_fields": [
            "bottleneck_variable_id",
            "relevance_variable_id",
            "marginal_information_gain",
            "robustness_receipt_id",
        ],
        "failure_mode": "latent relevance score treated as an exact rehydration witness",
    },
    {
        "id": "geometric_algebraic_error_subspaces",
        "source_examples": [
            "Geometry is All You Need: A Unified Taxonomy of Matrix and Tensor Factorization for Compression of Generative Language Models",
            "A General Error-Theoretical Analysis Framework for Constructing Compression Strategies",
            "Bridging Information-Theoretic and Geometric Compression in Language Models",
        ],
        "useful_shape": (
            "parameter/data compression can be expressed through intrinsic dimension, "
            "factorization geometry, or error subspace shape"
        ),
        "dd_use": "use geometry as route-feature coordinates and error-budget priors",
        "receipt_fields": [
            "intrinsic_dimension_estimate",
            "factorization_family_id",
            "error_subspace_shape_id",
            "layerwise_budget_vector",
        ],
        "failure_mode": "geometric compactness confused with source-byte compression",
    },
    {
        "id": "llm_understanding_compression_link",
        "source_examples": [
            "Lossless data compression by large models",
            "Semantic Compression with Large Language Models",
            "Language Modeling Is Compression",
            "Fundamental Limits of Prompt Compression: A Rate-Distortion Framework for Black-Box Language Models",
        ],
        "useful_shape": (
            "large learned models can improve compression by prediction/understanding, "
            "but introduce hallucination, context compression, and compute costs"
        ),
        "dd_use": "allow LLM routes as proposal/predictor engines with strict byte rehydration checks",
        "receipt_fields": [
            "model_id",
            "prompt_compression_ratio",
            "hallucination_guard_id",
            "context_rehydration_hash",
            "compute_budget_ms",
        ],
        "failure_mode": "LLM reconstruction is semantically plausible but byte-invalid",
    },
    {
        "id": "synonymity_and_semantic_arithmetic_coding",
        "source_examples": [
            "Semantic Arithmetic Coding Using Synonymous Mappings",
            "The Semantic Relations in LLMs: An Information-theoretic Compression Approach",
            "Preserving quality of information by using semantic relationships",
        ],
        "useful_shape": (
            "synonym or semantic-equivalence classes can reduce semantic code length "
            "when the equivalence relation is explicit"
        ),
        "dd_use": "map synonym classes to tokenbook proposals and residualize exact lexical choice",
        "receipt_fields": [
            "semantic_equivalence_class_id",
            "synonym_map_hash",
            "lexical_residual_bytes",
            "equivalence_ambiguity_count",
        ],
        "failure_mode": "semantic equivalence discards lexical bytes without residual repair",
    },
    {
        "id": "resource_constrained_semantic_limits",
        "source_examples": [
            "Compression Ratio Allocation for Probabilistic Semantic Communication With RSMA",
            "A Joint Communication and Computation Design for Probabilistic Semantic Communications",
            "Semantic Rate Distortion and Posterior Design: Compute Constraints, Multimodality, and Strategic Inference",
            "Semantic Communication with Side Information: A Rate-Distortion Perspective",
        ],
        "useful_shape": (
            "rate, compute, memory, side information, and multi-user allocation all "
            "change the achievable semantic limit"
        ),
        "dd_use": "make runtime, memory, sidecar, and witness budgets first-class route constraints",
        "receipt_fields": [
            "byte_budget",
            "compute_budget_ms",
            "memory_budget_bytes",
            "side_information_bytes",
            "allocation_policy_id",
        ],
        "failure_mode": "semantic route appears efficient only by ignoring compute or side-information cost",
    },
    {
        "id": "ambiguity_multimodality_and_generalization_gap",
        "source_examples": [
            "Compression Beyond Pixels: Semantic Compression with Multimodal Foundation Models",
            "Can Image Compression Rely on CLIP?",
            "Semantic Rate Distortion and Posterior Design: Compute Constraints, Multimodality, and Strategic Inference",
            "On the Fundamental Limits of LLMs at Scale",
        ],
        "useful_shape": (
            "ambiguity, polysemy, multimodality, hallucination, and retrieval fragility "
            "limit transfer from theoretical semantic rates to real systems"
        ),
        "dd_use": "force ambiguity packets and multimodal claim boundaries before promotion",
        "receipt_fields": [
            "ambiguity_class_id",
            "polysemy_count",
            "modality_set",
            "retrieval_fragility_score",
            "generalization_scope",
        ],
        "failure_mode": "semantic limit validated on narrow data is applied to a broader corpus slice",
    },
]


THEORETICAL_RECEIPT_RULES = {
    "ratio_schema": "source_bytes / compressed_total_bytes",
    "semantic_limit_status": "diagnostic unless byte_rehydration_hash matches",
    "promotion_rule": (
        "promote iff semantic-limit machinery only proposes, bounds, or budgets "
        "routes; exact residual lanes restore source bytes; decoded hash matches; "
        "and measured total bytes beat incumbent under explicit ratio_schema"
    ),
    "failure_rule": (
        "semantic entropy, rate-distortion, bottleneck, perceptual quality, LLM "
        "understanding, or synonymity without byte-exact restoration is diagnostic only"
    ),
}


def stable_hash(obj: Any) -> str:
    payload = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()


def build_receipt() -> dict[str, Any]:
    source_hash = stable_hash(CONSENSUS_SOURCE_SUMMARY)
    receipt: dict[str, Any] = {
        "schema": "semantic_compression_theoretical_limits_prior_v1",
        "generated_at": "2026-05-08T00:00:00+00:00",
        "source_summary": CONSENSUS_SOURCE_SUMMARY,
        "source_summary_hash": source_hash,
        "claim_boundary": (
            "Theoretical semantic-compression models define proposal and budget "
            "surfaces. They do not establish local byte compression unless the "
            "route has a local encode/decode/hash/byte-count receipt."
        ),
        "limit_families": LIMIT_FAMILIES,
        "theoretical_receipt_rules": THEORETICAL_RECEIPT_RULES,
        "dd_state_extension": [
            "semantic_source_model_id",
            "semantic_entropy_bound_bits",
            "semantic_distortion_metric_id",
            "rate_distortion_estimator_id",
            "bottleneck_variable_id",
            "marginal_information_gain",
            "intrinsic_dimension_estimate",
            "error_subspace_shape_id",
            "semantic_equivalence_class_id",
            "lexical_residual_bytes",
            "side_information_bytes",
            "compute_budget_ms",
            "ambiguity_class_id",
            "byte_rehydration_hash",
        ],
        "candidate_dd_edges": [
            "choose_semantic_source_model",
            "estimate_semantic_entropy_bound",
            "estimate_semantic_rate_distortion",
            "apply_information_bottleneck_rank",
            "emit_geometric_error_subspace",
            "propose_llm_predictor_route",
            "emit_synonym_class_tokenbook",
            "charge_side_information_budget",
            "record_ambiguity_packet",
            "emit_exact_residual_lane",
            "verify_byte_rehydration_hash",
            "reject_semantic_only_promotion",
        ],
    }
    receipt["receipt_hash"] = stable_hash(receipt)
    return receipt


def curriculum_records(receipt: dict[str, Any]) -> list[dict[str, Any]]:
    system = (
        "You are a semantic-compression limit router. Treat theory as a "
        "budget/proposal surface and byte-exact receipts as promotion authority."
    )
    records: list[dict[str, Any]] = []
    for family in receipt["limit_families"]:
        records.append(
            {
                "messages": [
                    {"role": "system", "content": system},
                    {
                        "role": "user",
                        "content": json.dumps(
                            {
                                "task": "route_semantic_compression_limit_family",
                                "family_id": family["id"],
                                "useful_shape": family["useful_shape"],
                                "dd_use": family["dd_use"],
                            },
                            ensure_ascii=False,
                        ),
                    },
                    {
                        "role": "assistant",
                        "content": json.dumps(
                            {
                                "selected": True,
                                "receipt_fields": family["receipt_fields"],
                                "failure_mode": family["failure_mode"],
                                "claim_boundary": "semantic-limit-prior-only",
                                "promotion_authority": "local encode/decode/hash/byte-count receipt",
                            },
                            ensure_ascii=False,
                        ),
                    },
                ]
            }
        )
    return records


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT)
    parser.add_argument("--curriculum", type=Path, default=DEFAULT_CURRICULUM)
    args = parser.parse_args()

    receipt = build_receipt()
    args.receipt.parent.mkdir(parents=True, exist_ok=True)
    args.receipt.write_text(json.dumps(receipt, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    with args.curriculum.open("w", encoding="utf-8") as handle:
        for record in curriculum_records(receipt):
            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
    print(json.dumps(receipt, indent=2, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())