Research-Stack/4-Infrastructure/shim/non_euclidean_semantic_kv_prior.py

#!/usr/bin/env python3
"""Non-Euclidean geometry, compression, and semantic KV-store prior.

This records the user's pasted Consensus thread as a bounded route prior.  The
source says the three themes are mostly separate in current research, so the
local extraction is an integration rule rather than a claim that the literature
already proves a unified compressor.
"""

from __future__ import annotations

import argparse
import hashlib
import json
from pathlib import Path
from typing import Any


DEFAULT_RECEIPT = Path("4-Infrastructure/shim/non_euclidean_semantic_kv_prior_receipt.json")
DEFAULT_CURRICULUM = Path("4-Infrastructure/shim/non_euclidean_semantic_kv_prior_curriculum.jsonl")


CONSENSUS_SOURCE_SUMMARY = {
    "thread_title": "Non-Euclidean Geometry Compression Methods",
    "prompt": "non euclidian approaches to geometry, compression and semantic key stores",
    "mode": "Pro",
    "search_count": 4,
    "reported_query_counts": {
        "compression_methods_and_semantic_key_value_stores": 17_900_000,
        "geometric_methods_in_non_euclidean_spaces_and_manifold_based_geometry": 3_000_000,
        "non_euclidean_geometry_hyperbolic_spherical_metric_geometry": 865_100,
    },
    "consensus_meter_question": (
        "Does non-Euclidean geometry improve compression efficiency in semantic key-value stores?"
    ),
    "claim_boundary": (
        "The thread says current work touches non-Euclidean geometry, classic "
        "KV-store compression, and semantic KV-cache compression mostly in "
        "separate lines. This prior extracts integration constraints only."
    ),
}


PRIOR_FAMILIES = [
    {
        "id": "riemannian_manifold_distortion",
        "source_examples": [
            "A Riemannian geometric framework for manifold learning of non-Euclidean data",
            "Survey of geometric optimization from Euclidean space to Riemannian manifolds",
            "Manifold learning in metric spaces",
        ],
        "useful_shape": (
            "learn or compare data on curved spaces using coordinate-invariant "
            "distortion and near-isometry checks"
        ),
        "compressor_mapping": "semantic keys live on a curved route manifold instead of a flat vector table",
        "receipt_fields": [
            "manifold_family_id",
            "chart_id",
            "coordinate_invariant_distortion",
            "near_isometry_status",
        ],
        "failure_mode": "curved embedding clusters keys but does not preserve byte rehydration",
    },
    {
        "id": "cartan_hadamard_optimal_transport",
        "source_examples": [
            "Sliced-Wasserstein Distances and Flows on Cartan-Hadamard Manifolds",
            "Spherical and hyperbolic embeddings of data",
            "Computationally tractable Riemannian manifolds for graph embeddings",
        ],
        "useful_shape": (
            "use non-positive curvature, hyperbolic/SPD geometry, and sliced "
            "Wasserstein flows for distribution-aware key movement"
        ),
        "compressor_mapping": "route token/key populations by geodesic transport rather than flat nearest neighbor only",
        "receipt_fields": [
            "curvature_class",
            "transport_plan_id",
            "sliced_wasserstein_score",
            "geodesic_owner_id",
        ],
        "failure_mode": "transport score improves retrieval geometry while sidecar cost exceeds byte gain",
    },
    {
        "id": "classic_kv_store_byte_compression",
        "source_examples": [
            "Requirements and Trade-Offs of Compression Techniques in Key-Value Stores: A Survey",
            "ZipKV: In-Memory Key-Value Store with Built-In Data Compression",
            "KallaxDB: A Table-less Hash-based Key-Value Store on Storage Hardware with Built-in Transparent Compression",
            "TinyEnc: Enabling Compressed and Encrypted Big Data Stores With Rich Query Support",
        ],
        "useful_shape": (
            "classic KV stores tune Snappy/LZ4/Zstd/Zlib, compaction, block size, "
            "selective compression, and read/write amplification"
        ),
        "compressor_mapping": "backend key store must expose actual byte counts and throughput tradeoffs",
        "receipt_fields": [
            "kv_backend_id",
            "codec_id",
            "block_granularity_bytes",
            "read_amplification",
            "write_amplification",
            "compressed_store_bytes",
        ],
        "failure_mode": "semantic key route ignores store codec overhead or compaction cost",
    },
    {
        "id": "semantic_chunk_anchor_kv_cache",
        "source_examples": [
            "ChunkKV: Semantic-Preserving KV Cache Compression for Efficient Long-Context LLM Inference",
            "FINCH: Prompt-guided Key-Value Cache Compression for Large Language Models",
            "Autoencoding-Free Context Compression for LLMs via Contextual Semantic Anchors",
            "ClusterKV: Manipulating LLM KV Cache in Semantic Space for Recallable Compression",
            "SentenceKV: Efficient LLM Inference via Sentence-Level Semantic KV Caching",
        ],
        "useful_shape": (
            "preserve semantic chunks, anchor tokens, sentence units, or recallable "
            "clusters under strict KV-cache budgets"
        ),
        "compressor_mapping": "chunk/anchor selection proposes tokenbook and sidecar lanes",
        "receipt_fields": [
            "semantic_chunk_id",
            "anchor_token_map_hash",
            "cluster_id",
            "recallability_score",
            "chunk_residual_bytes",
        ],
        "failure_mode": "semantic anchor reconstructs meaning but not source bytes",
    },
    {
        "id": "head_layer_importance_kv_cache",
        "source_examples": [
            "Dynamic Memory Compression: Retrofitting LLMs for Accelerated Inference",
            "RazorAttention: Efficient KV Cache Compression Through Retrieval Heads",
            "CompressKV: Semantic Retrieval Heads Know What Tokens are Not Important Before Generation",
            "HeadKV: A Head-Level KV Cache Compression Method with Integrated Retrieval and Reasoning",
            "MiniCache: KV Cache Compression in Depth Dimension for Large Language Models",
            "A Simple and Effective L2 Norm-Based Strategy for KV Cache Compression",
        ],
        "useful_shape": (
            "heads, layers, norms, importance, and diversity can rank what KV state "
            "to keep, merge, or evict"
        ),
        "compressor_mapping": "attention-derived importance becomes a DD feature coordinate, not a proof",
        "receipt_fields": [
            "attention_head_id",
            "layer_id",
            "importance_score",
            "diversity_score",
            "eviction_policy_id",
            "kv_budget_bytes",
        ],
        "failure_mode": "head/layer pruning breaks exact decode or retrieval receipt",
    },
    {
        "id": "value_aware_low_rank_kv_cache",
        "source_examples": [
            "GEAR: An Efficient KV Cache Compression Recipe for Near-Lossless Generative Inference of LLM",
            "Value-Guided KV Compression for LLMs via Approximated CUR Decomposition",
            "Palu: KV-Cache Compression with Low-Rank Projection",
            "LoRC: Low-Rank Compression for LLMs KV Cache with a Progressive Compression Strategy",
            "SVDq: 1.25-bit and 410x Key Cache Compression for LLM Attention",
        ],
        "useful_shape": (
            "low-rank, sparse correction, quantization, CUR/SVD, and value-guided "
            "decomposition approximate attention outputs"
        ),
        "compressor_mapping": "low-rank KV is a predictor sketch that needs exact residual authority",
        "receipt_fields": [
            "decomposition_family_id",
            "rank_budget",
            "quantization_bits",
            "sparse_correction_bytes",
            "value_guidance_hash",
        ],
        "failure_mode": "near-lossless KV approximation is treated as byte-exact",
    },
    {
        "id": "geometry_inspired_but_unproven_unification",
        "source_examples": [
            "Position: Beyond Euclidean - Foundation Models Should Embrace Non-Euclidean Geometries",
            "Beyond Euclid: an illustrated guide to modern machine learning with geometric, topological, and algebraic structures",
            "State of the Art of Graph Visualization in non-Euclidean Spaces",
        ],
        "useful_shape": (
            "non-Euclidean geometry may improve representation and retrieval, but "
            "the cited thread does not establish a single unified KV compressor"
        ),
        "compressor_mapping": "require an explicit bridge receipt between curved geometry and byte-store behavior",
        "receipt_fields": [
            "bridge_claim_id",
            "geometry_to_kv_mapping_id",
            "byte_store_receipt_id",
            "semantic_cache_receipt_id",
            "unification_status",
        ],
        "failure_mode": "geometry metaphor is promoted without a byte-store and semantic-cache bridge",
    },
]


LOCAL_TREEFIDDY_STATUS = {
    "status": "found_in_current_checkout",
    "model_map_entry": "3-Mathematical-Models/MATH_MODEL_MAP.tsv:102",
    "documentation": "6-Documentation/docs/semantics/TREE_FIDDY.md",
    "local_role": (
        "TREE(3) / Kruskal-style tree-sequence bound used as a state-space "
        "pruning shortcut and bounded archive depth guard"
    ),
    "compression_claim_boundary": (
        "Tree Fiddy can bound TreeKV route depth, owner routing, and archive "
        "receipts. It is not a hidden payload channel and does not prove byte "
        "compression by itself."
    ),
}


PRIORITY_WATCH_ITEMS = [
    {
        "id": "tinyenc_compressed_encrypted_kv_store",
        "source_examples": [
            "TinyEnc: Enabling Compressed and Encrypted Big Data Stores With Rich Query Support",
            "Encrypted and Compressed Key-Value Store With Pattern-Analysis Security in Cloud Systems",
            "Optimal Compression for Encrypted Key-Value Store in Cloud Systems",
        ],
        "why_pay_attention": (
            "TinyEnc sits on the byte-store side of the bridge: compression, "
            "encryption, and rich query support must be paid for in one receipt."
        ),
        "compressor_mapping": (
            "encrypted KV packet -> compressed store packet + query-support "
            "index + leakage/pattern guard + exact byte rehydration receipt"
        ),
        "receipt_fields": [
            "encryption_envelope_id",
            "cipher_suite_id",
            "query_support_class",
            "query_index_bytes",
            "pattern_leakage_guard_id",
            "compressed_encrypted_bytes",
            "plaintext_rehydration_hash",
        ],
        "promotion_guard": (
            "promote only if encryption envelope, query index, and compression "
            "container overhead are counted and plaintext bytes rehydrate exactly"
        ),
        "failure_mode": "query/encryption metadata hides byte debt or weakens the claim boundary",
    },
    {
        "id": "treekv_treefiddy_modification",
        "source_examples": [
            "TreeKV: Smooth Key-Value Cache Compression with Tree Structures",
            "Tree Fiddy: TREE(3) Combinatorial State Space Shortcut",
            "BHOCS: Bounded Hierarchical Cryptographic Space",
        ],
        "why_pay_attention": (
            "TreeKV already gives a tree-structured KV-cache route. Local "
            "Tree Fiddy can modify it into a bounded route spine with explicit "
            "depth, embedding, owner, and leaf-residual receipts."
        ),
        "compressor_mapping": (
            "TreeKV node -> Tree Fiddy bounded route spine -> deterministic "
            "subtree owner -> smooth merge receipt -> exact residual leaves"
        ),
        "receipt_fields": [
            "treekv_node_id",
            "treefiddy_spine_id",
            "tree_label_budget_k",
            "tree_depth_budget",
            "homeomorphic_embedding_guard",
            "subtree_owner_hash",
            "smooth_merge_receipt_id",
            "leaf_residual_bytes",
        ],
        "promotion_guard": (
            "promote only if Tree Fiddy bounds depth/branching, TreeKV smooth "
            "merges preserve decode reachability, and residual leaves restore "
            "the exact bytes"
        ),
        "failure_mode": "tree merge changes decode reachability or opens recursive repair",
    },
]


INTEGRATION_RULES = {
    "three_surface_model": {
        "curved_key_surface": "non-Euclidean manifold stores similarity, hierarchy, and geodesic owner routing",
        "byte_store_surface": "KV backend stores bytes with codec, compaction, and throughput receipts",
        "semantic_cache_surface": "LLM KV/cache route stores semantic anchors, heads, ranks, and residuals",
    },
    "promotion_rule": (
        "promote iff curved geometry only routes or clusters keys, KV-store byte "
        "compression is measured, semantic KV approximations carry exact residual "
        "repair, decoded hash matches source, and total bytes beat incumbent"
    ),
    "failure_rule": (
        "non-Euclidean retrieval, semantic cache recall, or KV throughput improvement "
        "without byte-exact rehydration is diagnostic only"
    ),
    "tinyenc_watch_rule": (
        "TinyEnc-style compressed encryption is relevant when query support, "
        "encryption envelope, compressed bytes, and leakage guards are all "
        "counted in the same byte-store receipt"
    ),
    "treekv_treefiddy_rule": (
        "TreeKV may be modified to use Tree Fiddy as a bounded tree-spine and "
        "homeomorphic-embedding guard, but Tree Fiddy remains a pruning and "
        "receipt primitive, not compression evidence"
    ),
}


def stable_hash(obj: Any) -> str:
    payload = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()


def build_receipt() -> dict[str, Any]:
    receipt: dict[str, Any] = {
        "schema": "non_euclidean_semantic_kv_prior_v1",
        "generated_at": "2026-05-08T00:00:00+00:00",
        "source_summary": CONSENSUS_SOURCE_SUMMARY,
        "source_summary_hash": stable_hash(CONSENSUS_SOURCE_SUMMARY),
        "claim_boundary": (
            "This is an integration prior. It records source families for curved "
            "geometry, KV-store compression, and semantic KV-cache compression; "
            "local encode/decode/hash/byte-count receipts remain authoritative."
        ),
        "prior_families": PRIOR_FAMILIES,
        "priority_watch_items": PRIORITY_WATCH_ITEMS,
        "local_treefiddy_status": LOCAL_TREEFIDDY_STATUS,
        "integration_rules": INTEGRATION_RULES,
        "dd_state_extension": [
            "manifold_family_id",
            "chart_id",
            "curvature_class",
            "geodesic_owner_id",
            "kv_backend_id",
            "codec_id",
            "block_granularity_bytes",
            "semantic_chunk_id",
            "anchor_token_map_hash",
            "attention_head_id",
            "importance_score",
            "decomposition_family_id",
            "rank_budget",
            "sparse_correction_bytes",
            "geometry_to_kv_mapping_id",
            "byte_store_receipt_id",
            "semantic_cache_receipt_id",
            "encryption_envelope_id",
            "query_support_class",
            "pattern_leakage_guard_id",
            "treekv_node_id",
            "treefiddy_spine_id",
            "tree_label_budget_k",
            "tree_depth_budget",
            "homeomorphic_embedding_guard",
            "subtree_owner_hash",
            "smooth_merge_receipt_id",
            "leaf_residual_bytes",
            "byte_rehydration_hash",
        ],
        "candidate_dd_edges": [
            "choose_curved_key_manifold",
            "assign_geodesic_owner",
            "measure_manifold_distortion",
            "choose_kv_backend_codec",
            "measure_kv_store_bytes",
            "emit_semantic_chunk_anchor",
            "rank_attention_heads",
            "apply_low_rank_kv_sketch",
            "emit_exact_kv_residual_lane",
            "bridge_geometry_to_byte_store",
            "charge_tinyenc_encryption_query_overhead",
            "open_treekv_treefiddy_spine",
            "bound_treefiddy_depth_and_embedding",
            "route_treefiddy_subtree_owner",
            "verify_treekv_smooth_merge_receipt",
            "emit_tree_leaf_exact_residuals",
            "verify_byte_rehydration_hash",
            "reject_geometry_only_kv_claim",
        ],
    }
    receipt["receipt_hash"] = stable_hash(receipt)
    return receipt


def curriculum_records(receipt: dict[str, Any]) -> list[dict[str, Any]]:
    system = (
        "You are a non-Euclidean semantic KV route controller. Keep geometry, "
        "KV-store bytes, and semantic cache receipts separate until a bridge is verified."
    )
    records: list[dict[str, Any]] = []
    for family in receipt["prior_families"]:
        records.append(
            {
                "messages": [
                    {"role": "system", "content": system},
                    {
                        "role": "user",
                        "content": json.dumps(
                            {
                                "task": "route_non_euclidean_semantic_kv_family",
                                "family_id": family["id"],
                                "useful_shape": family["useful_shape"],
                                "compressor_mapping": family["compressor_mapping"],
                            },
                            ensure_ascii=False,
                        ),
                    },
                    {
                        "role": "assistant",
                        "content": json.dumps(
                            {
                                "selected": True,
                                "receipt_fields": family["receipt_fields"],
                                "failure_mode": family["failure_mode"],
                                "claim_boundary": "integration-prior-only",
                                "promotion_authority": "local encode/decode/hash/byte-count receipt",
                            },
                            ensure_ascii=False,
                        ),
                    },
                ]
            }
        )
    for item in receipt["priority_watch_items"]:
        records.append(
            {
                "messages": [
                    {"role": "system", "content": system},
                    {
                        "role": "user",
                        "content": json.dumps(
                            {
                                "task": "route_priority_watch_item",
                                "watch_item_id": item["id"],
                                "why_pay_attention": item["why_pay_attention"],
                                "compressor_mapping": item["compressor_mapping"],
                            },
                            ensure_ascii=False,
                        ),
                    },
                    {
                        "role": "assistant",
                        "content": json.dumps(
                            {
                                "selected": True,
                                "receipt_fields": item["receipt_fields"],
                                "promotion_guard": item["promotion_guard"],
                                "failure_mode": item["failure_mode"],
                                "promotion_authority": "local encode/decode/hash/byte-count receipt",
                            },
                            ensure_ascii=False,
                        ),
                    },
                ]
            }
        )
    return records


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT)
    parser.add_argument("--curriculum", type=Path, default=DEFAULT_CURRICULUM)
    args = parser.parse_args()

    receipt = build_receipt()
    args.receipt.parent.mkdir(parents=True, exist_ok=True)
    args.receipt.write_text(json.dumps(receipt, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    with args.curriculum.open("w", encoding="utf-8") as handle:
        for record in curriculum_records(receipt):
            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
    print(json.dumps(receipt, indent=2, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())