#!/usr/bin/env python3 """Re-evaluate DD compression targets against current implementation receipts. This is a receipt-bearing target review. It does not recompress data. It reads the local decision-diagram, topology, route-card, KV, citation, and singular-chart receipts and classifies which configuration knobs are currently: * promotion-backed by measured bytes, * useful as pruning / control configuration, or * still only proposal surface until an encode/decode/hash evaluator exists. """ from __future__ import annotations import hashlib import json from pathlib import Path from typing import Any REPO = Path(__file__).resolve().parents[2] SHIM = REPO / "4-Infrastructure" / "shim" OUT = SHIM / "dd_target_implementation_reevaluation_receipt.json" CURRICULUM_OUT = SHIM / "dd_target_implementation_reevaluation_curriculum.jsonl" GENERATED_AT = "2026-05-08T00:00:00+00:00" HUTTER_ENWIK9_TARGET_BYTES = 109_685_197 RECEIPT_PATHS = { "dimensional_shell_dd_probe": SHIM / "dimensional_shell_dd_probe_receipt.json", "projectable_geometry_topology_model": SHIM / "projectable_geometry_topology_model_receipt.json", "alphaevolve_dd_experiment_card_runner": SHIM / "alphaevolve_dd_experiment_card_receipt.json", "non_euclidean_semantic_kv_prior": SHIM / "non_euclidean_semantic_kv_prior_receipt.json", "citation_math_function_distillation": SHIM / "citation_math_function_distillation_receipt.json", "singular_route_chart_equations": SHIM / "singular_route_chart_equations_receipt.json", "compression_ratio_rederivation": SHIM / "compression_ratio_rederivation_receipt.json", } def stable_json(obj: Any) -> str: return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def load_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) def rel(path: Path) -> str: return str(path.relative_to(REPO)) def receipt_hash(path: Path, data: dict[str, Any]) -> str: for key in ( "stable_topology_model_hash_sha256", "stable_shell_dd_hash_sha256", "receipt_hash", ): value = data.get(key) if isinstance(value, str): return value return sha256_bytes(path.read_bytes()) def source_receipts(receipts: dict[str, dict[str, Any]]) -> dict[str, Any]: return { name: { "path": rel(RECEIPT_PATHS[name]), "hash": receipt_hash(RECEIPT_PATHS[name], receipt), "schema": receipt.get("schema", "unknown"), } for name, receipt in receipts.items() } def class_counts(cards: dict[str, Any]) -> dict[str, int]: counts = cards.get("class_counts") if isinstance(counts, dict): return {str(key): int(value) for key, value in counts.items()} derived: dict[str, int] = {} for card in cards.get("cards", []): cls = str(card.get("applicability_class", "unknown")) derived[cls] = derived.get(cls, 0) + 1 return derived def current_best(shell: dict[str, Any], topology: dict[str, Any]) -> dict[str, Any]: shell_best = shell["summary"]["best_shell_adjusted_overall"] topology_best = topology["best_approach"]["selected_route"] return { "slice": topology_best["slice"], "route": topology["best_approach"]["route"], "byte_transform": f"{topology_best['transform']} -> {topology_best['codec']}", "source_bytes": topology_best["source_bytes"], "raw_baseline_bytes": topology_best["raw_baseline_bytes"], "xml_bz2_bytes": topology_best["compressed_bytes"], "topology_or_shell_witness_bytes": topology_best["topology_witness_bytes"], "modeled_total_bytes": topology_best["modeled_total_bytes"], "modeled_ratio": topology_best["modeled_ratio"], "remaining_margin_vs_raw_baseline_bytes": topology_best[ "gain_vs_raw_after_topology_bytes" ], "witness_budget_before_losing_raw_bytes": topology_best[ "overhead_budget_before_losing_raw" ], "projected_enwik9_total_bytes": shell_best[ "projected_enwik9_total_bytes" ], "projected_hutter_gap_bytes": shell_best[ "hutter_target_gap_bytes_projected_enwik9" ], "rehydration_status": shell_best["shell_status"], "lawful": bool(topology_best["lawful"] and not shell_best["shell_status"]["nan0"]), } def implementation_status(receipts: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: shell = receipts["dimensional_shell_dd_probe"] topology = receipts["projectable_geometry_topology_model"] cards = receipts["alphaevolve_dd_experiment_card_runner"] kv = receipts["non_euclidean_semantic_kv_prior"] citation = receipts["citation_math_function_distillation"] singular = receipts["singular_route_chart_equations"] ratio = receipts["compression_ratio_rederivation"] return [ { "id": "dimensional_shell_dd_probe", "status": "implemented_receipt_over_existing_measurements", "promotion_authority": "measured route receipt inherited from reversible approach; shell only adds bounded witness", "evidence": { "route_count": shell["summary"]["route_count"], "promoted_route_count": shell["summary"]["promoted_route_count"], "lower_bound_pruned_count": shell["summary"][ "lower_bound_pruned_count" ], "nan0_route_count": shell["summary"]["nan0_route_count"], "shell_witness_bytes": shell["dd_policy"][ "shell_witness_bytes_per_non_raw_route" ], }, "verdict": "promotable_when_margin_survives_witness", }, { "id": "projectable_geometry_topology_model", "status": "implemented_topology_witness_model", "promotion_authority": "existing xml_token -> bz2 byte measurement plus 16-byte witness budget", "evidence": topology["best_approach"]["selected_route"], "verdict": "current_best_control_plane", }, { "id": "alphaevolve_dd_experiment_card_runner", "status": "implemented_search_card_prior", "promotion_authority": "none until local candidate code runs through encode/decode/hash evaluator", "evidence": { "card_count": cards["card_count"], "class_counts": class_counts(cards), "full_program_collection": cards["validation"].get( "full_program_collections", False ), }, "verdict": "proposal_and_dashboard_surface", }, { "id": "citation_math_function_distillation", "status": "implemented_operator_distillation", "promotion_authority": "none; this is the route compiler algebra", "evidence": { "primary_function": citation["primary_function"]["name"], "algebra_count": len(citation["dd_function_algebra"]), "function_group_count": len(citation["function_groups"]), }, "verdict": "configuration_basis", }, { "id": "singular_route_chart_equations", "status": "implemented_equation_group", "promotion_authority": "none unless a finite chart decodes and hashes exact bytes under measured byte count", "evidence": { "primary_function": singular["singular_route_chart"][ "primary_function" ], "equation_count": len(singular["equations"]), "dd_state_field_count": len(singular["dd_state_extension"]), "receipt_hash": singular["receipt_hash"], }, "verdict": "fail_closed_chart_layer", }, { "id": "non_euclidean_semantic_kv_prior", "status": "implemented_prior_and_watchlist", "promotion_authority": "none until byte-store / KV-cache routes have exact rehydration receipts", "evidence": { "prior_family_count": len(kv["prior_families"]), "priority_watch_items": [ item["id"] for item in kv["priority_watch_items"] ], "treefiddy_status": kv["local_treefiddy_status"]["status"], }, "verdict": "proposal_surface_with_local_tree_bound", }, { "id": "compression_ratio_rederivation", "status": "implemented_ratio_audit", "promotion_authority": "ratio schema only; byte counts remain terminal authority", "evidence": { "claim_boundary": ratio["claim_boundary"], "finance_claim_lut_lawful": ratio["finance_claim_lut"]["lawful"], }, "verdict": "measurement_schema_guard", }, ] def configuration_axes(best: dict[str, Any]) -> list[dict[str, Any]]: remaining = int(best["remaining_margin_vs_raw_baseline_bytes"]) budget = int(best["witness_budget_before_losing_raw_bytes"]) return [ { "id": "topology_witness_bytes", "current_setting": 16, "admissible_range_for_current_best_bytes": [0, budget], "target_role": "bounded closure/control witness", "reevaluation": "keep at 16 unless a new encoder earns more payload savings", }, { "id": "operator_basis_B_R_F_T_A_E_V", "current_setting": "available_as_distilled_algebra", "target_role": "configure pruning, routing, folding, transport, allocation, repair, and verification", "reevaluation": "wire as evaluator stages; do not charge bytes unless an emitted receipt/witness is serialized", }, { "id": "singular_route_chart", "current_setting": "available_not_wired_to_byte_evaluator", "target_role": "contain singular/ambiguous/unbounded route regions", "reevaluation": ( "use on failing or ambiguous branches first; applying extra chart " f"bytes to the current best must stay under {remaining} bytes" ), }, { "id": "tinyenc_encrypted_kv_envelope", "current_setting": "watch_item_only", "target_role": "future compressed/encrypted byte-store route", "reevaluation": "diagnostic until compressed bytes, query index overhead, and plaintext hash are receipted", }, { "id": "treekv_treefiddy_spine", "current_setting": "local_treefiddy_found_prior_only", "target_role": "bounded tree route spine and owner/depth guard", "reevaluation": "proposal only until TreeKV merge preserves decode reachability and exact residual leaves", }, { "id": "alphaevolve_route_population", "current_setting": "experiment_card_shape_only", "target_role": "candidate generator and analysed/program counters", "reevaluation": "feed candidates into the DD; never promote gallery score or novelty without local byte receipt", }, ] def admissibility_verdicts(best: dict[str, Any]) -> list[dict[str, Any]]: remaining = int(best["remaining_margin_vs_raw_baseline_bytes"]) return [ { "route_or_overlay": "xml_token -> topology_witness_16b -> bz2", "verdict": "promote_current_incumbent_for_small_slice", "reason": "measured route survives exact shell/topology witness and beats raw+bz2 by 1105 bytes", }, { "route_or_overlay": "singular_chart_on_current_best", "verdict": "prune_unless_chart_bytes_are_paid_by_new_savings", "reason": f"any added chart receipt consumes the remaining {remaining}-byte margin", }, { "route_or_overlay": "B/R/F/T/A/E/V operator basis", "verdict": "admissible_as_evaluator_configuration", "reason": "operators organize route search and receipts, but do not themselves add byte evidence", }, { "route_or_overlay": "TreeKV + Tree Fiddy", "verdict": "diagnostic_until_encoder_exists", "reason": "local Tree Fiddy can bound tree depth, but no TreeKV byte route has been evaluated", }, { "route_or_overlay": "TinyEnc-style encrypted KV store", "verdict": "diagnostic_until_store_receipt_exists", "reason": "encryption, query index, and compression overhead must all be counted with plaintext rehydration", }, { "route_or_overlay": "AlphaEvolve route population", "verdict": "proposal_generator_only", "reason": "program and analysed counts are useful search controls, not compression evidence", }, ] def next_target(best: dict[str, Any]) -> dict[str, Any]: return { "target_name": "configurable_bounded_route_evaluator", "reframed_goal": "optimize route evaluation and pruning configuration before adding more serialized witness bytes", "why": ( "The current best has only 1105 bytes of margin after the 16-byte " "witness. New knobs should reject bad branches cheaply or propose " "new byte-saving transforms; they should not be blindly appended to " "the incumbent route." ), "minimum_config_fields": [ "slice_id", "candidate_route", "backend_codec", "topology_witness_bytes", "singular_chart_enabled", "singular_chart_bytes", "treefiddy_spine_enabled", "tinyenc_envelope_enabled", "ratio_schema", "incumbent_bytes", "lower_bound_bytes", "rehydration_hash", ], "first_safe_implementation_step": ( "Add a configuration-matrix wrapper over existing receipts that " "computes lower bounds and prunes overlays whose witness/chart/KV " "cost would erase the incumbent margin before recompression." ), "first_real_compression_step": ( "Evaluate one new payload-saving transform family at a time, starting " "with bounded corpus-resolution/fascicle or tokenbook variants, then " "charge any topology/singular/KV witness bytes in the terminal receipt." ), "hard_target_bytes_enwik9": HUTTER_ENWIK9_TARGET_BYTES, "diagnostic_target": "beat raw baseline per slice under explicit ratio_schema", "current_gap_to_hard_target_projected_enwik9_bytes": best[ "projected_hutter_gap_bytes" ], } def curriculum_lines(receipt: dict[str, Any]) -> list[dict[str, Any]]: lines: list[dict[str, Any]] = [] for item in receipt["implementation_status"]: lines.append( { "type": "implementation_status", "id": item["id"], "status": item["status"], "verdict": item["verdict"], } ) for item in receipt["configuration_axes"]: lines.append( { "type": "configuration_axis", "id": item["id"], "current_setting": item["current_setting"], "reevaluation": item["reevaluation"], } ) for item in receipt["admissibility_verdicts"]: lines.append( { "type": "admissibility_verdict", "id": item["route_or_overlay"], "verdict": item["verdict"], "reason": item["reason"], } ) return lines def build_receipt() -> dict[str, Any]: receipts = {name: load_json(path) for name, path in RECEIPT_PATHS.items()} best = current_best( receipts["dimensional_shell_dd_probe"], receipts["projectable_geometry_topology_model"], ) receipt: dict[str, Any] = { "schema": "dd_target_implementation_reevaluation_v1", "generated_at": GENERATED_AT, "source_receipts": source_receipts(receipts), "current_target": { "primary_machine": "bounded_exact_route_compiler", "hard_target_bytes_enwik9": HUTTER_ENWIK9_TARGET_BYTES, "diagnostic_target": "beat raw baseline per slice under explicit ratio_schema", "proof_surface": [ "exact decoded bytes", "rehydration hash", "measured compressed_total_bytes", "explicit ratio_schema", "bounded sidecar/witness/container/compute cost", "fail-closed NaN0 metadata holds", ], }, "current_best_implemented_route": best, "implementation_status": implementation_status(receipts), "configuration_axes": configuration_axes(best), "admissibility_verdicts": admissibility_verdicts(best), "reevaluated_next_target": next_target(best), "claim_boundary": ( "This is a target and implementation reevaluation over local receipts. " "It does not recompress data, prove optimality, or promote any route " "without an encode/decode/hash/byte-count receipt." ), } preimage = {key: value for key, value in receipt.items() if key != "receipt_hash"} receipt["receipt_hash"] = sha256_bytes(stable_json(preimage).encode("utf-8")) return receipt def main() -> None: receipt = build_receipt() OUT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") lines = curriculum_lines(receipt) CURRICULUM_OUT.write_text( "".join(json.dumps(line, sort_keys=True) + "\n" for line in lines), encoding="utf-8", ) print(json.dumps({ "receipt": rel(OUT), "curriculum": rel(CURRICULUM_OUT), "receipt_hash": receipt["receipt_hash"], "line_count": len(lines), "current_best": receipt["current_best_implemented_route"]["route"], "next_target": receipt["reevaluated_next_target"]["target_name"], }, indent=2, sort_keys=True)) if __name__ == "__main__": main()