#!/usr/bin/env python3 """Parquet to logogram efficiency accounting probe. This probe measures the user's proposed path: transcode sampled Parquet rows into a logogram/species-code style payload, then account for byte gains or losses with exact replay gates. It intentionally separates measured byte accounting from promotion claims: positive savings are fixture evidence only, and negative savings are still useful baseline values. It also tests the stronger hybrid hypothesis: keep Parquet as the physical storage substrate and add a compact logogram sidecar for schema lineage, prediction-cache routing, RAM-trace routing, and replay receipts. """ from __future__ import annotations import argparse import hashlib import importlib.util import json import math import sys from datetime import datetime, timezone from pathlib import Path from typing import Any import pyarrow as pa import pyarrow.parquet as pq REPO = Path(__file__).resolve().parents[2] V2_SCRIPT = REPO / "4-Infrastructure" / "shim" / "enwiki9_logogram_xml_dict_probe.py" OUT_DIR = REPO / "shared-data" / "data" / "parquet_logogram_efficiency" PAYLOAD_JSON = OUT_DIR / "parquet_logogram_efficiency.json" SUMMARY = OUT_DIR / "parquet_logogram_efficiency.md" RECEIPT = OUT_DIR / "parquet_logogram_efficiency_receipt.json" TIDDLER = ( REPO / "6-Documentation" / "tiddlywiki-local" / "wiki" / "tiddlers" / "Parquet Logogram Efficiency.tid" ) PROTOCOL_ID = "PQLOG1" SLICE_RECEIPT_ROOT_BYTES = 32 PROTOCOL_ID_BYTES = len(PROTOCOL_ID.encode("ascii")) SCHEMA_HASH_BYTES = 32 ROW_COUNT_BYTES = 8 DEFAULT_INPUTS = [ REPO / "3-Mathematical-Models" / "equations_parquet_tagged" / "mass_equations_unified.parquet", REPO / "3-Mathematical-Models" / "equations_parquet_tagged" / "equations_unified_9pattern.parquet", REPO / "shared-data" / "data" / "connectomes" / "openworm_parquet" / "summary.parquet", REPO / "shared-data" / "data" / "datasets" / "mathlib4_complete.parquet", ] def load_v2_module() -> Any: spec = importlib.util.spec_from_file_location("enwiki9_logogram_xml_dict_probe", V2_SCRIPT) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load logogram script: {V2_SCRIPT}") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module V2 = load_v2_module() def stable_json(obj: Any) -> str: return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def hash_obj(obj: Any) -> str: return sha256_bytes(stable_json(obj).encode("utf-8")) def file_hash(path: Path) -> str: return sha256_bytes(path.read_bytes()) def rel(path: Path) -> str: try: return str(path.relative_to(REPO)) except ValueError: return str(path) def clean_value(value: Any) -> Any: if isinstance(value, float): if math.isnan(value) or math.isinf(value): return None return value if isinstance(value, bytes): return {"__bytes_hex__": value.hex()} if isinstance(value, dict): return {str(k): clean_value(v) for k, v in sorted(value.items(), key=lambda item: str(item[0]))} if isinstance(value, (list, tuple)): return [clean_value(item) for item in value] return value def dictionary_payload() -> dict[str, list[str]]: return { "fixed": [tag.hex() for tag in V2.FIXED_TAGS], "pair": [tag.hex() for tag in V2.PAIR_TAGS], "attr": [tag.hex() for tag in V2.ATTR_TAGS], "motif": [tag.hex() for tag in V2.MOTIFS], } def encode_packet(data: bytes, name: str) -> dict[str, Any]: core, atoms = V2.encode(data) decoded = V2.decode_core(core) core_path = OUT_DIR / f"{name}.wlg2" core_path.write_bytes(core) atom_counts = {kind: sum(1 for atom in atoms if atom.kind == kind) for kind in sorted({atom.kind for atom in atoms})} packet_bytes = len(core) + SLICE_RECEIPT_ROOT_BYTES + PROTOCOL_ID_BYTES + SCHEMA_HASH_BYTES + ROW_COUNT_BYTES return { "core": rel(core_path), "core_bytes": len(core), "core_sha256": sha256_bytes(core), "packet_bytes": packet_bytes, "exact_replay": decoded == data, "atom_count": len(atoms), "atom_counts": atom_counts, } def exact_replay_pass(item: dict[str, Any]) -> bool: return bool( item["species_logogram"]["exact_replay"] and item["schema_hash_recomputes"] and item["row_count_matches"] and item["residual_bounded"] ) def route_for_column(name: str, field_type: str, distinct_count: int, row_count: int, null_count: int) -> str: lower = name.lower() if any(token in lower for token in ["hash", "sha", "id", "doi", "source", "extracted", "timestamp"]): return "ram_trace" if row_count and distinct_count <= max(8, row_count // 8): return "prediction_cache" if "bool" in field_type or distinct_count <= 2: return "prediction_cache" if null_count == row_count: return "prediction_cache" return "parquet_native" def column_sidecar(table: pa.Table, source: str, schema_hash: str, fields: list[dict[str, Any]]) -> tuple[bytes, dict[str, Any]]: row_count = table.num_rows columns: list[dict[str, Any]] = [] for field in fields: values = [clean_value(value) for value in table[field["name"]].to_pylist()] encoded_values = [stable_json(value) for value in values] distinct_count = len(set(encoded_values)) null_count = sum(1 for value in values if value is None) route = route_for_column(field["name"], field["type"], distinct_count, row_count, null_count) columns.append( { "name": field["name"], "type": field["type"], "nullable": field["nullable"], "distinct_count": distinct_count, "null_count": null_count, "route": route, } ) plan = { "prediction_cache_columns": [item["name"] for item in columns if item["route"] == "prediction_cache"], "ram_trace_columns": [item["name"] for item in columns if item["route"] == "ram_trace"], "parquet_native_columns": [item["name"] for item in columns if item["route"] == "parquet_native"], } sidecar = { "protocol": "PQLOG-HYBRID-SIDECAR-v1", "source": source, "schema_hash": schema_hash, "row_count": row_count, "columns": columns, "pathfinding_plan": plan, "exact_replay_gate": [ "parquet_sample_hash_recomputes", "schema_hash_recomputes", "row_count_matches", "sidecar_decode_replays", ], } return stable_json(sidecar).encode("utf-8"), sidecar def sample_table(path: Path, row_limit: int) -> pa.Table: parquet_file = pq.ParquetFile(path) table = parquet_file.read_row_group(0) return table.slice(0, min(row_limit, table.num_rows)) def write_sample_parquet(table: pa.Table, out_path: Path) -> str: for compression in ["zstd", "snappy", None]: try: pq.write_table(table, out_path, compression=compression) return str(compression or "none") except Exception: continue raise RuntimeError(f"could not write sample parquet: {out_path}") def schema_record(path: Path, parquet_file: pq.ParquetFile, table: pa.Table) -> dict[str, Any]: fields = [ {"name": field.name, "type": str(field.type), "nullable": field.nullable} for field in table.schema ] return { "source_path": rel(path), "source_bytes": path.stat().st_size, "source_sha256": file_hash(path), "source_rows": parquet_file.metadata.num_rows, "source_row_groups": parquet_file.metadata.num_row_groups, "source_columns": parquet_file.metadata.num_columns, "sample_rows": table.num_rows, "sample_columns": table.num_columns, "fields": fields, } def canonical_forms(table: pa.Table) -> tuple[bytes, bytes, list[str], list[dict[str, Any]]]: columns = table.column_names rows = [{key: clean_value(value) for key, value in row.items()} for row in table.to_pylist()] object_jsonl = b"".join((stable_json(row) + "\n").encode("utf-8") for row in rows) species_payload = { "schema": columns, "row_count": len(rows), "rows": [[clean_value(row.get(column)) for column in columns] for row in rows], } species_bytes = stable_json(species_payload).encode("utf-8") return object_jsonl, species_bytes, columns, rows def run_source(path: Path, row_limit: int) -> dict[str, Any]: parquet_file = pq.ParquetFile(path) table = sample_table(path, row_limit) safe_name = path.stem.replace(".", "_").replace("-", "_") sample_path = OUT_DIR / f"{safe_name}.sample.parquet" compression = write_sample_parquet(table, sample_path) object_bytes, species_bytes, columns, rows = canonical_forms(table) object_path = OUT_DIR / f"{safe_name}.canonical_rows.jsonl" species_path = OUT_DIR / f"{safe_name}.logogram_species_payload.json" object_path.write_bytes(object_bytes) species_path.write_bytes(species_bytes) object_packet = encode_packet(object_bytes, f"{safe_name}.object") species_packet = encode_packet(species_bytes, f"{safe_name}.species") schema = schema_record(path, parquet_file, table) schema_hash = hash_obj({"columns": columns, "fields": schema["fields"]}) sidecar_bytes, sidecar = column_sidecar(table, rel(path), schema_hash, schema["fields"]) sidecar_path = OUT_DIR / f"{safe_name}.hybrid_sidecar.json" sidecar_path.write_bytes(sidecar_bytes) sidecar_packet = encode_packet(sidecar_bytes, f"{safe_name}.hybrid_sidecar") sample_parquet_bytes = sample_path.stat().st_size species_global_bytes = species_packet["packet_bytes"] object_global_bytes = object_packet["packet_bytes"] hybrid_bytes = sample_parquet_bytes + sidecar_packet["packet_bytes"] gate_pass = ( species_packet["exact_replay"] and schema_hash == hash_obj({"columns": columns, "fields": schema["fields"]}) and len(rows) == table.num_rows and sidecar_packet["exact_replay"] ) if gate_pass: gate = "PASS_EXACT_REPLAY" else: gate = "FAIL_EXACT_REPLAY" delta_vs_parquet = sample_parquet_bytes - species_global_bytes delta_vs_object_canonical = len(object_bytes) - species_global_bytes species_payload_gain = len(object_bytes) - len(species_bytes) hybrid_delta_vs_parquet = sample_parquet_bytes - hybrid_bytes hybrid_materialization_avoidance = len(object_bytes) - hybrid_bytes return { "name": safe_name, "source": rel(path), "schema": schema, "schema_hash": schema_hash, "sample_parquet": rel(sample_path), "sample_parquet_bytes": sample_parquet_bytes, "sample_parquet_sha256": file_hash(sample_path), "sample_parquet_compression": compression, "canonical_object_rows": rel(object_path), "canonical_object_bytes": len(object_bytes), "canonical_object_sha256": sha256_bytes(object_bytes), "logogram_species_payload": rel(species_path), "logogram_species_payload_bytes": len(species_bytes), "logogram_species_payload_sha256": sha256_bytes(species_bytes), "object_logogram": object_packet, "species_logogram": species_packet, "hybrid_sidecar": { "path": rel(sidecar_path), "payload_bytes": len(sidecar_bytes), "payload_sha256": sha256_bytes(sidecar_bytes), "logogram": sidecar_packet, "route_counts": { route: sum(1 for item in sidecar["columns"] if item["route"] == route) for route in ["prediction_cache", "ram_trace", "parquet_native"] }, "pathfinding_plan": sidecar["pathfinding_plan"], }, "row_count_matches": len(rows) == table.num_rows, "schema_hash_recomputes": schema_hash == hash_obj({"columns": columns, "fields": schema["fields"]}), "residual_bounded": species_packet["exact_replay"] and sidecar_packet["exact_replay"], "exact_replay_gate": gate, "measured": { "delta_vs_sample_parquet_bytes": delta_vs_parquet, "gain_vs_sample_parquet_ratio": round(delta_vs_parquet / sample_parquet_bytes, 6) if sample_parquet_bytes else None, "delta_vs_object_canonical_bytes": delta_vs_object_canonical, "gain_vs_object_canonical_ratio": round(delta_vs_object_canonical / len(object_bytes), 6) if object_bytes else None, "schema_key_reuse_payload_gain_bytes": species_payload_gain, "schema_key_reuse_payload_gain_ratio": round(species_payload_gain / len(object_bytes), 6) if object_bytes else None, "hybrid_sidecar_overhead_bytes": sidecar_packet["packet_bytes"], "hybrid_total_bytes": hybrid_bytes, "hybrid_delta_vs_sample_parquet_bytes": hybrid_delta_vs_parquet, "hybrid_overhead_vs_sample_parquet_ratio": round(sidecar_packet["packet_bytes"] / sample_parquet_bytes, 6) if sample_parquet_bytes else None, "hybrid_materialization_avoidance_bytes": hybrid_materialization_avoidance, "hybrid_materialization_avoidance_ratio": round(hybrid_materialization_avoidance / len(object_bytes), 6) if object_bytes else None, }, "decision": "HOLD_PARQUET_LOGOGRAM_FIXTURE", } def aggregate(sources: list[dict[str, Any]], dictionary_bytes: int) -> dict[str, Any]: sample_parquet = sum(item["sample_parquet_bytes"] for item in sources) object_bytes = sum(item["canonical_object_bytes"] for item in sources) species_payload = sum(item["logogram_species_payload_bytes"] for item in sources) species_packet = sum(item["species_logogram"]["packet_bytes"] for item in sources) object_packet = sum(item["object_logogram"]["packet_bytes"] for item in sources) hybrid_sidecars = sum(item["hybrid_sidecar"]["logogram"]["packet_bytes"] for item in sources) global_species = species_packet + dictionary_bytes hybrid_total = sample_parquet + hybrid_sidecars + dictionary_bytes delta_parquet = sample_parquet - global_species delta_object = object_bytes - global_species payload_gain = object_bytes - species_payload hybrid_materialization = object_bytes - hybrid_total return { "source_count": len(sources), "all_exact_replay": bool(sources) and all(exact_replay_pass(item) for item in sources), "all_schema_hashes_recompute": bool(sources) and all(item["schema_hash_recomputes"] for item in sources), "all_row_counts_match": bool(sources) and all(item["row_count_matches"] for item in sources), "sample_parquet_bytes": sample_parquet, "canonical_object_bytes": object_bytes, "logogram_species_payload_bytes": species_payload, "object_logogram_packet_bytes": object_packet, "species_logogram_packet_bytes": species_packet, "hybrid_sidecar_packet_bytes": hybrid_sidecars, "dictionary_bytes": dictionary_bytes, "species_global_bytes": global_species, "hybrid_total_bytes": hybrid_total, "delta_vs_sample_parquet_bytes": delta_parquet, "gain_vs_sample_parquet_ratio": round(delta_parquet / sample_parquet, 6) if sample_parquet else None, "delta_vs_object_canonical_bytes": delta_object, "gain_vs_object_canonical_ratio": round(delta_object / object_bytes, 6) if object_bytes else None, "schema_key_reuse_payload_gain_bytes": payload_gain, "schema_key_reuse_payload_gain_ratio": round(payload_gain / object_bytes, 6) if object_bytes else None, "hybrid_delta_vs_sample_parquet_bytes": sample_parquet - hybrid_total, "hybrid_overhead_vs_sample_parquet_ratio": round((hybrid_sidecars + dictionary_bytes) / sample_parquet, 6) if sample_parquet else None, "hybrid_materialization_avoidance_bytes": hybrid_materialization, "hybrid_materialization_avoidance_ratio": round(hybrid_materialization / object_bytes, 6) if object_bytes else None, "hybrid_route_counts": { route: sum(item["hybrid_sidecar"]["route_counts"][route] for item in sources) for route in ["prediction_cache", "ram_trace", "parquet_native"] }, } def build_payload(paths: list[Path], row_limit: int) -> dict[str, Any]: OUT_DIR.mkdir(parents=True, exist_ok=True) existing = [path for path in paths if path.exists()] missing = [path for path in paths if not path.exists()] sources = [run_source(path, row_limit) for path in existing] dictionary_json = stable_json(dictionary_payload()).encode("utf-8") dictionary_bytes = len(dictionary_json) dictionary_hash = sha256_bytes(dictionary_json) payload = { "schema": "parquet_logogram_efficiency_probe_v1", "protocol_id": PROTOCOL_ID, "claim_boundary": ( "Parquet-to-logogram byte accounting only. This samples local Parquet files, " "canonicalizes rows, transcodes them into the existing WLG2 logogram core, " "and records exact replay plus byte deltas. It does not claim global " "compression, canonical enwik9 performance, or replacement of Parquet." ), "inputs": { "requested_paths": [rel(path) for path in paths], "existing_paths": [rel(path) for path in existing], "missing_paths": [rel(path) for path in missing], "row_limit_per_source": row_limit, }, "accounting_equations": { "parquet_logogram_transcode_efficiency": "E_pq_log=(bytes_sample_parquet-bytes_logogram_species_global)/bytes_sample_parquet", "columnar_lineage_logogram_gain": "G_lineage=(bytes_object_canonical-bytes_species_payload)/bytes_object_canonical", "parquet_logogram_exact_replay_gate": "G_pq_log=1[canonical_rows_decode]*1[schema_hash_recomputes]*1[row_count_matches]*1[residual_bounded]", "hybrid_parquet_logogram_sidecar_cost": "C_hybrid=(bytes_sidecar_packet+bytes_dictionary)/bytes_sample_parquet", "hybrid_materialization_avoidance_gain": "G_hybrid=(bytes_object_canonical-(bytes_sample_parquet+bytes_sidecar_packet+bytes_dictionary))/bytes_object_canonical", "hybrid_cache_trace_route_selector": "route(column)=argmax(U_prediction_cache,U_ram_trace,U_parquet_native)", }, "dictionary": { "source": rel(V2_SCRIPT), "bytes": dictionary_bytes, "sha256": dictionary_hash, }, "sources": sources, "aggregates": aggregate(sources, dictionary_bytes), "decision": ( "ADMIT_PARQUET_LOGOGRAM_EFFICIENCY_AS_HOLD_FIXTURE" if sources else "HOLD_NO_PARQUET_INPUTS" ), } payload["finding"] = ( "The useful accounting split is between Parquet bytes, full object-row canonical bytes, " "and logogram species-code bytes. Schema/key reuse can reduce the canonical row surface, " "but WLG2 packet plus dictionary costs must still beat Parquet before any compression gain is claimed. " "The hybrid path keeps Parquet as substrate and uses logograms as a sidecar for schema lineage, " "prediction-cache columns, RAM-trace columns, and exact replay gates." ) payload["payload_hash"] = hash_obj({k: v for k, v in payload.items() if k != "payload_hash"}) return payload def build_receipt(payload: dict[str, Any]) -> dict[str, Any]: receipt = { "schema": "parquet_logogram_efficiency_receipt_v1", "generated_at_utc": datetime.now(timezone.utc).isoformat(), "timestamp_role": "metadata_only", "generated_at_utc_included_in_receipt_hash": False, "payload_hash": payload["payload_hash"], "aggregates": payload["aggregates"], "source_hashes": {item["source"]: item["schema"]["source_sha256"] for item in payload["sources"]}, "sample_hashes": {item["source"]: item["sample_parquet_sha256"] for item in payload["sources"]}, "dictionary_sha256": payload["dictionary"]["sha256"], "decision": payload["decision"], "claim_boundary": payload["claim_boundary"], } receipt["receipt_hash"] = sha256_bytes( stable_json({k: v for k, v in receipt.items() if k not in {"receipt_hash", "generated_at_utc"}}).encode("utf-8") ) return receipt def write_summary(payload: dict[str, Any], receipt: dict[str, Any]) -> None: agg = payload["aggregates"] lines = [ "# Parquet Logogram Efficiency", "", f"Decision: `{payload['decision']}` ", f"Receipt hash: `{receipt['receipt_hash']}`", "", payload["claim_boundary"], "", "## Finding", "", payload["finding"], "", "## Aggregate", "", f"- Sources: `{agg['source_count']}`", f"- Exact replay: `{agg['all_exact_replay']}`", f"- Sample Parquet bytes: `{agg['sample_parquet_bytes']}`", f"- Object canonical bytes: `{agg['canonical_object_bytes']}`", f"- Logogram species global bytes: `{agg['species_global_bytes']}`", f"- Delta vs sample Parquet: `{agg['delta_vs_sample_parquet_bytes']}` ({agg['gain_vs_sample_parquet_ratio']})", f"- Delta vs object canonical: `{agg['delta_vs_object_canonical_bytes']}` ({agg['gain_vs_object_canonical_ratio']})", f"- Schema/key reuse payload gain: `{agg['schema_key_reuse_payload_gain_bytes']}` ({agg['schema_key_reuse_payload_gain_ratio']})", f"- Hybrid sidecar packet bytes: `{agg['hybrid_sidecar_packet_bytes']}`", f"- Hybrid total bytes: `{agg['hybrid_total_bytes']}`", f"- Hybrid overhead vs sample Parquet: `{agg['hybrid_delta_vs_sample_parquet_bytes']}` ({agg['hybrid_overhead_vs_sample_parquet_ratio']})", f"- Hybrid materialization avoidance: `{agg['hybrid_materialization_avoidance_bytes']}` ({agg['hybrid_materialization_avoidance_ratio']})", f"- Hybrid route counts: `{agg['hybrid_route_counts']}`", "", "## Equations", "", ] for name, equation in payload["accounting_equations"].items(): lines.append(f"- `{name}`: `{equation}`") lines.extend( [ "", "## Per Source", "", "| Source | Rows | Sample parquet | Object canonical | Species payload | Species packet | Delta vs parquet | Delta vs object | Gate |", "|---|---:|---:|---:|---:|---:|---:|---:|---|", ] ) for item in payload["sources"]: measured = item["measured"] lines.append( f"| `{item['source']}` | {item['schema']['sample_rows']} | {item['sample_parquet_bytes']} | " f"{item['canonical_object_bytes']} | {item['logogram_species_payload_bytes']} | " f"{item['species_logogram']['packet_bytes']} | {measured['delta_vs_sample_parquet_bytes']} | " f"{measured['delta_vs_object_canonical_bytes']} | {item['exact_replay_gate']} |" ) lines.extend(["", "## Receipt", "", f"`{rel(RECEIPT)}`"]) SUMMARY.write_text("\n".join(lines) + "\n", encoding="utf-8") def write_tiddler(payload: dict[str, Any], receipt: dict[str, Any]) -> None: agg = payload["aggregates"] lines = [ "title: Parquet Logogram Efficiency", "tags: Parquet Logogram Efficiency HOLD Receipt", "type: text/vnd.tiddlywiki", "", "! Parquet Logogram Efficiency", "", f"Decision: `{payload['decision']}`", "", f"Receipt hash: `{receipt['receipt_hash']}`", "", "!! Accounting", "", f"* `E_pq_log`: `{agg['gain_vs_sample_parquet_ratio']}`", f"* `G_lineage`: `{agg['schema_key_reuse_payload_gain_ratio']}`", f"* `C_hybrid`: `{agg['hybrid_overhead_vs_sample_parquet_ratio']}`", f"* `G_hybrid`: `{agg['hybrid_materialization_avoidance_ratio']}`", f"* Hybrid route counts: `{agg['hybrid_route_counts']}`", f"* Exact replay: `{agg['all_exact_replay']}`", "", "!! Equations", "", ] for name, equation in payload["accounting_equations"].items(): lines.append(f"* `{name}`: `{equation}`") lines.extend( [ "", "!! Boundary", "", payload["claim_boundary"], "", "!! Links", "", "* [[Combined Approach Equation Surface]]", "* [[TranscriptFormer Evolutionary Prior]]", "", f"Receipt: `{rel(RECEIPT)}`", ] ) TIDDLER.write_text("\n".join(lines) + "\n", encoding="utf-8") def main() -> None: parser = argparse.ArgumentParser(description="Account for Parquet to logogram transcode efficiency.") parser.add_argument("paths", nargs="*", type=Path) parser.add_argument("--row-limit", type=int, default=256) args = parser.parse_args() paths = args.paths or DEFAULT_INPUTS OUT_DIR.mkdir(parents=True, exist_ok=True) TIDDLER.parent.mkdir(parents=True, exist_ok=True) payload = build_payload(paths, args.row_limit) receipt = build_receipt(payload) PAYLOAD_JSON.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") RECEIPT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") write_summary(payload, receipt) write_tiddler(payload, receipt) print(json.dumps(receipt, indent=2, sort_keys=True)) if __name__ == "__main__": main()