diff --git a/4-Infrastructure/shim/blockchain_alternative_source_plan.py b/4-Infrastructure/shim/blockchain_alternative_source_plan.py new file mode 100644 index 00000000..87b258a5 --- /dev/null +++ b/4-Infrastructure/shim/blockchain_alternative_source_plan.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Create a receipt for alternative blockchain corpus sources. + +This is an admission/planning receipt, not a data-transfer receipt. It records +which public/research sources can replace throttled Blockchair dump pulls and +emits concrete command templates for the tools that can fetch them. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +DEFAULT_DESTINATION = "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10" + + +BIGQUERY_CRYPTO_DATASETS = { + "bitcoin_cash": { + "chain": "bitcoin-cash", + "dataset": "bigquery-public-data.crypto_bitcoin_cash", + "blockchair_status": "PARTIAL_HTTP_402", + }, + "dash": { + "chain": "dash", + "dataset": "bigquery-public-data.crypto_dash", + "blockchair_status": "PARTIAL_HTTP_402", + }, + "dogecoin": { + "chain": "dogecoin", + "dataset": "bigquery-public-data.crypto_dogecoin", + "blockchair_status": "PARTIAL_HTTP_402", + }, + "litecoin": { + "chain": "litecoin", + "dataset": "bigquery-public-data.crypto_litecoin", + "blockchair_status": "PARTIAL_HTTP_402", + }, + "zcash": { + "chain": "zcash", + "dataset": "bigquery-public-data.crypto_zcash", + "blockchair_status": "PARTIAL_HTTP_402", + }, + "ethereum_classic": { + "chain": "ethereum-classic", + "dataset": "bigquery-public-data.crypto_ethereum_classic", + "blockchair_status": "NOT_ATTEMPTED", + }, +} + + +BITCOIN_ETL_CHAINS = [ + "bitcoin", + "bitcoin_cash", + "bitcoin_gold", + "dogecoin", + "litecoin", + "dash", + "zcash", +] + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def stable_hash(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def command_status(name: str) -> dict[str, Any]: + path = shutil.which(name) + version = None + if path: + try: + version_command = [name, "version"] if name == "bq" else [name, "--version"] + proc = subprocess.run(version_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=15) + version = proc.stdout.splitlines()[0] if proc.stdout else None + except Exception as exc: # noqa: BLE001 - receipt diagnostic only. + version = f"VERSION_CHECK_FAILED: {exc}" + return {"available": bool(path), "path": path, "version": version} + + +def python_module_status(module: str) -> dict[str, Any]: + try: + __import__(module) + return {"available": True, "module": module} + except Exception as exc: # noqa: BLE001 - receipt diagnostic only. + return {"available": False, "module": module, "error": type(exc).__name__} + + +def bq_extract_commands(destination: str) -> list[dict[str, Any]]: + commands = [] + for source_id, meta in BIGQUERY_CRYPTO_DATASETS.items(): + dataset = meta["dataset"] + chain = meta["chain"] + remote = f"{destination.rstrip('/')}/public-datasets/google-bigquery/{chain}" + commands.append( + { + "source_id": source_id, + "chain": chain, + "dataset": dataset, + "tables_to_try_first": ["blocks", "transactions", "inputs", "outputs"], + "local_export_pattern": f"shared-data/data/blockchain_corpus/bigquery_exports/{chain}/{{table}}/*.parquet", + "drive_destination": remote, + "commands": [ + f"bq ls {dataset}", + ( + "bq extract --destination_format=PARQUET " + f"'{dataset}.{{table}}' " + f"'gs:///research-stack/blockchain-corpus/{chain}/{{table}}/*.parquet'" + ), + ( + "gcloud storage cp --recursive " + f"'gs:///research-stack/blockchain-corpus/{chain}/' " + f"'shared-data/data/blockchain_corpus/bigquery_exports/{chain}/'" + ), + ( + "rclone copy " + f"'shared-data/data/blockchain_corpus/bigquery_exports/{chain}/' " + f"'{remote}/'" + ), + ], + } + ) + return commands + + +def bitcoin_etl_commands(destination: str) -> list[dict[str, Any]]: + commands = [] + for chain in BITCOIN_ETL_CHAINS: + remote = f"{destination.rstrip('/')}/node-etl/bitcoin-etl/{chain}" + commands.append( + { + "chain": chain, + "requires": ["running_full_node_or_rpc_snapshot", "bitcoin-etl"], + "commands": [ + "python3 -m pip install --user bitcoin-etl", + ( + "bitcoinetl export_blocks_and_transactions " + "--start-block 0 --end-block " + f"--provider-uri http://:@127.0.0.1: --chain {chain} " + f"--blocks-output shared-data/data/blockchain_corpus/node_etl/{chain}/blocks.jsonl " + f"--transactions-output shared-data/data/blockchain_corpus/node_etl/{chain}/transactions.jsonl" + ), + f"rclone copy 'shared-data/data/blockchain_corpus/node_etl/{chain}/' '{remote}/'", + ], + } + ) + return commands + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", default=DEFAULT_DESTINATION) + parser.add_argument("--out", default="shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json") + args = parser.parse_args() + + receipt = { + "schema": "blockchain_alternative_source_plan_v0", + "created_utc": utc_now(), + "decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER", + "claim_boundary": ( + "This receipt identifies alternative public/research sources and tool readiness. " + "It does not prove data was exported from BigQuery, does not bypass source terms, " + "and does not claim decoded chain semantics." + ), + "destination": args.destination, + "source_candidates": [ + { + "name": "Google BigQuery Public Cryptocurrency Datasets", + "decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS", + "chains": list(BIGQUERY_CRYPTO_DATASETS.values()), + "why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash.", + "claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.", + "python_client_ready": python_module_status("google.cloud.bigquery").get("available"), + "source_urls": [ + "https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them", + "https://www.cloudskillsboost.google/focuses/8486?parent=catalog", + ], + }, + { + "name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots", + "decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED", + "chains": BITCOIN_ETL_CHAINS, + "why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps.", + "claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.", + "source_urls": ["https://github.com/blockchain-etl/bitcoin-etl"], + }, + { + "name": "Bitquery Cloud Data Dumps", + "decision": "EVALUATE_LICENSE_AND_COVERAGE", + "chains": ["bitcoin"], + "why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane.", + "claim_boundary": "Coverage and licensing must be checked before mirroring.", + "source_urls": ["https://docs.bitquery.io/docs/cloud/bitcoin/"], + }, + { + "name": "Kaggle/Hugging Face sampled datasets", + "decision": "HOLD_SAMPLE_ONLY", + "chains": ["bitcoin-cash", "dash", "dogecoin", "litecoin", "zcash"], + "why": "Useful for smoke tests and model fixtures, not complete chain mirrors.", + "claim_boundary": "Do not use for full-corpus claims.", + "source_urls": [ + "https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset", + "https://www.kaggle.com/datasets/amritpal333/crypto-mining-data", + ], + }, + ], + "local_tool_readiness": { + "commands": { + name: command_status(name) + for name in ["rclone", "bq", "gcloud", "gsutil", "kaggle", "duckdb"] + }, + "python_modules": { + module: python_module_status(module) + for module in ["requests", "boto3", "pyarrow", "pandas", "google.cloud.bigquery", "duckdb"] + }, + }, + "bigquery_export_commands": bq_extract_commands(args.destination), + "node_etl_commands": bitcoin_etl_commands(args.destination), + "next_gate": { + "decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET", + "required": [ + "Authenticate with a Google Cloud account allowed to query BigQuery public datasets.", + "Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.", + "Provide or create a GCS staging bucket for BigQuery extract jobs.", + "Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.", + "Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts.", + ], + }, + } + receipt["receipt_hash"] = stable_hash(receipt) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"decision": receipt["decision"], "out": str(out), "receipt_hash": receipt["receipt_hash"]})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/4-Infrastructure/shim/blockchain_bigquery_grabber.py b/4-Infrastructure/shim/blockchain_bigquery_grabber.py new file mode 100644 index 00000000..f6b6dd91 --- /dev/null +++ b/4-Infrastructure/shim/blockchain_bigquery_grabber.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""BigQuery grabber command surface for public cryptocurrency datasets. + +The script intentionally defaults to dry-run command emission. Actual BigQuery +exports require Google Cloud authentication and a GCS staging bucket. This keeps +the corpus lane receipt-bearing without embedding credentials or pretending +that a public dataset can be exported anonymously. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +DATASETS = { + "bitcoin-cash": "bigquery-public-data.crypto_bitcoin_cash", + "dash": "bigquery-public-data.crypto_dash", + "dogecoin": "bigquery-public-data.crypto_dogecoin", + "ethereum-classic": "bigquery-public-data.crypto_ethereum_classic", + "litecoin": "bigquery-public-data.crypto_litecoin", + "zcash": "bigquery-public-data.crypto_zcash", +} + +DEFAULT_TABLES = ["blocks", "transactions", "inputs", "outputs"] + + +def run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + + +def build_plan(chains: list[str], tables: list[str], gcs_bucket: str, drive_destination: str) -> dict[str, Any]: + entries = [] + for chain in chains: + dataset = DATASETS[chain] + chain_slug = chain + local_root = f"shared-data/data/blockchain_corpus/bigquery_exports/{chain_slug}" + remote_root = f"{drive_destination.rstrip('/')}/public-datasets/google-bigquery/{chain_slug}" + commands = [f"bq ls {dataset}"] + for table in tables: + commands.append( + "bq extract --destination_format=PARQUET " + f"'{dataset}.{table}' " + f"'gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/{table}/*.parquet'" + ) + commands.extend( + [ + ( + "gcloud storage cp --recursive " + f"'gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/' " + f"'{local_root}/'" + ), + f"rclone copy '{local_root}/' '{remote_root}/'", + ] + ) + entries.append( + { + "chain": chain, + "dataset": dataset, + "tables": tables, + "gcs_prefix": f"gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/", + "local_root": local_root, + "drive_destination": remote_root, + "commands": commands, + } + ) + return { + "schema": "blockchain_bigquery_grabber_plan_v0", + "claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.", + "tool_status": { + "bq": shutil.which("bq"), + "gcloud": shutil.which("gcloud"), + "gsutil": shutil.which("gsutil"), + "rclone": shutil.which("rclone"), + "python": sys.executable, + "python_bigquery_client": python_module_available("google.cloud.bigquery"), + "python_google_auth": python_module_available("google.auth"), + }, + "entries": entries, + } + + +def python_module_available(module: str) -> bool: + try: + __import__(module) + return True + except Exception: + return False + + +def list_with_python_client(entries: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]: + try: + from google.api_core.exceptions import GoogleAPIError # type: ignore + from google.auth.exceptions import DefaultCredentialsError # type: ignore + from google.cloud import bigquery # type: ignore + except Exception as exc: # noqa: BLE001 - receipt exact missing module. + return "HOLD_BIGQUERY_PYTHON_CLIENT_MISSING", [ + {"error": type(exc).__name__, "message": str(exc), "python": sys.executable} + ] + + try: + client = bigquery.Client() + except DefaultCredentialsError as exc: + return "HOLD_GOOGLE_ADC_MISSING", [ + { + "error": type(exc).__name__, + "message": str(exc).splitlines()[0], + "python": sys.executable, + "next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.", + } + ] + except Exception as exc: # noqa: BLE001 - receipt unexpected auth/client failures. + return "HOLD_BIGQUERY_CLIENT_INIT_FAILED", [ + {"error": type(exc).__name__, "message": str(exc), "python": sys.executable} + ] + + results = [] + ok = True + for entry in entries: + try: + tables = list(client.list_tables(entry["dataset"])) + results.append( + { + "chain": entry["chain"], + "dataset": entry["dataset"], + "tables": [{"table_id": table.table_id, "full_table_id": table.full_table_id} for table in tables], + } + ) + except GoogleAPIError as exc: + ok = False + results.append({"chain": entry["chain"], "dataset": entry["dataset"], "error": type(exc).__name__, "message": str(exc)}) + except Exception as exc: # noqa: BLE001 - receipt per-dataset failure. + ok = False + results.append({"chain": entry["chain"], "dataset": entry["dataset"], "error": type(exc).__name__, "message": str(exc)}) + return ("ADMIT_BIGQUERY_PYTHON_LIST_CHECK" if ok else "HOLD_BIGQUERY_PYTHON_LIST_CHECK_FAILED"), results + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--chains", nargs="+", default=list(DATASETS), choices=sorted(DATASETS)) + parser.add_argument("--tables", nargs="+", default=DEFAULT_TABLES) + parser.add_argument("--gcs-bucket", default="") + parser.add_argument( + "--drive-destination", + default="Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10", + ) + parser.add_argument("--out", default="shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json") + parser.add_argument("--execute-list-only", action="store_true", help="Run only bq ls commands to verify table access.") + parser.add_argument("--execute-python-list-only", action="store_true", help="Run table listing through google-cloud-bigquery.") + args = parser.parse_args() + + plan = build_plan(args.chains, args.tables, args.gcs_bucket, args.drive_destination) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if args.execute_list_only: + if not shutil.which("bq"): + print(json.dumps({"decision": "HOLD_BQ_CLI_MISSING", "out": str(out)})) + return 2 + results = [] + for entry in plan["entries"]: + proc = run(["bq", "ls", entry["dataset"]]) + results.append( + { + "chain": entry["chain"], + "dataset": entry["dataset"], + "returncode": proc.returncode, + "stdout": proc.stdout[-4000:], + "stderr": proc.stderr[-4000:], + } + ) + result_path = out.with_name(out.stem + "_list_results.json") + result_path.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8") + ok = all(item["returncode"] == 0 for item in results) + print(json.dumps({"decision": "ADMIT_BQ_LIST_CHECK" if ok else "HOLD_BQ_LIST_CHECK_FAILED", "out": str(out), "results": str(result_path)})) + return 0 if ok else 2 + + if args.execute_python_list_only: + decision, results = list_with_python_client(plan["entries"]) + result_path = out.with_name(out.stem + "_python_list_results.json") + result_path.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"decision": decision, "out": str(out), "results": str(result_path)})) + return 0 if decision.startswith("ADMIT") else 2 + + print(json.dumps({"decision": "ADMIT_BIGQUERY_GRABBER_DRY_RUN_PLAN", "out": str(out)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py b/4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py new file mode 100644 index 00000000..c671ca67 --- /dev/null +++ b/4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Emit a Layer-3 self-scanning plan for blockchain corpus receipts. + +The scheduler treats existing inventory and transfer receipts as the control +surface. It does not decode chain semantics or fetch more data. Its job is to +turn scan evidence into the next bounded scan actions. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REPO = Path(__file__).resolve().parents[2] +DEFAULT_ACCOUNT = REPO / "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" +DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json" + + +STORAGE_CHANNELS = { + "bitcoin": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion.", + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only.", + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness.", + }, + ], + "evm": [ + { + "channel": "calldata", + "scan_surface": "transaction input bytes", + "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow.", + }, + { + "channel": "event_logs", + "scan_surface": "topics and data fields", + "use_boundary": "Receipt emitted public events as storage-like data lanes.", + }, + { + "channel": "contract_bytecode", + "scan_surface": "creation/runtime bytecode", + "use_boundary": "Static payload/code-carrier diagnostic only.", + }, + { + "channel": "blob_or_da_payload", + "scan_surface": "data availability/blob lanes when present in source tables", + "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD.", + }, + ], + "memo": [ + { + "channel": "memo_or_message", + "scan_surface": "transaction memo/message/payload fields", + "use_boundary": "Public memo payload diagnostics only.", + } + ], +} + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def rel(path: str | Path) -> str: + p = Path(path) + try: + return str(p.relative_to(REPO)) + except ValueError: + return str(p) + + +def density_band(total_bytes: int, object_count: int) -> str: + if object_count <= 0: + return "empty" + bytes_per_object = total_bytes / object_count + if bytes_per_object >= 64 * 1024 * 1024: + return "heavy" + if bytes_per_object >= 8 * 1024 * 1024: + return "medium" + return "light" + + +def completion_state(item: dict[str, Any]) -> dict[str, Any]: + object_count = int(item.get("object_count") or 0) + remote_payload = int(item.get("remote_payload_count") or 0) + remote_parquet = int(item.get("remote_parquet_count") or 0) + completed = min(remote_payload, remote_parquet) + missing = max(object_count - completed, 0) + ratio = completed / object_count if object_count else 0.0 + return { + "completed_objects": completed, + "missing_objects": missing, + "remote_parquet_count": remote_parquet, + "remote_payload_count": remote_payload, + "completion_ratio": round(ratio, 9), + } + + +def scan_actions(item: dict[str, Any], state: dict[str, Any], band: str) -> list[dict[str, Any]]: + actions: list[dict[str, Any]] = [] + chain = item.get("chain") + table = item.get("table") + inventory = item.get("inventory") + + if state["missing_objects"]: + actions.append( + { + "action": "RETRY_MISSING_OBJECTS", + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py", + "inputs": {"inventory": inventory, "chain": chain, "table": table}, + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + } + ) + + actions.append( + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py", + "inputs": {"chain": chain, "table": table, "inventory": inventory}, + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + } + ) + + if band in {"medium", "heavy"}: + actions.append( + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py", + "inputs": {"chain": chain, "table": table, "sample_policy": "first_middle_last_partition"}, + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + } + ) + + actions.append( + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py", + "inputs": {"account_receipt": rel(DEFAULT_ACCOUNT)}, + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + } + ) + return actions + + +def chain_storage_family(chain: str | None) -> str: + if chain in {"ethereum", "ethereum-classic", "arbitrum", "base", "cronos"}: + return "evm" + if chain in {"bitcoin", "bitcoin-cash", "dash", "dogecoin", "litecoin", "zcash"}: + return "bitcoin" + if chain in {"xrp", "stellar", "ton", "aptos", "provenance"}: + return "memo" + return "unknown" + + +def storage_channel_actions(item: dict[str, Any], band: str) -> dict[str, Any]: + chain = item.get("chain") + family = chain_storage_family(chain) + channels = STORAGE_CHANNELS.get(family, []) + actions = [] + if channels: + actions.append( + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise.", + "inputs": { + "chain": chain, + "family": family, + "table": item.get("table"), + "inventory": item.get("inventory"), + "density_band": band, + }, + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + } + ) + return { + "family": family, + "channels": channels, + "actions": actions, + "claim_boundary": ( + "Storage-channel model is for detection, accounting, and receipt routing. " + "It does not recommend publishing payloads on-chain, bypassing moderation, " + "hiding data, or using public ledgers as private storage." + ), + } + + +def build_receipt(account_path: Path) -> dict[str, Any]: + account = json.loads(account_path.read_text(encoding="utf-8")) + entries = [] + for item in account.get("source_inventories", []): + total_bytes = int(item.get("total_listed_bytes") or 0) + object_count = int(item.get("object_count") or 0) + state = completion_state(item) + band = density_band(total_bytes, object_count) + entries.append( + { + "chain": item.get("chain"), + "table": item.get("table"), + "dataset": item.get("dataset"), + "inventory": item.get("inventory"), + "inventory_hash": item.get("inventory_hash"), + "object_count": object_count, + "total_listed_bytes": total_bytes, + "bytes_per_object": round(total_bytes / object_count, 3) if object_count else 0.0, + "density_band": band, + "storage_channel_model": storage_channel_actions(item, band), + "completion": state, + "scan_actions": scan_actions(item, state, band), + } + ) + + entries.sort(key=lambda row: (row["completion"]["missing_objects"] > 0, row["total_listed_bytes"]), reverse=True) + payload = { + "schema": "blockchain_l3_self_scan_scheduler_v0", + "created_utc": now_iso(), + "claim_boundary": ( + "Layer-3 scan scheduling receipt only. It uses existing inventory and transfer receipts " + "to choose next bounded probes. It does not decode chain semantics, claim compression gain, " + "claim market prediction, or fetch additional corpus bytes." + ), + "layer3_interpretation": { + "old_label": "L3 Bitstream", + "scanner_label": "L3 executable scan policy", + "core_rule": "scan_receipts_t -> route_policy_t_plus_1", + "feedback_equation": "P_{t+1}=L3(R_t,A_t,C_t); R_{t+1}=scan(P_{t+1})", + "storage_rule": "onchain_payload_surfaces_t -> storage_channel_receipts_t_plus_1", + }, + "global_storage_boundary": ( + "Public blockchains can store data, but this scheduler only models storage-bearing " + "surfaces for detection, provenance, and compression-route diagnostics. Bulk payload " + "reconstruction and payload publication remain outside this receipt." + ), + "account_receipt": rel(account_path), + "account_receipt_hash": account.get("receipt_hash"), + "source_count": len(entries), + "frontier": entries, + "decision": "ADMIT_L3_SELF_SCAN_POLICY", + } + payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"})) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--account", type=Path, default=DEFAULT_ACCOUNT) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + + receipt = build_receipt(args.account) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/4-Infrastructure/shim/nuvmap_chain_protocol_plan.py b/4-Infrastructure/shim/nuvmap_chain_protocol_plan.py new file mode 100644 index 00000000..e4dcdb2e --- /dev/null +++ b/4-Infrastructure/shim/nuvmap_chain_protocol_plan.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Create a protocol receipt for a dedicated NUVMAP chain. + +NUVMAP is treated here as a receipt chain for metaprobe/waveprobe outputs. This +is not a token, investment, mainnet, or compression-result claim. It specifies +what a block would carry so scanner state can be stored and replayed without +using third-party chains as an accidental storage layer. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REPO = Path(__file__).resolve().parents[2] +DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json" + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def rel(path: Path) -> str: + try: + return str(path.relative_to(REPO)) + except ValueError: + return str(path) + + +def build_receipt() -> dict[str, Any]: + payload: dict[str, Any] = { + "schema": "nuvmap_chain_protocol_plan_v0", + "created_utc": now_iso(), + "claim_boundary": ( + "Protocol design receipt only. This does not create a blockchain, token, market, " + "consensus network, compression result, or Hutter Prize claim. It defines a local/devnet " + "receipt substrate for metaprobe and waveprobe state." + ), + "purpose": { + "name": "NUVMAP", + "expanded_role": "Numerical Universe Vector Map receipt chain", + "core_use": "Store replayable probe-state commitments so scanning can route itself over time.", + "anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.", + }, + "chain_model": { + "deployment_stage": "LOCAL_DEVNET_ONLY", + "economic_model": "NO_TOKEN_NO_MARKET", + "consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set", + "finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional", + "data_availability": "on-chain summaries plus content-addressed off-chain sidecars", + }, + "block_shape": { + "header": { + "parent_hash": "sha256(previous_block)", + "height": "u64", + "created_utc": "iso8601", + "scanner_policy_hash": "sha256(l3_policy)", + "state_root": "merkle_root(receipt_state)", + "sidecar_manifest_root": "merkle_root(sidecar_hashes)", + }, + "body": { + "waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries", + "metaprobe_receipts": "bounded list of route/context/curriculum summaries", + "storage_channel_receipts": "detected on-chain storage surfaces from source chains", + "scan_actions": "next bounded scan plan emitted by L3 scheduler", + "negative_controls": "required controls before promotion of any route", + }, + "forbidden_body_fields": [ + "private keys", + "wallet credentials", + "bulk copyrighted payloads", + "payloads intended to hide from moderation or provenance", + "financial promotion metadata", + ], + }, + "probe_mapping": { + "metadata_blitter": { + "input": "fixed-size metadata words from chain/block/window records", + "output": "sortable route keys and packed metric words", + "storage": "blitter shader hash, parameter receipt, key/metric buffer hashes", + }, + "waveprobe": { + "input": "byte windows, block fields, density neighborhoods, eigenvalue spectra", + "output": "local signal vector plus residual/curvature summary", + "storage": "hash, vector summary, source window receipt, not raw bulk by default", + }, + "metaprobe": { + "input": "domain priors, route families, previous scan receipts", + "output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints", + "storage": "decision receipt, route vector, source receipt backlinks", + }, + "l3_scheduler": { + "input": "current receipt frontier", + "output": "next scan frontier", + "storage": "scanner policy hash and action list", + }, + }, + "radix_bin_model": { + "metaphor": "hyper_soliton_search", + "claim_boundary": "Radix bins are route basins, not semantic labels.", + "fields": { + "hash8": "phase seed / local identity packet", + "density7": "amplitude / byte pressure", + "delta7": "shock or torsion gradient", + "zero6": "void / low-information throat", + "flags4": "boundary condition", + }, + "stability_rule": "Promote only bins that persist across salts, windows, and negative controls.", + }, + "minimal_transaction_types": [ + { + "type": "BLITTER_BIN_RECEIPT", + "required_fields": ["shader_hash", "params_hash", "source_hash", "key_buffer_hash", "metric_buffer_hash", "decision"], + }, + { + "type": "WAVE_RECEIPT", + "required_fields": ["source_hash", "window_descriptor", "feature_vector_hash", "decision"], + }, + { + "type": "META_RECEIPT", + "required_fields": ["prior_id", "route_family", "evidence_hash", "decision"], + }, + { + "type": "STORAGE_SURFACE_RECEIPT", + "required_fields": ["source_chain", "channel", "carrier_descriptor", "payload_hash_or_null", "decision"], + }, + { + "type": "SCAN_POLICY_UPDATE", + "required_fields": ["previous_policy_hash", "new_policy_hash", "action_merkle_root", "decision"], + }, + { + "type": "NEGATIVE_CONTROL", + "required_fields": ["control_kind", "source_hash", "result_hash", "decision"], + }, + ], + "decision_law": { + "ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.", + "HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.", + "QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation.", + }, + "next_implementation_steps": [ + "Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.", + "Add a verifier that recomputes the block hash and merkle roots.", + "Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.", + "Run negative controls before any compression-route promotion.", + "Only after local fixture replay works, consider a small append-only local devnet.", + ], + "decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION", + } + payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"})) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + + receipt = build_receipt() + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py b/4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py new file mode 100644 index 00000000..c3ef92e3 --- /dev/null +++ b/4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Emit a bounded GPU-burst check plan for NUVMAP probe validation. + +The plan is intended for a short rented GPU session. It avoids wallets, mining, +public-chain writes, and long-lived services. The output is a receipt/checklist +for replaying local fixtures, checking CPU/GPU parity, and collecting bounded +waveprobe/metaprobe evidence. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REPO = Path(__file__).resolve().parents[2] +DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json" + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def rel(path: Path) -> str: + try: + return str(path.relative_to(REPO)) + except ValueError: + return str(path) + + +def build_receipt(max_hours: float) -> dict[str, Any]: + payload: dict[str, Any] = { + "schema": "nuvmap_gpu_burst_check_plan_v0", + "created_utc": now_iso(), + "claim_boundary": ( + "GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, " + "not a token launch, not a compression result, and not a hardware acceleration claim until " + "the listed receipts are produced." + ), + "timebox_hours": max_hours, + "forbidden_inputs": [ + "wallet private keys", + "exchange credentials", + "mainnet transaction signing material", + "unbounded copyrighted payload mirrors", + ], + "required_outputs": [ + "environment_receipt.json", + "cpu_gpu_parity_receipt.json", + "waveprobe_batch_receipt.json", + "metaprobe_batch_receipt.json", + "negative_controls_receipt.json", + "cost_and_shutdown_receipt.json", + ], + "burst_phases": [ + { + "phase": "P0_ENVIRONMENT_SNAPSHOT", + "target_minutes": 10, + "checks": [ + "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv", + "python -c 'import torch; print(torch.cuda.is_available())' if torch is installed", + "git rev-parse HEAD and git status --short for provenance", + ], + "decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE", + }, + { + "phase": "P1_FIXTURE_REPLAY", + "target_minutes": 25, + "checks": [ + "Verify NUVMAP fixture block hash and merkle roots.", + "Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.", + "Reject any fixture with missing source hashes.", + ], + "decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD", + }, + { + "phase": "P2_CPU_GPU_PARITY", + "target_minutes": 35, + "checks": [ + "Run the same wave/vector kernel on CPU and GPU.", + "Compare Q16.16 or declared float tolerance residuals.", + "Emit max_abs_error, mean_abs_error, and mismatch examples.", + ], + "decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED", + }, + { + "phase": "P3_WAVEPROBE_BATCH", + "target_minutes": 45, + "checks": [ + "Run bounded byte-window/eigen/density probes over sampled blockchain shards.", + "Store only feature hashes and summaries unless payload admission exists.", + "Emit route candidates as HOLD until negative controls pass.", + ], + "decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION", + }, + { + "phase": "P4_METAPROBE_AND_CONTROLS", + "target_minutes": 45, + "checks": [ + "Run metaprobe route selection over waveprobe outputs.", + "Run shuffled window, shuffled chain label, and random-byte controls.", + "Require controls before any Hutter/logogram feedback promotion.", + ], + "decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL", + }, + { + "phase": "P5_COST_AND_SHUTDOWN", + "target_minutes": 20, + "checks": [ + "Write final receipt bundle.", + "Sync receipts and small summaries to Drive.", + "Record instance type, wall time, estimated cost, and shutdown confirmation.", + ], + "decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT", + }, + ], + "promotion_gates": { + "hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.", + "compression_feedback": "HOLD until byte-exact baseline matrix exists.", + "nuvmap_chain": "HOLD until local fixture block replay and verifier exist.", + }, + "decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION", + } + payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"})) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hours", type=float, default=3.0) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + + receipt = build_receipt(args.hours) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/5-Applications/scripts/nuvmap_metadata_blitter.md b/5-Applications/scripts/nuvmap_metadata_blitter.md new file mode 100644 index 00000000..65f2e8b0 --- /dev/null +++ b/5-Applications/scripts/nuvmap_metadata_blitter.md @@ -0,0 +1,133 @@ +# NUVMAP Metadata Blitter + +`nuvmap_metadata_blitter.wgsl` is a deliberately small WebGPU compute pass for +metadata sorting. It treats the GPU as a dataport fabric: one SIMD lane group +reads fixed-size byte/word windows and emits compact route keys for later sort, +waveprobe, metaprobe, or NUVMAP receipt steps. + +## Contract + +Input: + +| Buffer | Meaning | +| --- | --- | +| `source_words` | Packed `u32` metadata words from block/window/shard records | +| `BlitParams` | Record count, record width, stride, offset, salt, flags | + +Output: + +| Buffer | Meaning | +| --- | --- | +| `key_buffer` | One sortable `u32` route key per record | +| `metric_buffer` | One packed `u32` metrics word per record | + +The pass does not decompress, classify semantic meaning, publish data on-chain, +or claim compression improvement. + +## Key Word + +`key_buffer[i]` packs: + +| Bits | Field | Meaning | +| --- | --- | --- | +| 31..24 | `hash8` | salted local hash prefix | +| 23..17 | `density7` | one-bit density estimate | +| 16..10 | `delta7` | adjacent-word XOR delta pressure | +| 9..4 | `zero6` | zero-byte share | +| 3..0 | `flags4` | route flags | + +This is designed for a follow-up radix/sort pass. Sorting by this key groups +similar metadata windows without pretending the groups are semantic classes. + +## Metric Word + +`metric_buffer[i]` packs: + +| Bits | Field | Meaning | +| --- | --- | --- | +| 31..26 | `class0` | byte count for `00xxxxxx` | +| 25..20 | `class1` | byte count for `01xxxxxx` | +| 19..14 | `class2` | byte count for `10xxxxxx` | +| 13..8 | `class3` | byte count for `11xxxxxx` | +| 7..1 | `high_bit_share7` | same density estimate used in the key | +| 0 | `valid` | one if at least one word was scanned | + +The four byte classes are cheap enough to compute per lane and useful enough to +seed later waveprobe/metaprobe decisions. + +## Dispatch Shape + +The shader uses: + +```wgsl +@compute @workgroup_size(256, 1, 1) +``` + +Dispatch count: + +```text +ceil(record_count / 256) +``` + +Each invocation scans at most `64` words. Larger records should be presented as +multiple fixed windows so the kernel stays predictable and massively shardable. + +## NUVMAP Use + +The intended receipt loop is: + +```text +metadata shard + -> blitter key/metric buffers + -> bounded sort/group + -> waveprobe/metaprobe sample selection + -> NUVMAP receipt + -> L3 self-scan scheduler +``` + +Promotion stays bounded: + +```text +ADMIT: deterministic key/metric emission with replayable source window +HOLD: route meaning, compression gain, or semantic label +QUARANTINE: hash mismatch, unsafe payload class, or missing provenance +``` + +## Hyper-Soliton Radix Bins + +The clean metaphor for the radix bins is the hyper-soliton search surface. + +A radix bin is not a semantic label. It is a temporary, stable packet of similar +metadata pressure: + +```text +blitter key field + -> radix bin + -> route basin + -> waveprobe sample + -> metaprobe decision +``` + +In the hyper-soliton view: + +| Radix Object | Hyper-Soliton Analogue | +| --- | --- | +| `hash8` | phase seed / local identity packet | +| `density7` | amplitude / byte pressure | +| `delta7` | shock or torsion gradient | +| `zero6` | void / low-information throat | +| `flags4` | boundary condition | +| radix bin | persistent soliton basin | +| bin split | soliton fission | +| bin merge | soliton collision | +| unstable bin | dispersive residue / HOLD | + +That gives the scanner a useful rule: + +```text +probe bins that persist across salts, windows, and controls; +hold bins that only exist under one projection. +``` + +So the GPU blitter is the high-throughput flow step, radix grouping is the +soliton-basin finder, and NUVMAP receipts record which basins survived replay. diff --git a/5-Applications/scripts/nuvmap_metadata_blitter.wgsl b/5-Applications/scripts/nuvmap_metadata_blitter.wgsl new file mode 100644 index 00000000..464d9bdc --- /dev/null +++ b/5-Applications/scripts/nuvmap_metadata_blitter.wgsl @@ -0,0 +1,136 @@ +// NUVMAP metadata blitter for WebGPU. +// +// One invocation scans one fixed-size metadata record/window and emits: +// key_buffer[i] = sortable route key +// metric_buffer[i] = packed byte/delta/hash metrics +// +// The shader is intentionally tiny and SIMD-friendly. It does not decompress, +// classify semantic meaning, or claim compression gain. It only turns byte-ish +// metadata windows into deterministic route keys for later sorting/probing. + +struct BlitParams { + record_count: u32, + words_per_record: u32, + stride_words: u32, + source_offset_words: u32, + route_salt: u32, + flags: u32, +} + +@group(0) @binding(0) var source_words: array; +@group(0) @binding(1) var key_buffer: array; +@group(0) @binding(2) var metric_buffer: array; +@group(0) @binding(3) var params: BlitParams; + +const WORDS_HARD_CAP: u32 = 64u; + +fn popcount32(v: u32) -> u32 { + return countOneBits(v); +} + +fn byte_class_counts(word: u32) -> vec4 { + var counts = vec4(0u, 0u, 0u, 0u); + for (var lane = 0u; lane < 4u; lane = lane + 1u) { + let b = (word >> (lane * 8u)) & 0xFFu; + let cls = b >> 6u; + if (cls == 0u) { + counts.x = counts.x + 1u; + } else if (cls == 1u) { + counts.y = counts.y + 1u; + } else if (cls == 2u) { + counts.z = counts.z + 1u; + } else { + counts.w = counts.w + 1u; + } + } + return counts; +} + +fn mix32(x: u32) -> u32 { + var h = x; + h = h ^ (h >> 16u); + h = h * 0x7FEB352Du; + h = h ^ (h >> 15u); + h = h * 0x846CA68Bu; + h = h ^ (h >> 16u); + return h; +} + +fn pack_key(hash8: u32, density7: u32, delta7: u32, zero6: u32, flags4: u32) -> u32 { + return ((hash8 & 0xFFu) << 24u) | + ((density7 & 0x7Fu) << 17u) | + ((delta7 & 0x7Fu) << 10u) | + ((zero6 & 0x3Fu) << 4u) | + (flags4 & 0xFu); +} + +fn pack_metric(class_counts: vec4, high_bit_share7: u32, words_seen: u32) -> u32 { + let c0 = min(class_counts.x, 0x3Fu); + let c1 = min(class_counts.y, 0x3Fu); + let c2 = min(class_counts.z, 0x3Fu); + let c3 = min(class_counts.w, 0x3Fu); + return (c0 << 26u) | + (c1 << 20u) | + (c2 << 14u) | + (c3 << 8u) | + ((high_bit_share7 & 0x7Fu) << 1u) | + min(words_seen, 1u); +} + +@compute @workgroup_size(256, 1, 1) +fn main(@builtin(global_invocation_id) gid: vec3) { + let record_index = gid.x; + if (record_index >= params.record_count) { + return; + } + + let words_to_scan = min(params.words_per_record, WORDS_HARD_CAP); + let base = params.source_offset_words + record_index * params.stride_words; + var hash_acc = mix32(record_index ^ params.route_salt); + var one_bits = 0u; + var zero_bytes = 0u; + var delta_acc = 0u; + var class_counts = vec4(0u, 0u, 0u, 0u); + var previous = 0u; + + for (var i = 0u; i < words_to_scan; i = i + 1u) { + let word = source_words[base + i]; + hash_acc = mix32(hash_acc ^ word ^ (i * 0x9E3779B9u)); + one_bits = one_bits + popcount32(word); + class_counts = class_counts + byte_class_counts(word); + + for (var lane = 0u; lane < 4u; lane = lane + 1u) { + let b = (word >> (lane * 8u)) & 0xFFu; + if (b == 0u) { + zero_bytes = zero_bytes + 1u; + } + } + + if (i > 0u) { + delta_acc = delta_acc + popcount32(word ^ previous); + } + previous = word; + } + + let total_bits = max(words_to_scan * 32u, 1u); + let total_bytes = max(words_to_scan * 4u, 1u); + let high_bit_share7 = min((one_bits * 127u) / total_bits, 127u); + let density7 = high_bit_share7; + let delta7 = min((delta_acc * 127u) / total_bits, 127u); + let zero6 = min((zero_bytes * 63u) / total_bytes, 63u); + let hash8 = hash_acc >> 24u; + + var flags4 = params.flags & 0xFu; + if (zero6 > 48u) { + flags4 = flags4 | 0x1u; + } + if (delta7 > 96u) { + flags4 = flags4 | 0x2u; + } + if (density7 < 16u || density7 > 112u) { + flags4 = flags4 | 0x4u; + } + + key_buffer[record_index] = pack_key(hash8, density7, delta7, zero6, flags4); + metric_buffer[record_index] = pack_metric(class_counts, high_bit_share7, words_to_scan); +} diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/Blockchain L3 Self-Scan Scheduler.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Blockchain L3 Self-Scan Scheduler.tid new file mode 100644 index 00000000..036ecfab --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Blockchain L3 Self-Scan Scheduler.tid @@ -0,0 +1,91 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack Blockchain L3 Scheduler Receipts NUVMAP HOLD +title: Blockchain L3 Self-Scan Scheduler +type: text/vnd.tiddlywiki + +! Blockchain L3 Self-Scan Scheduler + +Status: `ADMIT_L3_SELF_SCAN_POLICY` + +The blockchain L3 self-scan scheduler treats scan receipts as the control +surface for the next scan. This reframes L3 from only "bitstream build" into an +executable scan policy: + +```text +P_{t+1}=L3(R_t,A_t,C_t) +R_{t+1}=scan(P_{t+1}) +``` + +Where: + +```text +R_t = current receipt frontier +A_t = available scan actions +C_t = cost / completion / claim-boundary constraints +P_t = next scan policy +``` + +!! Runner + +```text +4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py +``` + +Receipt: + +```text +shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json +``` + +Current receipt hash: + +```text +d9d8c49369c700e0749e467615bf0ab87fc180b5f0d5a31788d2886a1014632a +``` + +!! Storage Channel Awareness + +The scheduler now recognizes that blockchains are not just event streams. They +are also public storage substrates. + +Detected storage surfaces include: + +| Family | Surfaces | +|---|---| +| Bitcoin-like | `op_return`, witness/inscription-like payloads, coinbase tags | +| EVM | calldata, logs/events, bytecode, blob / DA lanes | +| Memo/message chains | memo, message, payload fields | + +Boundary: + +```text +onchain_payload_surfaces_t -> storage_channel_receipts_t+1 +``` + +This is for detection, accounting, provenance, and compression-route +diagnostics. It is not a recommendation to publish payloads on public chains or +use ledgers as private storage. + +!! Actions + +The scheduler emits bounded next actions: + +* `RETRY_MISSING_OBJECTS` +* `RUN_HEADER_OR_BLOCK_PATTERN_PROBE` +* `RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE` +* `RUN_ONCHAIN_STORAGE_CHANNEL_PROBE` +* `REFRESH_SELF_SCAN_SCHEDULER` + +!! Claim Boundary + +The scheduler does not decode chain semantics, prove market behavior, prove +compression gain, or fetch additional bytes by itself. It chooses the next +bounded probe from existing receipts. + +Related: + +* [[NUVMAP Turbulence Stabilization Index]] +* [[NUVMAP Chain Protocol Plan]] +* [[NUVMAP Metadata Blitter]] +* [[Hyper-Soliton Radix Bins]] diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/Hyper-Soliton Radix Bins.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Hyper-Soliton Radix Bins.tid new file mode 100644 index 00000000..288ba72c --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Hyper-Soliton Radix Bins.tid @@ -0,0 +1,79 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack NUVMAP Soliton RadixBins GPU Waveprobe Metaprobe HOLD +title: Hyper-Soliton Radix Bins +type: text/vnd.tiddlywiki + +! Hyper-Soliton Radix Bins + +Status: `METAPHOR_TO_ROUTE_MODEL` + +Hyper-soliton radix bins are the route-basin interpretation of the NUVMAP +metadata blitter output. + +The crucial rule: + +```text +radix bins are route basins, not semantic labels +``` + +!! Mapping + +| Radix Object | Hyper-Soliton Analogue | +|---|---| +| `hash8` | phase seed / local identity packet | +| `density7` | amplitude / byte pressure | +| `delta7` | shock or torsion gradient | +| `zero6` | void / low-information throat | +| `flags4` | boundary condition | +| radix bin | persistent soliton basin | +| bin split | soliton fission | +| bin merge | soliton collision | +| unstable bin | dispersive residue / HOLD | + +!! Stability Rule + +```text +probe bins that persist across salts, windows, and controls; +hold bins that only exist under one projection. +``` + +The bin becomes interesting only if it survives: + +* salt changes +* window shifts +* shuffled source controls +* random-byte controls +* chain-label shuffles +* CPU/GPU parity checks + +!! NUVMAP Receipt + +The NUVMAP protocol adds: + +```text +BLITTER_BIN_RECEIPT +``` + +Required fields: + +```text +shader_hash +params_hash +source_hash +key_buffer_hash +metric_buffer_hash +decision +``` + +!! Claim Boundary + +The bin is a routing hint. It is not a semantic proof, compression proof, market +signal, or chain behavior prediction. + +Related: + +* [[NUVMAP Metadata Blitter]] +* [[NUVMAP Chain Protocol Plan]] +* [[Topological Soliton Raytrace Tessellation NUVMAP Protocol]] +* [[Waveprobe]] diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Chain Protocol Plan.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Chain Protocol Plan.tid new file mode 100644 index 00000000..7d880e07 --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Chain Protocol Plan.tid @@ -0,0 +1,95 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack NUVMAP Blockchain ReceiptChain Waveprobe Metaprobe HOLD +title: NUVMAP Chain Protocol Plan +type: text/vnd.tiddlywiki + +! NUVMAP Chain Protocol Plan + +Status: `ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION` + +NUVMAP is currently a protocol plan for a dedicated receipt chain: + +```text +Numerical Universe Vector Map receipt chain +``` + +Its purpose is to store replayable probe-state commitments so scanning can route +itself over time without treating commodity public chains as accidental bulk +storage. + +!! Runner + +```text +4-Infrastructure/shim/nuvmap_chain_protocol_plan.py +``` + +Receipt: + +```text +shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json +``` + +Current receipt hash: + +```text +5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40 +``` + +!! Chain Model + +```text +deployment_stage: LOCAL_DEVNET_ONLY +economic_model: NO_TOKEN_NO_MARKET +finality_model: append-only merkle root +data_availability: on-chain summaries + content-addressed sidecars +``` + +!! Minimal Transaction Types + +* `BLITTER_BIN_RECEIPT` +* `WAVE_RECEIPT` +* `META_RECEIPT` +* `STORAGE_SURFACE_RECEIPT` +* `SCAN_POLICY_UPDATE` +* `NEGATIVE_CONTROL` + +!! Block Shape + +```text +header: + parent_hash + height + scanner_policy_hash + state_root + sidecar_manifest_root + +body: + blitter bin receipts + waveprobe receipts + metaprobe receipts + storage-channel receipts + scan actions + negative controls +``` + +Forbidden body fields: + +* private keys +* wallet credentials +* bulk copyrighted payloads +* hidden/evasive payloads +* financial promotion metadata + +!! Claim Boundary + +This does not create a blockchain, token, market, consensus network, +compression result, or Hutter Prize claim. Chain implementation remains HOLD +until local fixture block replay and verifier receipts exist. + +Related: + +* [[Blockchain L3 Self-Scan Scheduler]] +* [[NUVMAP Metadata Blitter]] +* [[Hyper-Soliton Radix Bins]] +* [[NUVMAP GPU Burst Check Plan]] diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP GPU Burst Check Plan.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP GPU Burst Check Plan.tid new file mode 100644 index 00000000..ed2576fe --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP GPU Burst Check Plan.tid @@ -0,0 +1,69 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack NUVMAP GPU BurstCheck HardwareWitness HOLD +title: NUVMAP GPU Burst Check Plan +type: text/vnd.tiddlywiki + +! NUVMAP GPU Burst Check Plan + +Status: `ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION` + +This is the time-boxed plan for renting a GPU briefly, approximately three +hours, to validate NUVMAP probe machinery. + +Runner: + +```text +4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py +``` + +Receipt: + +```text +shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json +``` + +Current receipt hash: + +```text +73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296 +``` + +!! Burst Phases + +```text +P0 environment snapshot +P1 NUVMAP fixture replay +P2 CPU/GPU parity +P3 waveprobe batch +P4 metaprobe + negative controls +P5 cost and shutdown receipt +``` + +!! Required Outputs + +* `environment_receipt.json` +* `cpu_gpu_parity_receipt.json` +* `waveprobe_batch_receipt.json` +* `metaprobe_batch_receipt.json` +* `negative_controls_receipt.json` +* `cost_and_shutdown_receipt.json` + +!! Forbidden Inputs + +* wallet private keys +* exchange credentials +* mainnet transaction signing material +* unbounded copyrighted payload mirrors + +!! Claim Boundary + +The GPU rental proves nothing by itself. Hardware witness requires visible GPU +state plus CPU/GPU parity receipts. Compression feedback remains HOLD until a +byte-exact baseline matrix exists. + +Related: + +* [[NUVMAP Metadata Blitter]] +* [[Blockchain L3 Self-Scan Scheduler]] +* [[NUVMAP Chain Protocol Plan]] diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Metadata Blitter.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Metadata Blitter.tid new file mode 100644 index 00000000..a0a4e99a --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Metadata Blitter.tid @@ -0,0 +1,73 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack NUVMAP GPU WebGPU Shader MetadataBlitter SIMD HOLD +title: NUVMAP Metadata Blitter +type: text/vnd.tiddlywiki + +! NUVMAP Metadata Blitter + +Status: `SHADER_PRIMITIVE_WITH_HOLD_BOUNDARIES` + +The NUVMAP metadata blitter is a small WebGPU compute shader that treats the GPU +as a dataport fabric for metadata sorting. + +Runner: + +```text +5-Applications/scripts/nuvmap_metadata_blitter.wgsl +``` + +Documentation: + +```text +5-Applications/scripts/nuvmap_metadata_blitter.md +``` + +!! Purpose + +```text +metadata shard + -> fixed-size word windows + -> SIMD-wide blitter pass + -> sortable route keys + -> packed metric words + -> radix grouping + -> waveprobe / metaprobe selection + -> NUVMAP receipts +``` + +The shader does not sort by itself. It emits deterministic keys and metrics that +a later radix/sort pass or CPU verifier can consume. + +!! Key Word + +```text +bits 31..24 hash8 phase seed +bits 23..17 density7 byte pressure +bits 16..10 delta7 shock / torsion gradient +bits 9..4 zero6 void / low-information throat +bits 3..0 flags4 boundary condition +``` + +!! Dispatch Shape + +```text +@compute @workgroup_size(256, 1, 1) +dispatch = ceil(record_count / 256) +``` + +Each invocation scans at most `64` words. Larger records should be split into +fixed windows so the kernel remains predictable and shardable. + +!! Claim Boundary + +This shader emits route keys and metrics only. It does not decompress, classify +semantic meaning, publish data on-chain, claim GPU acceleration, or claim +compression improvement. GPU claims require CPU/GPU parity receipts. + +Related: + +* [[Hyper-Soliton Radix Bins]] +* [[NUVMAP Chain Protocol Plan]] +* [[Blockchain L3 Self-Scan Scheduler]] +* [[NUVMAP GPU Burst Check Plan]] diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Turbulence Stabilization Index.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Turbulence Stabilization Index.tid new file mode 100644 index 00000000..f0b78a18 --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/NUVMAP Turbulence Stabilization Index.tid @@ -0,0 +1,75 @@ +created: 20260511000000000 +modified: 20260511000000000 +tags: ResearchStack NUVMAP Blockchain L3 Waveprobe Metaprobe GPU Turbulence Index +title: NUVMAP Turbulence Stabilization Index +type: text/vnd.tiddlywiki + +! NUVMAP Turbulence Stabilization Index + +Status: `STABILIZATION_INDEX` + +This index captures the current high-turbulence blockchain / GPU / NUVMAP +design surface so the stack does not have to re-derive it from chat context. + +!! Current Shape + +```text +public blockchain corpus + -> L3 self-scan scheduler + -> GPU metadata blitter + -> hyper-soliton radix bins + -> waveprobe / metaprobe routing + -> NUVMAP receipt chain + -> next scan frontier +``` + +The key change is that scanning is no longer only a data-ingest action. It is a +feedback loop: + +```text +scan_receipts_t -> route_policy_t+1 +``` + +!! Core Cards + +* [[Blockchain L3 Self-Scan Scheduler]] +* [[NUVMAP Chain Protocol Plan]] +* [[NUVMAP Metadata Blitter]] +* [[Hyper-Soliton Radix Bins]] +* [[NUVMAP GPU Burst Check Plan]] +* [[Waveprobe]] +* [[Topological Soliton Raytrace Tessellation NUVMAP Protocol]] +* [[Hutter JPEG XL Starfield Eigenprobe First Sweep]] + +!! Durable Artifacts + +```text +4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py +4-Infrastructure/shim/nuvmap_chain_protocol_plan.py +4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py +5-Applications/scripts/nuvmap_metadata_blitter.wgsl +5-Applications/scripts/nuvmap_metadata_blitter.md +``` + +Receipts: + +```text +shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json +shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json +shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json +``` + +!! Claim Boundary + +This cluster is protocol and routing infrastructure. It does not claim: + +* Hutter Prize progress +* compression gain +* blockchain market prediction +* mainnet readiness +* token or economic design +* semantic meaning from radix bins +* GPU acceleration without parity receipts + +Every promotion requires replayable source windows, negative controls, receipt +hashes, and explicit HOLD/ADMIT/QUARANTINE decisions. diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/Topological Soliton Raytrace Tessellation NUVMAP Protocol.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Topological Soliton Raytrace Tessellation NUVMAP Protocol.tid new file mode 100644 index 00000000..51ea5520 --- /dev/null +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Topological Soliton Raytrace Tessellation NUVMAP Protocol.tid @@ -0,0 +1,81 @@ +created: 20260510000000000 +modified: 20260510000000000 +tags: ResearchStack Soliton Raytrace Tessellation NUVMAP TreeFiddy Protocol +title: Topological Soliton Raytrace Tessellation NUVMAP Protocol +type: text/vnd.tiddlywiki + +! Topological Soliton Raytrace Tessellation NUVMAP Protocol + +Status: `CANDIDATE_PROTOCOL_WITH_HOLD_BOUNDARIES` + +Seed: + +```text +Topological solitons > ray-traced tessellation > Tree Fiddy > NUVMAP results +``` + +!! Core Pipeline + +```text +topological soliton + -> invariant field identity + -> ray-traced local intersections + -> tessellated cells + -> Tree Fiddy bounded recursion + -> NUVMAP address/projection result + -> replay receipt +``` + +The move is to turn persistent topology into bounded addressable geometry. +Solitons supply identity. Ray tracing supplies observation. Tessellation makes +the surface finite. Tree Fiddy prevents runaway refinement. NUVMAP stores the +address/projection result. + +!! Admission Rule + +```text +ADMIT iff: + invariant_present + projection_present + tessellation_finite + tree_fiddy_depth <= declared_bound + nuvmap_address_valid + replay_residual <= epsilon +``` + +!! Why It Matters + +This bridges field-heavy soliton math into the compiler/address layer: + +```text +field identity + -> sampled intersections + -> finite cells + -> bounded refinement + -> addressable receipt +``` + +It does not treat a ray-rendered view as proof. The rendered/tessellated view is +only a projection. The receipt closes only if replay residual stays bounded. + +!! HOLD Boundary + +HOLD remains on: + +* physical control of solitons +* device-readiness +* ray rendering as proof of topology +* infinite refinement +* unbounded recursion +* NUVMAP address results without replay residual and hash receipts + +Related: + +* [[Soliton N-Space Path]] +* [[Tessellated Triangle Flow Migration]] +* [[Tree Fiddy Monster Assignment]] +* [[NUVMAP Turbulence Stabilization Index]] +* [[NUVMAP Chain Protocol Plan]] +* [[Hyper-Soliton Radix Bins]] +* `6-Documentation/docs/topological_soliton_raytrace_tessellation_nuvmap_protocol_2026-05-10.md` +* `shared-data/data/stack_solidification/topological_soliton_raytrace_tessellation_nuvmap_receipt.json` diff --git a/6-Documentation/tiddlywiki-local/wiki/tiddlers/Waveprobe.tid b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Waveprobe.tid index d4bbd8f5..5c2ef4ee 100644 --- a/6-Documentation/tiddlywiki-local/wiki/tiddlers/Waveprobe.tid +++ b/6-Documentation/tiddlywiki-local/wiki/tiddlers/Waveprobe.tid @@ -12,3 +12,6 @@ WaveProbe is a manifold-probing system that emits and measures wavefronts across * [[Topological State Machine]] * [[Manifold Flow]] * [[Semantic RG Flow]] +* [[NUVMAP Turbulence Stabilization Index]] +* [[Blockchain L3 Self-Scan Scheduler]] +* [[NUVMAP Metadata Blitter]] diff --git a/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json b/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json new file mode 100644 index 00000000..0a3e25c2 --- /dev/null +++ b/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json @@ -0,0 +1,374 @@ +{ + "bigquery_export_commands": [ + { + "chain": "bitcoin-cash", + "commands": [ + "bq ls bigquery-public-data.crypto_bitcoin_cash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.{table}' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'" + ], + "dataset": "bigquery-public-data.crypto_bitcoin_cash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/{table}/*.parquet", + "source_id": "bitcoin_cash", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "dash", + "commands": [ + "bq ls bigquery-public-data.crypto_dash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.{table}' 'gs:///research-stack/blockchain-corpus/dash/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'" + ], + "dataset": "bigquery-public-data.crypto_dash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dash/{table}/*.parquet", + "source_id": "dash", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "dogecoin", + "commands": [ + "bq ls bigquery-public-data.crypto_dogecoin", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.{table}' 'gs:///research-stack/blockchain-corpus/dogecoin/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'" + ], + "dataset": "bigquery-public-data.crypto_dogecoin", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/{table}/*.parquet", + "source_id": "dogecoin", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "litecoin", + "commands": [ + "bq ls bigquery-public-data.crypto_litecoin", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.{table}' 'gs:///research-stack/blockchain-corpus/litecoin/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'" + ], + "dataset": "bigquery-public-data.crypto_litecoin", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin/{table}/*.parquet", + "source_id": "litecoin", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "zcash", + "commands": [ + "bq ls bigquery-public-data.crypto_zcash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.{table}' 'gs:///research-stack/blockchain-corpus/zcash/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'" + ], + "dataset": "bigquery-public-data.crypto_zcash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/zcash/{table}/*.parquet", + "source_id": "zcash", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "ethereum-classic", + "commands": [ + "bq ls bigquery-public-data.crypto_ethereum_classic", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.{table}' 'gs:///research-stack/blockchain-corpus/ethereum-classic/{table}/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'" + ], + "dataset": "bigquery-public-data.crypto_ethereum_classic", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic", + "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/{table}/*.parquet", + "source_id": "ethereum_classic", + "tables_to_try_first": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + } + ], + "claim_boundary": "This receipt identifies alternative public/research sources and tool readiness. It does not prove data was exported from BigQuery, does not bypass source terms, and does not claim decoded chain semantics.", + "created_utc": "2026-05-11T17:18:18Z", + "decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER", + "destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10", + "local_tool_readiness": { + "commands": { + "bq": { + "available": true, + "path": "/home/allaun/.local/bin/bq", + "version": "This is BigQuery CLI 2.1.31" + }, + "duckdb": { + "available": false, + "path": null, + "version": null + }, + "gcloud": { + "available": true, + "path": "/home/allaun/.local/bin/gcloud", + "version": "Google Cloud SDK 567.0.0" + }, + "gsutil": { + "available": true, + "path": "/home/allaun/.local/bin/gsutil", + "version": "gsutil version: 5.37" + }, + "kaggle": { + "available": false, + "path": null, + "version": null + }, + "rclone": { + "available": true, + "path": "/usr/bin/rclone", + "version": "rclone v1.74.0" + } + }, + "python_modules": { + "boto3": { + "available": false, + "error": "ModuleNotFoundError", + "module": "boto3" + }, + "duckdb": { + "available": false, + "error": "ModuleNotFoundError", + "module": "duckdb" + }, + "google.cloud.bigquery": { + "available": true, + "module": "google.cloud.bigquery" + }, + "pandas": { + "available": true, + "module": "pandas" + }, + "pyarrow": { + "available": true, + "module": "pyarrow" + }, + "requests": { + "available": true, + "module": "requests" + } + } + }, + "next_gate": { + "decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET", + "required": [ + "Authenticate with a Google Cloud account allowed to query BigQuery public datasets.", + "Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.", + "Provide or create a GCS staging bucket for BigQuery extract jobs.", + "Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.", + "Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts." + ] + }, + "node_etl_commands": [ + { + "chain": "bitcoin", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "bitcoin_cash", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin_cash --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_cash/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "bitcoin_gold", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin_gold --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_gold/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "dogecoin", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain dogecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/dogecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dogecoin/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dogecoin/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "litecoin", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain litecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/litecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/litecoin/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/litecoin/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "dash", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain dash --blocks-output shared-data/data/blockchain_corpus/node_etl/dash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dash/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dash/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + }, + { + "chain": "zcash", + "commands": [ + "python3 -m pip install --user bitcoin-etl", + "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain zcash --blocks-output shared-data/data/blockchain_corpus/node_etl/zcash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/zcash/transactions.jsonl", + "rclone copy 'shared-data/data/blockchain_corpus/node_etl/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/zcash/'" + ], + "requires": [ + "running_full_node_or_rpc_snapshot", + "bitcoin-etl" + ] + } + ], + "receipt_hash": "3327d7c4e0f98a984522f4f66e0bbe3afc66dc933a86a97b6a3c6a29baf3572f", + "schema": "blockchain_alternative_source_plan_v0", + "source_candidates": [ + { + "chains": [ + { + "blockchair_status": "PARTIAL_HTTP_402", + "chain": "bitcoin-cash", + "dataset": "bigquery-public-data.crypto_bitcoin_cash" + }, + { + "blockchair_status": "PARTIAL_HTTP_402", + "chain": "dash", + "dataset": "bigquery-public-data.crypto_dash" + }, + { + "blockchair_status": "PARTIAL_HTTP_402", + "chain": "dogecoin", + "dataset": "bigquery-public-data.crypto_dogecoin" + }, + { + "blockchair_status": "PARTIAL_HTTP_402", + "chain": "litecoin", + "dataset": "bigquery-public-data.crypto_litecoin" + }, + { + "blockchair_status": "PARTIAL_HTTP_402", + "chain": "zcash", + "dataset": "bigquery-public-data.crypto_zcash" + }, + { + "blockchair_status": "NOT_ATTEMPTED", + "chain": "ethereum-classic", + "dataset": "bigquery-public-data.crypto_ethereum_classic" + } + ], + "claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.", + "decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS", + "name": "Google BigQuery Public Cryptocurrency Datasets", + "python_client_ready": true, + "source_urls": [ + "https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them", + "https://www.cloudskillsboost.google/focuses/8486?parent=catalog" + ], + "why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash." + }, + { + "chains": [ + "bitcoin", + "bitcoin_cash", + "bitcoin_gold", + "dogecoin", + "litecoin", + "dash", + "zcash" + ], + "claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.", + "decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED", + "name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots", + "source_urls": [ + "https://github.com/blockchain-etl/bitcoin-etl" + ], + "why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps." + }, + { + "chains": [ + "bitcoin" + ], + "claim_boundary": "Coverage and licensing must be checked before mirroring.", + "decision": "EVALUATE_LICENSE_AND_COVERAGE", + "name": "Bitquery Cloud Data Dumps", + "source_urls": [ + "https://docs.bitquery.io/docs/cloud/bitcoin/" + ], + "why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane." + }, + { + "chains": [ + "bitcoin-cash", + "dash", + "dogecoin", + "litecoin", + "zcash" + ], + "claim_boundary": "Do not use for full-corpus claims.", + "decision": "HOLD_SAMPLE_ONLY", + "name": "Kaggle/Hugging Face sampled datasets", + "source_urls": [ + "https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset", + "https://www.kaggle.com/datasets/amritpal333/crypto-mining-data" + ], + "why": "Useful for smoke tests and model fixtures, not complete chain mirrors." + } + ] +} diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json new file mode 100644 index 00000000..15a3c548 --- /dev/null +++ b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json @@ -0,0 +1,147 @@ +{ + "claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.", + "entries": [ + { + "chain": "bitcoin-cash", + "commands": [ + "bq ls bigquery-public-data.crypto_bitcoin_cash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.blocks' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.transactions' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.inputs' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.outputs' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'" + ], + "dataset": "bigquery-public-data.crypto_bitcoin_cash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/bitcoin-cash/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "dash", + "commands": [ + "bq ls bigquery-public-data.crypto_dash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.blocks' 'gs:///research-stack/blockchain-corpus/dash/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.transactions' 'gs:///research-stack/blockchain-corpus/dash/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.inputs' 'gs:///research-stack/blockchain-corpus/dash/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.outputs' 'gs:///research-stack/blockchain-corpus/dash/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'" + ], + "dataset": "bigquery-public-data.crypto_dash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/dash/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dash", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "dogecoin", + "commands": [ + "bq ls bigquery-public-data.crypto_dogecoin", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.blocks' 'gs:///research-stack/blockchain-corpus/dogecoin/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.transactions' 'gs:///research-stack/blockchain-corpus/dogecoin/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.inputs' 'gs:///research-stack/blockchain-corpus/dogecoin/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.outputs' 'gs:///research-stack/blockchain-corpus/dogecoin/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'" + ], + "dataset": "bigquery-public-data.crypto_dogecoin", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/dogecoin/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "ethereum-classic", + "commands": [ + "bq ls bigquery-public-data.crypto_ethereum_classic", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.blocks' 'gs:///research-stack/blockchain-corpus/ethereum-classic/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.transactions' 'gs:///research-stack/blockchain-corpus/ethereum-classic/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.inputs' 'gs:///research-stack/blockchain-corpus/ethereum-classic/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.outputs' 'gs:///research-stack/blockchain-corpus/ethereum-classic/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'" + ], + "dataset": "bigquery-public-data.crypto_ethereum_classic", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/ethereum-classic/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "litecoin", + "commands": [ + "bq ls bigquery-public-data.crypto_litecoin", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.blocks' 'gs:///research-stack/blockchain-corpus/litecoin/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.transactions' 'gs:///research-stack/blockchain-corpus/litecoin/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.inputs' 'gs:///research-stack/blockchain-corpus/litecoin/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.outputs' 'gs:///research-stack/blockchain-corpus/litecoin/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'" + ], + "dataset": "bigquery-public-data.crypto_litecoin", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/litecoin/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + }, + { + "chain": "zcash", + "commands": [ + "bq ls bigquery-public-data.crypto_zcash", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.blocks' 'gs:///research-stack/blockchain-corpus/zcash/blocks/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.transactions' 'gs:///research-stack/blockchain-corpus/zcash/transactions/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.inputs' 'gs:///research-stack/blockchain-corpus/zcash/inputs/*.parquet'", + "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.outputs' 'gs:///research-stack/blockchain-corpus/zcash/outputs/*.parquet'", + "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'", + "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'" + ], + "dataset": "bigquery-public-data.crypto_zcash", + "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash", + "gcs_prefix": "gs:///research-stack/blockchain-corpus/zcash/", + "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/zcash", + "tables": [ + "blocks", + "transactions", + "inputs", + "outputs" + ] + } + ], + "schema": "blockchain_bigquery_grabber_plan_v0", + "tool_status": { + "bq": "/home/allaun/.local/bin/bq", + "gcloud": "/home/allaun/.local/bin/gcloud", + "gsutil": "/home/allaun/.local/bin/gsutil", + "python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python", + "python_bigquery_client": true, + "python_google_auth": true, + "rclone": "/usr/bin/rclone" + } +} diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json new file mode 100644 index 00000000..9f48168b --- /dev/null +++ b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json @@ -0,0 +1,8 @@ +[ + { + "error": "DefaultCredentialsError", + "message": "Your default credentials were not found. To set up Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information.", + "next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.", + "python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python" + } +] diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json new file mode 100644 index 00000000..2261cf4d --- /dev/null +++ b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json @@ -0,0 +1 @@ +{"decision": "HOLD_GOOGLE_ADC_MISSING", "out": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json", "results": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json"} diff --git a/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json b/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json new file mode 100644 index 00000000..7d9025d5 --- /dev/null +++ b/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json @@ -0,0 +1,1171 @@ +{ + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json", + "account_receipt_hash": "7f01aec8b1ee6531712f2c0d82f349d44ccd3d6ab0102387c1dcc35cd18a9683", + "claim_boundary": "Layer-3 scan scheduling receipt only. It uses existing inventory and transfer receipts to choose next bounded probes. It does not decode chain semantics, claim compression gain, claim market prediction, or fetch additional corpus bytes.", + "created_utc": "2026-05-11T17:39:53Z", + "decision": "ADMIT_L3_SELF_SCAN_POLICY", + "frontier": [ + { + "bytes_per_object": 4767427.881, + "chain": "ethereum", + "completion": { + "completed_objects": 3937, + "completion_ratio": 0.999746064, + "missing_objects": 1, + "remote_parquet_count": 3937, + "remote_payload_count": 3937 + }, + "dataset": "aws-public-blockchain", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", + "inventory_hash": "1edc316f0e250ea56e9e4d9958db559af28c717cc53c6fdd6fcde717ab876bda", + "object_count": 3938, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "ethereum", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "ethereum", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "ethereum", + "density_band": "light", + "family": "evm", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "calldata", + "scan_surface": "transaction input bytes", + "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." + }, + { + "channel": "event_logs", + "scan_surface": "topics and data fields", + "use_boundary": "Receipt emitted public events as storage-like data lanes." + }, + { + "channel": "contract_bytecode", + "scan_surface": "creation/runtime bytecode", + "use_boundary": "Static payload/code-carrier diagnostic only." + }, + { + "channel": "blob_or_da_payload", + "scan_surface": "data availability/blob lanes when present in source tables", + "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "evm" + }, + "table": "blocks", + "total_listed_bytes": 18774130997 + }, + { + "bytes_per_object": 0.0, + "chain": "bitcoin-cash", + "completion": { + "completed_objects": 0, + "completion_ratio": 0.0, + "missing_objects": 100, + "remote_parquet_count": 0, + "remote_payload_count": 0 + }, + "dataset": "blockchair_dumps", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", + "inventory_hash": "67c7b090be906e72933ec4d958ff8690856b3487a5ec8ca3eb78e17458ad6815", + "object_count": 100, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "bitcoin-cash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "bitcoin-cash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "bitcoin-cash", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 0 + }, + { + "bytes_per_object": 0.0, + "chain": "dash", + "completion": { + "completed_objects": 0, + "completion_ratio": 0.0, + "missing_objects": 100, + "remote_parquet_count": 0, + "remote_payload_count": 0 + }, + "dataset": "blockchair_dumps", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", + "inventory_hash": "6e5449f73399c45838e139386c34d36cebf1b8c2ce855a7619dbe4b7d51a26a0", + "object_count": 100, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "dash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "dash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "dash", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 0 + }, + { + "bytes_per_object": 0.0, + "chain": "dogecoin", + "completion": { + "completed_objects": 0, + "completion_ratio": 0.0, + "missing_objects": 100, + "remote_parquet_count": 0, + "remote_payload_count": 0 + }, + "dataset": "blockchair_dumps", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", + "inventory_hash": "b4765f59840c9c4ed35b4863652a91eb66956a3d1930bb41053a646c621df298", + "object_count": 100, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "dogecoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "dogecoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "dogecoin", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 0 + }, + { + "bytes_per_object": 0.0, + "chain": "litecoin", + "completion": { + "completed_objects": 0, + "completion_ratio": 0.0, + "missing_objects": 100, + "remote_parquet_count": 0, + "remote_payload_count": 0 + }, + "dataset": "blockchair_dumps", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", + "inventory_hash": "70037496b226b387b6fac749a699791dc38a7a817e05b6be0cfe85c63bde55bc", + "object_count": 100, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "litecoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "litecoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "litecoin", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 0 + }, + { + "bytes_per_object": 0.0, + "chain": "zcash", + "completion": { + "completed_objects": 0, + "completion_ratio": 0.0, + "missing_objects": 100, + "remote_parquet_count": 0, + "remote_payload_count": 0 + }, + "dataset": "blockchair_dumps", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", + "inventory_hash": "4f5c828c07792f8be026891585954bc2cf08ea13f0068d6ba01c6f8042a4b245", + "object_count": 100, + "scan_actions": [ + { + "action": "RETRY_MISSING_OBJECTS", + "decision": "HOLD_UNTIL_RETRY_RECEIPT", + "inputs": { + "chain": "zcash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Remote payload/parquet count is below inventory object count.", + "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" + }, + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "zcash", + "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "zcash", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 0 + }, + { + "bytes_per_object": 68716662.373, + "chain": "arbitrum", + "completion": { + "completed_objects": 3200, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 3200, + "remote_payload_count": 3200 + }, + "dataset": "aws-public-blockchain", + "density_band": "heavy", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", + "inventory_hash": "2d5dd25747c8384c06b4f453060c71166cb472beb398f16ef9b2914cecf96230", + "object_count": 3200, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "arbitrum", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + "inputs": { + "chain": "arbitrum", + "sample_policy": "first_middle_last_partition", + "table": "blocks" + }, + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "arbitrum", + "density_band": "heavy", + "family": "evm", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "calldata", + "scan_surface": "transaction input bytes", + "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." + }, + { + "channel": "event_logs", + "scan_surface": "topics and data fields", + "use_boundary": "Receipt emitted public events as storage-like data lanes." + }, + { + "channel": "contract_bytecode", + "scan_surface": "creation/runtime bytecode", + "use_boundary": "Static payload/code-carrier diagnostic only." + }, + { + "channel": "blob_or_da_payload", + "scan_surface": "data availability/blob lanes when present in source tables", + "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "evm" + }, + "table": "blocks", + "total_listed_bytes": 219893319594 + }, + { + "bytes_per_object": 45683299.611, + "chain": "aptos", + "completion": { + "completed_objects": 1665, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 1665, + "remote_payload_count": 1665 + }, + "dataset": "aws-public-blockchain", + "density_band": "medium", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", + "inventory_hash": "962c88cf0ebc0ceec4aed0b52855e40d82be50cf180790eda96eb0b7484f66ce", + "object_count": 1665, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "aptos", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + "inputs": { + "chain": "aptos", + "sample_policy": "first_middle_last_partition", + "table": "blocks" + }, + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "aptos", + "density_band": "medium", + "family": "memo", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "memo_or_message", + "scan_surface": "transaction memo/message/payload fields", + "use_boundary": "Public memo payload diagnostics only." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "memo" + }, + "table": "blocks", + "total_listed_bytes": 76062693853 + }, + { + "bytes_per_object": 22105222.309, + "chain": "provenance", + "completion": { + "completed_objects": 2014, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 2014, + "remote_payload_count": 2014 + }, + "dataset": "aws-public-blockchain", + "density_band": "medium", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", + "inventory_hash": "454acd030b9f66326ba4585711c9bc507e98a618c6905d2f09598a0ec9f71648", + "object_count": 2014, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "provenance", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + "inputs": { + "chain": "provenance", + "sample_policy": "first_middle_last_partition", + "table": "blocks" + }, + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "provenance", + "density_band": "medium", + "family": "memo", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "memo_or_message", + "scan_surface": "transaction memo/message/payload fields", + "use_boundary": "Public memo payload diagnostics only." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "memo" + }, + "table": "blocks", + "total_listed_bytes": 44519917731 + }, + { + "bytes_per_object": 4324610.932, + "chain": "ton", + "completion": { + "completed_objects": 9327, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 9327, + "remote_payload_count": 9327 + }, + "dataset": "aws-public-blockchain", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", + "inventory_hash": "9366e8758b563c2c1219c3eaba209b30b9eaf54e5779c53fa65f23956312dda4", + "object_count": 9327, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "ton", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "ton", + "density_band": "light", + "family": "memo", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "memo_or_message", + "scan_surface": "transaction memo/message/payload fields", + "use_boundary": "Public memo payload diagnostics only." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "memo" + }, + "table": "blocks", + "total_listed_bytes": 40335646161 + }, + { + "bytes_per_object": 29268320.978, + "chain": "base", + "completion": { + "completed_objects": 1215, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 1215, + "remote_payload_count": 1215 + }, + "dataset": "aws-public-blockchain", + "density_band": "medium", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", + "inventory_hash": "d3121c9b9c4856e97d14ea855adab9fef2237c3f509cbdc6213a797420a3cb83", + "object_count": 1215, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "base", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + "inputs": { + "chain": "base", + "sample_policy": "first_middle_last_partition", + "table": "blocks" + }, + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "base", + "density_band": "medium", + "family": "evm", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "calldata", + "scan_surface": "transaction input bytes", + "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." + }, + { + "channel": "event_logs", + "scan_surface": "topics and data fields", + "use_boundary": "Receipt emitted public events as storage-like data lanes." + }, + { + "channel": "contract_bytecode", + "scan_surface": "creation/runtime bytecode", + "use_boundary": "Static payload/code-carrier diagnostic only." + }, + { + "channel": "blob_or_da_payload", + "scan_surface": "data availability/blob lanes when present in source tables", + "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "evm" + }, + "table": "blocks", + "total_listed_bytes": 35561009988 + }, + { + "bytes_per_object": 6635676.137, + "chain": "xrp", + "completion": { + "completed_objects": 5032, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 5032, + "remote_payload_count": 5032 + }, + "dataset": "aws-public-blockchain", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", + "inventory_hash": "c7904177ada7fea02aeef391e3eb78e69fec846bda02205aabfc51c79fa75fc8", + "object_count": 5032, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "xrp", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "xrp", + "density_band": "light", + "family": "memo", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "memo_or_message", + "scan_surface": "transaction memo/message/payload fields", + "use_boundary": "Public memo payload diagnostics only." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "memo" + }, + "table": "blocks", + "total_listed_bytes": 33390722320 + }, + { + "bytes_per_object": 11077089.183, + "chain": "cronos", + "completion": { + "completed_objects": 1537, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 1537, + "remote_payload_count": 1537 + }, + "dataset": "aws-public-blockchain", + "density_band": "medium", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", + "inventory_hash": "f5fe32ec4ddc02600fd6da03a2837ffdb43a1dd7cdfc09f5472cd440a54707e7", + "object_count": 1537, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "cronos", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", + "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", + "inputs": { + "chain": "cronos", + "sample_policy": "first_middle_last_partition", + "table": "blocks" + }, + "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", + "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "cronos", + "density_band": "medium", + "family": "evm", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "calldata", + "scan_surface": "transaction input bytes", + "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." + }, + { + "channel": "event_logs", + "scan_surface": "topics and data fields", + "use_boundary": "Receipt emitted public events as storage-like data lanes." + }, + { + "channel": "contract_bytecode", + "scan_surface": "creation/runtime bytecode", + "use_boundary": "Static payload/code-carrier diagnostic only." + }, + { + "channel": "blob_or_da_payload", + "scan_surface": "data availability/blob lanes when present in source tables", + "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "evm" + }, + "table": "blocks", + "total_listed_bytes": 17025486075 + }, + { + "bytes_per_object": 40756.121, + "chain": "bitcoin", + "completion": { + "completed_objects": 9303, + "completion_ratio": 1.0, + "missing_objects": 0, + "remote_parquet_count": 9303, + "remote_payload_count": 9303 + }, + "dataset": "aws-public-blockchain", + "density_band": "light", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", + "inventory_hash": "cf537aa53a3e23e71499ed8eddcf080ab08178e2db9754d085a30b4f79a6334e", + "object_count": 9303, + "scan_actions": [ + { + "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", + "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", + "inputs": { + "chain": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", + "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" + }, + { + "action": "REFRESH_SELF_SCAN_SCHEDULER", + "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", + "inputs": { + "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" + }, + "reason": "The scan output becomes the next L3 control input.", + "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" + } + ], + "storage_channel_model": { + "actions": [ + { + "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", + "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", + "inputs": { + "chain": "bitcoin", + "density_band": "light", + "family": "bitcoin", + "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", + "table": "blocks" + }, + "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." + } + ], + "channels": [ + { + "channel": "op_return", + "scan_surface": "transaction outputs / scriptPubKey", + "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." + }, + { + "channel": "witness_inscription_like_payload", + "scan_surface": "segwit witness data", + "use_boundary": "Treat as public immutable payload evidence only." + }, + { + "channel": "coinbase_tag", + "scan_surface": "coinbase transaction script/witness fields", + "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." + } + ], + "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", + "family": "bitcoin" + }, + "table": "blocks", + "total_listed_bytes": 379154195 + } + ], + "global_storage_boundary": "Public blockchains can store data, but this scheduler only models storage-bearing surfaces for detection, provenance, and compression-route diagnostics. Bulk payload reconstruction and payload publication remain outside this receipt.", + "layer3_interpretation": { + "core_rule": "scan_receipts_t -> route_policy_t_plus_1", + "feedback_equation": "P_{t+1}=L3(R_t,A_t,C_t); R_{t+1}=scan(P_{t+1})", + "old_label": "L3 Bitstream", + "scanner_label": "L3 executable scan policy", + "storage_rule": "onchain_payload_surfaces_t -> storage_channel_receipts_t_plus_1" + }, + "receipt_hash": "d9d8c49369c700e0749e467615bf0ab87fc180b5f0d5a31788d2886a1014632a", + "schema": "blockchain_l3_self_scan_scheduler_v0", + "source_count": 14 +} diff --git a/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json b/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json new file mode 100644 index 00000000..4f441a93 --- /dev/null +++ b/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json @@ -0,0 +1,149 @@ +{ + "block_shape": { + "body": { + "metaprobe_receipts": "bounded list of route/context/curriculum summaries", + "negative_controls": "required controls before promotion of any route", + "scan_actions": "next bounded scan plan emitted by L3 scheduler", + "storage_channel_receipts": "detected on-chain storage surfaces from source chains", + "waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries" + }, + "forbidden_body_fields": [ + "private keys", + "wallet credentials", + "bulk copyrighted payloads", + "payloads intended to hide from moderation or provenance", + "financial promotion metadata" + ], + "header": { + "created_utc": "iso8601", + "height": "u64", + "parent_hash": "sha256(previous_block)", + "scanner_policy_hash": "sha256(l3_policy)", + "sidecar_manifest_root": "merkle_root(sidecar_hashes)", + "state_root": "merkle_root(receipt_state)" + } + }, + "chain_model": { + "consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set", + "data_availability": "on-chain summaries plus content-addressed off-chain sidecars", + "deployment_stage": "LOCAL_DEVNET_ONLY", + "economic_model": "NO_TOKEN_NO_MARKET", + "finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional" + }, + "claim_boundary": "Protocol design receipt only. This does not create a blockchain, token, market, consensus network, compression result, or Hutter Prize claim. It defines a local/devnet receipt substrate for metaprobe and waveprobe state.", + "created_utc": "2026-05-11T17:45:10Z", + "decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION", + "decision_law": { + "ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.", + "HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.", + "QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation." + }, + "minimal_transaction_types": [ + { + "required_fields": [ + "shader_hash", + "params_hash", + "source_hash", + "key_buffer_hash", + "metric_buffer_hash", + "decision" + ], + "type": "BLITTER_BIN_RECEIPT" + }, + { + "required_fields": [ + "source_hash", + "window_descriptor", + "feature_vector_hash", + "decision" + ], + "type": "WAVE_RECEIPT" + }, + { + "required_fields": [ + "prior_id", + "route_family", + "evidence_hash", + "decision" + ], + "type": "META_RECEIPT" + }, + { + "required_fields": [ + "source_chain", + "channel", + "carrier_descriptor", + "payload_hash_or_null", + "decision" + ], + "type": "STORAGE_SURFACE_RECEIPT" + }, + { + "required_fields": [ + "previous_policy_hash", + "new_policy_hash", + "action_merkle_root", + "decision" + ], + "type": "SCAN_POLICY_UPDATE" + }, + { + "required_fields": [ + "control_kind", + "source_hash", + "result_hash", + "decision" + ], + "type": "NEGATIVE_CONTROL" + } + ], + "next_implementation_steps": [ + "Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.", + "Add a verifier that recomputes the block hash and merkle roots.", + "Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.", + "Run negative controls before any compression-route promotion.", + "Only after local fixture replay works, consider a small append-only local devnet." + ], + "probe_mapping": { + "l3_scheduler": { + "input": "current receipt frontier", + "output": "next scan frontier", + "storage": "scanner policy hash and action list" + }, + "metadata_blitter": { + "input": "fixed-size metadata words from chain/block/window records", + "output": "sortable route keys and packed metric words", + "storage": "blitter shader hash, parameter receipt, key/metric buffer hashes" + }, + "metaprobe": { + "input": "domain priors, route families, previous scan receipts", + "output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints", + "storage": "decision receipt, route vector, source receipt backlinks" + }, + "waveprobe": { + "input": "byte windows, block fields, density neighborhoods, eigenvalue spectra", + "output": "local signal vector plus residual/curvature summary", + "storage": "hash, vector summary, source window receipt, not raw bulk by default" + } + }, + "purpose": { + "anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.", + "core_use": "Store replayable probe-state commitments so scanning can route itself over time.", + "expanded_role": "Numerical Universe Vector Map receipt chain", + "name": "NUVMAP" + }, + "radix_bin_model": { + "claim_boundary": "Radix bins are route basins, not semantic labels.", + "fields": { + "delta7": "shock or torsion gradient", + "density7": "amplitude / byte pressure", + "flags4": "boundary condition", + "hash8": "phase seed / local identity packet", + "zero6": "void / low-information throat" + }, + "metaphor": "hyper_soliton_search", + "stability_rule": "Promote only bins that persist across salts, windows, and negative controls." + }, + "receipt_hash": "5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40", + "schema": "nuvmap_chain_protocol_plan_v0" +} diff --git a/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json b/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json new file mode 100644 index 00000000..a6a5e7d2 --- /dev/null +++ b/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json @@ -0,0 +1,89 @@ +{ + "burst_phases": [ + { + "checks": [ + "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv", + "python -c 'import torch; print(torch.cuda.is_available())' if torch is installed", + "git rev-parse HEAD and git status --short for provenance" + ], + "decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE", + "phase": "P0_ENVIRONMENT_SNAPSHOT", + "target_minutes": 10 + }, + { + "checks": [ + "Verify NUVMAP fixture block hash and merkle roots.", + "Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.", + "Reject any fixture with missing source hashes." + ], + "decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD", + "phase": "P1_FIXTURE_REPLAY", + "target_minutes": 25 + }, + { + "checks": [ + "Run the same wave/vector kernel on CPU and GPU.", + "Compare Q16.16 or declared float tolerance residuals.", + "Emit max_abs_error, mean_abs_error, and mismatch examples." + ], + "decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED", + "phase": "P2_CPU_GPU_PARITY", + "target_minutes": 35 + }, + { + "checks": [ + "Run bounded byte-window/eigen/density probes over sampled blockchain shards.", + "Store only feature hashes and summaries unless payload admission exists.", + "Emit route candidates as HOLD until negative controls pass." + ], + "decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION", + "phase": "P3_WAVEPROBE_BATCH", + "target_minutes": 45 + }, + { + "checks": [ + "Run metaprobe route selection over waveprobe outputs.", + "Run shuffled window, shuffled chain label, and random-byte controls.", + "Require controls before any Hutter/logogram feedback promotion." + ], + "decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL", + "phase": "P4_METAPROBE_AND_CONTROLS", + "target_minutes": 45 + }, + { + "checks": [ + "Write final receipt bundle.", + "Sync receipts and small summaries to Drive.", + "Record instance type, wall time, estimated cost, and shutdown confirmation." + ], + "decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT", + "phase": "P5_COST_AND_SHUTDOWN", + "target_minutes": 20 + } + ], + "claim_boundary": "GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, not a token launch, not a compression result, and not a hardware acceleration claim until the listed receipts are produced.", + "created_utc": "2026-05-11T17:41:19Z", + "decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION", + "forbidden_inputs": [ + "wallet private keys", + "exchange credentials", + "mainnet transaction signing material", + "unbounded copyrighted payload mirrors" + ], + "promotion_gates": { + "compression_feedback": "HOLD until byte-exact baseline matrix exists.", + "hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.", + "nuvmap_chain": "HOLD until local fixture block replay and verifier exist." + }, + "receipt_hash": "73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296", + "required_outputs": [ + "environment_receipt.json", + "cpu_gpu_parity_receipt.json", + "waveprobe_batch_receipt.json", + "metaprobe_batch_receipt.json", + "negative_controls_receipt.json", + "cost_and_shutdown_receipt.json" + ], + "schema": "nuvmap_gpu_burst_check_plan_v0", + "timebox_hours": 3.0 +}