From 5cfa8f0898dad5fee356204d43071bde0148cf83 Mon Sep 17 00:00:00 2001 From: allaun Date: Mon, 22 Jun 2026 04:00:32 -0500 Subject: [PATCH] chore(cleanup): remove ingested corpus artifacts and scratch scripts - Delete 4-Infrastructure/shim/lean_corpus/ (downloaded third-party corpus). - Delete shared-data/data/blockchain_corpus/ receipts (already ingested/offloaded). - Delete scratch/ bulk-download scripts (superseded by arxiv_crossref_stream.py). - Clean __pycache__ directories outside .venv. - Also fix populate_ene_tables.py to pipe SQL via stdin instead of shell escaping for neon-64gb psql execution. --- scratch/bulk_download_arxiv.py | 93 -- scratch/download_open_papers.py | 108 -- scratch/extract_and_deduplicate.py | 177 --- scripts/populate_ene_tables.py | 6 +- ...chain_alternative_source_plan_receipt.json | 374 ------ .../blockchain_bigquery_grabber_plan.json | 147 --- ...uery_grabber_plan_python_list_results.json | 8 - ...n_bigquery_grabber_python_list.stdout.json | 1 - ...kchain_l3_self_scan_scheduler_receipt.json | 1171 ----------------- .../nuvmap_chain_protocol_plan_receipt.json | 149 --- .../nuvmap_gpu_burst_check_plan_receipt.json | 89 -- 11 files changed, 3 insertions(+), 2320 deletions(-) delete mode 100755 scratch/bulk_download_arxiv.py delete mode 100755 scratch/download_open_papers.py delete mode 100644 scratch/extract_and_deduplicate.py delete mode 100644 shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json delete mode 100644 shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json delete mode 100644 shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json delete mode 100644 shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json delete mode 100644 shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json delete mode 100644 shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json delete mode 100644 shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json diff --git a/scratch/bulk_download_arxiv.py b/scratch/bulk_download_arxiv.py deleted file mode 100755 index 4138c942..00000000 --- a/scratch/bulk_download_arxiv.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import sys -import time -import urllib.request -import ssl - -ssl_context = ssl._create_unverified_context() - -def get_arxiv_links(url): - """ - Fetch the arXiv page and extract all PDF links. - """ - headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"} - req = urllib.request.Request(url, headers=headers) - try: - with urllib.request.urlopen(req, context=ssl_context) as response: - html = response.read().decode('utf-8') - # Look for /pdf/YYMM.NNNNN or similar patterns - links = re.findall(r'/pdf/[0-9]{4}\.[0-9]{4,5}', html) - # De-duplicate while preserving order - seen = set() - unique_links = [] - for link in links: - if link not in seen: - seen.add(link) - unique_links.append(link) - return unique_links - except Exception as e: - print(f"Error fetching arXiv page: {e}", file=sys.stderr) - return [] - -def main(): - arxiv_list_url = "https://arxiv.org/list/math/recent?skip=0&show=2000" - output_dir = "./arxiv_papers" - os.makedirs(output_dir, exist_ok=True) - - print("Fetching recent math papers list from arXiv...") - pdf_paths = get_arxiv_links(arxiv_list_url) - total_papers = len(pdf_paths) - - if total_papers == 0: - print("No papers found. Exiting.") - return - - print(f"Found {total_papers} papers to download.") - print(f"Starting sequential download with a 4-second delay to avoid rate limits...") - print(f"Target directory: {output_dir}\n") - - downloaded_count = 0 - skipped_count = 0 - failed_count = 0 - - for idx, path in enumerate(pdf_paths): - paper_id = path.split('/')[-1] - filename = f"{paper_id}.pdf" - dest_path = os.path.join(output_dir, filename) - pdf_url = f"https://arxiv.org{path}" - - # Resume capability: check if file already exists and is non-empty - if os.path.exists(dest_path) and os.path.getsize(dest_path) > 10000: - print(f"[{idx+1}/{total_papers}] Skipping {paper_id} (already downloaded)") - skipped_count += 1 - continue - - print(f"[{idx+1}/{total_papers}] Downloading {paper_id} from {pdf_url}...") - - headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"} - req = urllib.request.Request(pdf_url, headers=headers) - - try: - with urllib.request.urlopen(req, context=ssl_context) as response: - with open(dest_path, 'wb') as f: - f.write(response.read()) - print(f"[{idx+1}/{total_papers}] Successfully saved {paper_id}") - downloaded_count += 1 - # Sleep to prevent IP block - time.sleep(4) - except Exception as e: - print(f"[{idx+1}/{total_papers}] Failed to download {paper_id}: {e}", file=sys.stderr) - failed_count += 1 - # Sleep a bit longer on failure - time.sleep(10) - - print("\n=== Download Session Finished ===") - print(f"Total Papers: {total_papers}") - print(f"Downloaded: {downloaded_count}") - print(f"Skipped: {skipped_count}") - print(f"Failed: {failed_count}") - -if __name__ == "__main__": - main() diff --git a/scratch/download_open_papers.py b/scratch/download_open_papers.py deleted file mode 100755 index 02f61fc4..00000000 --- a/scratch/download_open_papers.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import time -import urllib.request -import json -import urllib.parse -import ssl - -# Bypass SSL verification if needed (common in local environments/proxies) -ssl_context = ssl._create_unverified_context() - -def openalex_search(query, polite_email="allaunthefox@gmail.com"): - """ - Search OpenAlex works for a given query and return a list of matching papers with PDF links. - """ - headers = { - "User-Agent": f"OpenAlexDownloader/1.0 (mailto:{polite_email})" - } - encoded_query = urllib.parse.quote(query) - url = f"https://api.openalex.org/works?search={encoded_query}&per-page=5" - - req = urllib.request.Request(url, headers=headers) - try: - with urllib.request.urlopen(req, context=ssl_context) as response: - data = json.loads(response.read().decode()) - results = data.get("results", []) - return results - except Exception as e: - print(f"Error querying OpenAlex for '{query}': {e}", file=sys.stderr) - return [] - -def download_pdf(pdf_url, output_path, polite_email="allaunthefox@gmail.com"): - """ - Download a PDF file from a given URL to the output path. - """ - headers = { - "User-Agent": f"OpenAlexDownloader/1.0 (mailto:{polite_email})" - } - req = urllib.request.Request(pdf_url, headers=headers) - try: - with urllib.request.urlopen(req, context=ssl_context) as response: - with open(output_path, 'wb') as f: - f.write(response.read()) - return True - except Exception as e: - print(f"Failed to download from {pdf_url}: {e}", file=sys.stderr) - return False - -def main(): - if len(sys.argv) < 2: - print("Usage: python3 download_open_papers.py [output_directory]") - print("Example: python3 download_open_papers.py \"inverse matrices Gauss-Jordan\" ./papers") - sys.exit(1) - - query = sys.argv[1] - output_dir = sys.argv[2] if len(sys.argv) > 2 else "./downloaded_papers" - os.makedirs(output_dir, exist_ok=True) - - print(f"Searching OpenAlex for: '{query}'...") - works = openalex_search(query) - - if not works: - print("No works found.") - return - - downloaded_count = 0 - for idx, work in enumerate(works): - title = work.get("display_name", "untitled") - doi = work.get("doi") - - # Clean title for filename - safe_title = "".join([c if c.isalnum() or c in " -_" else "_" for c in title])[:60].strip() - work_id = work.get("id", "").split("/")[-1] - filename = f"{safe_title}_{work_id}.pdf" - output_path = os.path.join(output_dir, filename) - - # Try to find a PDF link - pdf_url = None - best_location = work.get("best_oa_location") - if best_location and best_location.get("pdf_url"): - pdf_url = best_location.get("pdf_url") - else: - # Check alternative locations - for loc in work.get("locations", []): - if loc.get("pdf_url"): - pdf_url = loc.get("pdf_url") - break - - if pdf_url: - print(f"\nFound Open Access PDF for: \"{title}\"") - print(f"URL: {pdf_url}") - print(f"Downloading to: {output_path}...") - - success = download_pdf(pdf_url, output_path) - if success: - print("Download complete!") - downloaded_count += 1 - - # Safe sequential delay to respect rate limits - time.sleep(3) - else: - print(f"\nWork found but no Open Access PDF link available: \"{title}\" (DOI: {doi})") - - print(f"\nDone. Successfully downloaded {downloaded_count} paper(s) to '{output_dir}'.") - -if __name__ == "__main__": - main() diff --git a/scratch/extract_and_deduplicate.py b/scratch/extract_and_deduplicate.py deleted file mode 100644 index 08bd596e..00000000 --- a/scratch/extract_and_deduplicate.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import zipfile -import subprocess -import hashlib -import random -from pathlib import Path - -# Paths -GDRIVE_DIR = Path("/home/allaun/gdrive") -TAKEOUT_DIR = GDRIVE_DIR / "Takeout" -TARGET_DIR = Path("/home/allaun/Takeout_extracted") - -def get_gdrive_md5(gdrive_rel_path): - """Get the MD5 hash of a file on Google Drive using rclone md5sum.""" - rclone_path = f"gdrive:{gdrive_rel_path}" - try: - res = subprocess.run( - ["rclone", "md5sum", rclone_path], - capture_output=True, - text=True, - timeout=10 - ) - if res.returncode == 0 and res.stdout.strip(): - parts = res.stdout.strip().split() - if parts: - return parts[0].strip() - except subprocess.TimeoutExpired: - pass - except Exception as e: - print(f"Error querying rclone md5sum: {e}") - return None - -def check_file_duplicate(zip_ref, info): - """Check if a file in the zip is a duplicate of a file in Google Drive.""" - path_parts = Path(info.filename).parts - if len(path_parts) > 1 and path_parts[0] == "Takeout": - if len(path_parts) > 2 and path_parts[1] == "Drive": - gdrive_rel = Path(*path_parts[2:]) - else: - gdrive_rel = Path(*path_parts[1:]) - else: - gdrive_rel = Path(info.filename) - - gdrive_file = GDRIVE_DIR / gdrive_rel - - if gdrive_file.exists(): - try: - gdrive_size = gdrive_file.stat().st_size - if gdrive_size == info.file_size: - # Sizes match, verify with MD5 - remote_md5 = get_gdrive_md5(str(gdrive_rel)) - if remote_md5: - hasher = hashlib.md5() - with zip_ref.open(info) as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - zip_md5 = hasher.hexdigest() - if zip_md5 == remote_md5: - return True, gdrive_rel - except Exception as e: - print(f"Error checking {gdrive_file}: {e}") - - return False, gdrive_rel - -def run_metaprobe(zip_path, zip_ref, infolist): - """ - Run a windowed metaprobe (sampling test) on the ZIP. - Returns True if the entire ZIP is highly likely to be 100% redundant. - """ - files_only = [info for info in infolist if not info.is_dir()] - total_files = len(files_only) - - if total_files == 0: - return True # Empty zip is redundant/trivial - - # Sample size: 5% of files, min 10, max 30 - sample_size = min(max(int(total_files * 0.05), 10), 30) - sample_size = min(sample_size, total_files) - - random.seed(42) # Deterministic sampling - sample = random.sample(files_only, sample_size) - - print(f" [Metaprobe] Sampling {sample_size}/{total_files} files for redundancy...") - - duplicates = 0 - for idx, info in enumerate(sample): - is_dup, rel_path = check_file_duplicate(zip_ref, info) - if is_dup: - duplicates += 1 - else: - print(f" [Metaprobe] Unique file found: {info.filename}") - return False # Found a unique file, must process this zip - - # If all sampled files are duplicates - if duplicates == sample_size: - print(f" [Metaprobe] 100% redundancy in sample window. Trusting and SKIPPING zip.") - return True - - return False - -def process_zip(zip_path, dry_run=False): - print(f"\nProcessing {zip_path.name}...") - try: - with zipfile.ZipFile(zip_path, 'r') as zip_ref: - infolist = zip_ref.infolist() - - # Run metaprobe sampling check first - is_redundant = run_metaprobe(zip_path, zip_ref, infolist) - if is_redundant: - print(f"SKIPPED (Metaprobe: 100% redundant): {zip_path.name}") - return - - # If not redundant, proceed with extracting unique files - print(f" Metaprobe failed (unique files present). Running full extraction...") - total_files = len(infolist) - redundant_count = 0 - unique_count = 0 - skipped_dirs = 0 - - for idx, info in enumerate(infolist): - if info.is_dir(): - skipped_dirs += 1 - continue - - is_duplicate, gdrive_rel = check_file_duplicate(zip_ref, info) - - if is_duplicate: - redundant_count += 1 - if idx % 100 == 0 or idx == total_files - 1: - print(f" Progress: Checked {idx+1}/{total_files} | Duplicates: {redundant_count} | Unique: {unique_count}") - else: - unique_count += 1 - target_file = TARGET_DIR / info.filename - print(f" [UNIQUE] {info.filename} -> {target_file}") - if not dry_run: - target_file.parent.mkdir(parents=True, exist_ok=True) - with zip_ref.open(info) as source, open(target_file, "wb") as target: - target.write(source.read()) - - print(f"Finished {zip_path.name}: Duplicates: {redundant_count}, Unique: {unique_count}") - except Exception as e: - print(f"Error reading zip {zip_path.name}: {e}") - -def main(): - dry_run = "--dry-run" in sys.argv - single_zip = None - for arg in sys.argv[1:]: - if arg.endswith(".zip"): - single_zip = arg - break - - if dry_run: - print("Running in DRY RUN mode.") - - TARGET_DIR.mkdir(parents=True, exist_ok=True) - - if single_zip: - zip_path = TAKEOUT_DIR / single_zip - if zip_path.exists(): - process_zip(zip_path, dry_run=dry_run) - else: - print(f"Zip file {zip_path} not found.") - sys.exit(1) - else: - # Find all zip files - zips = sorted(list(TAKEOUT_DIR.glob("*.zip"))) - if not zips: - print(f"No zip files found in {TAKEOUT_DIR}") - sys.exit(1) - print(f"Found {len(zips)} zip files to process.") - for zip_path in zips: - process_zip(zip_path, dry_run=dry_run) - -if __name__ == "__main__": - main() diff --git a/scripts/populate_ene_tables.py b/scripts/populate_ene_tables.py index ed6e1d8b..904954d6 100644 --- a/scripts/populate_ene_tables.py +++ b/scripts/populate_ene_tables.py @@ -49,13 +49,13 @@ GRAPH_KEYWORDS = {"graph", "gossip", "surface", "network", "neighborhood", "topo # ── Helpers ─────────────────────────────────────────────────────────────────── def neon(sql: str, dry_run: bool = False, timeout: int = 60) -> str: - """Run SQL on neon-64gb ene DB, return stdout.""" + """Run SQL on neon-64gb ene DB, return stdout. Uses stdin pipe to avoid shell escaping.""" if dry_run: return "DRY_RUN" r = subprocess.run( ["ssh", NEON_HOST, - f"podman exec {CONTAINER} psql -U postgres -d {DB} -t -A -c \"{sql}\""], - capture_output=True, text=True, timeout=timeout, + f"podman exec -i {CONTAINER} psql -U postgres -d {DB} -t -A"], + input=sql, capture_output=True, text=True, timeout=timeout, ) if r.returncode != 0: print(f" NEON ERR: {r.stderr[:200]}", file=sys.stderr) diff --git a/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json b/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json deleted file mode 100644 index 0a3e25c2..00000000 --- a/shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "bigquery_export_commands": [ - { - "chain": "bitcoin-cash", - "commands": [ - "bq ls bigquery-public-data.crypto_bitcoin_cash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.{table}' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'" - ], - "dataset": "bigquery-public-data.crypto_bitcoin_cash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/{table}/*.parquet", - "source_id": "bitcoin_cash", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "dash", - "commands": [ - "bq ls bigquery-public-data.crypto_dash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.{table}' 'gs:///research-stack/blockchain-corpus/dash/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'" - ], - "dataset": "bigquery-public-data.crypto_dash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dash/{table}/*.parquet", - "source_id": "dash", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "dogecoin", - "commands": [ - "bq ls bigquery-public-data.crypto_dogecoin", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.{table}' 'gs:///research-stack/blockchain-corpus/dogecoin/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'" - ], - "dataset": "bigquery-public-data.crypto_dogecoin", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/{table}/*.parquet", - "source_id": "dogecoin", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "litecoin", - "commands": [ - "bq ls bigquery-public-data.crypto_litecoin", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.{table}' 'gs:///research-stack/blockchain-corpus/litecoin/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'" - ], - "dataset": "bigquery-public-data.crypto_litecoin", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin/{table}/*.parquet", - "source_id": "litecoin", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "zcash", - "commands": [ - "bq ls bigquery-public-data.crypto_zcash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.{table}' 'gs:///research-stack/blockchain-corpus/zcash/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'" - ], - "dataset": "bigquery-public-data.crypto_zcash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/zcash/{table}/*.parquet", - "source_id": "zcash", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "ethereum-classic", - "commands": [ - "bq ls bigquery-public-data.crypto_ethereum_classic", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.{table}' 'gs:///research-stack/blockchain-corpus/ethereum-classic/{table}/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'" - ], - "dataset": "bigquery-public-data.crypto_ethereum_classic", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic", - "local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/{table}/*.parquet", - "source_id": "ethereum_classic", - "tables_to_try_first": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - } - ], - "claim_boundary": "This receipt identifies alternative public/research sources and tool readiness. It does not prove data was exported from BigQuery, does not bypass source terms, and does not claim decoded chain semantics.", - "created_utc": "2026-05-11T17:18:18Z", - "decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER", - "destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10", - "local_tool_readiness": { - "commands": { - "bq": { - "available": true, - "path": "/home/allaun/.local/bin/bq", - "version": "This is BigQuery CLI 2.1.31" - }, - "duckdb": { - "available": false, - "path": null, - "version": null - }, - "gcloud": { - "available": true, - "path": "/home/allaun/.local/bin/gcloud", - "version": "Google Cloud SDK 567.0.0" - }, - "gsutil": { - "available": true, - "path": "/home/allaun/.local/bin/gsutil", - "version": "gsutil version: 5.37" - }, - "kaggle": { - "available": false, - "path": null, - "version": null - }, - "rclone": { - "available": true, - "path": "/usr/bin/rclone", - "version": "rclone v1.74.0" - } - }, - "python_modules": { - "boto3": { - "available": false, - "error": "ModuleNotFoundError", - "module": "boto3" - }, - "duckdb": { - "available": false, - "error": "ModuleNotFoundError", - "module": "duckdb" - }, - "google.cloud.bigquery": { - "available": true, - "module": "google.cloud.bigquery" - }, - "pandas": { - "available": true, - "module": "pandas" - }, - "pyarrow": { - "available": true, - "module": "pyarrow" - }, - "requests": { - "available": true, - "module": "requests" - } - } - }, - "next_gate": { - "decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET", - "required": [ - "Authenticate with a Google Cloud account allowed to query BigQuery public datasets.", - "Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.", - "Provide or create a GCS staging bucket for BigQuery extract jobs.", - "Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.", - "Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts." - ] - }, - "node_etl_commands": [ - { - "chain": "bitcoin", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "bitcoin_cash", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin_cash --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_cash/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "bitcoin_gold", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain bitcoin_gold --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_gold/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "dogecoin", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain dogecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/dogecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dogecoin/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dogecoin/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "litecoin", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain litecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/litecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/litecoin/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/litecoin/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "dash", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain dash --blocks-output shared-data/data/blockchain_corpus/node_etl/dash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dash/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dash/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - }, - { - "chain": "zcash", - "commands": [ - "python3 -m pip install --user bitcoin-etl", - "bitcoinetl export_blocks_and_transactions --start-block 0 --end-block --provider-uri http://:@127.0.0.1: --chain zcash --blocks-output shared-data/data/blockchain_corpus/node_etl/zcash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/zcash/transactions.jsonl", - "rclone copy 'shared-data/data/blockchain_corpus/node_etl/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/zcash/'" - ], - "requires": [ - "running_full_node_or_rpc_snapshot", - "bitcoin-etl" - ] - } - ], - "receipt_hash": "3327d7c4e0f98a984522f4f66e0bbe3afc66dc933a86a97b6a3c6a29baf3572f", - "schema": "blockchain_alternative_source_plan_v0", - "source_candidates": [ - { - "chains": [ - { - "blockchair_status": "PARTIAL_HTTP_402", - "chain": "bitcoin-cash", - "dataset": "bigquery-public-data.crypto_bitcoin_cash" - }, - { - "blockchair_status": "PARTIAL_HTTP_402", - "chain": "dash", - "dataset": "bigquery-public-data.crypto_dash" - }, - { - "blockchair_status": "PARTIAL_HTTP_402", - "chain": "dogecoin", - "dataset": "bigquery-public-data.crypto_dogecoin" - }, - { - "blockchair_status": "PARTIAL_HTTP_402", - "chain": "litecoin", - "dataset": "bigquery-public-data.crypto_litecoin" - }, - { - "blockchair_status": "PARTIAL_HTTP_402", - "chain": "zcash", - "dataset": "bigquery-public-data.crypto_zcash" - }, - { - "blockchair_status": "NOT_ATTEMPTED", - "chain": "ethereum-classic", - "dataset": "bigquery-public-data.crypto_ethereum_classic" - } - ], - "claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.", - "decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS", - "name": "Google BigQuery Public Cryptocurrency Datasets", - "python_client_ready": true, - "source_urls": [ - "https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them", - "https://www.cloudskillsboost.google/focuses/8486?parent=catalog" - ], - "why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash." - }, - { - "chains": [ - "bitcoin", - "bitcoin_cash", - "bitcoin_gold", - "dogecoin", - "litecoin", - "dash", - "zcash" - ], - "claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.", - "decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED", - "name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots", - "source_urls": [ - "https://github.com/blockchain-etl/bitcoin-etl" - ], - "why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps." - }, - { - "chains": [ - "bitcoin" - ], - "claim_boundary": "Coverage and licensing must be checked before mirroring.", - "decision": "EVALUATE_LICENSE_AND_COVERAGE", - "name": "Bitquery Cloud Data Dumps", - "source_urls": [ - "https://docs.bitquery.io/docs/cloud/bitcoin/" - ], - "why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane." - }, - { - "chains": [ - "bitcoin-cash", - "dash", - "dogecoin", - "litecoin", - "zcash" - ], - "claim_boundary": "Do not use for full-corpus claims.", - "decision": "HOLD_SAMPLE_ONLY", - "name": "Kaggle/Hugging Face sampled datasets", - "source_urls": [ - "https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset", - "https://www.kaggle.com/datasets/amritpal333/crypto-mining-data" - ], - "why": "Useful for smoke tests and model fixtures, not complete chain mirrors." - } - ] -} diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json deleted file mode 100644 index 15a3c548..00000000 --- a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.", - "entries": [ - { - "chain": "bitcoin-cash", - "commands": [ - "bq ls bigquery-public-data.crypto_bitcoin_cash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.blocks' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.transactions' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.inputs' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.outputs' 'gs:///research-stack/blockchain-corpus/bitcoin-cash/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'" - ], - "dataset": "bigquery-public-data.crypto_bitcoin_cash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/bitcoin-cash/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "dash", - "commands": [ - "bq ls bigquery-public-data.crypto_dash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.blocks' 'gs:///research-stack/blockchain-corpus/dash/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.transactions' 'gs:///research-stack/blockchain-corpus/dash/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.inputs' 'gs:///research-stack/blockchain-corpus/dash/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.outputs' 'gs:///research-stack/blockchain-corpus/dash/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'" - ], - "dataset": "bigquery-public-data.crypto_dash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/dash/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dash", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "dogecoin", - "commands": [ - "bq ls bigquery-public-data.crypto_dogecoin", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.blocks' 'gs:///research-stack/blockchain-corpus/dogecoin/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.transactions' 'gs:///research-stack/blockchain-corpus/dogecoin/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.inputs' 'gs:///research-stack/blockchain-corpus/dogecoin/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.outputs' 'gs:///research-stack/blockchain-corpus/dogecoin/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'" - ], - "dataset": "bigquery-public-data.crypto_dogecoin", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/dogecoin/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "ethereum-classic", - "commands": [ - "bq ls bigquery-public-data.crypto_ethereum_classic", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.blocks' 'gs:///research-stack/blockchain-corpus/ethereum-classic/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.transactions' 'gs:///research-stack/blockchain-corpus/ethereum-classic/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.inputs' 'gs:///research-stack/blockchain-corpus/ethereum-classic/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.outputs' 'gs:///research-stack/blockchain-corpus/ethereum-classic/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'" - ], - "dataset": "bigquery-public-data.crypto_ethereum_classic", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/ethereum-classic/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "litecoin", - "commands": [ - "bq ls bigquery-public-data.crypto_litecoin", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.blocks' 'gs:///research-stack/blockchain-corpus/litecoin/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.transactions' 'gs:///research-stack/blockchain-corpus/litecoin/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.inputs' 'gs:///research-stack/blockchain-corpus/litecoin/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.outputs' 'gs:///research-stack/blockchain-corpus/litecoin/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'" - ], - "dataset": "bigquery-public-data.crypto_litecoin", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/litecoin/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - }, - { - "chain": "zcash", - "commands": [ - "bq ls bigquery-public-data.crypto_zcash", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.blocks' 'gs:///research-stack/blockchain-corpus/zcash/blocks/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.transactions' 'gs:///research-stack/blockchain-corpus/zcash/transactions/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.inputs' 'gs:///research-stack/blockchain-corpus/zcash/inputs/*.parquet'", - "bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.outputs' 'gs:///research-stack/blockchain-corpus/zcash/outputs/*.parquet'", - "gcloud storage cp --recursive 'gs:///research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'", - "rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'" - ], - "dataset": "bigquery-public-data.crypto_zcash", - "drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash", - "gcs_prefix": "gs:///research-stack/blockchain-corpus/zcash/", - "local_root": "shared-data/data/blockchain_corpus/bigquery_exports/zcash", - "tables": [ - "blocks", - "transactions", - "inputs", - "outputs" - ] - } - ], - "schema": "blockchain_bigquery_grabber_plan_v0", - "tool_status": { - "bq": "/home/allaun/.local/bin/bq", - "gcloud": "/home/allaun/.local/bin/gcloud", - "gsutil": "/home/allaun/.local/bin/gsutil", - "python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python", - "python_bigquery_client": true, - "python_google_auth": true, - "rclone": "/usr/bin/rclone" - } -} diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json deleted file mode 100644 index 9f48168b..00000000 --- a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "error": "DefaultCredentialsError", - "message": "Your default credentials were not found. To set up Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information.", - "next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.", - "python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python" - } -] diff --git a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json b/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json deleted file mode 100644 index 2261cf4d..00000000 --- a/shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_python_list.stdout.json +++ /dev/null @@ -1 +0,0 @@ -{"decision": "HOLD_GOOGLE_ADC_MISSING", "out": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json", "results": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json"} diff --git a/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json b/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json deleted file mode 100644 index 7d9025d5..00000000 --- a/shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json +++ /dev/null @@ -1,1171 +0,0 @@ -{ - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json", - "account_receipt_hash": "7f01aec8b1ee6531712f2c0d82f349d44ccd3d6ab0102387c1dcc35cd18a9683", - "claim_boundary": "Layer-3 scan scheduling receipt only. It uses existing inventory and transfer receipts to choose next bounded probes. It does not decode chain semantics, claim compression gain, claim market prediction, or fetch additional corpus bytes.", - "created_utc": "2026-05-11T17:39:53Z", - "decision": "ADMIT_L3_SELF_SCAN_POLICY", - "frontier": [ - { - "bytes_per_object": 4767427.881, - "chain": "ethereum", - "completion": { - "completed_objects": 3937, - "completion_ratio": 0.999746064, - "missing_objects": 1, - "remote_parquet_count": 3937, - "remote_payload_count": 3937 - }, - "dataset": "aws-public-blockchain", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", - "inventory_hash": "1edc316f0e250ea56e9e4d9958db559af28c717cc53c6fdd6fcde717ab876bda", - "object_count": 3938, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "ethereum", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "ethereum", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "ethereum", - "density_band": "light", - "family": "evm", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_eth_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "calldata", - "scan_surface": "transaction input bytes", - "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." - }, - { - "channel": "event_logs", - "scan_surface": "topics and data fields", - "use_boundary": "Receipt emitted public events as storage-like data lanes." - }, - { - "channel": "contract_bytecode", - "scan_surface": "creation/runtime bytecode", - "use_boundary": "Static payload/code-carrier diagnostic only." - }, - { - "channel": "blob_or_da_payload", - "scan_surface": "data availability/blob lanes when present in source tables", - "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "evm" - }, - "table": "blocks", - "total_listed_bytes": 18774130997 - }, - { - "bytes_per_object": 0.0, - "chain": "bitcoin-cash", - "completion": { - "completed_objects": 0, - "completion_ratio": 0.0, - "missing_objects": 100, - "remote_parquet_count": 0, - "remote_payload_count": 0 - }, - "dataset": "blockchair_dumps", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", - "inventory_hash": "67c7b090be906e72933ec4d958ff8690856b3487a5ec8ca3eb78e17458ad6815", - "object_count": 100, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "bitcoin-cash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "bitcoin-cash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "bitcoin-cash", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_bitcoin-cash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 0 - }, - { - "bytes_per_object": 0.0, - "chain": "dash", - "completion": { - "completed_objects": 0, - "completion_ratio": 0.0, - "missing_objects": 100, - "remote_parquet_count": 0, - "remote_payload_count": 0 - }, - "dataset": "blockchair_dumps", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", - "inventory_hash": "6e5449f73399c45838e139386c34d36cebf1b8c2ce855a7619dbe4b7d51a26a0", - "object_count": 100, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "dash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "dash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "dash", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 0 - }, - { - "bytes_per_object": 0.0, - "chain": "dogecoin", - "completion": { - "completed_objects": 0, - "completion_ratio": 0.0, - "missing_objects": 100, - "remote_parquet_count": 0, - "remote_payload_count": 0 - }, - "dataset": "blockchair_dumps", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", - "inventory_hash": "b4765f59840c9c4ed35b4863652a91eb66956a3d1930bb41053a646c621df298", - "object_count": 100, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "dogecoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "dogecoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "dogecoin", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_dogecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 0 - }, - { - "bytes_per_object": 0.0, - "chain": "litecoin", - "completion": { - "completed_objects": 0, - "completion_ratio": 0.0, - "missing_objects": 100, - "remote_parquet_count": 0, - "remote_payload_count": 0 - }, - "dataset": "blockchair_dumps", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", - "inventory_hash": "70037496b226b387b6fac749a699791dc38a7a817e05b6be0cfe85c63bde55bc", - "object_count": 100, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "litecoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "litecoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "litecoin", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_litecoin_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 0 - }, - { - "bytes_per_object": 0.0, - "chain": "zcash", - "completion": { - "completed_objects": 0, - "completion_ratio": 0.0, - "missing_objects": 100, - "remote_parquet_count": 0, - "remote_payload_count": 0 - }, - "dataset": "blockchair_dumps", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", - "inventory_hash": "4f5c828c07792f8be026891585954bc2cf08ea13f0068d6ba01c6f8042a4b245", - "object_count": 100, - "scan_actions": [ - { - "action": "RETRY_MISSING_OBJECTS", - "decision": "HOLD_UNTIL_RETRY_RECEIPT", - "inputs": { - "chain": "zcash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Remote payload/parquet count is below inventory object count.", - "script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py" - }, - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "zcash", - "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "zcash", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/blockchair_zcash_blocks_inventory_first100.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 0 - }, - { - "bytes_per_object": 68716662.373, - "chain": "arbitrum", - "completion": { - "completed_objects": 3200, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 3200, - "remote_payload_count": 3200 - }, - "dataset": "aws-public-blockchain", - "density_band": "heavy", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", - "inventory_hash": "2d5dd25747c8384c06b4f453060c71166cb472beb398f16ef9b2914cecf96230", - "object_count": 3200, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "arbitrum", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", - "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", - "inputs": { - "chain": "arbitrum", - "sample_policy": "first_middle_last_partition", - "table": "blocks" - }, - "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", - "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "arbitrum", - "density_band": "heavy", - "family": "evm", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_arbitrum_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "calldata", - "scan_surface": "transaction input bytes", - "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." - }, - { - "channel": "event_logs", - "scan_surface": "topics and data fields", - "use_boundary": "Receipt emitted public events as storage-like data lanes." - }, - { - "channel": "contract_bytecode", - "scan_surface": "creation/runtime bytecode", - "use_boundary": "Static payload/code-carrier diagnostic only." - }, - { - "channel": "blob_or_da_payload", - "scan_surface": "data availability/blob lanes when present in source tables", - "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "evm" - }, - "table": "blocks", - "total_listed_bytes": 219893319594 - }, - { - "bytes_per_object": 45683299.611, - "chain": "aptos", - "completion": { - "completed_objects": 1665, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 1665, - "remote_payload_count": 1665 - }, - "dataset": "aws-public-blockchain", - "density_band": "medium", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", - "inventory_hash": "962c88cf0ebc0ceec4aed0b52855e40d82be50cf180790eda96eb0b7484f66ce", - "object_count": 1665, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "aptos", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", - "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", - "inputs": { - "chain": "aptos", - "sample_policy": "first_middle_last_partition", - "table": "blocks" - }, - "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", - "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "aptos", - "density_band": "medium", - "family": "memo", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_aptos_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "memo_or_message", - "scan_surface": "transaction memo/message/payload fields", - "use_boundary": "Public memo payload diagnostics only." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "memo" - }, - "table": "blocks", - "total_listed_bytes": 76062693853 - }, - { - "bytes_per_object": 22105222.309, - "chain": "provenance", - "completion": { - "completed_objects": 2014, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 2014, - "remote_payload_count": 2014 - }, - "dataset": "aws-public-blockchain", - "density_band": "medium", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", - "inventory_hash": "454acd030b9f66326ba4585711c9bc507e98a618c6905d2f09598a0ec9f71648", - "object_count": 2014, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "provenance", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", - "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", - "inputs": { - "chain": "provenance", - "sample_policy": "first_middle_last_partition", - "table": "blocks" - }, - "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", - "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "provenance", - "density_band": "medium", - "family": "memo", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_provenance_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "memo_or_message", - "scan_surface": "transaction memo/message/payload fields", - "use_boundary": "Public memo payload diagnostics only." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "memo" - }, - "table": "blocks", - "total_listed_bytes": 44519917731 - }, - { - "bytes_per_object": 4324610.932, - "chain": "ton", - "completion": { - "completed_objects": 9327, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 9327, - "remote_payload_count": 9327 - }, - "dataset": "aws-public-blockchain", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", - "inventory_hash": "9366e8758b563c2c1219c3eaba209b30b9eaf54e5779c53fa65f23956312dda4", - "object_count": 9327, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "ton", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "ton", - "density_band": "light", - "family": "memo", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_ton_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "memo_or_message", - "scan_surface": "transaction memo/message/payload fields", - "use_boundary": "Public memo payload diagnostics only." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "memo" - }, - "table": "blocks", - "total_listed_bytes": 40335646161 - }, - { - "bytes_per_object": 29268320.978, - "chain": "base", - "completion": { - "completed_objects": 1215, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 1215, - "remote_payload_count": 1215 - }, - "dataset": "aws-public-blockchain", - "density_band": "medium", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", - "inventory_hash": "d3121c9b9c4856e97d14ea855adab9fef2237c3f509cbdc6213a797420a3cb83", - "object_count": 1215, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "base", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", - "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", - "inputs": { - "chain": "base", - "sample_policy": "first_middle_last_partition", - "table": "blocks" - }, - "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", - "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "base", - "density_band": "medium", - "family": "evm", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_base_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "calldata", - "scan_surface": "transaction input bytes", - "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." - }, - { - "channel": "event_logs", - "scan_surface": "topics and data fields", - "use_boundary": "Receipt emitted public events as storage-like data lanes." - }, - { - "channel": "contract_bytecode", - "scan_surface": "creation/runtime bytecode", - "use_boundary": "Static payload/code-carrier diagnostic only." - }, - { - "channel": "blob_or_da_payload", - "scan_surface": "data availability/blob lanes when present in source tables", - "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "evm" - }, - "table": "blocks", - "total_listed_bytes": 35561009988 - }, - { - "bytes_per_object": 6635676.137, - "chain": "xrp", - "completion": { - "completed_objects": 5032, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 5032, - "remote_payload_count": 5032 - }, - "dataset": "aws-public-blockchain", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", - "inventory_hash": "c7904177ada7fea02aeef391e3eb78e69fec846bda02205aabfc51c79fa75fc8", - "object_count": 5032, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "xrp", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "xrp", - "density_band": "light", - "family": "memo", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_xrp_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "memo_or_message", - "scan_surface": "transaction memo/message/payload fields", - "use_boundary": "Public memo payload diagnostics only." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "memo" - }, - "table": "blocks", - "total_listed_bytes": 33390722320 - }, - { - "bytes_per_object": 11077089.183, - "chain": "cronos", - "completion": { - "completed_objects": 1537, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 1537, - "remote_payload_count": 1537 - }, - "dataset": "aws-public-blockchain", - "density_band": "medium", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", - "inventory_hash": "f5fe32ec4ddc02600fd6da03a2837ffdb43a1dd7cdfc09f5472cd440a54707e7", - "object_count": 1537, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "cronos", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE", - "decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK", - "inputs": { - "chain": "cronos", - "sample_policy": "first_middle_last_partition", - "table": "blocks" - }, - "reason": "Large average objects should be sampled through feature/density probes before broader transfer.", - "script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "cronos", - "density_band": "medium", - "family": "evm", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_cronos_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "calldata", - "scan_surface": "transaction input bytes", - "use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow." - }, - { - "channel": "event_logs", - "scan_surface": "topics and data fields", - "use_boundary": "Receipt emitted public events as storage-like data lanes." - }, - { - "channel": "contract_bytecode", - "scan_surface": "creation/runtime bytecode", - "use_boundary": "Static payload/code-carrier diagnostic only." - }, - { - "channel": "blob_or_da_payload", - "scan_surface": "data availability/blob lanes when present in source tables", - "use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "evm" - }, - "table": "blocks", - "total_listed_bytes": 17025486075 - }, - { - "bytes_per_object": 40756.121, - "chain": "bitcoin", - "completion": { - "completed_objects": 9303, - "completion_ratio": 1.0, - "missing_objects": 0, - "remote_parquet_count": 9303, - "remote_payload_count": 9303 - }, - "dataset": "aws-public-blockchain", - "density_band": "light", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", - "inventory_hash": "cf537aa53a3e23e71499ed8eddcf080ab08178e2db9754d085a30b4f79a6334e", - "object_count": 9303, - "scan_actions": [ - { - "action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE", - "decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN", - "inputs": { - "chain": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.", - "script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py" - }, - { - "action": "REFRESH_SELF_SCAN_SCHEDULER", - "decision": "ADMIT_RECEIPT_FEEDBACK_LOOP", - "inputs": { - "account_receipt": "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json" - }, - "reason": "The scan output becomes the next L3 control input.", - "script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py" - } - ], - "storage_channel_model": { - "actions": [ - { - "action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE", - "decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE", - "inputs": { - "chain": "bitcoin", - "density_band": "light", - "family": "bitcoin", - "inventory": "shared-data/data/blockchain_corpus/aws_public_blockchain_btc_blocks_inventory_full.json", - "table": "blocks" - }, - "reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise." - } - ], - "channels": [ - { - "channel": "op_return", - "scan_surface": "transaction outputs / scriptPubKey", - "use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion." - }, - { - "channel": "witness_inscription_like_payload", - "scan_surface": "segwit witness data", - "use_boundary": "Treat as public immutable payload evidence only." - }, - { - "channel": "coinbase_tag", - "scan_surface": "coinbase transaction script/witness fields", - "use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness." - } - ], - "claim_boundary": "Storage-channel model is for detection, accounting, and receipt routing. It does not recommend publishing payloads on-chain, bypassing moderation, hiding data, or using public ledgers as private storage.", - "family": "bitcoin" - }, - "table": "blocks", - "total_listed_bytes": 379154195 - } - ], - "global_storage_boundary": "Public blockchains can store data, but this scheduler only models storage-bearing surfaces for detection, provenance, and compression-route diagnostics. Bulk payload reconstruction and payload publication remain outside this receipt.", - "layer3_interpretation": { - "core_rule": "scan_receipts_t -> route_policy_t_plus_1", - "feedback_equation": "P_{t+1}=L3(R_t,A_t,C_t); R_{t+1}=scan(P_{t+1})", - "old_label": "L3 Bitstream", - "scanner_label": "L3 executable scan policy", - "storage_rule": "onchain_payload_surfaces_t -> storage_channel_receipts_t_plus_1" - }, - "receipt_hash": "d9d8c49369c700e0749e467615bf0ab87fc180b5f0d5a31788d2886a1014632a", - "schema": "blockchain_l3_self_scan_scheduler_v0", - "source_count": 14 -} diff --git a/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json b/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json deleted file mode 100644 index 4f441a93..00000000 --- a/shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "block_shape": { - "body": { - "metaprobe_receipts": "bounded list of route/context/curriculum summaries", - "negative_controls": "required controls before promotion of any route", - "scan_actions": "next bounded scan plan emitted by L3 scheduler", - "storage_channel_receipts": "detected on-chain storage surfaces from source chains", - "waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries" - }, - "forbidden_body_fields": [ - "private keys", - "wallet credentials", - "bulk copyrighted payloads", - "payloads intended to hide from moderation or provenance", - "financial promotion metadata" - ], - "header": { - "created_utc": "iso8601", - "height": "u64", - "parent_hash": "sha256(previous_block)", - "scanner_policy_hash": "sha256(l3_policy)", - "sidecar_manifest_root": "merkle_root(sidecar_hashes)", - "state_root": "merkle_root(receipt_state)" - } - }, - "chain_model": { - "consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set", - "data_availability": "on-chain summaries plus content-addressed off-chain sidecars", - "deployment_stage": "LOCAL_DEVNET_ONLY", - "economic_model": "NO_TOKEN_NO_MARKET", - "finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional" - }, - "claim_boundary": "Protocol design receipt only. This does not create a blockchain, token, market, consensus network, compression result, or Hutter Prize claim. It defines a local/devnet receipt substrate for metaprobe and waveprobe state.", - "created_utc": "2026-05-11T17:45:10Z", - "decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION", - "decision_law": { - "ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.", - "HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.", - "QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation." - }, - "minimal_transaction_types": [ - { - "required_fields": [ - "shader_hash", - "params_hash", - "source_hash", - "key_buffer_hash", - "metric_buffer_hash", - "decision" - ], - "type": "BLITTER_BIN_RECEIPT" - }, - { - "required_fields": [ - "source_hash", - "window_descriptor", - "feature_vector_hash", - "decision" - ], - "type": "WAVE_RECEIPT" - }, - { - "required_fields": [ - "prior_id", - "route_family", - "evidence_hash", - "decision" - ], - "type": "META_RECEIPT" - }, - { - "required_fields": [ - "source_chain", - "channel", - "carrier_descriptor", - "payload_hash_or_null", - "decision" - ], - "type": "STORAGE_SURFACE_RECEIPT" - }, - { - "required_fields": [ - "previous_policy_hash", - "new_policy_hash", - "action_merkle_root", - "decision" - ], - "type": "SCAN_POLICY_UPDATE" - }, - { - "required_fields": [ - "control_kind", - "source_hash", - "result_hash", - "decision" - ], - "type": "NEGATIVE_CONTROL" - } - ], - "next_implementation_steps": [ - "Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.", - "Add a verifier that recomputes the block hash and merkle roots.", - "Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.", - "Run negative controls before any compression-route promotion.", - "Only after local fixture replay works, consider a small append-only local devnet." - ], - "probe_mapping": { - "l3_scheduler": { - "input": "current receipt frontier", - "output": "next scan frontier", - "storage": "scanner policy hash and action list" - }, - "metadata_blitter": { - "input": "fixed-size metadata words from chain/block/window records", - "output": "sortable route keys and packed metric words", - "storage": "blitter shader hash, parameter receipt, key/metric buffer hashes" - }, - "metaprobe": { - "input": "domain priors, route families, previous scan receipts", - "output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints", - "storage": "decision receipt, route vector, source receipt backlinks" - }, - "waveprobe": { - "input": "byte windows, block fields, density neighborhoods, eigenvalue spectra", - "output": "local signal vector plus residual/curvature summary", - "storage": "hash, vector summary, source window receipt, not raw bulk by default" - } - }, - "purpose": { - "anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.", - "core_use": "Store replayable probe-state commitments so scanning can route itself over time.", - "expanded_role": "Numerical Universe Vector Map receipt chain", - "name": "NUVMAP" - }, - "radix_bin_model": { - "claim_boundary": "Radix bins are route basins, not semantic labels.", - "fields": { - "delta7": "shock or torsion gradient", - "density7": "amplitude / byte pressure", - "flags4": "boundary condition", - "hash8": "phase seed / local identity packet", - "zero6": "void / low-information throat" - }, - "metaphor": "hyper_soliton_search", - "stability_rule": "Promote only bins that persist across salts, windows, and negative controls." - }, - "receipt_hash": "5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40", - "schema": "nuvmap_chain_protocol_plan_v0" -} diff --git a/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json b/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json deleted file mode 100644 index a6a5e7d2..00000000 --- a/shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "burst_phases": [ - { - "checks": [ - "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv", - "python -c 'import torch; print(torch.cuda.is_available())' if torch is installed", - "git rev-parse HEAD and git status --short for provenance" - ], - "decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE", - "phase": "P0_ENVIRONMENT_SNAPSHOT", - "target_minutes": 10 - }, - { - "checks": [ - "Verify NUVMAP fixture block hash and merkle roots.", - "Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.", - "Reject any fixture with missing source hashes." - ], - "decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD", - "phase": "P1_FIXTURE_REPLAY", - "target_minutes": 25 - }, - { - "checks": [ - "Run the same wave/vector kernel on CPU and GPU.", - "Compare Q16.16 or declared float tolerance residuals.", - "Emit max_abs_error, mean_abs_error, and mismatch examples." - ], - "decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED", - "phase": "P2_CPU_GPU_PARITY", - "target_minutes": 35 - }, - { - "checks": [ - "Run bounded byte-window/eigen/density probes over sampled blockchain shards.", - "Store only feature hashes and summaries unless payload admission exists.", - "Emit route candidates as HOLD until negative controls pass." - ], - "decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION", - "phase": "P3_WAVEPROBE_BATCH", - "target_minutes": 45 - }, - { - "checks": [ - "Run metaprobe route selection over waveprobe outputs.", - "Run shuffled window, shuffled chain label, and random-byte controls.", - "Require controls before any Hutter/logogram feedback promotion." - ], - "decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL", - "phase": "P4_METAPROBE_AND_CONTROLS", - "target_minutes": 45 - }, - { - "checks": [ - "Write final receipt bundle.", - "Sync receipts and small summaries to Drive.", - "Record instance type, wall time, estimated cost, and shutdown confirmation." - ], - "decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT", - "phase": "P5_COST_AND_SHUTDOWN", - "target_minutes": 20 - } - ], - "claim_boundary": "GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, not a token launch, not a compression result, and not a hardware acceleration claim until the listed receipts are produced.", - "created_utc": "2026-05-11T17:41:19Z", - "decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION", - "forbidden_inputs": [ - "wallet private keys", - "exchange credentials", - "mainnet transaction signing material", - "unbounded copyrighted payload mirrors" - ], - "promotion_gates": { - "compression_feedback": "HOLD until byte-exact baseline matrix exists.", - "hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.", - "nuvmap_chain": "HOLD until local fixture block replay and verifier exist." - }, - "receipt_hash": "73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296", - "required_outputs": [ - "environment_receipt.json", - "cpu_gpu_parity_receipt.json", - "waveprobe_batch_receipt.json", - "metaprobe_batch_receipt.json", - "negative_controls_receipt.json", - "cost_and_shutdown_receipt.json" - ], - "schema": "nuvmap_gpu_burst_check_plan_v0", - "timebox_hours": 3.0 -}