mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
Add NUVMAP scan scheduling receipts
This commit is contained in:
parent
600c96f179
commit
f2d75ea7be
22 changed files with 3836 additions and 0 deletions
249
4-Infrastructure/shim/blockchain_alternative_source_plan.py
Normal file
249
4-Infrastructure/shim/blockchain_alternative_source_plan.py
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Create a receipt for alternative blockchain corpus sources.
|
||||
|
||||
This is an admission/planning receipt, not a data-transfer receipt. It records
|
||||
which public/research sources can replace throttled Blockchair dump pulls and
|
||||
emits concrete command templates for the tools that can fetch them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_DESTINATION = "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10"
|
||||
|
||||
|
||||
BIGQUERY_CRYPTO_DATASETS = {
|
||||
"bitcoin_cash": {
|
||||
"chain": "bitcoin-cash",
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
},
|
||||
"dash": {
|
||||
"chain": "dash",
|
||||
"dataset": "bigquery-public-data.crypto_dash",
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
},
|
||||
"dogecoin": {
|
||||
"chain": "dogecoin",
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin",
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
},
|
||||
"litecoin": {
|
||||
"chain": "litecoin",
|
||||
"dataset": "bigquery-public-data.crypto_litecoin",
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
},
|
||||
"zcash": {
|
||||
"chain": "zcash",
|
||||
"dataset": "bigquery-public-data.crypto_zcash",
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
},
|
||||
"ethereum_classic": {
|
||||
"chain": "ethereum-classic",
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"blockchair_status": "NOT_ATTEMPTED",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
BITCOIN_ETL_CHAINS = [
|
||||
"bitcoin",
|
||||
"bitcoin_cash",
|
||||
"bitcoin_gold",
|
||||
"dogecoin",
|
||||
"litecoin",
|
||||
"dash",
|
||||
"zcash",
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def stable_hash(value: Any) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def command_status(name: str) -> dict[str, Any]:
|
||||
path = shutil.which(name)
|
||||
version = None
|
||||
if path:
|
||||
try:
|
||||
version_command = [name, "version"] if name == "bq" else [name, "--version"]
|
||||
proc = subprocess.run(version_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=15)
|
||||
version = proc.stdout.splitlines()[0] if proc.stdout else None
|
||||
except Exception as exc: # noqa: BLE001 - receipt diagnostic only.
|
||||
version = f"VERSION_CHECK_FAILED: {exc}"
|
||||
return {"available": bool(path), "path": path, "version": version}
|
||||
|
||||
|
||||
def python_module_status(module: str) -> dict[str, Any]:
|
||||
try:
|
||||
__import__(module)
|
||||
return {"available": True, "module": module}
|
||||
except Exception as exc: # noqa: BLE001 - receipt diagnostic only.
|
||||
return {"available": False, "module": module, "error": type(exc).__name__}
|
||||
|
||||
|
||||
def bq_extract_commands(destination: str) -> list[dict[str, Any]]:
|
||||
commands = []
|
||||
for source_id, meta in BIGQUERY_CRYPTO_DATASETS.items():
|
||||
dataset = meta["dataset"]
|
||||
chain = meta["chain"]
|
||||
remote = f"{destination.rstrip('/')}/public-datasets/google-bigquery/{chain}"
|
||||
commands.append(
|
||||
{
|
||||
"source_id": source_id,
|
||||
"chain": chain,
|
||||
"dataset": dataset,
|
||||
"tables_to_try_first": ["blocks", "transactions", "inputs", "outputs"],
|
||||
"local_export_pattern": f"shared-data/data/blockchain_corpus/bigquery_exports/{chain}/{{table}}/*.parquet",
|
||||
"drive_destination": remote,
|
||||
"commands": [
|
||||
f"bq ls {dataset}",
|
||||
(
|
||||
"bq extract --destination_format=PARQUET "
|
||||
f"'{dataset}.{{table}}' "
|
||||
f"'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/{chain}/{{table}}/*.parquet'"
|
||||
),
|
||||
(
|
||||
"gcloud storage cp --recursive "
|
||||
f"'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/{chain}/' "
|
||||
f"'shared-data/data/blockchain_corpus/bigquery_exports/{chain}/'"
|
||||
),
|
||||
(
|
||||
"rclone copy "
|
||||
f"'shared-data/data/blockchain_corpus/bigquery_exports/{chain}/' "
|
||||
f"'{remote}/'"
|
||||
),
|
||||
],
|
||||
}
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def bitcoin_etl_commands(destination: str) -> list[dict[str, Any]]:
|
||||
commands = []
|
||||
for chain in BITCOIN_ETL_CHAINS:
|
||||
remote = f"{destination.rstrip('/')}/node-etl/bitcoin-etl/{chain}"
|
||||
commands.append(
|
||||
{
|
||||
"chain": chain,
|
||||
"requires": ["running_full_node_or_rpc_snapshot", "bitcoin-etl"],
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
(
|
||||
"bitcoinetl export_blocks_and_transactions "
|
||||
"--start-block 0 --end-block <END_BLOCK> "
|
||||
f"--provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain {chain} "
|
||||
f"--blocks-output shared-data/data/blockchain_corpus/node_etl/{chain}/blocks.jsonl "
|
||||
f"--transactions-output shared-data/data/blockchain_corpus/node_etl/{chain}/transactions.jsonl"
|
||||
),
|
||||
f"rclone copy 'shared-data/data/blockchain_corpus/node_etl/{chain}/' '{remote}/'",
|
||||
],
|
||||
}
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--destination", default=DEFAULT_DESTINATION)
|
||||
parser.add_argument("--out", default="shared-data/data/blockchain_corpus/blockchain_alternative_source_plan_receipt.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
receipt = {
|
||||
"schema": "blockchain_alternative_source_plan_v0",
|
||||
"created_utc": utc_now(),
|
||||
"decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER",
|
||||
"claim_boundary": (
|
||||
"This receipt identifies alternative public/research sources and tool readiness. "
|
||||
"It does not prove data was exported from BigQuery, does not bypass source terms, "
|
||||
"and does not claim decoded chain semantics."
|
||||
),
|
||||
"destination": args.destination,
|
||||
"source_candidates": [
|
||||
{
|
||||
"name": "Google BigQuery Public Cryptocurrency Datasets",
|
||||
"decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS",
|
||||
"chains": list(BIGQUERY_CRYPTO_DATASETS.values()),
|
||||
"why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash.",
|
||||
"claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.",
|
||||
"python_client_ready": python_module_status("google.cloud.bigquery").get("available"),
|
||||
"source_urls": [
|
||||
"https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them",
|
||||
"https://www.cloudskillsboost.google/focuses/8486?parent=catalog",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots",
|
||||
"decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED",
|
||||
"chains": BITCOIN_ETL_CHAINS,
|
||||
"why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps.",
|
||||
"claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.",
|
||||
"source_urls": ["https://github.com/blockchain-etl/bitcoin-etl"],
|
||||
},
|
||||
{
|
||||
"name": "Bitquery Cloud Data Dumps",
|
||||
"decision": "EVALUATE_LICENSE_AND_COVERAGE",
|
||||
"chains": ["bitcoin"],
|
||||
"why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane.",
|
||||
"claim_boundary": "Coverage and licensing must be checked before mirroring.",
|
||||
"source_urls": ["https://docs.bitquery.io/docs/cloud/bitcoin/"],
|
||||
},
|
||||
{
|
||||
"name": "Kaggle/Hugging Face sampled datasets",
|
||||
"decision": "HOLD_SAMPLE_ONLY",
|
||||
"chains": ["bitcoin-cash", "dash", "dogecoin", "litecoin", "zcash"],
|
||||
"why": "Useful for smoke tests and model fixtures, not complete chain mirrors.",
|
||||
"claim_boundary": "Do not use for full-corpus claims.",
|
||||
"source_urls": [
|
||||
"https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset",
|
||||
"https://www.kaggle.com/datasets/amritpal333/crypto-mining-data",
|
||||
],
|
||||
},
|
||||
],
|
||||
"local_tool_readiness": {
|
||||
"commands": {
|
||||
name: command_status(name)
|
||||
for name in ["rclone", "bq", "gcloud", "gsutil", "kaggle", "duckdb"]
|
||||
},
|
||||
"python_modules": {
|
||||
module: python_module_status(module)
|
||||
for module in ["requests", "boto3", "pyarrow", "pandas", "google.cloud.bigquery", "duckdb"]
|
||||
},
|
||||
},
|
||||
"bigquery_export_commands": bq_extract_commands(args.destination),
|
||||
"node_etl_commands": bitcoin_etl_commands(args.destination),
|
||||
"next_gate": {
|
||||
"decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET",
|
||||
"required": [
|
||||
"Authenticate with a Google Cloud account allowed to query BigQuery public datasets.",
|
||||
"Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
"Provide or create a GCS staging bucket for BigQuery extract jobs.",
|
||||
"Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.",
|
||||
"Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts.",
|
||||
],
|
||||
},
|
||||
}
|
||||
receipt["receipt_hash"] = stable_hash(receipt)
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"decision": receipt["decision"], "out": str(out), "receipt_hash": receipt["receipt_hash"]}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
196
4-Infrastructure/shim/blockchain_bigquery_grabber.py
Normal file
196
4-Infrastructure/shim/blockchain_bigquery_grabber.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
#!/usr/bin/env python3
|
||||
"""BigQuery grabber command surface for public cryptocurrency datasets.
|
||||
|
||||
The script intentionally defaults to dry-run command emission. Actual BigQuery
|
||||
exports require Google Cloud authentication and a GCS staging bucket. This keeps
|
||||
the corpus lane receipt-bearing without embedding credentials or pretending
|
||||
that a public dataset can be exported anonymously.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DATASETS = {
|
||||
"bitcoin-cash": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"dash": "bigquery-public-data.crypto_dash",
|
||||
"dogecoin": "bigquery-public-data.crypto_dogecoin",
|
||||
"ethereum-classic": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"litecoin": "bigquery-public-data.crypto_litecoin",
|
||||
"zcash": "bigquery-public-data.crypto_zcash",
|
||||
}
|
||||
|
||||
DEFAULT_TABLES = ["blocks", "transactions", "inputs", "outputs"]
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
|
||||
|
||||
|
||||
def build_plan(chains: list[str], tables: list[str], gcs_bucket: str, drive_destination: str) -> dict[str, Any]:
|
||||
entries = []
|
||||
for chain in chains:
|
||||
dataset = DATASETS[chain]
|
||||
chain_slug = chain
|
||||
local_root = f"shared-data/data/blockchain_corpus/bigquery_exports/{chain_slug}"
|
||||
remote_root = f"{drive_destination.rstrip('/')}/public-datasets/google-bigquery/{chain_slug}"
|
||||
commands = [f"bq ls {dataset}"]
|
||||
for table in tables:
|
||||
commands.append(
|
||||
"bq extract --destination_format=PARQUET "
|
||||
f"'{dataset}.{table}' "
|
||||
f"'gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/{table}/*.parquet'"
|
||||
)
|
||||
commands.extend(
|
||||
[
|
||||
(
|
||||
"gcloud storage cp --recursive "
|
||||
f"'gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/' "
|
||||
f"'{local_root}/'"
|
||||
),
|
||||
f"rclone copy '{local_root}/' '{remote_root}/'",
|
||||
]
|
||||
)
|
||||
entries.append(
|
||||
{
|
||||
"chain": chain,
|
||||
"dataset": dataset,
|
||||
"tables": tables,
|
||||
"gcs_prefix": f"gs://{gcs_bucket}/research-stack/blockchain-corpus/{chain_slug}/",
|
||||
"local_root": local_root,
|
||||
"drive_destination": remote_root,
|
||||
"commands": commands,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema": "blockchain_bigquery_grabber_plan_v0",
|
||||
"claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.",
|
||||
"tool_status": {
|
||||
"bq": shutil.which("bq"),
|
||||
"gcloud": shutil.which("gcloud"),
|
||||
"gsutil": shutil.which("gsutil"),
|
||||
"rclone": shutil.which("rclone"),
|
||||
"python": sys.executable,
|
||||
"python_bigquery_client": python_module_available("google.cloud.bigquery"),
|
||||
"python_google_auth": python_module_available("google.auth"),
|
||||
},
|
||||
"entries": entries,
|
||||
}
|
||||
|
||||
|
||||
def python_module_available(module: str) -> bool:
|
||||
try:
|
||||
__import__(module)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def list_with_python_client(entries: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
|
||||
try:
|
||||
from google.api_core.exceptions import GoogleAPIError # type: ignore
|
||||
from google.auth.exceptions import DefaultCredentialsError # type: ignore
|
||||
from google.cloud import bigquery # type: ignore
|
||||
except Exception as exc: # noqa: BLE001 - receipt exact missing module.
|
||||
return "HOLD_BIGQUERY_PYTHON_CLIENT_MISSING", [
|
||||
{"error": type(exc).__name__, "message": str(exc), "python": sys.executable}
|
||||
]
|
||||
|
||||
try:
|
||||
client = bigquery.Client()
|
||||
except DefaultCredentialsError as exc:
|
||||
return "HOLD_GOOGLE_ADC_MISSING", [
|
||||
{
|
||||
"error": type(exc).__name__,
|
||||
"message": str(exc).splitlines()[0],
|
||||
"python": sys.executable,
|
||||
"next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
}
|
||||
]
|
||||
except Exception as exc: # noqa: BLE001 - receipt unexpected auth/client failures.
|
||||
return "HOLD_BIGQUERY_CLIENT_INIT_FAILED", [
|
||||
{"error": type(exc).__name__, "message": str(exc), "python": sys.executable}
|
||||
]
|
||||
|
||||
results = []
|
||||
ok = True
|
||||
for entry in entries:
|
||||
try:
|
||||
tables = list(client.list_tables(entry["dataset"]))
|
||||
results.append(
|
||||
{
|
||||
"chain": entry["chain"],
|
||||
"dataset": entry["dataset"],
|
||||
"tables": [{"table_id": table.table_id, "full_table_id": table.full_table_id} for table in tables],
|
||||
}
|
||||
)
|
||||
except GoogleAPIError as exc:
|
||||
ok = False
|
||||
results.append({"chain": entry["chain"], "dataset": entry["dataset"], "error": type(exc).__name__, "message": str(exc)})
|
||||
except Exception as exc: # noqa: BLE001 - receipt per-dataset failure.
|
||||
ok = False
|
||||
results.append({"chain": entry["chain"], "dataset": entry["dataset"], "error": type(exc).__name__, "message": str(exc)})
|
||||
return ("ADMIT_BIGQUERY_PYTHON_LIST_CHECK" if ok else "HOLD_BIGQUERY_PYTHON_LIST_CHECK_FAILED"), results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--chains", nargs="+", default=list(DATASETS), choices=sorted(DATASETS))
|
||||
parser.add_argument("--tables", nargs="+", default=DEFAULT_TABLES)
|
||||
parser.add_argument("--gcs-bucket", default="<YOUR_GCS_BUCKET>")
|
||||
parser.add_argument(
|
||||
"--drive-destination",
|
||||
default="Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10",
|
||||
)
|
||||
parser.add_argument("--out", default="shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json")
|
||||
parser.add_argument("--execute-list-only", action="store_true", help="Run only bq ls commands to verify table access.")
|
||||
parser.add_argument("--execute-python-list-only", action="store_true", help="Run table listing through google-cloud-bigquery.")
|
||||
args = parser.parse_args()
|
||||
|
||||
plan = build_plan(args.chains, args.tables, args.gcs_bucket, args.drive_destination)
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
if args.execute_list_only:
|
||||
if not shutil.which("bq"):
|
||||
print(json.dumps({"decision": "HOLD_BQ_CLI_MISSING", "out": str(out)}))
|
||||
return 2
|
||||
results = []
|
||||
for entry in plan["entries"]:
|
||||
proc = run(["bq", "ls", entry["dataset"]])
|
||||
results.append(
|
||||
{
|
||||
"chain": entry["chain"],
|
||||
"dataset": entry["dataset"],
|
||||
"returncode": proc.returncode,
|
||||
"stdout": proc.stdout[-4000:],
|
||||
"stderr": proc.stderr[-4000:],
|
||||
}
|
||||
)
|
||||
result_path = out.with_name(out.stem + "_list_results.json")
|
||||
result_path.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
ok = all(item["returncode"] == 0 for item in results)
|
||||
print(json.dumps({"decision": "ADMIT_BQ_LIST_CHECK" if ok else "HOLD_BQ_LIST_CHECK_FAILED", "out": str(out), "results": str(result_path)}))
|
||||
return 0 if ok else 2
|
||||
|
||||
if args.execute_python_list_only:
|
||||
decision, results = list_with_python_client(plan["entries"])
|
||||
result_path = out.with_name(out.stem + "_python_list_results.json")
|
||||
result_path.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"decision": decision, "out": str(out), "results": str(result_path)}))
|
||||
return 0 if decision.startswith("ADMIT") else 2
|
||||
|
||||
print(json.dumps({"decision": "ADMIT_BIGQUERY_GRABBER_DRY_RUN_PLAN", "out": str(out)}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
283
4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py
Normal file
283
4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Emit a Layer-3 self-scanning plan for blockchain corpus receipts.
|
||||
|
||||
The scheduler treats existing inventory and transfer receipts as the control
|
||||
surface. It does not decode chain semantics or fetch more data. Its job is to
|
||||
turn scan evidence into the next bounded scan actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_ACCOUNT = REPO / "shared-data/data/blockchain_corpus/blockchain_full_account_live_receipt.json"
|
||||
DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json"
|
||||
|
||||
|
||||
STORAGE_CHANNELS = {
|
||||
"bitcoin": [
|
||||
{
|
||||
"channel": "op_return",
|
||||
"scan_surface": "transaction outputs / scriptPubKey",
|
||||
"use_boundary": "Detect and receipt embedded payload surfaces; do not publish or optimize arbitrary payload insertion.",
|
||||
},
|
||||
{
|
||||
"channel": "witness_inscription_like_payload",
|
||||
"scan_surface": "segwit witness data",
|
||||
"use_boundary": "Treat as public immutable payload evidence only.",
|
||||
},
|
||||
{
|
||||
"channel": "coinbase_tag",
|
||||
"scan_surface": "coinbase transaction script/witness fields",
|
||||
"use_boundary": "Pool tags and receipt markers only; no attribution claim without external witness.",
|
||||
},
|
||||
],
|
||||
"evm": [
|
||||
{
|
||||
"channel": "calldata",
|
||||
"scan_surface": "transaction input bytes",
|
||||
"use_boundary": "Route by byte density and selector shape only; no private-key, exploit, or evasion workflow.",
|
||||
},
|
||||
{
|
||||
"channel": "event_logs",
|
||||
"scan_surface": "topics and data fields",
|
||||
"use_boundary": "Receipt emitted public events as storage-like data lanes.",
|
||||
},
|
||||
{
|
||||
"channel": "contract_bytecode",
|
||||
"scan_surface": "creation/runtime bytecode",
|
||||
"use_boundary": "Static payload/code-carrier diagnostic only.",
|
||||
},
|
||||
{
|
||||
"channel": "blob_or_da_payload",
|
||||
"scan_surface": "data availability/blob lanes when present in source tables",
|
||||
"use_boundary": "Cost-bounded metadata scan first; bulk payload fetch remains HOLD.",
|
||||
},
|
||||
],
|
||||
"memo": [
|
||||
{
|
||||
"channel": "memo_or_message",
|
||||
"scan_surface": "transaction memo/message/payload fields",
|
||||
"use_boundary": "Public memo payload diagnostics only.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def stable_json(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
|
||||
|
||||
|
||||
def sha256_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def rel(path: str | Path) -> str:
|
||||
p = Path(path)
|
||||
try:
|
||||
return str(p.relative_to(REPO))
|
||||
except ValueError:
|
||||
return str(p)
|
||||
|
||||
|
||||
def density_band(total_bytes: int, object_count: int) -> str:
|
||||
if object_count <= 0:
|
||||
return "empty"
|
||||
bytes_per_object = total_bytes / object_count
|
||||
if bytes_per_object >= 64 * 1024 * 1024:
|
||||
return "heavy"
|
||||
if bytes_per_object >= 8 * 1024 * 1024:
|
||||
return "medium"
|
||||
return "light"
|
||||
|
||||
|
||||
def completion_state(item: dict[str, Any]) -> dict[str, Any]:
|
||||
object_count = int(item.get("object_count") or 0)
|
||||
remote_payload = int(item.get("remote_payload_count") or 0)
|
||||
remote_parquet = int(item.get("remote_parquet_count") or 0)
|
||||
completed = min(remote_payload, remote_parquet)
|
||||
missing = max(object_count - completed, 0)
|
||||
ratio = completed / object_count if object_count else 0.0
|
||||
return {
|
||||
"completed_objects": completed,
|
||||
"missing_objects": missing,
|
||||
"remote_parquet_count": remote_parquet,
|
||||
"remote_payload_count": remote_payload,
|
||||
"completion_ratio": round(ratio, 9),
|
||||
}
|
||||
|
||||
|
||||
def scan_actions(item: dict[str, Any], state: dict[str, Any], band: str) -> list[dict[str, Any]]:
|
||||
actions: list[dict[str, Any]] = []
|
||||
chain = item.get("chain")
|
||||
table = item.get("table")
|
||||
inventory = item.get("inventory")
|
||||
|
||||
if state["missing_objects"]:
|
||||
actions.append(
|
||||
{
|
||||
"action": "RETRY_MISSING_OBJECTS",
|
||||
"reason": "Remote payload/parquet count is below inventory object count.",
|
||||
"script": "4-Infrastructure/shim/blockchain_public_dataset_transfer.py",
|
||||
"inputs": {"inventory": inventory, "chain": chain, "table": table},
|
||||
"decision": "HOLD_UNTIL_RETRY_RECEIPT",
|
||||
}
|
||||
)
|
||||
|
||||
actions.append(
|
||||
{
|
||||
"action": "RUN_HEADER_OR_BLOCK_PATTERN_PROBE",
|
||||
"reason": "Use deterministic numeric residuals as route priors before byte-heavy scans.",
|
||||
"script": "4-Infrastructure/shim/blockchain_header_pattern_probe.py",
|
||||
"inputs": {"chain": chain, "table": table, "inventory": inventory},
|
||||
"decision": "ADMIT_BOUNDED_ROUTE_PRIOR_SCAN",
|
||||
}
|
||||
)
|
||||
|
||||
if band in {"medium", "heavy"}:
|
||||
actions.append(
|
||||
{
|
||||
"action": "RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE",
|
||||
"reason": "Large average objects should be sampled through feature/density probes before broader transfer.",
|
||||
"script": "4-Infrastructure/shim/parquet_logogram_eigenprobe.py",
|
||||
"inputs": {"chain": chain, "table": table, "sample_policy": "first_middle_last_partition"},
|
||||
"decision": "ADMIT_SAMPLE_ONLY_HOLD_BULK",
|
||||
}
|
||||
)
|
||||
|
||||
actions.append(
|
||||
{
|
||||
"action": "REFRESH_SELF_SCAN_SCHEDULER",
|
||||
"reason": "The scan output becomes the next L3 control input.",
|
||||
"script": "4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py",
|
||||
"inputs": {"account_receipt": rel(DEFAULT_ACCOUNT)},
|
||||
"decision": "ADMIT_RECEIPT_FEEDBACK_LOOP",
|
||||
}
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
def chain_storage_family(chain: str | None) -> str:
|
||||
if chain in {"ethereum", "ethereum-classic", "arbitrum", "base", "cronos"}:
|
||||
return "evm"
|
||||
if chain in {"bitcoin", "bitcoin-cash", "dash", "dogecoin", "litecoin", "zcash"}:
|
||||
return "bitcoin"
|
||||
if chain in {"xrp", "stellar", "ton", "aptos", "provenance"}:
|
||||
return "memo"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def storage_channel_actions(item: dict[str, Any], band: str) -> dict[str, Any]:
|
||||
chain = item.get("chain")
|
||||
family = chain_storage_family(chain)
|
||||
channels = STORAGE_CHANNELS.get(family, [])
|
||||
actions = []
|
||||
if channels:
|
||||
actions.append(
|
||||
{
|
||||
"action": "RUN_ONCHAIN_STORAGE_CHANNEL_PROBE",
|
||||
"reason": "Blockchain records can contain intentional public data carriers; scan them as storage surfaces before treating bytes as neutral noise.",
|
||||
"inputs": {
|
||||
"chain": chain,
|
||||
"family": family,
|
||||
"table": item.get("table"),
|
||||
"inventory": item.get("inventory"),
|
||||
"density_band": band,
|
||||
},
|
||||
"decision": "ADMIT_STORAGE_SURFACE_SCAN_HOLD_PAYLOAD_USE",
|
||||
}
|
||||
)
|
||||
return {
|
||||
"family": family,
|
||||
"channels": channels,
|
||||
"actions": actions,
|
||||
"claim_boundary": (
|
||||
"Storage-channel model is for detection, accounting, and receipt routing. "
|
||||
"It does not recommend publishing payloads on-chain, bypassing moderation, "
|
||||
"hiding data, or using public ledgers as private storage."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_receipt(account_path: Path) -> dict[str, Any]:
|
||||
account = json.loads(account_path.read_text(encoding="utf-8"))
|
||||
entries = []
|
||||
for item in account.get("source_inventories", []):
|
||||
total_bytes = int(item.get("total_listed_bytes") or 0)
|
||||
object_count = int(item.get("object_count") or 0)
|
||||
state = completion_state(item)
|
||||
band = density_band(total_bytes, object_count)
|
||||
entries.append(
|
||||
{
|
||||
"chain": item.get("chain"),
|
||||
"table": item.get("table"),
|
||||
"dataset": item.get("dataset"),
|
||||
"inventory": item.get("inventory"),
|
||||
"inventory_hash": item.get("inventory_hash"),
|
||||
"object_count": object_count,
|
||||
"total_listed_bytes": total_bytes,
|
||||
"bytes_per_object": round(total_bytes / object_count, 3) if object_count else 0.0,
|
||||
"density_band": band,
|
||||
"storage_channel_model": storage_channel_actions(item, band),
|
||||
"completion": state,
|
||||
"scan_actions": scan_actions(item, state, band),
|
||||
}
|
||||
)
|
||||
|
||||
entries.sort(key=lambda row: (row["completion"]["missing_objects"] > 0, row["total_listed_bytes"]), reverse=True)
|
||||
payload = {
|
||||
"schema": "blockchain_l3_self_scan_scheduler_v0",
|
||||
"created_utc": now_iso(),
|
||||
"claim_boundary": (
|
||||
"Layer-3 scan scheduling receipt only. It uses existing inventory and transfer receipts "
|
||||
"to choose next bounded probes. It does not decode chain semantics, claim compression gain, "
|
||||
"claim market prediction, or fetch additional corpus bytes."
|
||||
),
|
||||
"layer3_interpretation": {
|
||||
"old_label": "L3 Bitstream",
|
||||
"scanner_label": "L3 executable scan policy",
|
||||
"core_rule": "scan_receipts_t -> route_policy_t_plus_1",
|
||||
"feedback_equation": "P_{t+1}=L3(R_t,A_t,C_t); R_{t+1}=scan(P_{t+1})",
|
||||
"storage_rule": "onchain_payload_surfaces_t -> storage_channel_receipts_t_plus_1",
|
||||
},
|
||||
"global_storage_boundary": (
|
||||
"Public blockchains can store data, but this scheduler only models storage-bearing "
|
||||
"surfaces for detection, provenance, and compression-route diagnostics. Bulk payload "
|
||||
"reconstruction and payload publication remain outside this receipt."
|
||||
),
|
||||
"account_receipt": rel(account_path),
|
||||
"account_receipt_hash": account.get("receipt_hash"),
|
||||
"source_count": len(entries),
|
||||
"frontier": entries,
|
||||
"decision": "ADMIT_L3_SELF_SCAN_POLICY",
|
||||
}
|
||||
payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"}))
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--account", type=Path, default=DEFAULT_ACCOUNT)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
receipt = build_receipt(args.account)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
180
4-Infrastructure/shim/nuvmap_chain_protocol_plan.py
Normal file
180
4-Infrastructure/shim/nuvmap_chain_protocol_plan.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Create a protocol receipt for a dedicated NUVMAP chain.
|
||||
|
||||
NUVMAP is treated here as a receipt chain for metaprobe/waveprobe outputs. This
|
||||
is not a token, investment, mainnet, or compression-result claim. It specifies
|
||||
what a block would carry so scanner state can be stored and replayed without
|
||||
using third-party chains as an accidental storage layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json"
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def stable_json(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
|
||||
|
||||
|
||||
def sha256_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def rel(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(REPO))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def build_receipt() -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"schema": "nuvmap_chain_protocol_plan_v0",
|
||||
"created_utc": now_iso(),
|
||||
"claim_boundary": (
|
||||
"Protocol design receipt only. This does not create a blockchain, token, market, "
|
||||
"consensus network, compression result, or Hutter Prize claim. It defines a local/devnet "
|
||||
"receipt substrate for metaprobe and waveprobe state."
|
||||
),
|
||||
"purpose": {
|
||||
"name": "NUVMAP",
|
||||
"expanded_role": "Numerical Universe Vector Map receipt chain",
|
||||
"core_use": "Store replayable probe-state commitments so scanning can route itself over time.",
|
||||
"anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.",
|
||||
},
|
||||
"chain_model": {
|
||||
"deployment_stage": "LOCAL_DEVNET_ONLY",
|
||||
"economic_model": "NO_TOKEN_NO_MARKET",
|
||||
"consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set",
|
||||
"finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional",
|
||||
"data_availability": "on-chain summaries plus content-addressed off-chain sidecars",
|
||||
},
|
||||
"block_shape": {
|
||||
"header": {
|
||||
"parent_hash": "sha256(previous_block)",
|
||||
"height": "u64",
|
||||
"created_utc": "iso8601",
|
||||
"scanner_policy_hash": "sha256(l3_policy)",
|
||||
"state_root": "merkle_root(receipt_state)",
|
||||
"sidecar_manifest_root": "merkle_root(sidecar_hashes)",
|
||||
},
|
||||
"body": {
|
||||
"waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries",
|
||||
"metaprobe_receipts": "bounded list of route/context/curriculum summaries",
|
||||
"storage_channel_receipts": "detected on-chain storage surfaces from source chains",
|
||||
"scan_actions": "next bounded scan plan emitted by L3 scheduler",
|
||||
"negative_controls": "required controls before promotion of any route",
|
||||
},
|
||||
"forbidden_body_fields": [
|
||||
"private keys",
|
||||
"wallet credentials",
|
||||
"bulk copyrighted payloads",
|
||||
"payloads intended to hide from moderation or provenance",
|
||||
"financial promotion metadata",
|
||||
],
|
||||
},
|
||||
"probe_mapping": {
|
||||
"metadata_blitter": {
|
||||
"input": "fixed-size metadata words from chain/block/window records",
|
||||
"output": "sortable route keys and packed metric words",
|
||||
"storage": "blitter shader hash, parameter receipt, key/metric buffer hashes",
|
||||
},
|
||||
"waveprobe": {
|
||||
"input": "byte windows, block fields, density neighborhoods, eigenvalue spectra",
|
||||
"output": "local signal vector plus residual/curvature summary",
|
||||
"storage": "hash, vector summary, source window receipt, not raw bulk by default",
|
||||
},
|
||||
"metaprobe": {
|
||||
"input": "domain priors, route families, previous scan receipts",
|
||||
"output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints",
|
||||
"storage": "decision receipt, route vector, source receipt backlinks",
|
||||
},
|
||||
"l3_scheduler": {
|
||||
"input": "current receipt frontier",
|
||||
"output": "next scan frontier",
|
||||
"storage": "scanner policy hash and action list",
|
||||
},
|
||||
},
|
||||
"radix_bin_model": {
|
||||
"metaphor": "hyper_soliton_search",
|
||||
"claim_boundary": "Radix bins are route basins, not semantic labels.",
|
||||
"fields": {
|
||||
"hash8": "phase seed / local identity packet",
|
||||
"density7": "amplitude / byte pressure",
|
||||
"delta7": "shock or torsion gradient",
|
||||
"zero6": "void / low-information throat",
|
||||
"flags4": "boundary condition",
|
||||
},
|
||||
"stability_rule": "Promote only bins that persist across salts, windows, and negative controls.",
|
||||
},
|
||||
"minimal_transaction_types": [
|
||||
{
|
||||
"type": "BLITTER_BIN_RECEIPT",
|
||||
"required_fields": ["shader_hash", "params_hash", "source_hash", "key_buffer_hash", "metric_buffer_hash", "decision"],
|
||||
},
|
||||
{
|
||||
"type": "WAVE_RECEIPT",
|
||||
"required_fields": ["source_hash", "window_descriptor", "feature_vector_hash", "decision"],
|
||||
},
|
||||
{
|
||||
"type": "META_RECEIPT",
|
||||
"required_fields": ["prior_id", "route_family", "evidence_hash", "decision"],
|
||||
},
|
||||
{
|
||||
"type": "STORAGE_SURFACE_RECEIPT",
|
||||
"required_fields": ["source_chain", "channel", "carrier_descriptor", "payload_hash_or_null", "decision"],
|
||||
},
|
||||
{
|
||||
"type": "SCAN_POLICY_UPDATE",
|
||||
"required_fields": ["previous_policy_hash", "new_policy_hash", "action_merkle_root", "decision"],
|
||||
},
|
||||
{
|
||||
"type": "NEGATIVE_CONTROL",
|
||||
"required_fields": ["control_kind", "source_hash", "result_hash", "decision"],
|
||||
},
|
||||
],
|
||||
"decision_law": {
|
||||
"ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.",
|
||||
"HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.",
|
||||
"QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation.",
|
||||
},
|
||||
"next_implementation_steps": [
|
||||
"Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.",
|
||||
"Add a verifier that recomputes the block hash and merkle roots.",
|
||||
"Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.",
|
||||
"Run negative controls before any compression-route promotion.",
|
||||
"Only after local fixture replay works, consider a small append-only local devnet.",
|
||||
],
|
||||
"decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION",
|
||||
}
|
||||
payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"}))
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
receipt = build_receipt()
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
154
4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py
Normal file
154
4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Emit a bounded GPU-burst check plan for NUVMAP probe validation.
|
||||
|
||||
The plan is intended for a short rented GPU session. It avoids wallets, mining,
|
||||
public-chain writes, and long-lived services. The output is a receipt/checklist
|
||||
for replaying local fixtures, checking CPU/GPU parity, and collecting bounded
|
||||
waveprobe/metaprobe evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_OUT = REPO / "shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json"
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def stable_json(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
|
||||
|
||||
|
||||
def sha256_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def rel(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(REPO))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def build_receipt(max_hours: float) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"schema": "nuvmap_gpu_burst_check_plan_v0",
|
||||
"created_utc": now_iso(),
|
||||
"claim_boundary": (
|
||||
"GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, "
|
||||
"not a token launch, not a compression result, and not a hardware acceleration claim until "
|
||||
"the listed receipts are produced."
|
||||
),
|
||||
"timebox_hours": max_hours,
|
||||
"forbidden_inputs": [
|
||||
"wallet private keys",
|
||||
"exchange credentials",
|
||||
"mainnet transaction signing material",
|
||||
"unbounded copyrighted payload mirrors",
|
||||
],
|
||||
"required_outputs": [
|
||||
"environment_receipt.json",
|
||||
"cpu_gpu_parity_receipt.json",
|
||||
"waveprobe_batch_receipt.json",
|
||||
"metaprobe_batch_receipt.json",
|
||||
"negative_controls_receipt.json",
|
||||
"cost_and_shutdown_receipt.json",
|
||||
],
|
||||
"burst_phases": [
|
||||
{
|
||||
"phase": "P0_ENVIRONMENT_SNAPSHOT",
|
||||
"target_minutes": 10,
|
||||
"checks": [
|
||||
"nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv",
|
||||
"python -c 'import torch; print(torch.cuda.is_available())' if torch is installed",
|
||||
"git rev-parse HEAD and git status --short for provenance",
|
||||
],
|
||||
"decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE",
|
||||
},
|
||||
{
|
||||
"phase": "P1_FIXTURE_REPLAY",
|
||||
"target_minutes": 25,
|
||||
"checks": [
|
||||
"Verify NUVMAP fixture block hash and merkle roots.",
|
||||
"Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.",
|
||||
"Reject any fixture with missing source hashes.",
|
||||
],
|
||||
"decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD",
|
||||
},
|
||||
{
|
||||
"phase": "P2_CPU_GPU_PARITY",
|
||||
"target_minutes": 35,
|
||||
"checks": [
|
||||
"Run the same wave/vector kernel on CPU and GPU.",
|
||||
"Compare Q16.16 or declared float tolerance residuals.",
|
||||
"Emit max_abs_error, mean_abs_error, and mismatch examples.",
|
||||
],
|
||||
"decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED",
|
||||
},
|
||||
{
|
||||
"phase": "P3_WAVEPROBE_BATCH",
|
||||
"target_minutes": 45,
|
||||
"checks": [
|
||||
"Run bounded byte-window/eigen/density probes over sampled blockchain shards.",
|
||||
"Store only feature hashes and summaries unless payload admission exists.",
|
||||
"Emit route candidates as HOLD until negative controls pass.",
|
||||
],
|
||||
"decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION",
|
||||
},
|
||||
{
|
||||
"phase": "P4_METAPROBE_AND_CONTROLS",
|
||||
"target_minutes": 45,
|
||||
"checks": [
|
||||
"Run metaprobe route selection over waveprobe outputs.",
|
||||
"Run shuffled window, shuffled chain label, and random-byte controls.",
|
||||
"Require controls before any Hutter/logogram feedback promotion.",
|
||||
],
|
||||
"decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL",
|
||||
},
|
||||
{
|
||||
"phase": "P5_COST_AND_SHUTDOWN",
|
||||
"target_minutes": 20,
|
||||
"checks": [
|
||||
"Write final receipt bundle.",
|
||||
"Sync receipts and small summaries to Drive.",
|
||||
"Record instance type, wall time, estimated cost, and shutdown confirmation.",
|
||||
],
|
||||
"decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT",
|
||||
},
|
||||
],
|
||||
"promotion_gates": {
|
||||
"hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.",
|
||||
"compression_feedback": "HOLD until byte-exact baseline matrix exists.",
|
||||
"nuvmap_chain": "HOLD until local fixture block replay and verifier exist.",
|
||||
},
|
||||
"decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION",
|
||||
}
|
||||
payload["receipt_hash"] = sha256_text(stable_json({k: v for k, v in payload.items() if k != "receipt_hash"}))
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--hours", type=float, default=3.0)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
receipt = build_receipt(args.hours)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps({"decision": receipt["decision"], "out": rel(args.out), "receipt_hash": receipt["receipt_hash"]}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
133
5-Applications/scripts/nuvmap_metadata_blitter.md
Normal file
133
5-Applications/scripts/nuvmap_metadata_blitter.md
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
# NUVMAP Metadata Blitter
|
||||
|
||||
`nuvmap_metadata_blitter.wgsl` is a deliberately small WebGPU compute pass for
|
||||
metadata sorting. It treats the GPU as a dataport fabric: one SIMD lane group
|
||||
reads fixed-size byte/word windows and emits compact route keys for later sort,
|
||||
waveprobe, metaprobe, or NUVMAP receipt steps.
|
||||
|
||||
## Contract
|
||||
|
||||
Input:
|
||||
|
||||
| Buffer | Meaning |
|
||||
| --- | --- |
|
||||
| `source_words` | Packed `u32` metadata words from block/window/shard records |
|
||||
| `BlitParams` | Record count, record width, stride, offset, salt, flags |
|
||||
|
||||
Output:
|
||||
|
||||
| Buffer | Meaning |
|
||||
| --- | --- |
|
||||
| `key_buffer` | One sortable `u32` route key per record |
|
||||
| `metric_buffer` | One packed `u32` metrics word per record |
|
||||
|
||||
The pass does not decompress, classify semantic meaning, publish data on-chain,
|
||||
or claim compression improvement.
|
||||
|
||||
## Key Word
|
||||
|
||||
`key_buffer[i]` packs:
|
||||
|
||||
| Bits | Field | Meaning |
|
||||
| --- | --- | --- |
|
||||
| 31..24 | `hash8` | salted local hash prefix |
|
||||
| 23..17 | `density7` | one-bit density estimate |
|
||||
| 16..10 | `delta7` | adjacent-word XOR delta pressure |
|
||||
| 9..4 | `zero6` | zero-byte share |
|
||||
| 3..0 | `flags4` | route flags |
|
||||
|
||||
This is designed for a follow-up radix/sort pass. Sorting by this key groups
|
||||
similar metadata windows without pretending the groups are semantic classes.
|
||||
|
||||
## Metric Word
|
||||
|
||||
`metric_buffer[i]` packs:
|
||||
|
||||
| Bits | Field | Meaning |
|
||||
| --- | --- | --- |
|
||||
| 31..26 | `class0` | byte count for `00xxxxxx` |
|
||||
| 25..20 | `class1` | byte count for `01xxxxxx` |
|
||||
| 19..14 | `class2` | byte count for `10xxxxxx` |
|
||||
| 13..8 | `class3` | byte count for `11xxxxxx` |
|
||||
| 7..1 | `high_bit_share7` | same density estimate used in the key |
|
||||
| 0 | `valid` | one if at least one word was scanned |
|
||||
|
||||
The four byte classes are cheap enough to compute per lane and useful enough to
|
||||
seed later waveprobe/metaprobe decisions.
|
||||
|
||||
## Dispatch Shape
|
||||
|
||||
The shader uses:
|
||||
|
||||
```wgsl
|
||||
@compute @workgroup_size(256, 1, 1)
|
||||
```
|
||||
|
||||
Dispatch count:
|
||||
|
||||
```text
|
||||
ceil(record_count / 256)
|
||||
```
|
||||
|
||||
Each invocation scans at most `64` words. Larger records should be presented as
|
||||
multiple fixed windows so the kernel stays predictable and massively shardable.
|
||||
|
||||
## NUVMAP Use
|
||||
|
||||
The intended receipt loop is:
|
||||
|
||||
```text
|
||||
metadata shard
|
||||
-> blitter key/metric buffers
|
||||
-> bounded sort/group
|
||||
-> waveprobe/metaprobe sample selection
|
||||
-> NUVMAP receipt
|
||||
-> L3 self-scan scheduler
|
||||
```
|
||||
|
||||
Promotion stays bounded:
|
||||
|
||||
```text
|
||||
ADMIT: deterministic key/metric emission with replayable source window
|
||||
HOLD: route meaning, compression gain, or semantic label
|
||||
QUARANTINE: hash mismatch, unsafe payload class, or missing provenance
|
||||
```
|
||||
|
||||
## Hyper-Soliton Radix Bins
|
||||
|
||||
The clean metaphor for the radix bins is the hyper-soliton search surface.
|
||||
|
||||
A radix bin is not a semantic label. It is a temporary, stable packet of similar
|
||||
metadata pressure:
|
||||
|
||||
```text
|
||||
blitter key field
|
||||
-> radix bin
|
||||
-> route basin
|
||||
-> waveprobe sample
|
||||
-> metaprobe decision
|
||||
```
|
||||
|
||||
In the hyper-soliton view:
|
||||
|
||||
| Radix Object | Hyper-Soliton Analogue |
|
||||
| --- | --- |
|
||||
| `hash8` | phase seed / local identity packet |
|
||||
| `density7` | amplitude / byte pressure |
|
||||
| `delta7` | shock or torsion gradient |
|
||||
| `zero6` | void / low-information throat |
|
||||
| `flags4` | boundary condition |
|
||||
| radix bin | persistent soliton basin |
|
||||
| bin split | soliton fission |
|
||||
| bin merge | soliton collision |
|
||||
| unstable bin | dispersive residue / HOLD |
|
||||
|
||||
That gives the scanner a useful rule:
|
||||
|
||||
```text
|
||||
probe bins that persist across salts, windows, and controls;
|
||||
hold bins that only exist under one projection.
|
||||
```
|
||||
|
||||
So the GPU blitter is the high-throughput flow step, radix grouping is the
|
||||
soliton-basin finder, and NUVMAP receipts record which basins survived replay.
|
||||
136
5-Applications/scripts/nuvmap_metadata_blitter.wgsl
Normal file
136
5-Applications/scripts/nuvmap_metadata_blitter.wgsl
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// NUVMAP metadata blitter for WebGPU.
|
||||
//
|
||||
// One invocation scans one fixed-size metadata record/window and emits:
|
||||
// key_buffer[i] = sortable route key
|
||||
// metric_buffer[i] = packed byte/delta/hash metrics
|
||||
//
|
||||
// The shader is intentionally tiny and SIMD-friendly. It does not decompress,
|
||||
// classify semantic meaning, or claim compression gain. It only turns byte-ish
|
||||
// metadata windows into deterministic route keys for later sorting/probing.
|
||||
|
||||
struct BlitParams {
|
||||
record_count: u32,
|
||||
words_per_record: u32,
|
||||
stride_words: u32,
|
||||
source_offset_words: u32,
|
||||
route_salt: u32,
|
||||
flags: u32,
|
||||
}
|
||||
|
||||
@group(0) @binding(0) var<storage, read> source_words: array<u32>;
|
||||
@group(0) @binding(1) var<storage, read_write> key_buffer: array<u32>;
|
||||
@group(0) @binding(2) var<storage, read_write> metric_buffer: array<u32>;
|
||||
@group(0) @binding(3) var<uniform> params: BlitParams;
|
||||
|
||||
const WORDS_HARD_CAP: u32 = 64u;
|
||||
|
||||
fn popcount32(v: u32) -> u32 {
|
||||
return countOneBits(v);
|
||||
}
|
||||
|
||||
fn byte_class_counts(word: u32) -> vec4<u32> {
|
||||
var counts = vec4<u32>(0u, 0u, 0u, 0u);
|
||||
for (var lane = 0u; lane < 4u; lane = lane + 1u) {
|
||||
let b = (word >> (lane * 8u)) & 0xFFu;
|
||||
let cls = b >> 6u;
|
||||
if (cls == 0u) {
|
||||
counts.x = counts.x + 1u;
|
||||
} else if (cls == 1u) {
|
||||
counts.y = counts.y + 1u;
|
||||
} else if (cls == 2u) {
|
||||
counts.z = counts.z + 1u;
|
||||
} else {
|
||||
counts.w = counts.w + 1u;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
fn mix32(x: u32) -> u32 {
|
||||
var h = x;
|
||||
h = h ^ (h >> 16u);
|
||||
h = h * 0x7FEB352Du;
|
||||
h = h ^ (h >> 15u);
|
||||
h = h * 0x846CA68Bu;
|
||||
h = h ^ (h >> 16u);
|
||||
return h;
|
||||
}
|
||||
|
||||
fn pack_key(hash8: u32, density7: u32, delta7: u32, zero6: u32, flags4: u32) -> u32 {
|
||||
return ((hash8 & 0xFFu) << 24u) |
|
||||
((density7 & 0x7Fu) << 17u) |
|
||||
((delta7 & 0x7Fu) << 10u) |
|
||||
((zero6 & 0x3Fu) << 4u) |
|
||||
(flags4 & 0xFu);
|
||||
}
|
||||
|
||||
fn pack_metric(class_counts: vec4<u32>, high_bit_share7: u32, words_seen: u32) -> u32 {
|
||||
let c0 = min(class_counts.x, 0x3Fu);
|
||||
let c1 = min(class_counts.y, 0x3Fu);
|
||||
let c2 = min(class_counts.z, 0x3Fu);
|
||||
let c3 = min(class_counts.w, 0x3Fu);
|
||||
return (c0 << 26u) |
|
||||
(c1 << 20u) |
|
||||
(c2 << 14u) |
|
||||
(c3 << 8u) |
|
||||
((high_bit_share7 & 0x7Fu) << 1u) |
|
||||
min(words_seen, 1u);
|
||||
}
|
||||
|
||||
@compute @workgroup_size(256, 1, 1)
|
||||
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
||||
let record_index = gid.x;
|
||||
if (record_index >= params.record_count) {
|
||||
return;
|
||||
}
|
||||
|
||||
let words_to_scan = min(params.words_per_record, WORDS_HARD_CAP);
|
||||
let base = params.source_offset_words + record_index * params.stride_words;
|
||||
var hash_acc = mix32(record_index ^ params.route_salt);
|
||||
var one_bits = 0u;
|
||||
var zero_bytes = 0u;
|
||||
var delta_acc = 0u;
|
||||
var class_counts = vec4<u32>(0u, 0u, 0u, 0u);
|
||||
var previous = 0u;
|
||||
|
||||
for (var i = 0u; i < words_to_scan; i = i + 1u) {
|
||||
let word = source_words[base + i];
|
||||
hash_acc = mix32(hash_acc ^ word ^ (i * 0x9E3779B9u));
|
||||
one_bits = one_bits + popcount32(word);
|
||||
class_counts = class_counts + byte_class_counts(word);
|
||||
|
||||
for (var lane = 0u; lane < 4u; lane = lane + 1u) {
|
||||
let b = (word >> (lane * 8u)) & 0xFFu;
|
||||
if (b == 0u) {
|
||||
zero_bytes = zero_bytes + 1u;
|
||||
}
|
||||
}
|
||||
|
||||
if (i > 0u) {
|
||||
delta_acc = delta_acc + popcount32(word ^ previous);
|
||||
}
|
||||
previous = word;
|
||||
}
|
||||
|
||||
let total_bits = max(words_to_scan * 32u, 1u);
|
||||
let total_bytes = max(words_to_scan * 4u, 1u);
|
||||
let high_bit_share7 = min((one_bits * 127u) / total_bits, 127u);
|
||||
let density7 = high_bit_share7;
|
||||
let delta7 = min((delta_acc * 127u) / total_bits, 127u);
|
||||
let zero6 = min((zero_bytes * 63u) / total_bytes, 63u);
|
||||
let hash8 = hash_acc >> 24u;
|
||||
|
||||
var flags4 = params.flags & 0xFu;
|
||||
if (zero6 > 48u) {
|
||||
flags4 = flags4 | 0x1u;
|
||||
}
|
||||
if (delta7 > 96u) {
|
||||
flags4 = flags4 | 0x2u;
|
||||
}
|
||||
if (density7 < 16u || density7 > 112u) {
|
||||
flags4 = flags4 | 0x4u;
|
||||
}
|
||||
|
||||
key_buffer[record_index] = pack_key(hash8, density7, delta7, zero6, flags4);
|
||||
metric_buffer[record_index] = pack_metric(class_counts, high_bit_share7, words_to_scan);
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack Blockchain L3 Scheduler Receipts NUVMAP HOLD
|
||||
title: Blockchain L3 Self-Scan Scheduler
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! Blockchain L3 Self-Scan Scheduler
|
||||
|
||||
Status: `ADMIT_L3_SELF_SCAN_POLICY`
|
||||
|
||||
The blockchain L3 self-scan scheduler treats scan receipts as the control
|
||||
surface for the next scan. This reframes L3 from only "bitstream build" into an
|
||||
executable scan policy:
|
||||
|
||||
```text
|
||||
P_{t+1}=L3(R_t,A_t,C_t)
|
||||
R_{t+1}=scan(P_{t+1})
|
||||
```
|
||||
|
||||
Where:
|
||||
|
||||
```text
|
||||
R_t = current receipt frontier
|
||||
A_t = available scan actions
|
||||
C_t = cost / completion / claim-boundary constraints
|
||||
P_t = next scan policy
|
||||
```
|
||||
|
||||
!! Runner
|
||||
|
||||
```text
|
||||
4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py
|
||||
```
|
||||
|
||||
Receipt:
|
||||
|
||||
```text
|
||||
shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json
|
||||
```
|
||||
|
||||
Current receipt hash:
|
||||
|
||||
```text
|
||||
d9d8c49369c700e0749e467615bf0ab87fc180b5f0d5a31788d2886a1014632a
|
||||
```
|
||||
|
||||
!! Storage Channel Awareness
|
||||
|
||||
The scheduler now recognizes that blockchains are not just event streams. They
|
||||
are also public storage substrates.
|
||||
|
||||
Detected storage surfaces include:
|
||||
|
||||
| Family | Surfaces |
|
||||
|---|---|
|
||||
| Bitcoin-like | `op_return`, witness/inscription-like payloads, coinbase tags |
|
||||
| EVM | calldata, logs/events, bytecode, blob / DA lanes |
|
||||
| Memo/message chains | memo, message, payload fields |
|
||||
|
||||
Boundary:
|
||||
|
||||
```text
|
||||
onchain_payload_surfaces_t -> storage_channel_receipts_t+1
|
||||
```
|
||||
|
||||
This is for detection, accounting, provenance, and compression-route
|
||||
diagnostics. It is not a recommendation to publish payloads on public chains or
|
||||
use ledgers as private storage.
|
||||
|
||||
!! Actions
|
||||
|
||||
The scheduler emits bounded next actions:
|
||||
|
||||
* `RETRY_MISSING_OBJECTS`
|
||||
* `RUN_HEADER_OR_BLOCK_PATTERN_PROBE`
|
||||
* `RUN_PARQUET_LOGOGRAM_EIGENPROBE_SAMPLE`
|
||||
* `RUN_ONCHAIN_STORAGE_CHANNEL_PROBE`
|
||||
* `REFRESH_SELF_SCAN_SCHEDULER`
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
The scheduler does not decode chain semantics, prove market behavior, prove
|
||||
compression gain, or fetch additional bytes by itself. It chooses the next
|
||||
bounded probe from existing receipts.
|
||||
|
||||
Related:
|
||||
|
||||
* [[NUVMAP Turbulence Stabilization Index]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
* [[Hyper-Soliton Radix Bins]]
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack NUVMAP Soliton RadixBins GPU Waveprobe Metaprobe HOLD
|
||||
title: Hyper-Soliton Radix Bins
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! Hyper-Soliton Radix Bins
|
||||
|
||||
Status: `METAPHOR_TO_ROUTE_MODEL`
|
||||
|
||||
Hyper-soliton radix bins are the route-basin interpretation of the NUVMAP
|
||||
metadata blitter output.
|
||||
|
||||
The crucial rule:
|
||||
|
||||
```text
|
||||
radix bins are route basins, not semantic labels
|
||||
```
|
||||
|
||||
!! Mapping
|
||||
|
||||
| Radix Object | Hyper-Soliton Analogue |
|
||||
|---|---|
|
||||
| `hash8` | phase seed / local identity packet |
|
||||
| `density7` | amplitude / byte pressure |
|
||||
| `delta7` | shock or torsion gradient |
|
||||
| `zero6` | void / low-information throat |
|
||||
| `flags4` | boundary condition |
|
||||
| radix bin | persistent soliton basin |
|
||||
| bin split | soliton fission |
|
||||
| bin merge | soliton collision |
|
||||
| unstable bin | dispersive residue / HOLD |
|
||||
|
||||
!! Stability Rule
|
||||
|
||||
```text
|
||||
probe bins that persist across salts, windows, and controls;
|
||||
hold bins that only exist under one projection.
|
||||
```
|
||||
|
||||
The bin becomes interesting only if it survives:
|
||||
|
||||
* salt changes
|
||||
* window shifts
|
||||
* shuffled source controls
|
||||
* random-byte controls
|
||||
* chain-label shuffles
|
||||
* CPU/GPU parity checks
|
||||
|
||||
!! NUVMAP Receipt
|
||||
|
||||
The NUVMAP protocol adds:
|
||||
|
||||
```text
|
||||
BLITTER_BIN_RECEIPT
|
||||
```
|
||||
|
||||
Required fields:
|
||||
|
||||
```text
|
||||
shader_hash
|
||||
params_hash
|
||||
source_hash
|
||||
key_buffer_hash
|
||||
metric_buffer_hash
|
||||
decision
|
||||
```
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
The bin is a routing hint. It is not a semantic proof, compression proof, market
|
||||
signal, or chain behavior prediction.
|
||||
|
||||
Related:
|
||||
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
* [[Topological Soliton Raytrace Tessellation NUVMAP Protocol]]
|
||||
* [[Waveprobe]]
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack NUVMAP Blockchain ReceiptChain Waveprobe Metaprobe HOLD
|
||||
title: NUVMAP Chain Protocol Plan
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! NUVMAP Chain Protocol Plan
|
||||
|
||||
Status: `ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION`
|
||||
|
||||
NUVMAP is currently a protocol plan for a dedicated receipt chain:
|
||||
|
||||
```text
|
||||
Numerical Universe Vector Map receipt chain
|
||||
```
|
||||
|
||||
Its purpose is to store replayable probe-state commitments so scanning can route
|
||||
itself over time without treating commodity public chains as accidental bulk
|
||||
storage.
|
||||
|
||||
!! Runner
|
||||
|
||||
```text
|
||||
4-Infrastructure/shim/nuvmap_chain_protocol_plan.py
|
||||
```
|
||||
|
||||
Receipt:
|
||||
|
||||
```text
|
||||
shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json
|
||||
```
|
||||
|
||||
Current receipt hash:
|
||||
|
||||
```text
|
||||
5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40
|
||||
```
|
||||
|
||||
!! Chain Model
|
||||
|
||||
```text
|
||||
deployment_stage: LOCAL_DEVNET_ONLY
|
||||
economic_model: NO_TOKEN_NO_MARKET
|
||||
finality_model: append-only merkle root
|
||||
data_availability: on-chain summaries + content-addressed sidecars
|
||||
```
|
||||
|
||||
!! Minimal Transaction Types
|
||||
|
||||
* `BLITTER_BIN_RECEIPT`
|
||||
* `WAVE_RECEIPT`
|
||||
* `META_RECEIPT`
|
||||
* `STORAGE_SURFACE_RECEIPT`
|
||||
* `SCAN_POLICY_UPDATE`
|
||||
* `NEGATIVE_CONTROL`
|
||||
|
||||
!! Block Shape
|
||||
|
||||
```text
|
||||
header:
|
||||
parent_hash
|
||||
height
|
||||
scanner_policy_hash
|
||||
state_root
|
||||
sidecar_manifest_root
|
||||
|
||||
body:
|
||||
blitter bin receipts
|
||||
waveprobe receipts
|
||||
metaprobe receipts
|
||||
storage-channel receipts
|
||||
scan actions
|
||||
negative controls
|
||||
```
|
||||
|
||||
Forbidden body fields:
|
||||
|
||||
* private keys
|
||||
* wallet credentials
|
||||
* bulk copyrighted payloads
|
||||
* hidden/evasive payloads
|
||||
* financial promotion metadata
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
This does not create a blockchain, token, market, consensus network,
|
||||
compression result, or Hutter Prize claim. Chain implementation remains HOLD
|
||||
until local fixture block replay and verifier receipts exist.
|
||||
|
||||
Related:
|
||||
|
||||
* [[Blockchain L3 Self-Scan Scheduler]]
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
* [[Hyper-Soliton Radix Bins]]
|
||||
* [[NUVMAP GPU Burst Check Plan]]
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack NUVMAP GPU BurstCheck HardwareWitness HOLD
|
||||
title: NUVMAP GPU Burst Check Plan
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! NUVMAP GPU Burst Check Plan
|
||||
|
||||
Status: `ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION`
|
||||
|
||||
This is the time-boxed plan for renting a GPU briefly, approximately three
|
||||
hours, to validate NUVMAP probe machinery.
|
||||
|
||||
Runner:
|
||||
|
||||
```text
|
||||
4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py
|
||||
```
|
||||
|
||||
Receipt:
|
||||
|
||||
```text
|
||||
shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json
|
||||
```
|
||||
|
||||
Current receipt hash:
|
||||
|
||||
```text
|
||||
73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296
|
||||
```
|
||||
|
||||
!! Burst Phases
|
||||
|
||||
```text
|
||||
P0 environment snapshot
|
||||
P1 NUVMAP fixture replay
|
||||
P2 CPU/GPU parity
|
||||
P3 waveprobe batch
|
||||
P4 metaprobe + negative controls
|
||||
P5 cost and shutdown receipt
|
||||
```
|
||||
|
||||
!! Required Outputs
|
||||
|
||||
* `environment_receipt.json`
|
||||
* `cpu_gpu_parity_receipt.json`
|
||||
* `waveprobe_batch_receipt.json`
|
||||
* `metaprobe_batch_receipt.json`
|
||||
* `negative_controls_receipt.json`
|
||||
* `cost_and_shutdown_receipt.json`
|
||||
|
||||
!! Forbidden Inputs
|
||||
|
||||
* wallet private keys
|
||||
* exchange credentials
|
||||
* mainnet transaction signing material
|
||||
* unbounded copyrighted payload mirrors
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
The GPU rental proves nothing by itself. Hardware witness requires visible GPU
|
||||
state plus CPU/GPU parity receipts. Compression feedback remains HOLD until a
|
||||
byte-exact baseline matrix exists.
|
||||
|
||||
Related:
|
||||
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
* [[Blockchain L3 Self-Scan Scheduler]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack NUVMAP GPU WebGPU Shader MetadataBlitter SIMD HOLD
|
||||
title: NUVMAP Metadata Blitter
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! NUVMAP Metadata Blitter
|
||||
|
||||
Status: `SHADER_PRIMITIVE_WITH_HOLD_BOUNDARIES`
|
||||
|
||||
The NUVMAP metadata blitter is a small WebGPU compute shader that treats the GPU
|
||||
as a dataport fabric for metadata sorting.
|
||||
|
||||
Runner:
|
||||
|
||||
```text
|
||||
5-Applications/scripts/nuvmap_metadata_blitter.wgsl
|
||||
```
|
||||
|
||||
Documentation:
|
||||
|
||||
```text
|
||||
5-Applications/scripts/nuvmap_metadata_blitter.md
|
||||
```
|
||||
|
||||
!! Purpose
|
||||
|
||||
```text
|
||||
metadata shard
|
||||
-> fixed-size word windows
|
||||
-> SIMD-wide blitter pass
|
||||
-> sortable route keys
|
||||
-> packed metric words
|
||||
-> radix grouping
|
||||
-> waveprobe / metaprobe selection
|
||||
-> NUVMAP receipts
|
||||
```
|
||||
|
||||
The shader does not sort by itself. It emits deterministic keys and metrics that
|
||||
a later radix/sort pass or CPU verifier can consume.
|
||||
|
||||
!! Key Word
|
||||
|
||||
```text
|
||||
bits 31..24 hash8 phase seed
|
||||
bits 23..17 density7 byte pressure
|
||||
bits 16..10 delta7 shock / torsion gradient
|
||||
bits 9..4 zero6 void / low-information throat
|
||||
bits 3..0 flags4 boundary condition
|
||||
```
|
||||
|
||||
!! Dispatch Shape
|
||||
|
||||
```text
|
||||
@compute @workgroup_size(256, 1, 1)
|
||||
dispatch = ceil(record_count / 256)
|
||||
```
|
||||
|
||||
Each invocation scans at most `64` words. Larger records should be split into
|
||||
fixed windows so the kernel remains predictable and shardable.
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
This shader emits route keys and metrics only. It does not decompress, classify
|
||||
semantic meaning, publish data on-chain, claim GPU acceleration, or claim
|
||||
compression improvement. GPU claims require CPU/GPU parity receipts.
|
||||
|
||||
Related:
|
||||
|
||||
* [[Hyper-Soliton Radix Bins]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
* [[Blockchain L3 Self-Scan Scheduler]]
|
||||
* [[NUVMAP GPU Burst Check Plan]]
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
created: 20260511000000000
|
||||
modified: 20260511000000000
|
||||
tags: ResearchStack NUVMAP Blockchain L3 Waveprobe Metaprobe GPU Turbulence Index
|
||||
title: NUVMAP Turbulence Stabilization Index
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! NUVMAP Turbulence Stabilization Index
|
||||
|
||||
Status: `STABILIZATION_INDEX`
|
||||
|
||||
This index captures the current high-turbulence blockchain / GPU / NUVMAP
|
||||
design surface so the stack does not have to re-derive it from chat context.
|
||||
|
||||
!! Current Shape
|
||||
|
||||
```text
|
||||
public blockchain corpus
|
||||
-> L3 self-scan scheduler
|
||||
-> GPU metadata blitter
|
||||
-> hyper-soliton radix bins
|
||||
-> waveprobe / metaprobe routing
|
||||
-> NUVMAP receipt chain
|
||||
-> next scan frontier
|
||||
```
|
||||
|
||||
The key change is that scanning is no longer only a data-ingest action. It is a
|
||||
feedback loop:
|
||||
|
||||
```text
|
||||
scan_receipts_t -> route_policy_t+1
|
||||
```
|
||||
|
||||
!! Core Cards
|
||||
|
||||
* [[Blockchain L3 Self-Scan Scheduler]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
* [[Hyper-Soliton Radix Bins]]
|
||||
* [[NUVMAP GPU Burst Check Plan]]
|
||||
* [[Waveprobe]]
|
||||
* [[Topological Soliton Raytrace Tessellation NUVMAP Protocol]]
|
||||
* [[Hutter JPEG XL Starfield Eigenprobe First Sweep]]
|
||||
|
||||
!! Durable Artifacts
|
||||
|
||||
```text
|
||||
4-Infrastructure/shim/blockchain_l3_self_scan_scheduler.py
|
||||
4-Infrastructure/shim/nuvmap_chain_protocol_plan.py
|
||||
4-Infrastructure/shim/nuvmap_gpu_burst_check_plan.py
|
||||
5-Applications/scripts/nuvmap_metadata_blitter.wgsl
|
||||
5-Applications/scripts/nuvmap_metadata_blitter.md
|
||||
```
|
||||
|
||||
Receipts:
|
||||
|
||||
```text
|
||||
shared-data/data/blockchain_corpus/blockchain_l3_self_scan_scheduler_receipt.json
|
||||
shared-data/data/blockchain_corpus/nuvmap_chain_protocol_plan_receipt.json
|
||||
shared-data/data/blockchain_corpus/nuvmap_gpu_burst_check_plan_receipt.json
|
||||
```
|
||||
|
||||
!! Claim Boundary
|
||||
|
||||
This cluster is protocol and routing infrastructure. It does not claim:
|
||||
|
||||
* Hutter Prize progress
|
||||
* compression gain
|
||||
* blockchain market prediction
|
||||
* mainnet readiness
|
||||
* token or economic design
|
||||
* semantic meaning from radix bins
|
||||
* GPU acceleration without parity receipts
|
||||
|
||||
Every promotion requires replayable source windows, negative controls, receipt
|
||||
hashes, and explicit HOLD/ADMIT/QUARANTINE decisions.
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
created: 20260510000000000
|
||||
modified: 20260510000000000
|
||||
tags: ResearchStack Soliton Raytrace Tessellation NUVMAP TreeFiddy Protocol
|
||||
title: Topological Soliton Raytrace Tessellation NUVMAP Protocol
|
||||
type: text/vnd.tiddlywiki
|
||||
|
||||
! Topological Soliton Raytrace Tessellation NUVMAP Protocol
|
||||
|
||||
Status: `CANDIDATE_PROTOCOL_WITH_HOLD_BOUNDARIES`
|
||||
|
||||
Seed:
|
||||
|
||||
```text
|
||||
Topological solitons > ray-traced tessellation > Tree Fiddy > NUVMAP results
|
||||
```
|
||||
|
||||
!! Core Pipeline
|
||||
|
||||
```text
|
||||
topological soliton
|
||||
-> invariant field identity
|
||||
-> ray-traced local intersections
|
||||
-> tessellated cells
|
||||
-> Tree Fiddy bounded recursion
|
||||
-> NUVMAP address/projection result
|
||||
-> replay receipt
|
||||
```
|
||||
|
||||
The move is to turn persistent topology into bounded addressable geometry.
|
||||
Solitons supply identity. Ray tracing supplies observation. Tessellation makes
|
||||
the surface finite. Tree Fiddy prevents runaway refinement. NUVMAP stores the
|
||||
address/projection result.
|
||||
|
||||
!! Admission Rule
|
||||
|
||||
```text
|
||||
ADMIT iff:
|
||||
invariant_present
|
||||
projection_present
|
||||
tessellation_finite
|
||||
tree_fiddy_depth <= declared_bound
|
||||
nuvmap_address_valid
|
||||
replay_residual <= epsilon
|
||||
```
|
||||
|
||||
!! Why It Matters
|
||||
|
||||
This bridges field-heavy soliton math into the compiler/address layer:
|
||||
|
||||
```text
|
||||
field identity
|
||||
-> sampled intersections
|
||||
-> finite cells
|
||||
-> bounded refinement
|
||||
-> addressable receipt
|
||||
```
|
||||
|
||||
It does not treat a ray-rendered view as proof. The rendered/tessellated view is
|
||||
only a projection. The receipt closes only if replay residual stays bounded.
|
||||
|
||||
!! HOLD Boundary
|
||||
|
||||
HOLD remains on:
|
||||
|
||||
* physical control of solitons
|
||||
* device-readiness
|
||||
* ray rendering as proof of topology
|
||||
* infinite refinement
|
||||
* unbounded recursion
|
||||
* NUVMAP address results without replay residual and hash receipts
|
||||
|
||||
Related:
|
||||
|
||||
* [[Soliton N-Space Path]]
|
||||
* [[Tessellated Triangle Flow Migration]]
|
||||
* [[Tree Fiddy Monster Assignment]]
|
||||
* [[NUVMAP Turbulence Stabilization Index]]
|
||||
* [[NUVMAP Chain Protocol Plan]]
|
||||
* [[Hyper-Soliton Radix Bins]]
|
||||
* `6-Documentation/docs/topological_soliton_raytrace_tessellation_nuvmap_protocol_2026-05-10.md`
|
||||
* `shared-data/data/stack_solidification/topological_soliton_raytrace_tessellation_nuvmap_receipt.json`
|
||||
|
|
@ -12,3 +12,6 @@ WaveProbe is a manifold-probing system that emits and measures wavefronts across
|
|||
* [[Topological State Machine]]
|
||||
* [[Manifold Flow]]
|
||||
* [[Semantic RG Flow]]
|
||||
* [[NUVMAP Turbulence Stabilization Index]]
|
||||
* [[Blockchain L3 Self-Scan Scheduler]]
|
||||
* [[NUVMAP Metadata Blitter]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,374 @@
|
|||
{
|
||||
"bigquery_export_commands": [
|
||||
{
|
||||
"chain": "bitcoin-cash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_bitcoin_cash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/{table}/*.parquet",
|
||||
"source_id": "bitcoin_cash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dash/{table}/*.parquet",
|
||||
"source_id": "dash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dogecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/{table}/*.parquet",
|
||||
"source_id": "dogecoin",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_litecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_litecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin/{table}/*.parquet",
|
||||
"source_id": "litecoin",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_zcash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_zcash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/zcash/{table}/*.parquet",
|
||||
"source_id": "zcash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "ethereum-classic",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_ethereum_classic",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/{table}/*.parquet",
|
||||
"source_id": "ethereum_classic",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
}
|
||||
],
|
||||
"claim_boundary": "This receipt identifies alternative public/research sources and tool readiness. It does not prove data was exported from BigQuery, does not bypass source terms, and does not claim decoded chain semantics.",
|
||||
"created_utc": "2026-05-11T17:18:18Z",
|
||||
"decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER",
|
||||
"destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10",
|
||||
"local_tool_readiness": {
|
||||
"commands": {
|
||||
"bq": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/bq",
|
||||
"version": "This is BigQuery CLI 2.1.31"
|
||||
},
|
||||
"duckdb": {
|
||||
"available": false,
|
||||
"path": null,
|
||||
"version": null
|
||||
},
|
||||
"gcloud": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/gcloud",
|
||||
"version": "Google Cloud SDK 567.0.0"
|
||||
},
|
||||
"gsutil": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/gsutil",
|
||||
"version": "gsutil version: 5.37"
|
||||
},
|
||||
"kaggle": {
|
||||
"available": false,
|
||||
"path": null,
|
||||
"version": null
|
||||
},
|
||||
"rclone": {
|
||||
"available": true,
|
||||
"path": "/usr/bin/rclone",
|
||||
"version": "rclone v1.74.0"
|
||||
}
|
||||
},
|
||||
"python_modules": {
|
||||
"boto3": {
|
||||
"available": false,
|
||||
"error": "ModuleNotFoundError",
|
||||
"module": "boto3"
|
||||
},
|
||||
"duckdb": {
|
||||
"available": false,
|
||||
"error": "ModuleNotFoundError",
|
||||
"module": "duckdb"
|
||||
},
|
||||
"google.cloud.bigquery": {
|
||||
"available": true,
|
||||
"module": "google.cloud.bigquery"
|
||||
},
|
||||
"pandas": {
|
||||
"available": true,
|
||||
"module": "pandas"
|
||||
},
|
||||
"pyarrow": {
|
||||
"available": true,
|
||||
"module": "pyarrow"
|
||||
},
|
||||
"requests": {
|
||||
"available": true,
|
||||
"module": "requests"
|
||||
}
|
||||
}
|
||||
},
|
||||
"next_gate": {
|
||||
"decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET",
|
||||
"required": [
|
||||
"Authenticate with a Google Cloud account allowed to query BigQuery public datasets.",
|
||||
"Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
"Provide or create a GCS staging bucket for BigQuery extract jobs.",
|
||||
"Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.",
|
||||
"Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts."
|
||||
]
|
||||
},
|
||||
"node_etl_commands": [
|
||||
{
|
||||
"chain": "bitcoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "bitcoin_cash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin_cash --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_cash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "bitcoin_gold",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin_gold --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_gold/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain dogecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/dogecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dogecoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dogecoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain litecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/litecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/litecoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/litecoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain dash --blocks-output shared-data/data/blockchain_corpus/node_etl/dash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain zcash --blocks-output shared-data/data/blockchain_corpus/node_etl/zcash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/zcash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/zcash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
}
|
||||
],
|
||||
"receipt_hash": "3327d7c4e0f98a984522f4f66e0bbe3afc66dc933a86a97b6a3c6a29baf3572f",
|
||||
"schema": "blockchain_alternative_source_plan_v0",
|
||||
"source_candidates": [
|
||||
{
|
||||
"chains": [
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "bitcoin-cash",
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "dash",
|
||||
"dataset": "bigquery-public-data.crypto_dash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "dogecoin",
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "litecoin",
|
||||
"dataset": "bigquery-public-data.crypto_litecoin"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "zcash",
|
||||
"dataset": "bigquery-public-data.crypto_zcash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "NOT_ATTEMPTED",
|
||||
"chain": "ethereum-classic",
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic"
|
||||
}
|
||||
],
|
||||
"claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.",
|
||||
"decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS",
|
||||
"name": "Google BigQuery Public Cryptocurrency Datasets",
|
||||
"python_client_ready": true,
|
||||
"source_urls": [
|
||||
"https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them",
|
||||
"https://www.cloudskillsboost.google/focuses/8486?parent=catalog"
|
||||
],
|
||||
"why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin",
|
||||
"bitcoin_cash",
|
||||
"bitcoin_gold",
|
||||
"dogecoin",
|
||||
"litecoin",
|
||||
"dash",
|
||||
"zcash"
|
||||
],
|
||||
"claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.",
|
||||
"decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED",
|
||||
"name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots",
|
||||
"source_urls": [
|
||||
"https://github.com/blockchain-etl/bitcoin-etl"
|
||||
],
|
||||
"why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin"
|
||||
],
|
||||
"claim_boundary": "Coverage and licensing must be checked before mirroring.",
|
||||
"decision": "EVALUATE_LICENSE_AND_COVERAGE",
|
||||
"name": "Bitquery Cloud Data Dumps",
|
||||
"source_urls": [
|
||||
"https://docs.bitquery.io/docs/cloud/bitcoin/"
|
||||
],
|
||||
"why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin-cash",
|
||||
"dash",
|
||||
"dogecoin",
|
||||
"litecoin",
|
||||
"zcash"
|
||||
],
|
||||
"claim_boundary": "Do not use for full-corpus claims.",
|
||||
"decision": "HOLD_SAMPLE_ONLY",
|
||||
"name": "Kaggle/Hugging Face sampled datasets",
|
||||
"source_urls": [
|
||||
"https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset",
|
||||
"https://www.kaggle.com/datasets/amritpal333/crypto-mining-data"
|
||||
],
|
||||
"why": "Useful for smoke tests and model fixtures, not complete chain mirrors."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
{
|
||||
"claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.",
|
||||
"entries": [
|
||||
{
|
||||
"chain": "bitcoin-cash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_bitcoin_cash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dogecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "ethereum-classic",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_ethereum_classic",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_litecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_litecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_zcash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_zcash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/zcash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
}
|
||||
],
|
||||
"schema": "blockchain_bigquery_grabber_plan_v0",
|
||||
"tool_status": {
|
||||
"bq": "/home/allaun/.local/bin/bq",
|
||||
"gcloud": "/home/allaun/.local/bin/gcloud",
|
||||
"gsutil": "/home/allaun/.local/bin/gsutil",
|
||||
"python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python",
|
||||
"python_bigquery_client": true,
|
||||
"python_google_auth": true,
|
||||
"rclone": "/usr/bin/rclone"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[
|
||||
{
|
||||
"error": "DefaultCredentialsError",
|
||||
"message": "Your default credentials were not found. To set up Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information.",
|
||||
"next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
"python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"decision": "HOLD_GOOGLE_ADC_MISSING", "out": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json", "results": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json"}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,149 @@
|
|||
{
|
||||
"block_shape": {
|
||||
"body": {
|
||||
"metaprobe_receipts": "bounded list of route/context/curriculum summaries",
|
||||
"negative_controls": "required controls before promotion of any route",
|
||||
"scan_actions": "next bounded scan plan emitted by L3 scheduler",
|
||||
"storage_channel_receipts": "detected on-chain storage surfaces from source chains",
|
||||
"waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries"
|
||||
},
|
||||
"forbidden_body_fields": [
|
||||
"private keys",
|
||||
"wallet credentials",
|
||||
"bulk copyrighted payloads",
|
||||
"payloads intended to hide from moderation or provenance",
|
||||
"financial promotion metadata"
|
||||
],
|
||||
"header": {
|
||||
"created_utc": "iso8601",
|
||||
"height": "u64",
|
||||
"parent_hash": "sha256(previous_block)",
|
||||
"scanner_policy_hash": "sha256(l3_policy)",
|
||||
"sidecar_manifest_root": "merkle_root(sidecar_hashes)",
|
||||
"state_root": "merkle_root(receipt_state)"
|
||||
}
|
||||
},
|
||||
"chain_model": {
|
||||
"consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set",
|
||||
"data_availability": "on-chain summaries plus content-addressed off-chain sidecars",
|
||||
"deployment_stage": "LOCAL_DEVNET_ONLY",
|
||||
"economic_model": "NO_TOKEN_NO_MARKET",
|
||||
"finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional"
|
||||
},
|
||||
"claim_boundary": "Protocol design receipt only. This does not create a blockchain, token, market, consensus network, compression result, or Hutter Prize claim. It defines a local/devnet receipt substrate for metaprobe and waveprobe state.",
|
||||
"created_utc": "2026-05-11T17:45:10Z",
|
||||
"decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION",
|
||||
"decision_law": {
|
||||
"ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.",
|
||||
"HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.",
|
||||
"QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation."
|
||||
},
|
||||
"minimal_transaction_types": [
|
||||
{
|
||||
"required_fields": [
|
||||
"shader_hash",
|
||||
"params_hash",
|
||||
"source_hash",
|
||||
"key_buffer_hash",
|
||||
"metric_buffer_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "BLITTER_BIN_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"source_hash",
|
||||
"window_descriptor",
|
||||
"feature_vector_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "WAVE_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"prior_id",
|
||||
"route_family",
|
||||
"evidence_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "META_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"source_chain",
|
||||
"channel",
|
||||
"carrier_descriptor",
|
||||
"payload_hash_or_null",
|
||||
"decision"
|
||||
],
|
||||
"type": "STORAGE_SURFACE_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"previous_policy_hash",
|
||||
"new_policy_hash",
|
||||
"action_merkle_root",
|
||||
"decision"
|
||||
],
|
||||
"type": "SCAN_POLICY_UPDATE"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"control_kind",
|
||||
"source_hash",
|
||||
"result_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "NEGATIVE_CONTROL"
|
||||
}
|
||||
],
|
||||
"next_implementation_steps": [
|
||||
"Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.",
|
||||
"Add a verifier that recomputes the block hash and merkle roots.",
|
||||
"Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.",
|
||||
"Run negative controls before any compression-route promotion.",
|
||||
"Only after local fixture replay works, consider a small append-only local devnet."
|
||||
],
|
||||
"probe_mapping": {
|
||||
"l3_scheduler": {
|
||||
"input": "current receipt frontier",
|
||||
"output": "next scan frontier",
|
||||
"storage": "scanner policy hash and action list"
|
||||
},
|
||||
"metadata_blitter": {
|
||||
"input": "fixed-size metadata words from chain/block/window records",
|
||||
"output": "sortable route keys and packed metric words",
|
||||
"storage": "blitter shader hash, parameter receipt, key/metric buffer hashes"
|
||||
},
|
||||
"metaprobe": {
|
||||
"input": "domain priors, route families, previous scan receipts",
|
||||
"output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints",
|
||||
"storage": "decision receipt, route vector, source receipt backlinks"
|
||||
},
|
||||
"waveprobe": {
|
||||
"input": "byte windows, block fields, density neighborhoods, eigenvalue spectra",
|
||||
"output": "local signal vector plus residual/curvature summary",
|
||||
"storage": "hash, vector summary, source window receipt, not raw bulk by default"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.",
|
||||
"core_use": "Store replayable probe-state commitments so scanning can route itself over time.",
|
||||
"expanded_role": "Numerical Universe Vector Map receipt chain",
|
||||
"name": "NUVMAP"
|
||||
},
|
||||
"radix_bin_model": {
|
||||
"claim_boundary": "Radix bins are route basins, not semantic labels.",
|
||||
"fields": {
|
||||
"delta7": "shock or torsion gradient",
|
||||
"density7": "amplitude / byte pressure",
|
||||
"flags4": "boundary condition",
|
||||
"hash8": "phase seed / local identity packet",
|
||||
"zero6": "void / low-information throat"
|
||||
},
|
||||
"metaphor": "hyper_soliton_search",
|
||||
"stability_rule": "Promote only bins that persist across salts, windows, and negative controls."
|
||||
},
|
||||
"receipt_hash": "5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40",
|
||||
"schema": "nuvmap_chain_protocol_plan_v0"
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
{
|
||||
"burst_phases": [
|
||||
{
|
||||
"checks": [
|
||||
"nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv",
|
||||
"python -c 'import torch; print(torch.cuda.is_available())' if torch is installed",
|
||||
"git rev-parse HEAD and git status --short for provenance"
|
||||
],
|
||||
"decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE",
|
||||
"phase": "P0_ENVIRONMENT_SNAPSHOT",
|
||||
"target_minutes": 10
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Verify NUVMAP fixture block hash and merkle roots.",
|
||||
"Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.",
|
||||
"Reject any fixture with missing source hashes."
|
||||
],
|
||||
"decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD",
|
||||
"phase": "P1_FIXTURE_REPLAY",
|
||||
"target_minutes": 25
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run the same wave/vector kernel on CPU and GPU.",
|
||||
"Compare Q16.16 or declared float tolerance residuals.",
|
||||
"Emit max_abs_error, mean_abs_error, and mismatch examples."
|
||||
],
|
||||
"decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED",
|
||||
"phase": "P2_CPU_GPU_PARITY",
|
||||
"target_minutes": 35
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run bounded byte-window/eigen/density probes over sampled blockchain shards.",
|
||||
"Store only feature hashes and summaries unless payload admission exists.",
|
||||
"Emit route candidates as HOLD until negative controls pass."
|
||||
],
|
||||
"decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION",
|
||||
"phase": "P3_WAVEPROBE_BATCH",
|
||||
"target_minutes": 45
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run metaprobe route selection over waveprobe outputs.",
|
||||
"Run shuffled window, shuffled chain label, and random-byte controls.",
|
||||
"Require controls before any Hutter/logogram feedback promotion."
|
||||
],
|
||||
"decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL",
|
||||
"phase": "P4_METAPROBE_AND_CONTROLS",
|
||||
"target_minutes": 45
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Write final receipt bundle.",
|
||||
"Sync receipts and small summaries to Drive.",
|
||||
"Record instance type, wall time, estimated cost, and shutdown confirmation."
|
||||
],
|
||||
"decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT",
|
||||
"phase": "P5_COST_AND_SHUTDOWN",
|
||||
"target_minutes": 20
|
||||
}
|
||||
],
|
||||
"claim_boundary": "GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, not a token launch, not a compression result, and not a hardware acceleration claim until the listed receipts are produced.",
|
||||
"created_utc": "2026-05-11T17:41:19Z",
|
||||
"decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION",
|
||||
"forbidden_inputs": [
|
||||
"wallet private keys",
|
||||
"exchange credentials",
|
||||
"mainnet transaction signing material",
|
||||
"unbounded copyrighted payload mirrors"
|
||||
],
|
||||
"promotion_gates": {
|
||||
"compression_feedback": "HOLD until byte-exact baseline matrix exists.",
|
||||
"hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.",
|
||||
"nuvmap_chain": "HOLD until local fixture block replay and verifier exist."
|
||||
},
|
||||
"receipt_hash": "73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296",
|
||||
"required_outputs": [
|
||||
"environment_receipt.json",
|
||||
"cpu_gpu_parity_receipt.json",
|
||||
"waveprobe_batch_receipt.json",
|
||||
"metaprobe_batch_receipt.json",
|
||||
"negative_controls_receipt.json",
|
||||
"cost_and_shutdown_receipt.json"
|
||||
],
|
||||
"schema": "nuvmap_gpu_burst_check_plan_v0",
|
||||
"timebox_hours": 3.0
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue