Research-Stack/4-Infrastructure/shim/bitcoin_header_jsonl_export.py

#!/usr/bin/env python3
"""Export Bitcoin active-chain headers as newline-delimited JSON.

This exporter is designed for pruned Bitcoin Core nodes. A pruned node cannot
serve every historical block body, but it can still expose active-chain header
metadata through the block index. That makes this lane useful for pattern
studies from genesis to the current validated height without claiming full
historical block-body coverage.
"""

from __future__ import annotations

import argparse
import base64
import hashlib
import json
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


REPO = Path(__file__).resolve().parents[2]
DATA_DIR = REPO / "shared-data/data/blockchain_corpus"
DEFAULT_RPC_URL = "http://127.0.0.1:8332"


def now_iso() -> str:
    return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")


def load_rpc_auth(conf_path: Path) -> tuple[str, str]:
    rpcuser = ""
    rpcpassword = ""
    for raw_line in conf_path.expanduser().read_text().splitlines():
        line = raw_line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, value = line.split("=", 1)
        if key == "rpcuser":
            rpcuser = value
        elif key == "rpcpassword":
            rpcpassword = value
    if not rpcuser or not rpcpassword:
        raise SystemExit(f"missing rpcuser/rpcpassword in {conf_path}")
    return rpcuser, rpcpassword


class BitcoinRpc:
    def __init__(self, url: str, rpcuser: str, rpcpassword: str, timeout: int) -> None:
        token = base64.b64encode(f"{rpcuser}:{rpcpassword}".encode()).decode()
        self.url = url
        self.timeout = timeout
        self.headers = {
            "Authorization": f"Basic {token}",
            "Content-Type": "application/json",
        }
        self.next_id = 1

    def call(self, method: str, params: list[Any] | None = None) -> Any:
        return self.batch([(method, params or [])])[0]

    def batch(self, calls: list[tuple[str, list[Any]]]) -> list[Any]:
        request_items = []
        ids: list[int] = []
        for method, params in calls:
            request_id = self.next_id
            self.next_id += 1
            ids.append(request_id)
            request_items.append(
                {
                    "jsonrpc": "1.0",
                    "id": request_id,
                    "method": method,
                    "params": params,
                }
            )
        payload = json.dumps(request_items).encode()
        req = urllib.request.Request(self.url, data=payload, headers=self.headers, method="POST")
        try:
            with urllib.request.urlopen(req, timeout=self.timeout) as response:
                response_payload = response.read()
        except urllib.error.URLError as exc:
            raise SystemExit(f"bitcoin rpc failed: {exc}") from exc

        decoded = json.loads(response_payload)
        by_id = {item["id"]: item for item in decoded}
        results = []
        for request_id in ids:
            item = by_id[request_id]
            if item.get("error"):
                raise SystemExit(f"bitcoin rpc error for id {request_id}: {item['error']}")
            results.append(item.get("result"))
        return results


def stable_json_line(payload: dict[str, Any]) -> bytes:
    return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n"


def export_headers(args: argparse.Namespace) -> dict[str, Any]:
    rpcuser, rpcpassword = load_rpc_auth(args.bitcoin_conf)
    rpc = BitcoinRpc(args.rpc_url, rpcuser, rpcpassword, args.timeout)
    chain_info = rpc.call("getblockchaininfo")
    source_height = int(chain_info[args.height_source])
    end_height = source_height if args.end_height is None else min(args.end_height, source_height)
    start_height = args.start_height
    if args.max_heights is not None:
        end_height = min(end_height, start_height + args.max_heights - 1)
    if end_height < start_height:
        raise SystemExit("end height is before start height")

    created = now_iso()
    stream_hash = hashlib.sha256()
    record_count = 0
    first_hash = None
    last_hash = None
    started = time.time()

    for batch_start in range(start_height, end_height + 1, args.batch_size):
        batch_end = min(end_height, batch_start + args.batch_size - 1)
        heights = list(range(batch_start, batch_end + 1))
        hashes = rpc.batch([("getblockhash", [height]) for height in heights])
        headers = rpc.batch([("getblockheader", [block_hash, True]) for block_hash in hashes])

        for height, block_hash, header in zip(heights, hashes, headers, strict=True):
            record = {
                "schema": "bitcoin_header_jsonl_v0",
                "chain": "bitcoin",
                "height": height,
                "hash": block_hash,
                "confirmations": header.get("confirmations"),
                "version": header.get("version"),
                "versionHex": header.get("versionHex"),
                "merkleroot": header.get("merkleroot"),
                "time": header.get("time"),
                "mediantime": header.get("mediantime"),
                "nonce": header.get("nonce"),
                "bits": header.get("bits"),
                "difficulty": header.get("difficulty"),
                "chainwork": header.get("chainwork"),
                "nTx": header.get("nTx"),
                "previousblockhash": header.get("previousblockhash"),
                "nextblockhash": header.get("nextblockhash"),
            }
            line = stable_json_line(record)
            args.output.buffer.write(line)
            stream_hash.update(line)
            record_count += 1
            first_hash = first_hash or block_hash
            last_hash = block_hash

        if args.progress_every and record_count % args.progress_every < len(heights):
            print(
                f"exported={record_count} height={batch_end} elapsed_s={time.time() - started:.1f}",
                file=sys.stderr,
                flush=True,
            )

    args.output.flush()
    receipt = {
        "schema": "bitcoin_header_jsonl_export_receipt_v0",
        "created_utc": created,
        "claim_boundary": "Active-chain Bitcoin header JSONL export from local Bitcoin Core RPC. This proves header metadata coverage for the exported height interval, not full historical block-body coverage.",
        "bitcoin_conf": str(args.bitcoin_conf.expanduser()),
        "rpc_url": args.rpc_url,
        "height_source": args.height_source,
        "chain_info": {
            "chain": chain_info.get("chain"),
            "blocks": chain_info.get("blocks"),
            "headers": chain_info.get("headers"),
            "bestblockhash": chain_info.get("bestblockhash"),
            "verificationprogress": chain_info.get("verificationprogress"),
            "initialblockdownload": chain_info.get("initialblockdownload"),
            "pruned": chain_info.get("pruned"),
            "pruneheight": chain_info.get("pruneheight"),
        },
        "start_height": start_height,
        "end_height": end_height,
        "record_count": record_count,
        "first_hash": first_hash,
        "last_hash": last_hash,
        "jsonl_sha256": stream_hash.hexdigest(),
        "decision": "ADMIT_BITCOIN_HEADER_INTERVAL_EXPORT",
    }
    if args.receipt_path:
        args.receipt_path.parent.mkdir(parents=True, exist_ok=True)
        args.receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
    return receipt


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--bitcoin-conf", type=Path, default=Path.home() / ".bitcoin/bitcoin.conf")
    parser.add_argument("--rpc-url", default=DEFAULT_RPC_URL)
    parser.add_argument("--timeout", type=int, default=30)
    parser.add_argument("--start-height", type=int, default=0)
    parser.add_argument("--end-height", type=int, default=None)
    parser.add_argument("--height-source", choices=["blocks", "headers"], default="blocks")
    parser.add_argument("--batch-size", type=int, default=128)
    parser.add_argument("--max-heights", type=int, default=None)
    parser.add_argument("--progress-every", type=int, default=10000)
    parser.add_argument("--receipt-path", type=Path, default=None)
    parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout)
    args = parser.parse_args()

    if args.receipt_path is None:
        DATA_DIR.mkdir(parents=True, exist_ok=True)
        stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
        args.receipt_path = DATA_DIR / f"bitcoin_header_jsonl_export_receipt_{stamp}.json"

    receipt = export_headers(args)
    print(json.dumps(receipt, indent=2, sort_keys=True), file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())