Research-Stack/4-Infrastructure/shim/blockchain_full_account_receipt.py

#!/usr/bin/env python3
"""Build an aggregate accounting receipt for blockchain corpus streaming.

This is an accounting surface only. It records source inventories, completed
transfer receipts, and optional remote object counts. It does not decode chain
semantics or claim complete cryptocurrency coverage beyond the listed source
objects.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import pathlib
import subprocess
from datetime import datetime, timezone
from typing import Any


def utc_now() -> str:
    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")


def sha256_json(value: Any) -> str:
    payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
    return hashlib.sha256(payload).hexdigest()


def load_json(path: pathlib.Path) -> dict[str, Any]:
    with path.open("r", encoding="utf-8") as handle:
        return json.load(handle)


def rclone_count(remote: str, suffix: str | None = None) -> int | None:
    try:
        proc = subprocess.run(
            ["rclone", "lsf", remote, "--recursive"],
            check=False,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            text=True,
            timeout=90,
        )
    except (FileNotFoundError, subprocess.TimeoutExpired):
        return None
    if proc.returncode != 0:
        return None
    lines = [
        line
        for line in proc.stdout.splitlines()
        if line and not line.endswith("/") and not line.startswith("receipts/") and "/receipts/" not in line
    ]
    if suffix:
        lines = [line for line in lines if line.endswith(suffix)]
    return len(lines)


def summarize_inventory(path: pathlib.Path) -> dict[str, Any]:
    doc = load_json(path)
    objects = doc.get("objects", [])
    payload_object_count = sum(1 for item in objects if not str(item.get("key", "")).endswith("/"))
    parquet_object_count = sum(1 for item in objects if str(item.get("key", "")).endswith(".parquet"))
    total_bytes = doc.get("total_listed_bytes")
    if total_bytes is None:
        total_bytes = doc.get("total_listed_bytes_known")
    return {
        "chain": doc.get("chain"),
        "claim_boundary": doc.get("claim_boundary"),
        "dataset": doc.get("dataset"),
        "decision": doc.get("decision"),
        "directory_url": doc.get("directory_url"),
        "inventory": str(path),
        "inventory_hash": doc.get("inventory_hash"),
        "object_count": doc.get("object_count"),
        "payload_object_count": payload_object_count,
        "parquet_object_count": parquet_object_count,
        "prefix": doc.get("prefix"),
        "table": doc.get("table"),
        "total_listed_bytes": total_bytes,
    }


def summarize_transfer(path: pathlib.Path) -> dict[str, Any]:
    doc = load_json(path)
    summary = doc.get("summary", {})
    return {
        "chain": doc.get("chain"),
        "dataset": doc.get("dataset"),
        "decision": doc.get("decision"),
        "receipt": str(path),
        "receipt_hash": doc.get("receipt_hash"),
        "table": doc.get("table"),
        "quarantine_count": summary.get("quarantine_count"),
        "selected_object_count": summary.get("selected_object_count"),
        "size_mismatch_count": summary.get("size_mismatch_count"),
        "successful_or_dry_run_count": summary.get("successful_or_dry_run_count"),
        "total_observed_bytes": summary.get("total_observed_bytes"),
    }


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--data-dir", default="shared-data/data/blockchain_corpus")
    parser.add_argument("--destination", required=True)
    parser.add_argument("--out", required=True)
    parser.add_argument("--include-remote-counts", action="store_true")
    parser.add_argument("--running", action="append", default=[])
    args = parser.parse_args()

    data_dir = pathlib.Path(args.data_dir)
    aws_full_inventories = [
        summarize_inventory(path)
        for path in sorted(data_dir.glob("aws_public_blockchain_*_blocks_inventory_full.json"))
    ]
    bounded_inventories = [
        summarize_inventory(path)
        for path in sorted(data_dir.glob("blockchair_*_blocks_inventory_first100.json"))
    ]
    inventories = aws_full_inventories + bounded_inventories
    transfers = []
    for path in sorted(data_dir.glob("*transfer*receipt.json")):
        doc = load_json(path)
        summary = doc.get("summary", {})
        if not doc.get("dataset") or not doc.get("chain") or "selected_object_count" not in summary:
            continue
        transfers.append(summarize_transfer(path))

    inventory_by_key = {
        (entry["dataset"], entry["chain"], entry["table"]): entry for entry in inventories
    }
    transferred_by_key: dict[tuple[Any, Any, Any], int] = {}
    quarantined_by_key: dict[tuple[Any, Any, Any], int] = {}
    bytes_by_key: dict[tuple[Any, Any, Any], int] = {}
    for entry in transfers:
        key = (entry["dataset"], entry["chain"], entry["table"])
        transferred_by_key[key] = transferred_by_key.get(key, 0) + int(entry.get("successful_or_dry_run_count") or 0)
        quarantined_by_key[key] = quarantined_by_key.get(key, 0) + int(entry.get("quarantine_count") or 0)
        bytes_by_key[key] = bytes_by_key.get(key, 0) + int(entry.get("total_observed_bytes") or 0)

    chains = []
    for key, inv in sorted(inventory_by_key.items(), key=lambda item: str(item[0])):
        remote = f"{args.destination}/{inv['dataset']}/{inv['chain']}/{inv['table']}"
        entry = {
            **inv,
            "completed_transfer_objects": transferred_by_key.get(key, 0),
            "completed_transfer_bytes": bytes_by_key.get(key, 0),
            "completed_quarantine_objects": quarantined_by_key.get(key, 0),
            "remote_parquet_count": rclone_count(remote) if args.include_remote_counts else None,
            "remote_payload_count": rclone_count(remote, None) if args.include_remote_counts else None,
            "remote_prefix": remote,
        }
        chains.append(entry)

    all_transfer_totals = {
        "quarantine_objects": sum(int(entry.get("quarantine_count") or 0) for entry in transfers),
        "selected_objects": sum(int(entry.get("selected_object_count") or 0) for entry in transfers),
        "size_mismatch_objects": sum(int(entry.get("size_mismatch_count") or 0) for entry in transfers),
        "successful_or_dry_run_objects": sum(int(entry.get("successful_or_dry_run_count") or 0) for entry in transfers),
        "total_observed_bytes": sum(int(entry.get("total_observed_bytes") or 0) for entry in transfers),
    }

    decision = "ADMIT_BLOCKCHAIN_CORPUS_ACCOUNTING"
    if args.running:
        decision = "ADMIT_BLOCKCHAIN_CORPUS_ACCOUNTING_WITH_RUNNING_TRANSFERS"

    receipt = {
        "claim_boundary": (
            "Full accounting receipt for currently listed public blockchain corpus sources. "
            "This records inventory coverage, transfer receipts, remote counts when requested, "
            "and running transfer labels. It does not prove decoded chain semantics, price/market "
            "coverage, private dataset coverage, or compression results."
        ),
        "created_utc": utc_now(),
        "decision": decision,
        "destination": args.destination,
        "aws_full_inventoried_chain_table_count": len(aws_full_inventories),
        "bounded_inventoried_chain_table_count": len(bounded_inventories),
        "inventoried_chain_table_count": len(inventories),
        "running_transfers": args.running,
        "schema": "blockchain_full_account_receipt_v0",
        "source_inventories": chains,
        "transfer_receipts": transfers,
        "totals": {
            "all_transfer_receipts": all_transfer_totals,
            "completed_quarantine_objects": sum(entry["completed_quarantine_objects"] for entry in chains),
            "completed_transfer_bytes": sum(entry["completed_transfer_bytes"] for entry in chains),
            "completed_transfer_objects": sum(entry["completed_transfer_objects"] for entry in chains),
            "inventoried_objects": sum(int(entry.get("object_count") or 0) for entry in chains),
            "inventoried_payload_objects": sum(int(entry.get("payload_object_count") or 0) for entry in chains),
            "inventoried_parquet_objects": sum(int(entry.get("parquet_object_count") or 0) for entry in chains),
            "inventoried_total_listed_bytes": sum(int(entry.get("total_listed_bytes") or 0) for entry in chains),
        },
    }
    receipt["receipt_hash"] = sha256_json(receipt)

    out = pathlib.Path(args.out)
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    print(json.dumps({"decision": receipt["decision"], "out": str(out), "receipt_hash": receipt["receipt_hash"]}))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())