mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
241 lines
10 KiB
Python
241 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Transfer inventoried public blockchain objects to Google Drive.
|
|
|
|
The script reads an inventory emitted by blockchain_public_dataset_inventory.py,
|
|
downloads each listed object, streams it to an rclone destination, and emits a
|
|
receipt with source metadata plus observed SHA-256 hashes. It is intentionally
|
|
object-preserving for Parquet snapshots: the payload object is the row group
|
|
container, while the receipt is the replay surface.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
DATA_DIR = REPO / "shared-data/data/blockchain_corpus"
|
|
DEFAULT_DESTINATION = "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets"
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
|
|
|
|
|
def stable_json(value: Any) -> str:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
|
|
|
|
|
|
def sha256_text(value: str) -> str:
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def run(cmd: list[str], input_bytes: bytes | None = None, timeout: int = 300) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
cmd,
|
|
input=input_bytes,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def rcat(remote_path: str, payload: bytes, timeout: int) -> tuple[bool, str]:
|
|
proc = run(["rclone", "rcat", remote_path], input_bytes=payload, timeout=timeout)
|
|
message = (proc.stderr or proc.stdout).decode(errors="replace").strip()
|
|
return proc.returncode == 0, message
|
|
|
|
|
|
def copyto(local_path: Path, remote_path: str, timeout: int) -> tuple[bool, str]:
|
|
proc = run(["rclone", "copyto", str(local_path), remote_path], timeout=timeout)
|
|
message = (proc.stderr or proc.stdout).decode(errors="replace").strip()
|
|
return proc.returncode == 0, message
|
|
|
|
|
|
def fetch_object_to_file(url: str, target: Path, timeout: int) -> tuple[int, str]:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "ResearchStackBlockchainTransfer/0"})
|
|
h = hashlib.sha256()
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
with target.open("wb") as handle:
|
|
byte_count = 0
|
|
while True:
|
|
chunk = response.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
h.update(chunk)
|
|
handle.write(chunk)
|
|
byte_count += len(chunk)
|
|
return byte_count, h.hexdigest()
|
|
|
|
|
|
def safe_remote_key(key: str) -> str:
|
|
return key.strip("/").replace("//", "/")
|
|
|
|
|
|
def transfer_one(index: int, item: dict[str, Any], remote_prefix: str, args: argparse.Namespace) -> dict[str, Any]:
|
|
source_url = item["url"]
|
|
key = safe_remote_key(item["key"])
|
|
if key.endswith("/"):
|
|
return {
|
|
"object_index": index,
|
|
"source_key": item.get("key"),
|
|
"source_url": source_url,
|
|
"source_size": item.get("size"),
|
|
"source_etag": item.get("etag"),
|
|
"drive_path": f"{remote_prefix}/{key}",
|
|
"upload_status": "ADMIT_SKIPPED_DIRECTORY_MARKER",
|
|
"upload_message": "Directory marker skipped so object payloads can occupy the prefix.",
|
|
"size_matches_inventory": None if item.get("size") is None else item.get("size") in {0, "0"},
|
|
}
|
|
remote_path = f"{remote_prefix}/{key}"
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="rs_blockchain_transfer_") as tmpdir:
|
|
local_path = Path(tmpdir) / Path(key).name
|
|
byte_count, observed_hash = fetch_object_to_file(source_url, local_path, args.timeout)
|
|
upload_ok, upload_message = (False, "HOLD_DRY_RUN_ONLY")
|
|
if args.execute:
|
|
upload_ok, upload_message = copyto(local_path, remote_path, args.upload_timeout)
|
|
status = "ADMIT_TRANSFERRED_TO_GDRIVE" if upload_ok else ("HOLD_DRY_RUN_ONLY" if not args.execute else "QUARANTINE_UPLOAD_FAILED")
|
|
size_matches = None if item.get("size") is None else byte_count == item.get("size")
|
|
return {
|
|
"object_index": index,
|
|
"source_key": item.get("key"),
|
|
"source_url": source_url,
|
|
"source_size": item.get("size"),
|
|
"source_etag": item.get("etag"),
|
|
"observed_byte_count": byte_count,
|
|
"observed_sha256": observed_hash,
|
|
"drive_path": remote_path,
|
|
"upload_status": status,
|
|
"upload_message": upload_message,
|
|
"size_matches_inventory": size_matches,
|
|
}
|
|
except Exception as exc: # noqa: BLE001 - receipt every per-object failure.
|
|
return {
|
|
"object_index": index,
|
|
"source_key": item.get("key"),
|
|
"source_url": source_url,
|
|
"upload_status": "QUARANTINE_TRANSFER_EXCEPTION",
|
|
"error": str(exc),
|
|
}
|
|
|
|
|
|
def transfer(args: argparse.Namespace) -> dict[str, Any]:
|
|
inventory = json.loads(args.inventory.read_text())
|
|
all_objects = inventory.get("objects", [])
|
|
objects = all_objects[args.start_index :]
|
|
if args.max_objects is not None:
|
|
objects = objects[: args.max_objects]
|
|
remote_prefix = f"{args.destination.rstrip('/')}/{inventory.get('dataset', 'dataset')}/{inventory.get('chain', 'chain')}/{inventory.get('table', 'table')}"
|
|
|
|
records: list[dict[str, Any]] = []
|
|
if args.workers <= 1:
|
|
for index, item in enumerate(objects):
|
|
record = transfer_one(index, item, remote_prefix, args)
|
|
records.append(record)
|
|
if str(record.get("upload_status", "")).startswith("QUARANTINE") and not args.keep_going:
|
|
break
|
|
else:
|
|
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
|
futures = {
|
|
executor.submit(transfer_one, index, item, remote_prefix, args): index
|
|
for index, item in enumerate(objects)
|
|
}
|
|
for future in as_completed(futures):
|
|
record = future.result()
|
|
records.append(record)
|
|
if str(record.get("upload_status", "")).startswith("QUARANTINE") and not args.keep_going:
|
|
break
|
|
records.sort(key=lambda record: int(record.get("object_index", 0)))
|
|
|
|
total_bytes = sum(record.get("observed_byte_count", 0) for record in records)
|
|
ok_count = sum(
|
|
1
|
|
for record in records
|
|
if record.get("upload_status") in {"ADMIT_TRANSFERRED_TO_GDRIVE", "HOLD_DRY_RUN_ONLY"}
|
|
)
|
|
|
|
receipt = {
|
|
"schema": "blockchain_public_dataset_transfer_receipt_v0",
|
|
"created_utc": now_iso(),
|
|
"claim_boundary": "Public dataset object transfer receipt only. This proves bytes fetched from listed URLs and optional Drive upload status; it does not prove full dataset coverage, Parquet decoding, or chain semantics.",
|
|
"inventory": str(args.inventory.relative_to(REPO)) if args.inventory.is_relative_to(REPO) else str(args.inventory),
|
|
"inventory_hash": inventory.get("inventory_hash"),
|
|
"dataset": inventory.get("dataset"),
|
|
"chain": inventory.get("chain"),
|
|
"table": inventory.get("table"),
|
|
"source_prefix": inventory.get("prefix"),
|
|
"destination": args.destination,
|
|
"remote_prefix": remote_prefix,
|
|
"execute": args.execute,
|
|
"selection": {
|
|
"inventory_object_count": len(all_objects),
|
|
"start_index": args.start_index,
|
|
"max_objects": args.max_objects,
|
|
"workers": args.workers,
|
|
},
|
|
"summary": {
|
|
"selected_object_count": len(objects),
|
|
"successful_or_dry_run_count": ok_count,
|
|
"total_observed_bytes": total_bytes,
|
|
"size_mismatch_count": sum(1 for record in records if record.get("size_matches_inventory") is False),
|
|
"size_unknown_count": sum(1 for record in records if record.get("size_matches_inventory") is None),
|
|
"skipped_directory_marker_count": sum(1 for record in records if record.get("upload_status") == "ADMIT_SKIPPED_DIRECTORY_MARKER"),
|
|
"quarantine_count": sum(1 for record in records if str(record.get("upload_status", "")).startswith("QUARANTINE")),
|
|
},
|
|
"objects": records,
|
|
}
|
|
decision = "ADMIT_PUBLIC_DATASET_TRANSFER_TO_GDRIVE" if args.execute else "HOLD_PUBLIC_DATASET_TRANSFER_DRY_RUN"
|
|
if receipt["summary"]["quarantine_count"]:
|
|
decision = "QUARANTINE_PUBLIC_DATASET_TRANSFER_PARTIAL"
|
|
receipt["decision"] = decision
|
|
receipt["receipt_hash"] = sha256_text(stable_json({k: v for k, v in receipt.items() if k != "receipt_hash"}))
|
|
|
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
out = args.out or DATA_DIR / f"blockchain_public_dataset_transfer_receipt_{inventory.get('chain', 'chain')}_{inventory.get('table', 'table')}_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}.json"
|
|
out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
|
receipt["receipt_path"] = str(out.relative_to(REPO)) if out.is_relative_to(REPO) else str(out)
|
|
|
|
if args.execute:
|
|
receipt_remote = f"{remote_prefix}/receipts/{out.name}"
|
|
ok, message = rcat(receipt_remote, out.read_bytes(), args.upload_timeout)
|
|
receipt["receipt_upload"] = {
|
|
"drive_path": receipt_remote,
|
|
"ok": ok,
|
|
"message": message,
|
|
}
|
|
out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
|
return receipt
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--inventory", type=Path, required=True)
|
|
parser.add_argument("--destination", default=DEFAULT_DESTINATION)
|
|
parser.add_argument("--max-objects", type=int, default=None)
|
|
parser.add_argument("--start-index", type=int, default=0)
|
|
parser.add_argument("--timeout", type=int, default=60)
|
|
parser.add_argument("--upload-timeout", type=int, default=600)
|
|
parser.add_argument("--workers", type=int, default=1)
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument("--keep-going", action="store_true")
|
|
parser.add_argument("--out", type=Path, default=None)
|
|
args = parser.parse_args()
|
|
receipt = transfer(args)
|
|
print(json.dumps(receipt, indent=2, sort_keys=True))
|
|
return 0 if not str(receipt.get("decision", "")).startswith("QUARANTINE") else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|