mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-30 18:56:16 +00:00
chore(cleanup): remove ingested corpus artifacts and scratch scripts
- Delete 4-Infrastructure/shim/lean_corpus/ (downloaded third-party corpus). - Delete shared-data/data/blockchain_corpus/ receipts (already ingested/offloaded). - Delete scratch/ bulk-download scripts (superseded by arxiv_crossref_stream.py). - Clean __pycache__ directories outside .venv. - Also fix populate_ene_tables.py to pipe SQL via stdin instead of shell escaping for neon-64gb psql execution.
This commit is contained in:
parent
e02786e310
commit
5cfa8f0898
11 changed files with 3 additions and 2320 deletions
|
|
@ -1,93 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import ssl
|
||||
|
||||
ssl_context = ssl._create_unverified_context()
|
||||
|
||||
def get_arxiv_links(url):
|
||||
"""
|
||||
Fetch the arXiv page and extract all PDF links.
|
||||
"""
|
||||
headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ssl_context) as response:
|
||||
html = response.read().decode('utf-8')
|
||||
# Look for /pdf/YYMM.NNNNN or similar patterns
|
||||
links = re.findall(r'/pdf/[0-9]{4}\.[0-9]{4,5}', html)
|
||||
# De-duplicate while preserving order
|
||||
seen = set()
|
||||
unique_links = []
|
||||
for link in links:
|
||||
if link not in seen:
|
||||
seen.add(link)
|
||||
unique_links.append(link)
|
||||
return unique_links
|
||||
except Exception as e:
|
||||
print(f"Error fetching arXiv page: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
def main():
|
||||
arxiv_list_url = "https://arxiv.org/list/math/recent?skip=0&show=2000"
|
||||
output_dir = "./arxiv_papers"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
print("Fetching recent math papers list from arXiv...")
|
||||
pdf_paths = get_arxiv_links(arxiv_list_url)
|
||||
total_papers = len(pdf_paths)
|
||||
|
||||
if total_papers == 0:
|
||||
print("No papers found. Exiting.")
|
||||
return
|
||||
|
||||
print(f"Found {total_papers} papers to download.")
|
||||
print(f"Starting sequential download with a 4-second delay to avoid rate limits...")
|
||||
print(f"Target directory: {output_dir}\n")
|
||||
|
||||
downloaded_count = 0
|
||||
skipped_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for idx, path in enumerate(pdf_paths):
|
||||
paper_id = path.split('/')[-1]
|
||||
filename = f"{paper_id}.pdf"
|
||||
dest_path = os.path.join(output_dir, filename)
|
||||
pdf_url = f"https://arxiv.org{path}"
|
||||
|
||||
# Resume capability: check if file already exists and is non-empty
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 10000:
|
||||
print(f"[{idx+1}/{total_papers}] Skipping {paper_id} (already downloaded)")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
print(f"[{idx+1}/{total_papers}] Downloading {paper_id} from {pdf_url}...")
|
||||
|
||||
headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"}
|
||||
req = urllib.request.Request(pdf_url, headers=headers)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ssl_context) as response:
|
||||
with open(dest_path, 'wb') as f:
|
||||
f.write(response.read())
|
||||
print(f"[{idx+1}/{total_papers}] Successfully saved {paper_id}")
|
||||
downloaded_count += 1
|
||||
# Sleep to prevent IP block
|
||||
time.sleep(4)
|
||||
except Exception as e:
|
||||
print(f"[{idx+1}/{total_papers}] Failed to download {paper_id}: {e}", file=sys.stderr)
|
||||
failed_count += 1
|
||||
# Sleep a bit longer on failure
|
||||
time.sleep(10)
|
||||
|
||||
print("\n=== Download Session Finished ===")
|
||||
print(f"Total Papers: {total_papers}")
|
||||
print(f"Downloaded: {downloaded_count}")
|
||||
print(f"Skipped: {skipped_count}")
|
||||
print(f"Failed: {failed_count}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import json
|
||||
import urllib.parse
|
||||
import ssl
|
||||
|
||||
# Bypass SSL verification if needed (common in local environments/proxies)
|
||||
ssl_context = ssl._create_unverified_context()
|
||||
|
||||
def openalex_search(query, polite_email="allaunthefox@gmail.com"):
|
||||
"""
|
||||
Search OpenAlex works for a given query and return a list of matching papers with PDF links.
|
||||
"""
|
||||
headers = {
|
||||
"User-Agent": f"OpenAlexDownloader/1.0 (mailto:{polite_email})"
|
||||
}
|
||||
encoded_query = urllib.parse.quote(query)
|
||||
url = f"https://api.openalex.org/works?search={encoded_query}&per-page=5"
|
||||
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ssl_context) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
results = data.get("results", [])
|
||||
return results
|
||||
except Exception as e:
|
||||
print(f"Error querying OpenAlex for '{query}': {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
def download_pdf(pdf_url, output_path, polite_email="allaunthefox@gmail.com"):
|
||||
"""
|
||||
Download a PDF file from a given URL to the output path.
|
||||
"""
|
||||
headers = {
|
||||
"User-Agent": f"OpenAlexDownloader/1.0 (mailto:{polite_email})"
|
||||
}
|
||||
req = urllib.request.Request(pdf_url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ssl_context) as response:
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(response.read())
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Failed to download from {pdf_url}: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 download_open_papers.py <query_or_arxiv_id> [output_directory]")
|
||||
print("Example: python3 download_open_papers.py \"inverse matrices Gauss-Jordan\" ./papers")
|
||||
sys.exit(1)
|
||||
|
||||
query = sys.argv[1]
|
||||
output_dir = sys.argv[2] if len(sys.argv) > 2 else "./downloaded_papers"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
print(f"Searching OpenAlex for: '{query}'...")
|
||||
works = openalex_search(query)
|
||||
|
||||
if not works:
|
||||
print("No works found.")
|
||||
return
|
||||
|
||||
downloaded_count = 0
|
||||
for idx, work in enumerate(works):
|
||||
title = work.get("display_name", "untitled")
|
||||
doi = work.get("doi")
|
||||
|
||||
# Clean title for filename
|
||||
safe_title = "".join([c if c.isalnum() or c in " -_" else "_" for c in title])[:60].strip()
|
||||
work_id = work.get("id", "").split("/")[-1]
|
||||
filename = f"{safe_title}_{work_id}.pdf"
|
||||
output_path = os.path.join(output_dir, filename)
|
||||
|
||||
# Try to find a PDF link
|
||||
pdf_url = None
|
||||
best_location = work.get("best_oa_location")
|
||||
if best_location and best_location.get("pdf_url"):
|
||||
pdf_url = best_location.get("pdf_url")
|
||||
else:
|
||||
# Check alternative locations
|
||||
for loc in work.get("locations", []):
|
||||
if loc.get("pdf_url"):
|
||||
pdf_url = loc.get("pdf_url")
|
||||
break
|
||||
|
||||
if pdf_url:
|
||||
print(f"\nFound Open Access PDF for: \"{title}\"")
|
||||
print(f"URL: {pdf_url}")
|
||||
print(f"Downloading to: {output_path}...")
|
||||
|
||||
success = download_pdf(pdf_url, output_path)
|
||||
if success:
|
||||
print("Download complete!")
|
||||
downloaded_count += 1
|
||||
|
||||
# Safe sequential delay to respect rate limits
|
||||
time.sleep(3)
|
||||
else:
|
||||
print(f"\nWork found but no Open Access PDF link available: \"{title}\" (DOI: {doi})")
|
||||
|
||||
print(f"\nDone. Successfully downloaded {downloaded_count} paper(s) to '{output_dir}'.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,177 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
import subprocess
|
||||
import hashlib
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
# Paths
|
||||
GDRIVE_DIR = Path("/home/allaun/gdrive")
|
||||
TAKEOUT_DIR = GDRIVE_DIR / "Takeout"
|
||||
TARGET_DIR = Path("/home/allaun/Takeout_extracted")
|
||||
|
||||
def get_gdrive_md5(gdrive_rel_path):
|
||||
"""Get the MD5 hash of a file on Google Drive using rclone md5sum."""
|
||||
rclone_path = f"gdrive:{gdrive_rel_path}"
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["rclone", "md5sum", rclone_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
)
|
||||
if res.returncode == 0 and res.stdout.strip():
|
||||
parts = res.stdout.strip().split()
|
||||
if parts:
|
||||
return parts[0].strip()
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"Error querying rclone md5sum: {e}")
|
||||
return None
|
||||
|
||||
def check_file_duplicate(zip_ref, info):
|
||||
"""Check if a file in the zip is a duplicate of a file in Google Drive."""
|
||||
path_parts = Path(info.filename).parts
|
||||
if len(path_parts) > 1 and path_parts[0] == "Takeout":
|
||||
if len(path_parts) > 2 and path_parts[1] == "Drive":
|
||||
gdrive_rel = Path(*path_parts[2:])
|
||||
else:
|
||||
gdrive_rel = Path(*path_parts[1:])
|
||||
else:
|
||||
gdrive_rel = Path(info.filename)
|
||||
|
||||
gdrive_file = GDRIVE_DIR / gdrive_rel
|
||||
|
||||
if gdrive_file.exists():
|
||||
try:
|
||||
gdrive_size = gdrive_file.stat().st_size
|
||||
if gdrive_size == info.file_size:
|
||||
# Sizes match, verify with MD5
|
||||
remote_md5 = get_gdrive_md5(str(gdrive_rel))
|
||||
if remote_md5:
|
||||
hasher = hashlib.md5()
|
||||
with zip_ref.open(info) as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
zip_md5 = hasher.hexdigest()
|
||||
if zip_md5 == remote_md5:
|
||||
return True, gdrive_rel
|
||||
except Exception as e:
|
||||
print(f"Error checking {gdrive_file}: {e}")
|
||||
|
||||
return False, gdrive_rel
|
||||
|
||||
def run_metaprobe(zip_path, zip_ref, infolist):
|
||||
"""
|
||||
Run a windowed metaprobe (sampling test) on the ZIP.
|
||||
Returns True if the entire ZIP is highly likely to be 100% redundant.
|
||||
"""
|
||||
files_only = [info for info in infolist if not info.is_dir()]
|
||||
total_files = len(files_only)
|
||||
|
||||
if total_files == 0:
|
||||
return True # Empty zip is redundant/trivial
|
||||
|
||||
# Sample size: 5% of files, min 10, max 30
|
||||
sample_size = min(max(int(total_files * 0.05), 10), 30)
|
||||
sample_size = min(sample_size, total_files)
|
||||
|
||||
random.seed(42) # Deterministic sampling
|
||||
sample = random.sample(files_only, sample_size)
|
||||
|
||||
print(f" [Metaprobe] Sampling {sample_size}/{total_files} files for redundancy...")
|
||||
|
||||
duplicates = 0
|
||||
for idx, info in enumerate(sample):
|
||||
is_dup, rel_path = check_file_duplicate(zip_ref, info)
|
||||
if is_dup:
|
||||
duplicates += 1
|
||||
else:
|
||||
print(f" [Metaprobe] Unique file found: {info.filename}")
|
||||
return False # Found a unique file, must process this zip
|
||||
|
||||
# If all sampled files are duplicates
|
||||
if duplicates == sample_size:
|
||||
print(f" [Metaprobe] 100% redundancy in sample window. Trusting and SKIPPING zip.")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def process_zip(zip_path, dry_run=False):
|
||||
print(f"\nProcessing {zip_path.name}...")
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
||||
infolist = zip_ref.infolist()
|
||||
|
||||
# Run metaprobe sampling check first
|
||||
is_redundant = run_metaprobe(zip_path, zip_ref, infolist)
|
||||
if is_redundant:
|
||||
print(f"SKIPPED (Metaprobe: 100% redundant): {zip_path.name}")
|
||||
return
|
||||
|
||||
# If not redundant, proceed with extracting unique files
|
||||
print(f" Metaprobe failed (unique files present). Running full extraction...")
|
||||
total_files = len(infolist)
|
||||
redundant_count = 0
|
||||
unique_count = 0
|
||||
skipped_dirs = 0
|
||||
|
||||
for idx, info in enumerate(infolist):
|
||||
if info.is_dir():
|
||||
skipped_dirs += 1
|
||||
continue
|
||||
|
||||
is_duplicate, gdrive_rel = check_file_duplicate(zip_ref, info)
|
||||
|
||||
if is_duplicate:
|
||||
redundant_count += 1
|
||||
if idx % 100 == 0 or idx == total_files - 1:
|
||||
print(f" Progress: Checked {idx+1}/{total_files} | Duplicates: {redundant_count} | Unique: {unique_count}")
|
||||
else:
|
||||
unique_count += 1
|
||||
target_file = TARGET_DIR / info.filename
|
||||
print(f" [UNIQUE] {info.filename} -> {target_file}")
|
||||
if not dry_run:
|
||||
target_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zip_ref.open(info) as source, open(target_file, "wb") as target:
|
||||
target.write(source.read())
|
||||
|
||||
print(f"Finished {zip_path.name}: Duplicates: {redundant_count}, Unique: {unique_count}")
|
||||
except Exception as e:
|
||||
print(f"Error reading zip {zip_path.name}: {e}")
|
||||
|
||||
def main():
|
||||
dry_run = "--dry-run" in sys.argv
|
||||
single_zip = None
|
||||
for arg in sys.argv[1:]:
|
||||
if arg.endswith(".zip"):
|
||||
single_zip = arg
|
||||
break
|
||||
|
||||
if dry_run:
|
||||
print("Running in DRY RUN mode.")
|
||||
|
||||
TARGET_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if single_zip:
|
||||
zip_path = TAKEOUT_DIR / single_zip
|
||||
if zip_path.exists():
|
||||
process_zip(zip_path, dry_run=dry_run)
|
||||
else:
|
||||
print(f"Zip file {zip_path} not found.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Find all zip files
|
||||
zips = sorted(list(TAKEOUT_DIR.glob("*.zip")))
|
||||
if not zips:
|
||||
print(f"No zip files found in {TAKEOUT_DIR}")
|
||||
sys.exit(1)
|
||||
print(f"Found {len(zips)} zip files to process.")
|
||||
for zip_path in zips:
|
||||
process_zip(zip_path, dry_run=dry_run)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -49,13 +49,13 @@ GRAPH_KEYWORDS = {"graph", "gossip", "surface", "network", "neighborhood", "topo
|
|||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def neon(sql: str, dry_run: bool = False, timeout: int = 60) -> str:
|
||||
"""Run SQL on neon-64gb ene DB, return stdout."""
|
||||
"""Run SQL on neon-64gb ene DB, return stdout. Uses stdin pipe to avoid shell escaping."""
|
||||
if dry_run:
|
||||
return "DRY_RUN"
|
||||
r = subprocess.run(
|
||||
["ssh", NEON_HOST,
|
||||
f"podman exec {CONTAINER} psql -U postgres -d {DB} -t -A -c \"{sql}\""],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
f"podman exec -i {CONTAINER} psql -U postgres -d {DB} -t -A"],
|
||||
input=sql, capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
print(f" NEON ERR: {r.stderr[:200]}", file=sys.stderr)
|
||||
|
|
|
|||
|
|
@ -1,374 +0,0 @@
|
|||
{
|
||||
"bigquery_export_commands": [
|
||||
{
|
||||
"chain": "bitcoin-cash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_bitcoin_cash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/{table}/*.parquet",
|
||||
"source_id": "bitcoin_cash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dash/{table}/*.parquet",
|
||||
"source_id": "dash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dogecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/{table}/*.parquet",
|
||||
"source_id": "dogecoin",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_litecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_litecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin/{table}/*.parquet",
|
||||
"source_id": "litecoin",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_zcash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_zcash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/zcash/{table}/*.parquet",
|
||||
"source_id": "zcash",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "ethereum-classic",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_ethereum_classic",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.{table}' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/{table}/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic",
|
||||
"local_export_pattern": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/{table}/*.parquet",
|
||||
"source_id": "ethereum_classic",
|
||||
"tables_to_try_first": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
}
|
||||
],
|
||||
"claim_boundary": "This receipt identifies alternative public/research sources and tool readiness. It does not prove data was exported from BigQuery, does not bypass source terms, and does not claim decoded chain semantics.",
|
||||
"created_utc": "2026-05-11T17:18:18Z",
|
||||
"decision": "ADMIT_ALTERNATIVE_SOURCE_PLAN_HOLD_TRANSFER",
|
||||
"destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10",
|
||||
"local_tool_readiness": {
|
||||
"commands": {
|
||||
"bq": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/bq",
|
||||
"version": "This is BigQuery CLI 2.1.31"
|
||||
},
|
||||
"duckdb": {
|
||||
"available": false,
|
||||
"path": null,
|
||||
"version": null
|
||||
},
|
||||
"gcloud": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/gcloud",
|
||||
"version": "Google Cloud SDK 567.0.0"
|
||||
},
|
||||
"gsutil": {
|
||||
"available": true,
|
||||
"path": "/home/allaun/.local/bin/gsutil",
|
||||
"version": "gsutil version: 5.37"
|
||||
},
|
||||
"kaggle": {
|
||||
"available": false,
|
||||
"path": null,
|
||||
"version": null
|
||||
},
|
||||
"rclone": {
|
||||
"available": true,
|
||||
"path": "/usr/bin/rclone",
|
||||
"version": "rclone v1.74.0"
|
||||
}
|
||||
},
|
||||
"python_modules": {
|
||||
"boto3": {
|
||||
"available": false,
|
||||
"error": "ModuleNotFoundError",
|
||||
"module": "boto3"
|
||||
},
|
||||
"duckdb": {
|
||||
"available": false,
|
||||
"error": "ModuleNotFoundError",
|
||||
"module": "duckdb"
|
||||
},
|
||||
"google.cloud.bigquery": {
|
||||
"available": true,
|
||||
"module": "google.cloud.bigquery"
|
||||
},
|
||||
"pandas": {
|
||||
"available": true,
|
||||
"module": "pandas"
|
||||
},
|
||||
"pyarrow": {
|
||||
"available": true,
|
||||
"module": "pyarrow"
|
||||
},
|
||||
"requests": {
|
||||
"available": true,
|
||||
"module": "requests"
|
||||
}
|
||||
}
|
||||
},
|
||||
"next_gate": {
|
||||
"decision": "HOLD_UNTIL_GOOGLE_ADC_AND_GCS_BUCKET",
|
||||
"required": [
|
||||
"Authenticate with a Google Cloud account allowed to query BigQuery public datasets.",
|
||||
"Use either gcloud application-default login or GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
"Provide or create a GCS staging bucket for BigQuery extract jobs.",
|
||||
"Run the Python BigQuery list check against each candidate dataset and emit table inventory receipt.",
|
||||
"Export tables to Parquet, copy locally, then rclone to Drive with SHA receipts."
|
||||
]
|
||||
},
|
||||
"node_etl_commands": [
|
||||
{
|
||||
"chain": "bitcoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "bitcoin_cash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin_cash --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_cash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "bitcoin_gold",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain bitcoin_gold --blocks-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/bitcoin_gold/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/bitcoin_gold/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain dogecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/dogecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dogecoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dogecoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain litecoin --blocks-output shared-data/data/blockchain_corpus/node_etl/litecoin/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/litecoin/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/litecoin/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain dash --blocks-output shared-data/data/blockchain_corpus/node_etl/dash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/dash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/dash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"python3 -m pip install --user bitcoin-etl",
|
||||
"bitcoinetl export_blocks_and_transactions --start-block 0 --end-block <END_BLOCK> --provider-uri http://<rpc_user>:<rpc_pass>@127.0.0.1:<rpc_port> --chain zcash --blocks-output shared-data/data/blockchain_corpus/node_etl/zcash/blocks.jsonl --transactions-output shared-data/data/blockchain_corpus/node_etl/zcash/transactions.jsonl",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/node_etl/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/node-etl/bitcoin-etl/zcash/'"
|
||||
],
|
||||
"requires": [
|
||||
"running_full_node_or_rpc_snapshot",
|
||||
"bitcoin-etl"
|
||||
]
|
||||
}
|
||||
],
|
||||
"receipt_hash": "3327d7c4e0f98a984522f4f66e0bbe3afc66dc933a86a97b6a3c6a29baf3572f",
|
||||
"schema": "blockchain_alternative_source_plan_v0",
|
||||
"source_candidates": [
|
||||
{
|
||||
"chains": [
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "bitcoin-cash",
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "dash",
|
||||
"dataset": "bigquery-public-data.crypto_dash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "dogecoin",
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "litecoin",
|
||||
"dataset": "bigquery-public-data.crypto_litecoin"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "PARTIAL_HTTP_402",
|
||||
"chain": "zcash",
|
||||
"dataset": "bigquery-public-data.crypto_zcash"
|
||||
},
|
||||
{
|
||||
"blockchair_status": "NOT_ATTEMPTED",
|
||||
"chain": "ethereum-classic",
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic"
|
||||
}
|
||||
],
|
||||
"claim_boundary": "Requires Google Cloud authentication and a GCS staging bucket before local/Drive export.",
|
||||
"decision": "PREFERRED_FOR_BLOCKCHAIR_GAPS",
|
||||
"name": "Google BigQuery Public Cryptocurrency Datasets",
|
||||
"python_client_ready": true,
|
||||
"source_urls": [
|
||||
"https://cloud.google.com/blog/products/data-analytics/introducing-six-new-cryptocurrencies-in-bigquery-public-datasets-and-how-to-analyze-them",
|
||||
"https://www.cloudskillsboost.google/focuses/8486?parent=catalog"
|
||||
],
|
||||
"why": "Covers Bitcoin-derived chains that Blockchair throttled: Bitcoin Cash, Dash, Dogecoin, Litecoin, and Zcash."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin",
|
||||
"bitcoin_cash",
|
||||
"bitcoin_gold",
|
||||
"dogecoin",
|
||||
"litecoin",
|
||||
"dash",
|
||||
"zcash"
|
||||
],
|
||||
"claim_boundary": "Requires per-chain node data/RPC and enough local storage; slower but self-verifiable.",
|
||||
"decision": "FALLBACK_IF_BIGQUERY_EXPORT_BLOCKED",
|
||||
"name": "Blockchain ETL / bitcoin-etl from full nodes or snapshots",
|
||||
"source_urls": [
|
||||
"https://github.com/blockchain-etl/bitcoin-etl"
|
||||
],
|
||||
"why": "Can export Bitcoin-like chains from local RPC nodes or grabbed node snapshots without Blockchair dumps."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin"
|
||||
],
|
||||
"claim_boundary": "Coverage and licensing must be checked before mirroring.",
|
||||
"decision": "EVALUATE_LICENSE_AND_COVERAGE",
|
||||
"name": "Bitquery Cloud Data Dumps",
|
||||
"source_urls": [
|
||||
"https://docs.bitquery.io/docs/cloud/bitcoin/"
|
||||
],
|
||||
"why": "Offers Parquet dump patterns and cloud data products; useful as a schema/tooling reference and possible paid/free lane."
|
||||
},
|
||||
{
|
||||
"chains": [
|
||||
"bitcoin-cash",
|
||||
"dash",
|
||||
"dogecoin",
|
||||
"litecoin",
|
||||
"zcash"
|
||||
],
|
||||
"claim_boundary": "Do not use for full-corpus claims.",
|
||||
"decision": "HOLD_SAMPLE_ONLY",
|
||||
"name": "Kaggle/Hugging Face sampled datasets",
|
||||
"source_urls": [
|
||||
"https://huggingface.co/datasets/Omarrran/CryptoXChain_500K_Multi_Network_Blockchain_Transaction_Dataset",
|
||||
"https://www.kaggle.com/datasets/amritpal333/crypto-mining-data"
|
||||
],
|
||||
"why": "Useful for smoke tests and model fixtures, not complete chain mirrors."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,147 +0,0 @@
|
|||
{
|
||||
"claim_boundary": "Dry-run command plan only unless --execute is used with authenticated Google Cloud tooling.",
|
||||
"entries": [
|
||||
{
|
||||
"chain": "bitcoin-cash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_bitcoin_cash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_bitcoin_cash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/' 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_bitcoin_cash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/bitcoin-cash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/bitcoin-cash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/bitcoin-cash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/' 'shared-data/data/blockchain_corpus/bigquery_exports/dash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "dogecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_dogecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_dogecoin.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/dogecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_dogecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/dogecoin",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/dogecoin/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/dogecoin",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "ethereum-classic",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_ethereum_classic",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_ethereum_classic.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/' 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_ethereum_classic",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/ethereum-classic",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/ethereum-classic/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/ethereum-classic",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "litecoin",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_litecoin",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_litecoin.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/' 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/litecoin/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_litecoin",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/litecoin",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/litecoin/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/litecoin",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"chain": "zcash",
|
||||
"commands": [
|
||||
"bq ls bigquery-public-data.crypto_zcash",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.blocks' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/blocks/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.transactions' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/transactions/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.inputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/inputs/*.parquet'",
|
||||
"bq extract --destination_format=PARQUET 'bigquery-public-data.crypto_zcash.outputs' 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/outputs/*.parquet'",
|
||||
"gcloud storage cp --recursive 'gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/' 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/'",
|
||||
"rclone copy 'shared-data/data/blockchain_corpus/bigquery_exports/zcash/' 'Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash/'"
|
||||
],
|
||||
"dataset": "bigquery-public-data.crypto_zcash",
|
||||
"drive_destination": "Gdrive:topological_storage/research-stack/blockchain-corpus/seed-2026-05-10/public-datasets/google-bigquery/zcash",
|
||||
"gcs_prefix": "gs://<YOUR_GCS_BUCKET>/research-stack/blockchain-corpus/zcash/",
|
||||
"local_root": "shared-data/data/blockchain_corpus/bigquery_exports/zcash",
|
||||
"tables": [
|
||||
"blocks",
|
||||
"transactions",
|
||||
"inputs",
|
||||
"outputs"
|
||||
]
|
||||
}
|
||||
],
|
||||
"schema": "blockchain_bigquery_grabber_plan_v0",
|
||||
"tool_status": {
|
||||
"bq": "/home/allaun/.local/bin/bq",
|
||||
"gcloud": "/home/allaun/.local/bin/gcloud",
|
||||
"gsutil": "/home/allaun/.local/bin/gsutil",
|
||||
"python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python",
|
||||
"python_bigquery_client": true,
|
||||
"python_google_auth": true,
|
||||
"rclone": "/usr/bin/rclone"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
[
|
||||
{
|
||||
"error": "DefaultCredentialsError",
|
||||
"message": "Your default credentials were not found. To set up Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information.",
|
||||
"next_step": "Run gcloud auth application-default login or provide GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
"python": "/home/allaun/.local/share/research-stack/blockchain-bigquery-venv/bin/python"
|
||||
}
|
||||
]
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"decision": "HOLD_GOOGLE_ADC_MISSING", "out": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan.json", "results": "shared-data/data/blockchain_corpus/blockchain_bigquery_grabber_plan_python_list_results.json"}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,149 +0,0 @@
|
|||
{
|
||||
"block_shape": {
|
||||
"body": {
|
||||
"metaprobe_receipts": "bounded list of route/context/curriculum summaries",
|
||||
"negative_controls": "required controls before promotion of any route",
|
||||
"scan_actions": "next bounded scan plan emitted by L3 scheduler",
|
||||
"storage_channel_receipts": "detected on-chain storage surfaces from source chains",
|
||||
"waveprobe_receipts": "bounded list of wave/eigen/density diagnostic summaries"
|
||||
},
|
||||
"forbidden_body_fields": [
|
||||
"private keys",
|
||||
"wallet credentials",
|
||||
"bulk copyrighted payloads",
|
||||
"payloads intended to hide from moderation or provenance",
|
||||
"financial promotion metadata"
|
||||
],
|
||||
"header": {
|
||||
"created_utc": "iso8601",
|
||||
"height": "u64",
|
||||
"parent_hash": "sha256(previous_block)",
|
||||
"scanner_policy_hash": "sha256(l3_policy)",
|
||||
"sidecar_manifest_root": "merkle_root(sidecar_hashes)",
|
||||
"state_root": "merkle_root(receipt_state)"
|
||||
}
|
||||
},
|
||||
"chain_model": {
|
||||
"consensus_candidate": "single-writer_receipt_log_then_multisig_validator_set",
|
||||
"data_availability": "on-chain summaries plus content-addressed off-chain sidecars",
|
||||
"deployment_stage": "LOCAL_DEVNET_ONLY",
|
||||
"economic_model": "NO_TOKEN_NO_MARKET",
|
||||
"finality_model": "append-only_merkle_root_with_periodic_external_anchor_optional"
|
||||
},
|
||||
"claim_boundary": "Protocol design receipt only. This does not create a blockchain, token, market, consensus network, compression result, or Hutter Prize claim. It defines a local/devnet receipt substrate for metaprobe and waveprobe state.",
|
||||
"created_utc": "2026-05-11T17:45:10Z",
|
||||
"decision": "ADMIT_NUVMAP_PROTOCOL_PLAN_HOLD_CHAIN_IMPLEMENTATION",
|
||||
"decision_law": {
|
||||
"ADMIT": "All required hashes replay, source windows exist, and negative controls do not collapse the signal.",
|
||||
"HOLD": "Evidence is incomplete, cost is unknown, payload surface is unverified, or object-level provenance is missing.",
|
||||
"QUARANTINE": "Hash mismatch, unsafe payload class, missing provenance, or claim boundary violation."
|
||||
},
|
||||
"minimal_transaction_types": [
|
||||
{
|
||||
"required_fields": [
|
||||
"shader_hash",
|
||||
"params_hash",
|
||||
"source_hash",
|
||||
"key_buffer_hash",
|
||||
"metric_buffer_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "BLITTER_BIN_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"source_hash",
|
||||
"window_descriptor",
|
||||
"feature_vector_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "WAVE_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"prior_id",
|
||||
"route_family",
|
||||
"evidence_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "META_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"source_chain",
|
||||
"channel",
|
||||
"carrier_descriptor",
|
||||
"payload_hash_or_null",
|
||||
"decision"
|
||||
],
|
||||
"type": "STORAGE_SURFACE_RECEIPT"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"previous_policy_hash",
|
||||
"new_policy_hash",
|
||||
"action_merkle_root",
|
||||
"decision"
|
||||
],
|
||||
"type": "SCAN_POLICY_UPDATE"
|
||||
},
|
||||
{
|
||||
"required_fields": [
|
||||
"control_kind",
|
||||
"source_hash",
|
||||
"result_hash",
|
||||
"decision"
|
||||
],
|
||||
"type": "NEGATIVE_CONTROL"
|
||||
}
|
||||
],
|
||||
"next_implementation_steps": [
|
||||
"Create a JSON fixture block with one WAVE_RECEIPT, one META_RECEIPT, and one SCAN_POLICY_UPDATE.",
|
||||
"Add a verifier that recomputes the block hash and merkle roots.",
|
||||
"Feed the current blockchain_l3_self_scan_scheduler receipt into the first fixture block.",
|
||||
"Run negative controls before any compression-route promotion.",
|
||||
"Only after local fixture replay works, consider a small append-only local devnet."
|
||||
],
|
||||
"probe_mapping": {
|
||||
"l3_scheduler": {
|
||||
"input": "current receipt frontier",
|
||||
"output": "next scan frontier",
|
||||
"storage": "scanner policy hash and action list"
|
||||
},
|
||||
"metadata_blitter": {
|
||||
"input": "fixed-size metadata words from chain/block/window records",
|
||||
"output": "sortable route keys and packed metric words",
|
||||
"storage": "blitter shader hash, parameter receipt, key/metric buffer hashes"
|
||||
},
|
||||
"metaprobe": {
|
||||
"input": "domain priors, route families, previous scan receipts",
|
||||
"output": "policy/context update and HOLD/ADMIT/QUARANTINE decision hints",
|
||||
"storage": "decision receipt, route vector, source receipt backlinks"
|
||||
},
|
||||
"waveprobe": {
|
||||
"input": "byte windows, block fields, density neighborhoods, eigenvalue spectra",
|
||||
"output": "local signal vector plus residual/curvature summary",
|
||||
"storage": "hash, vector summary, source window receipt, not raw bulk by default"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"anti_goal": "Do not use public commodity ledgers as arbitrary bulk storage when a purpose-built receipt chain is enough.",
|
||||
"core_use": "Store replayable probe-state commitments so scanning can route itself over time.",
|
||||
"expanded_role": "Numerical Universe Vector Map receipt chain",
|
||||
"name": "NUVMAP"
|
||||
},
|
||||
"radix_bin_model": {
|
||||
"claim_boundary": "Radix bins are route basins, not semantic labels.",
|
||||
"fields": {
|
||||
"delta7": "shock or torsion gradient",
|
||||
"density7": "amplitude / byte pressure",
|
||||
"flags4": "boundary condition",
|
||||
"hash8": "phase seed / local identity packet",
|
||||
"zero6": "void / low-information throat"
|
||||
},
|
||||
"metaphor": "hyper_soliton_search",
|
||||
"stability_rule": "Promote only bins that persist across salts, windows, and negative controls."
|
||||
},
|
||||
"receipt_hash": "5e51f55f88133e3eb61e4179bcd932fb58512db76ec46dbb37d4419db32ced40",
|
||||
"schema": "nuvmap_chain_protocol_plan_v0"
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
{
|
||||
"burst_phases": [
|
||||
{
|
||||
"checks": [
|
||||
"nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv",
|
||||
"python -c 'import torch; print(torch.cuda.is_available())' if torch is installed",
|
||||
"git rev-parse HEAD and git status --short for provenance"
|
||||
],
|
||||
"decision": "ADMIT_ENVIRONMENT_IF_GPU_VISIBLE",
|
||||
"phase": "P0_ENVIRONMENT_SNAPSHOT",
|
||||
"target_minutes": 10
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Verify NUVMAP fixture block hash and merkle roots.",
|
||||
"Replay blockchain_l3_self_scan_scheduler_receipt as first policy input.",
|
||||
"Reject any fixture with missing source hashes."
|
||||
],
|
||||
"decision": "ADMIT_FIXTURE_REPLAY_OR_HOLD",
|
||||
"phase": "P1_FIXTURE_REPLAY",
|
||||
"target_minutes": 25
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run the same wave/vector kernel on CPU and GPU.",
|
||||
"Compare Q16.16 or declared float tolerance residuals.",
|
||||
"Emit max_abs_error, mean_abs_error, and mismatch examples."
|
||||
],
|
||||
"decision": "ADMIT_GPU_WITNESS_ONLY_IF_PARITY_BOUNDED",
|
||||
"phase": "P2_CPU_GPU_PARITY",
|
||||
"target_minutes": 35
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run bounded byte-window/eigen/density probes over sampled blockchain shards.",
|
||||
"Store only feature hashes and summaries unless payload admission exists.",
|
||||
"Emit route candidates as HOLD until negative controls pass."
|
||||
],
|
||||
"decision": "ADMIT_WAVEPROBE_BATCH_HOLD_PROMOTION",
|
||||
"phase": "P3_WAVEPROBE_BATCH",
|
||||
"target_minutes": 45
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Run metaprobe route selection over waveprobe outputs.",
|
||||
"Run shuffled window, shuffled chain label, and random-byte controls.",
|
||||
"Require controls before any Hutter/logogram feedback promotion."
|
||||
],
|
||||
"decision": "ADMIT_METAPROBE_IF_CONTROLS_BOUND_SIGNAL",
|
||||
"phase": "P4_METAPROBE_AND_CONTROLS",
|
||||
"target_minutes": 45
|
||||
},
|
||||
{
|
||||
"checks": [
|
||||
"Write final receipt bundle.",
|
||||
"Sync receipts and small summaries to Drive.",
|
||||
"Record instance type, wall time, estimated cost, and shutdown confirmation."
|
||||
],
|
||||
"decision": "ADMIT_BURST_COMPLETE_ONLY_WITH_SHUTDOWN_RECEIPT",
|
||||
"phase": "P5_COST_AND_SHUTDOWN",
|
||||
"target_minutes": 20
|
||||
}
|
||||
],
|
||||
"claim_boundary": "GPU rental execution plan only. This is not crypto mining, not a blockchain mainnet, not a token launch, not a compression result, and not a hardware acceleration claim until the listed receipts are produced.",
|
||||
"created_utc": "2026-05-11T17:41:19Z",
|
||||
"decision": "ADMIT_GPU_BURST_PLAN_HOLD_EXECUTION",
|
||||
"forbidden_inputs": [
|
||||
"wallet private keys",
|
||||
"exchange credentials",
|
||||
"mainnet transaction signing material",
|
||||
"unbounded copyrighted payload mirrors"
|
||||
],
|
||||
"promotion_gates": {
|
||||
"compression_feedback": "HOLD until byte-exact baseline matrix exists.",
|
||||
"hardware_witness": "GPU visible plus CPU/GPU parity receipt, not just code execution.",
|
||||
"nuvmap_chain": "HOLD until local fixture block replay and verifier exist."
|
||||
},
|
||||
"receipt_hash": "73b6fab2800a94019247623b71082b54e80f71a819722ae2fb7bfc8fc6780296",
|
||||
"required_outputs": [
|
||||
"environment_receipt.json",
|
||||
"cpu_gpu_parity_receipt.json",
|
||||
"waveprobe_batch_receipt.json",
|
||||
"metaprobe_batch_receipt.json",
|
||||
"negative_controls_receipt.json",
|
||||
"cost_and_shutdown_receipt.json"
|
||||
],
|
||||
"schema": "nuvmap_gpu_burst_check_plan_v0",
|
||||
"timebox_hours": 3.0
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue