mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
e5fb0a5f4d
Lean: update Semantics modules, add new numerics/physics data files Hardware: update FPGA bitstreams (tangnano9k_uart_loopback) Infra: k3s-flake tests, netcup-vps configuration, VCN compute substrate Docs: ARCHITECTURE, specs, citation updates
99 lines
3.2 KiB
Python
99 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
bulk_process_hepdata.py — Process all downloaded HEPData through the RRC pipeline
|
|
|
|
This script:
|
|
1. Scans all CSV files in /tmp/hepdata-bulk
|
|
2. Extracts numerical data points
|
|
3. Builds event matrices for spectral analysis
|
|
4. Outputs JSON for Lean pipeline ingestion
|
|
|
|
Usage:
|
|
python3 bulk_process_hepData.py > all_physics_data.json
|
|
"""
|
|
|
|
import csv
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
def extract_numerical_data(filepath):
|
|
"""Extract all numerical values from a CSV file."""
|
|
data_points = []
|
|
try:
|
|
with open(filepath, 'r') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith('#'):
|
|
continue
|
|
parts = line.split(',')
|
|
values = []
|
|
for p in parts:
|
|
try:
|
|
v = float(p)
|
|
values.append(v)
|
|
except:
|
|
pass
|
|
if len(values) >= 2:
|
|
data_points.append(values)
|
|
except:
|
|
pass
|
|
return data_points
|
|
|
|
def process_record(record_dir):
|
|
"""Process all CSV files in a record directory."""
|
|
record_name = os.path.basename(record_dir)
|
|
all_data = []
|
|
|
|
for csv_file in sorted(Path(record_dir).glob("*.csv")):
|
|
data = extract_numerical_data(str(csv_file))
|
|
if data:
|
|
all_data.append({
|
|
"file": csv_file.name,
|
|
"points": data[:100] # Limit to 100 points per file
|
|
})
|
|
|
|
return {
|
|
"record": record_name,
|
|
"tables": len(all_data),
|
|
"data": all_data
|
|
}
|
|
|
|
def main():
|
|
bulk_dir = Path("/tmp/hepdata-bulk")
|
|
|
|
if not bulk_dir.exists():
|
|
print("Error: /tmp/hepdata-bulk not found", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
results = []
|
|
total_points = 0
|
|
|
|
for record_dir in sorted(bulk_dir.iterdir()):
|
|
if record_dir.is_dir() and record_dir.name.startswith("HEPData-"):
|
|
record_data = process_record(record_dir)
|
|
results.append(record_data)
|
|
|
|
# Count points
|
|
for table in record_data["data"]:
|
|
total_points += len(table["points"])
|
|
|
|
output = {
|
|
"source": "HEPData bulk download",
|
|
"records": len(results),
|
|
"total_data_points": total_points,
|
|
"coverage": {
|
|
"B_physics": sum(1 for r in results if any(x in r["record"] for x in ["ins14", "ins13", "ins15", "ins16", "ins17"])),
|
|
"Z_Higgs": sum(1 for r in results if any(x in r["record"] for x in ["ins11", "ins12", "ins134", "ins178", "ins167"])),
|
|
"Top_quark": sum(1 for r in results if any(x in r["record"] for x in ["ins160", "ins179", "ins136", "ins146", "ins315"])),
|
|
"Diboson": sum(1 for r in results if any(x in r["record"] for x in ["ins89", "ins218", "ins292", "ins276", "ins185", "ins119"])),
|
|
"eplus_eminus": sum(1 for r in results if any(x in r["record"] for x in ["ins21", "ins26", "ins177", "ins100", "ins99", "ins124"]))
|
|
},
|
|
"records": results[:50] # Limit output
|
|
}
|
|
|
|
print(json.dumps(output, indent=2))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|