mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-30 18:56:16 +00:00
aba1fba7ad
Security & correctness fixes from full adversarial review: Lean (7 fixes): - FixedPoint.lean: guard false theorem with n > 0 precondition - QFactor.lean: remove double-scaling error in energy decrease - AVMIsa/Step.lean: implement addSatQ16/subSatQ16 primitives - BraidEigensolid.lean: fix crossStep second output argument swap - SSMS.lean: complete ACI preservation proof (with rounding caveat) - HouseholderQR.lean: add n > 0 precondition to spectral theorem Verilog (7 fixes): - q16_lut_core.v: fix multiply shift (16 → 32 bits) - q16_lut_top.v: fix valid bit (0 → 1) - cff_accelerator.v: fix SHA-256 padding (len < 448 check) - research_stack_top.v: fix trigger aliasing (unique counters) - Blitter6502OISC_small.v: fix address width (15 → 16 bits) - spatial_hash_bram.v: add OOB write guard - tmr_oepi_safety_fsm.v: fix double-increment race WGSL (6 fixes): - shaders.wgsl: atomicAdd for concurrent writes - frustration_qubo.wgsl: double-buffer + CAS loop - braid_fft.wgsl: workgroupBarrier synchronization - burgers_scar_filter.wgsl: atomic E_bins array Rust (9 fixes): - thermodynamic.rs: Arc::from_raw → Arc::clone (double-free) - thermodynamic.rs: Box::into_raw → Box (leak) - tools/src/lib.rs: shell injection → shlex.quote - ene-node/src/lib.rs: LRU caps, constant-time HMAC, peer caps Python (6 fixes): - similarity/__init__.py: pickle.load → RestrictedUnpickler - AI-Feynman: torch.load → weights_only=True (14 calls) - fetch_arxiv.py, fetch_s2.py: eval → ast.literal_eval - topology.py: os.system → shutil.copy2 - SSH pipe: os.system → base64 pipe Build: lake build 3572 jobs, 0 errors
100 lines
3.4 KiB
Python
100 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
bulk_process_hepdata.py — Process all downloaded HEPData through the RRC pipeline
|
|
|
|
This script:
|
|
1. Scans all CSV files in /tmp/hepdata-bulk
|
|
2. Extracts numerical data points
|
|
3. Builds event matrices for spectral analysis
|
|
4. Outputs JSON for Lean pipeline ingestion
|
|
|
|
Usage:
|
|
python3 bulk_process_hepData.py > all_physics_data.json
|
|
"""
|
|
|
|
import csv
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
def extract_numerical_data(filepath):
|
|
"""Extract all numerical values from a CSV file."""
|
|
data_points = []
|
|
try:
|
|
with open(filepath, 'r') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith('#'):
|
|
continue
|
|
parts = line.split(',')
|
|
values = []
|
|
for p in parts:
|
|
try:
|
|
v = float(p)
|
|
values.append(v)
|
|
except (ValueError, TypeError, KeyError) as e:
|
|
logging.warning(f"Failed to parse value: {e}")
|
|
if len(values) >= 2:
|
|
data_points.append(values)
|
|
except (ValueError, TypeError, KeyError) as e:
|
|
logging.warning(f"Failed to read file {filepath}: {e}")
|
|
return data_points
|
|
|
|
def process_record(record_dir):
|
|
"""Process all CSV files in a record directory."""
|
|
record_name = os.path.basename(record_dir)
|
|
all_data = []
|
|
|
|
for csv_file in sorted(Path(record_dir).glob("*.csv")):
|
|
data = extract_numerical_data(str(csv_file))
|
|
if data:
|
|
all_data.append({
|
|
"file": csv_file.name,
|
|
"points": data[:100] # Limit to 100 points per file
|
|
})
|
|
|
|
return {
|
|
"record": record_name,
|
|
"tables": len(all_data),
|
|
"data": all_data
|
|
}
|
|
|
|
def main():
|
|
bulk_dir = Path("/tmp/hepdata-bulk")
|
|
|
|
if not bulk_dir.exists():
|
|
print("Error: /tmp/hepdata-bulk not found", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
results = []
|
|
total_points = 0
|
|
|
|
for record_dir in sorted(bulk_dir.iterdir()):
|
|
if record_dir.is_dir() and record_dir.name.startswith("HEPData-"):
|
|
record_data = process_record(record_dir)
|
|
results.append(record_data)
|
|
|
|
# Count points
|
|
for table in record_data["data"]:
|
|
total_points += len(table["points"])
|
|
|
|
output = {
|
|
"source": "HEPData bulk download",
|
|
"record_count": len(results),
|
|
"total_data_points": total_points,
|
|
"coverage": {
|
|
"B_physics": sum(1 for r in results if any(x in r["record"] for x in ["ins14", "ins13", "ins15", "ins16", "ins17"])),
|
|
"Z_Higgs": sum(1 for r in results if any(x in r["record"] for x in ["ins11", "ins12", "ins134", "ins178", "ins167"])),
|
|
"Top_quark": sum(1 for r in results if any(x in r["record"] for x in ["ins160", "ins179", "ins136", "ins146", "ins315"])),
|
|
"Diboson": sum(1 for r in results if any(x in r["record"] for x in ["ins89", "ins218", "ins292", "ins276", "ins185", "ins119"])),
|
|
"eplus_eminus": sum(1 for r in results if any(x in r["record"] for x in ["ins21", "ins26", "ins177", "ins100", "ins99", "ins124"]))
|
|
},
|
|
"records": results[:50] # Limit output
|
|
}
|
|
|
|
print(json.dumps(output, indent=2))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|