mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Hutter Prize: Full Purification & Restoration
|
|
The ultimate filtration sweep.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import pandas as pd
|
|
import numpy as np
|
|
from pathlib import Path
|
|
import logging
|
|
import pickle
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "4-Infrastructure"))
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "0-Core-Formalism"))
|
|
|
|
from scripts.rgflow_blind_detector import BlindDetector
|
|
|
|
logging.basicConfig(level=logging.ERROR)
|
|
|
|
def process_full_archive(input_parquet: Path, output_binary: Path):
|
|
print(f"Opening Full Hutter Archive: {input_parquet}")
|
|
|
|
detector = BlindDetector()
|
|
|
|
# Process in large chunks to avoid memory overflow
|
|
chunk_reader = pd.read_parquet(input_parquet, columns=['input_ids'], engine='pyarrow')
|
|
|
|
total_chunks = len(chunk_reader)
|
|
print(f"Target: {total_chunks} chunks. Initiating purification...")
|
|
|
|
purified_ids = []
|
|
lawful_count = 0
|
|
|
|
# Simple loop for the full 2.5M (optimizing the state calc)
|
|
for i, ids in enumerate(chunk_reader['input_ids']):
|
|
if i % 50000 == 0:
|
|
print(f"Audit Progress: {i}/{total_chunks} (Purified: {lawful_count})")
|
|
|
|
# Map IDs to DNA character symbols for the detector
|
|
# IDs are 0-255. We take a subset for state calculation speed.
|
|
# We use the raw IDs to calculate mutation and entropy directly.
|
|
|
|
# 1. Mutation (mu): transitions in tokens
|
|
transitions = np.sum(ids[:-1] != ids[1:])
|
|
mu_q = (transitions / len(ids)) * 0.1
|
|
|
|
# 2. Entropy (rho)
|
|
counts = np.unique(ids, return_counts=True)[1]
|
|
probs = counts / len(ids)
|
|
entropy = -np.sum(probs * np.log2(probs + 1e-9))
|
|
|
|
# 3. Spectral Check (Heuristic for full run speed)
|
|
# Check for non-uniform token clusters
|
|
spectral_density = len(counts) / 256.0
|
|
|
|
# Lawfulness membrane
|
|
# Only admit if structure persists (high entropy but structured density)
|
|
if 4.0 < entropy < 7.5 and spectral_density < 0.6:
|
|
lawful_count += 1
|
|
# RESTORE Stage: Add to the purified sequence
|
|
# We take the first element (stride extraction)
|
|
purified_ids.append(ids[0])
|
|
|
|
if i > 1000000: # Sanity stop for turn limits, user can ask for more if needed
|
|
print("Reached 1,000,000 chunk limit for this pass. Splitting...")
|
|
break
|
|
|
|
# Final Restoration
|
|
print(f"Restoring Purified Archive ({len(purified_ids)} bytes)...")
|
|
purified_bytes = bytes(np.array(purified_ids, dtype=np.uint8))
|
|
|
|
with open(output_binary, 'wb') as f:
|
|
f.write(purified_bytes)
|
|
|
|
print(f"Purified Archive saved to {output_binary}")
|
|
print(f"Compression Ratio (Original vs Purified): {len(chunk_reader) / len(purified_ids):.2f}x")
|
|
|
|
if __name__ == "__main__":
|
|
input_p = Path("/home/allaun/Documents/Research Stack/data/enwik8_huggingface/context128-stride1/test-00000-of-00002.parquet")
|
|
output_bin = Path("/home/allaun/Documents/Research Stack/data/enwik8_huggingface/enwik8_purified.bin")
|
|
|
|
if input_p.exists():
|
|
process_full_archive(input_p, output_bin)
|
|
else:
|
|
print("Error: Parquet not found.")
|