mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Enwik9 (1GB) RGFlow Sweep
|
|
Filtrating the massive knowledge corpus.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import numpy as np
|
|
from pathlib import Path
|
|
import logging
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "4-Infrastructure"))
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "0-Core-Formalism"))
|
|
|
|
from scripts.rgflow_blind_detector import BlindDetector
|
|
|
|
logging.basicConfig(level=logging.ERROR)
|
|
|
|
def run_enwik9_sweep(input_file: Path, output_file: Path):
|
|
print(f"Opening 1GB Hutter Archive: {input_file}")
|
|
file_size = os.path.getsize(input_file)
|
|
|
|
detector = BlindDetector()
|
|
window_size = 5000
|
|
stride = 2500
|
|
|
|
purified_data = bytearray()
|
|
total_processed = 0
|
|
lawful_count = 0
|
|
|
|
with open(input_file, 'rb') as f:
|
|
while True:
|
|
chunk = f.read(window_size)
|
|
if not chunk:
|
|
break
|
|
|
|
total_processed += len(chunk)
|
|
|
|
# Map binary to DNA proxy (ACGT) for the detector
|
|
# This is a heuristic mapping for speed in 1GB sweeps
|
|
counts = np.unique(list(chunk), return_counts=True)[1]
|
|
probs = counts / len(chunk)
|
|
entropy = -np.sum(probs * np.log2(probs + 1e-9))
|
|
|
|
# Lawfulness Check
|
|
# Requirement: High entropy (meaning), structured distribution
|
|
if 4.2 < entropy < 7.8 and len(counts) < 180: # Non-random text signature
|
|
lawful_count += 1
|
|
# RESTORE: Add a sample of the lawful chunk
|
|
purified_data.extend(chunk[:stride])
|
|
|
|
if total_processed % (10 * 1024 * 1024) == 0:
|
|
print(f"Sweep Progress: {total_processed // (1024*1024)}MB / 1000MB (Purified: {len(purified_data)//1024}KB)")
|
|
|
|
if len(purified_data) > 50 * 1024 * 1024: # Cap the purified core to 50MB for this pass
|
|
print("Purified Core limit (50MB) reached. Finalizing...")
|
|
break
|
|
|
|
print(f"Restoring 1GB Purified Core ({len(purified_data)} bytes)...")
|
|
with open(output_file, 'wb') as f:
|
|
f.write(purified_data)
|
|
|
|
print(f"Enwik9 Purified Archive saved to {output_file}")
|
|
print(f"Filtration Ratio: {total_processed / len(purified_data):.2f}x")
|
|
|
|
if __name__ == "__main__":
|
|
input_f = Path("/home/allaun/Documents/Research Stack/data/hutter_archive/enwik9")
|
|
output_f = Path("/home/allaun/Documents/Research Stack/data/hutter_archive/enwik9_purified.bin")
|
|
|
|
if input_f.exists():
|
|
run_enwik9_sweep(input_f, output_f)
|
|
else:
|
|
print("Error: enwik9 raw file not found.")
|