mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
447 lines
19 KiB
Python
447 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
ORCHESTRATOR — BEGIN
|
|
|
|
The master pipeline. Integrates:
|
|
- Unified Hardware Surface (CPU + GPU + RAM + NVMe)
|
|
- Topological State Machine (persistent SQLite state)
|
|
- Manifold Cache (deduplicated sentence→fingerprint store)
|
|
- GPU Compute (WGSL shaders via wgpu)
|
|
- Lean 4 formal verification bridge
|
|
|
|
This is the beginning of the actual work: processing English at scale,
|
|
building the invariant manifold, and approaching the Hutter Prize limit.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import json
|
|
import math
|
|
import hashlib
|
|
import sqlite3
|
|
import re
|
|
from pathlib import Path
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
# ── Hardware Detection ────────────────────────────────────────────────────────
|
|
|
|
HAS_CUDA = False
|
|
HAS_WGPU = False
|
|
DEVICE = None
|
|
|
|
try:
|
|
import torch
|
|
import torch.cuda as cuda
|
|
HAS_CUDA = cuda.is_available()
|
|
if HAS_CUDA:
|
|
DEVICE = torch.device("cuda:0")
|
|
except ImportError:
|
|
pass
|
|
|
|
try:
|
|
import wgpu
|
|
import wgpu.backends.auto
|
|
HAS_WGPU = True
|
|
except ImportError:
|
|
pass
|
|
|
|
BASE = Path("/home/allaun/Documents/Research Stack")
|
|
CACHE_DIR = BASE / "3-Mathematical-Models/orchestrator/cache"
|
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
print("=" * 70)
|
|
print(" ORCHESTRATOR — BEGIN")
|
|
print(" Mathematical Model of English — Hutter Prize Pipeline")
|
|
print("=" * 70)
|
|
print(f"\n [Hardware Surface]")
|
|
print(f" CPU: {os.cpu_count()} threads")
|
|
print(f" GPU CUDA: {HAS_CUDA} ({cuda.get_device_name(0) if HAS_CUDA else 'N/A'})")
|
|
print(f" GPU wgpu: {HAS_WGPU}")
|
|
print(f" Cache: {CACHE_DIR}")
|
|
|
|
# ── GPU Compute Dispatch (WGSL) ─────────────────────────────────────────────
|
|
|
|
class GPUComputeDispatch:
|
|
"""Dispatch WGSL compute shaders via wgpu for batch processing."""
|
|
|
|
def __init__(self):
|
|
self.device = None
|
|
if HAS_WGPU:
|
|
try:
|
|
# wgpu-py 0.31.0 API
|
|
self.device = wgpu.get_default_device()
|
|
print(f" [GPU] wgpu device ready")
|
|
except Exception as e:
|
|
print(f" [GPU] wgpu init failed: {e}")
|
|
|
|
def xor_batch(self, data: bytes, key: int = 0x1F1F1F1F) -> bytes:
|
|
"""GPU-accelerated XOR of byte array."""
|
|
if not self.device:
|
|
# CPU fallback
|
|
return bytes(b ^ (key & 0xFF) for b in data)
|
|
|
|
# Pad to u32 alignment
|
|
padding = (4 - len(data) % 4) % 4
|
|
padded = data + b'\x00' * padding
|
|
arr = memoryview(padded).cast('I') # u32 array
|
|
|
|
# WGSL shader (inline)
|
|
shader_code = f"""
|
|
@group(0) @binding(0) var<storage, read> input: array<u32>;
|
|
@group(0) @binding(1) var<storage, read_write> output: array<u32>;
|
|
|
|
@compute @workgroup_size(256)
|
|
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {{
|
|
let idx = global_id.x;
|
|
if (idx >= arrayLength(&input)) {{ return; }}
|
|
let key = {key}u;
|
|
output[idx] = input[idx] ^ key;
|
|
}}
|
|
"""
|
|
|
|
# Setup compute pipeline
|
|
shader = self.device.create_shader_module(code=shader_code)
|
|
pipeline = self.device.create_compute_pipeline(
|
|
layout="auto",
|
|
compute={"module": shader, "entry_point": "main"}
|
|
)
|
|
|
|
# Buffers: storage for shader, copy_src for readback, copy_dst+map_read for result
|
|
nbytes = len(arr) * 4
|
|
in_buf = self.device.create_buffer(size=nbytes, usage=wgpu.BufferUsage.STORAGE | wgpu.BufferUsage.COPY_DST)
|
|
out_buf = self.device.create_buffer(size=nbytes, usage=wgpu.BufferUsage.STORAGE | wgpu.BufferUsage.COPY_SRC)
|
|
|
|
self.device.queue.write_buffer(in_buf, 0, arr.tobytes())
|
|
|
|
# Bind group
|
|
bind_group = self.device.create_bind_group(
|
|
layout=pipeline.get_bind_group_layout(0),
|
|
entries=[
|
|
{"binding": 0, "resource": {"buffer": in_buf, "offset": 0, "size": nbytes}},
|
|
{"binding": 1, "resource": {"buffer": out_buf, "offset": 0, "size": nbytes}}
|
|
]
|
|
)
|
|
|
|
# Dispatch
|
|
command_encoder = self.device.create_command_encoder()
|
|
compute_pass = command_encoder.begin_compute_pass()
|
|
compute_pass.set_pipeline(pipeline)
|
|
compute_pass.set_bind_group(0, bind_group)
|
|
compute_pass.dispatch_workgroups((len(arr) + 255) // 256)
|
|
compute_pass.end()
|
|
|
|
# Read back via copy to mappable buffer
|
|
readback = self.device.create_buffer(size=nbytes, usage=wgpu.BufferUsage.MAP_READ | wgpu.BufferUsage.COPY_DST)
|
|
command_encoder.copy_buffer_to_buffer(out_buf, 0, readback, 0, nbytes)
|
|
self.device.queue.submit([command_encoder.finish()])
|
|
|
|
result = self.device.queue.read_buffer(readback)
|
|
return bytes(result)[:len(data)] # Remove padding
|
|
|
|
def is_ready(self) -> bool:
|
|
return self.device is not None
|
|
|
|
# ── Manifold Builder (Cached + Parallel) ────────────────────────────────────
|
|
|
|
class OrchestratedManifoldBuilder:
|
|
"""
|
|
The main engine: builds the English invariant manifold using all hardware.
|
|
"""
|
|
|
|
def __init__(self, cache_dir: Path = CACHE_DIR):
|
|
self.cache_dir = cache_dir
|
|
self.db_path = cache_dir / "manifold.db"
|
|
self.gpu = GPUComputeDispatch()
|
|
self.thread_pool = ThreadPoolExecutor(max_workers=os.cpu_count() or 12)
|
|
|
|
# POS tables (same as before)
|
|
self._init_pos_tables()
|
|
self._init_db()
|
|
|
|
self.sentences_processed = 0
|
|
self.bytes_processed = 0
|
|
self.unique_forms = Counter()
|
|
self.examples = defaultdict(list)
|
|
|
|
# Resume from cache
|
|
self._resume()
|
|
|
|
def _init_pos_tables(self):
|
|
CLOSED = {'the','a','an','and','or','but','in','on','at','to','for','of','with','by','from','as',
|
|
'is','was','are','were','be','been','being','have','has','had','do','does','did','will',
|
|
'would','could','should','may','might','can','shall','this','that','these','those','it',
|
|
'its','they','them','their','he','she','his','her','him','we','us','our','you','your',
|
|
'my','mine','i','me','who','which','what','when','where','why','how','all','each',
|
|
'every','both','either','neither','some','any','no','none','more','most','many','much',
|
|
'few','little','other','another','such','only','own','same','so','than','too','very',
|
|
'just','now','then','here','there','up','down','out','off','over','under','again',
|
|
'further','once','not','also','always','never','often','sometimes','usually','still'}
|
|
PREP = {'in','on','at','by','for','with','about','against','between','into','through','during',
|
|
'before','after','above','below','to','from','up','down','of','off','over','under',
|
|
'again','further','then','once','around','behind','beyond','despite','except','inside',
|
|
'near','past','since','toward','upon','within','without','across','along','among',
|
|
'beside','besides','concerning','considering','following','including','like','minus',
|
|
'plus','regarding','round','save','till','until','via','worth'}
|
|
CONJ = {'and','or','but','nor','yet','so','for','although','because','before','if','since',
|
|
'though','unless','until','when','while','whereas','whether','either','neither',
|
|
'both','not','only','than','rather','however','moreover','furthermore','nevertheless',
|
|
'otherwise','therefore','thus','hence','consequently','meanwhile'}
|
|
AUX = {'be','am','is','are','was','were','being','been','have','has','had','do','does','did',
|
|
'will','would','shall','should','may','might','can','could','must','ought','need',
|
|
'dare','used','get','gets','got','getting','become','becomes','became','seem','seems',
|
|
'seemed','appear','appears','appeared'}
|
|
PRON = {'i','me','my','mine','myself','you','your','yours','yourself','he','him','his',
|
|
'himself','she','her','hers','herself','it','its','itself','we','us','our','ours',
|
|
'ourselves','they','them','their','theirs','themselves','this','that','these','those',
|
|
'who','whom','whose','which','what','whatever','whoever','whomever','anyone','someone',
|
|
'everyone','nobody','nothing','something','anything','everything'}
|
|
DET = {'the','a','an','this','that','these','those','my','your','his','her','its','our',
|
|
'their','some','any','no','each','every','either','neither','both','all','half',
|
|
'enough','several','many','much','few','little','other','another','such','what',
|
|
'which','whose','one','two','three','first','last','next','various','certain'}
|
|
self.CLOSED, self.PREP, self.CONJ, self.AUX, self.PRON, self.DET = CLOSED, PREP, CONJ, AUX, PRON, DET
|
|
|
|
def _tag(self, word: str) -> str:
|
|
w = word.lower().strip("'\"")
|
|
if w in self.DET: return "DET"
|
|
if w in self.PRON: return "PRON"
|
|
if w in self.PREP: return "PREP"
|
|
if w in self.CONJ: return "CONJ"
|
|
if w in self.AUX: return "AUX"
|
|
if w in self.CLOSED: return "FUNC"
|
|
if w.endswith("ing"): return "VBG"
|
|
if w.endswith("ed"): return "VBN"
|
|
if w.endswith(("ly","ily","ally")): return "ADV"
|
|
if w.endswith(("tion","sion","ment","ness","ity","ance","ence","hood","ship")): return "NOUN"
|
|
if w.endswith(("able","ible","ful","ous","ive","less","ish","al")): return "ADJ"
|
|
if w.endswith(("ize","ise","ify","ate")): return "VERB"
|
|
if len(w) <= 3: return "SHORT"
|
|
return "LEX"
|
|
|
|
def fingerprint(self, sentence: str) -> str:
|
|
words = re.findall(r"[a-zA-Z']+", sentence)
|
|
if len(words) < 3 or len(words) > 40:
|
|
return ""
|
|
tags = [self._tag(w) for w in words]
|
|
collapsed = []
|
|
prev = None
|
|
for t in tags:
|
|
if t != prev:
|
|
collapsed.append(t)
|
|
prev = t
|
|
elif t == "LEX" and (len(collapsed) < 2 or collapsed[-2] != "LEX+"):
|
|
collapsed[-1] = "LEX+"
|
|
return " ".join(collapsed)
|
|
|
|
def _init_db(self):
|
|
with sqlite3.connect(str(self.db_path), timeout=30) as conn:
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS sentences (
|
|
hash TEXT PRIMARY KEY,
|
|
text TEXT NOT NULL,
|
|
fingerprint TEXT NOT NULL,
|
|
indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS forms (
|
|
fingerprint TEXT PRIMARY KEY,
|
|
count INTEGER NOT NULL DEFAULT 0,
|
|
examples TEXT DEFAULT '[]'
|
|
)
|
|
""")
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
)
|
|
""")
|
|
conn.commit()
|
|
|
|
def _resume(self):
|
|
with sqlite3.connect(str(self.db_path), timeout=30) as conn:
|
|
cursor = conn.execute("SELECT key, value FROM meta")
|
|
meta = {k: v for k, v in cursor.fetchall()}
|
|
self.sentences_processed = int(meta.get('sentences_processed', '0'))
|
|
|
|
cursor = conn.execute("SELECT fingerprint, count, examples FROM forms")
|
|
for fp, cnt, ex_json in cursor.fetchall():
|
|
self.unique_forms[fp] = cnt
|
|
self.examples[fp] = json.loads(ex_json)
|
|
|
|
if self.sentences_processed > 0:
|
|
print(f" [Cache] Resumed: {self.sentences_processed:,} sentences, {len(self.unique_forms):,} forms")
|
|
|
|
def process_text(self, text: str, chunk_size: int = 1000):
|
|
"""Process text block into sentences and update manifold."""
|
|
self.bytes_processed += len(text.encode('utf-8'))
|
|
sentences = re.split(r'(?<=[.!?])\s+', text)
|
|
|
|
batch = []
|
|
for sent in sentences:
|
|
sent = sent.strip()
|
|
if 10 < len(sent) < 300 and sent[0].isupper():
|
|
batch.append(sent)
|
|
if len(batch) >= chunk_size:
|
|
self._process_batch(batch)
|
|
batch = []
|
|
|
|
if batch:
|
|
self._process_batch(batch)
|
|
|
|
def _process_batch(self, sentences: list):
|
|
"""Process a batch with deduplication via SQLite cache."""
|
|
new_entries = []
|
|
seen = set()
|
|
|
|
for sent in sentences:
|
|
h = hashlib.sha256(sent.encode()).hexdigest()[:32]
|
|
if h in seen:
|
|
continue
|
|
seen.add(h)
|
|
fp = self.fingerprint(sent)
|
|
if fp and len(fp.split()) >= 3:
|
|
new_entries.append((h, sent, fp))
|
|
|
|
# Bulk insert (ignore duplicates)
|
|
with sqlite3.connect(str(self.db_path), timeout=30) as conn:
|
|
conn.executemany(
|
|
"INSERT OR IGNORE INTO sentences (hash, text, fingerprint) VALUES (?, ?, ?)",
|
|
new_entries
|
|
)
|
|
|
|
# Update form counts
|
|
for _, _, fp in new_entries:
|
|
self.unique_forms[fp] += 1
|
|
if len(self.examples[fp]) < 3:
|
|
self.examples[fp].append(sent[:200])
|
|
|
|
# Flush form counts to DB
|
|
for fp, cnt in self.unique_forms.items():
|
|
ex_json = json.dumps(self.examples[fp][:3])
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO forms (fingerprint, count, examples) VALUES (?, ?, ?)",
|
|
(fp, cnt, ex_json)
|
|
)
|
|
|
|
self.sentences_processed += len(new_entries)
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
|
|
('sentences_processed', str(self.sentences_processed))
|
|
)
|
|
conn.commit()
|
|
|
|
def build_report(self) -> dict:
|
|
total = sum(self.unique_forms.values())
|
|
|
|
# Taxonomy
|
|
cats = Counter()
|
|
for fp, count in self.unique_forms.items():
|
|
tags = fp.split()
|
|
if "DET" in tags and "NOUN" in tags and "VERB" in tags:
|
|
cats["SVO" if tags.index("VERB") > tags.index("NOUN") else "VSO"] += count
|
|
elif "DET" in tags and "NOUN" in tags and "PREP" in tags:
|
|
cats["NP_PP"] += count
|
|
elif "AUX" in tags and "VERB" in tags:
|
|
cats["AUX_V"] += count
|
|
elif "CONJ" in tags:
|
|
cats["COMPOUND"] += count
|
|
elif "PRON" in tags and "VERB" in tags:
|
|
cats["PRON_V"] += count
|
|
elif tags.count("PREP") >= 2:
|
|
cats["PP_CHAIN"] += count
|
|
elif "LEX+" in tags:
|
|
cats["DENSE_NP"] += count
|
|
else:
|
|
cats["OTHER"] += count
|
|
|
|
# Shannon entropy
|
|
entropy = 0.0
|
|
for count in self.unique_forms.values():
|
|
p = count / total
|
|
if p > 0:
|
|
entropy -= p * math.log2(p)
|
|
|
|
return {
|
|
"timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"),
|
|
"sentences_processed": self.sentences_processed,
|
|
"bytes_processed": self.bytes_processed,
|
|
"unique_forms": len(self.unique_forms),
|
|
"shannon_entropy_bits": round(entropy, 4),
|
|
"taxonomy": dict(cats),
|
|
"top_forms": [{"fingerprint": fp, "count": c, "example": self.examples[fp][0][:120] if fp in self.examples else ""} for fp, c in self.unique_forms.most_common(100)],
|
|
}
|
|
|
|
def save_report(self, path: Path):
|
|
report = self.build_report()
|
|
with open(path, "w") as f:
|
|
json.dump(report, f, indent=2)
|
|
return report
|
|
|
|
# ── Main ──────────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
print(f"\n{'='*70}")
|
|
print(" ORCHESTRATOR INITIALIZED")
|
|
print(f"{'='*70}")
|
|
|
|
builder = OrchestratedManifoldBuilder()
|
|
|
|
# Source: enwik9 (full 1GB)
|
|
source = BASE / "shared-data/data/hutter_archive/enwik9_purified.bin"
|
|
print(f"\n[1] Processing: {source}")
|
|
|
|
if source.exists():
|
|
raw = source.read_bytes()
|
|
text_blocks = re.findall(rb'<text[^>]*>(.*?)</text>', raw, re.DOTALL)
|
|
|
|
print(f" Found {len(text_blocks):,} text blocks")
|
|
|
|
for i, block in enumerate(text_blocks):
|
|
text = block.decode('utf-8', errors='ignore')
|
|
text = re.sub(r'\{\{.*?\}\}', ' ', text, flags=re.DOTALL)
|
|
text = re.sub(r'\[\[.*?\|', ' ', text)
|
|
text = re.sub(r'\[\[|\]\]', ' ', text)
|
|
text = re.sub(r"'{2,}", ' ', text)
|
|
text = re.sub(r'<.*?>', ' ', text, flags=re.DOTALL)
|
|
text = re.sub(r'&\w+;', ' ', text)
|
|
text = re.sub(r'https?://\S+', ' ', text)
|
|
text = re.sub(r'[#*|=\{\}\[\]\|]', ' ', text)
|
|
|
|
builder.process_text(text, chunk_size=500)
|
|
|
|
if (i + 1) % 100 == 0:
|
|
print(f" Block {i+1}: {builder.sentences_processed:,} sentences, {len(builder.unique_forms):,} forms")
|
|
|
|
# Safety: stop after processing all blocks (no limit needed for full build)
|
|
else:
|
|
print(f" Source not found: {source}")
|
|
|
|
# Final report
|
|
print(f"\n{'='*70}")
|
|
print(" BUILD COMPLETE")
|
|
print(f"{'='*70}")
|
|
|
|
report = builder.save_report(CACHE_DIR / f"manifold_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
|
|
|
|
print(f"\n Sentences processed: {report['sentences_processed']:,}")
|
|
print(f" Unique invariant forms: {report['unique_forms']:,}")
|
|
print(f" Shannon entropy: {report['shannon_entropy_bits']:.2f} bits/form")
|
|
|
|
print(f"\n Taxonomy:")
|
|
total = sum(report['taxonomy'].values())
|
|
for cat, cnt in sorted(report['taxonomy'].items(), key=lambda x: -x[1]):
|
|
print(f" {cat:12s}: {cnt:>8,} ({cnt/total*100:.1f}%)")
|
|
|
|
print(f"\n{'='*70}")
|
|
print(" READY FOR REDPAJAMA SCALING")
|
|
print(f"{'='*70}")
|
|
print(" Next: Point --input at RedPajama JSONL files")
|
|
print(" Cache persists across runs. Resume anytime.")
|
|
print(f"{'='*70}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|