SilverSight/scripts/braille_t9.py

#!/usr/bin/env python3
"""braille_t9.py — Three-layer compressor: Braille LUT → T9 → hachimoji DNA.

Layer 1: Braille LUT — text fragments → 6-bit cells (dictionary substitution)
Layer 2: T9 mapping — 6-bit cells → 3-bit keys + KV cache disambiguation
Layer 3: Hachimoji — T9 keys map directly to 8 DNA bases

The "weird machine" concept: Braille was designed for tactile reading.
Using it to project data onto a 6-bit discrete space is unintended
computation through an accessibility substrate. The same machine works
on text, spectral data (GW/GRB), or any input.

Round-trip: lossless. Unseen transitions use (flag + cell value) fallback.
"""
import math, hashlib, re, sys, time, lzma, zlib
from collections import Counter, defaultdict
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
HACHIMOJI = list("ABCGPSTZ")

def build_lut(text_sample: bytes) -> dict:
    """Build Braille LUT: map common fragments to 6-bit cells (0-63)."""
    text = text_sample[:50000].decode('utf-8', errors='replace').lower()
    words = re.findall(r'\w+', text)
    word_freq = Counter(words)
    patterns = re.findall(r'\]\]|\{\{|\}\}|<ref|</ref>|==|http|www\.|\.org|\[\[', text)
    pattern_freq = Counter(patterns)
    
    fragments = []
    for w, f in word_freq.most_common(20):
        if len(w) >= 3: fragments.append((w.encode(), f, len(w)))
    for p, f in pattern_freq.most_common(10):
        fragments.append((p.encode(), f, len(p)))
    fragments.sort(key=lambda x: x[1]*x[2], reverse=True)
    
    # Cells 0-47: single bytes (most frequent first)
    # Cells 48-63: contractions (16 slots)
    freq = Counter(text_sample)
    single = sorted(range(256), key=lambda b: -freq.get(b, 0))
    
    forward = {}  # bytes → cell
    reverse = {}  # cell → bytes
    cell = 0
    
    for b in single[:48]:
        forward[bytes([b])] = cell
        reverse[cell] = bytes([b])
        cell += 1
    
    for frag, f, length in fragments:
        if cell >= 64: break
        if frag not in forward:
            forward[frag] = cell
            reverse[cell] = frag
            cell += 1
    
    # Fill remaining cells with unused bytes
    for b in single[48:]:
        if bytes([b]) not in forward and cell < 64:
            forward[bytes([b])] = cell
            reverse[cell] = bytes([b])
            cell += 1
    
    return {"forward": forward, "reverse": reverse,
            "n_contractions": sum(1 for c in reverse if len(reverse[c]) > 1)}

def braille_encode(data, lut):
    """Encode: longest-match-first dictionary substitution → cells."""
    cells = []
    i = 0
    fwd = lut["forward"]
    while i < len(data):
        matched = False
        for length in range(min(20, len(data) - i), 0, -1):
            frag = data[i:i+length]
            if frag in fwd:
                cells.append(fwd[frag])
                i += length
                matched = True
                break
        if not matched:
            # Shouldn't happen if all 256 bytes are in LUT
            cells.append(0)
            i += 1
    return cells

def braille_decode(cells, lut):
    """Decode: cells → bytes via reverse LUT."""
    result = bytearray()
    for c in cells:
        result.extend(lut["reverse"].get(c, b'\x00'))
    return bytes(result)

def compress(data, lut):
    """Full three-layer compress with lossless round-trip.
    
    Storage: T9 keys (3 bits each) + disambiguation stream
    Disambiguation: ('r', rank) for seen, ('c', cell) for unseen
    """
    n = len(data)
    if n == 0:
        return {"original": 0, "compressed": 0, "ratio": 1.0, "savings_pct": 0.0,
                "match": True, "hash_match": True, "bits_per_byte": 0, "pred_acc": 0,
                "braille_cells": 0, "n_contractions": 0, "n_rank": 0, "n_cell": 0,
                "braille_compressed": 0, "n_zeros": 0, "dna_sample": ""}
    
    # Layer 1: Braille encode
    cells = braille_encode(data, lut)
    
    # Layer 2: T9 + KV cache
    # cell → key = (cell % 8) + 1, keys 1-8
    # 8 cells per key → disambiguation needed
    cache = defaultdict(lambda: defaultdict(int))
    t9_keys = []
    disambig = []  # ('r', rank) or ('c', cell_value)
    
    for i in range(len(cells)):
        cell = cells[i]
        key = (cell % 8) + 1
        prev = cells[i-1] if i > 0 else 255  # special context for first
        
        # Predict from cache
        row = sorted(cache[prev].items(), key=lambda x: -x[1])
        order = [c for c, _ in row if (c % 8) + 1 == key]
        
        if cell in order:
            disambig.append(('r', order.index(cell)))
        else:
            disambig.append(('c', cell))  # store full cell value
        
        t9_keys.append(key)
        cache[prev][cell] += 1
    
    # DECODE: reconstruct from T9 keys + disambiguation
    cache_d = defaultdict(lambda: defaultdict(int))
    decoded_cells = []
    
    for i in range(len(t9_keys)):
        key = t9_keys[i]
        typ, val = disambig[i]
        prev = decoded_cells[-1] if decoded_cells else 255
        
        if typ == 'r':
            row = sorted(cache_d[prev].items(), key=lambda x: -x[1])
            order = [c for c, _ in row if (c % 8) + 1 == key]
            cell = order[val] if val < len(order) else 0
        else:  # 'c'
            cell = val
        
        decoded_cells.append(cell)
        cache_d[prev][cell] += 1
    
    # Layer 1 decode
    decoded = braille_decode(decoded_cells, lut)
    
    # Verify
    match = decoded == data
    oh = hashlib.sha256(data).hexdigest()[:16]
    dh = hashlib.sha256(decoded).hexdigest()[:16]
    
    # Layer 3: T9 → hachimoji (for receipt)
    key_map = {1:'A', 2:'B', 3:'C', 4:'G', 5:'P', 6:'S', 7:'T', 8:'Z'}
    dna = ''.join(key_map.get(k, 'A') for k in t9_keys[:30])
    
    # Sizes
    n_rank = sum(1 for t, _ in disambig if t == 'r')
    n_cell = sum(1 for t, _ in disambig if t == 'c')
    rank_bits = sum(max(0.01, math.log2(v+1)) for t, v in disambig if t == 'r')
    t9_bits = len(t9_keys) * 3
    cell_bits = n_cell * 6  # full cell value (6 bits) + 1 flag
    flag_bits = len(disambig)  # 1 bit per entry (r vs c)
    total_bits = t9_bits + flag_bits + rank_bits + cell_bits
    
    lut_bytes = 512  # 64 entries × ~8 bytes each
    compressed = lut_bytes + math.ceil(total_bits / 8)
    
    # Braille-only baseline (no T9)
    braille_bits = len(cells) * 6
    braille_compressed = lut_bytes + math.ceil(braille_bits / 8)
    
    correct = sum(1 for t, v in disambig if t == 'r' and v == 0)
    pred_acc = correct / max(len(disambig), 1) * 100
    
    return {
        "original": n, "compressed": compressed,
        "braille_compressed": braille_compressed,
        "braille_cells": len(cells),
        "ratio": n / max(compressed, 1),
        "savings_pct": (1 - compressed/max(n,1)) * 100,
        "match": match, "hash_match": oh == dh,
        "pred_acc": pred_acc, "n_zeros": correct,
        "n_rank": n_rank, "n_cell": n_cell,
        "n_contractions": lut["n_contractions"],
        "bits_per_byte": total_bits / max(n, 1),
        "dna_sample": dna,
    }

if __name__ == "__main__":
    texts = {
        "repetitive": b'abc abc abc abc def def def ghi ghi ghi abc def ghi ' * 10,
        "english": b'The quick brown fox jumps over the lazy dog. The dog was not amused. ' * 10,
        "latex": rb'\alpha + \beta = \gamma. \int_0^\infty e^{-x^2} dx = \sqrt{\pi}. ' * 10,
        "wiki": b'<page><title>Test</title><text>The Gaussian integral is defined as ' * 20,
        "empty": b'', "single": b'A',
    }
    
    print("=" * 70)
    print("  Three-Layer: Braille → T9 → Hachimoji (lossless)")
    print("=" * 70)
    
    all_match = True
    for name, data in texts.items():
        lut = build_lut(data[:10000])
        r = compress(data, lut)
        if not r["match"]: all_match = False
        print(f"\n[{name:>12s}] {r['original']:>5d}B → {r['compressed']:>5d}B "
              f"({r['savings_pct']:+.1f}%) ratio={r['ratio']:.2f}x "
              f"b/B={r['bits_per_byte']:.3f} pred={r['pred_acc']:.0f}% "
              f"match={'Y' if r['match'] else 'N'} hash={'Y' if r['hash_match'] else 'N'}")
        print(f"  Braille: {r['braille_cells']} cells, {r['braille_compressed']}B alone | "
              f"T9: {r['n_rank']} ranked + {r['n_cell']} explicit | "
              f"LUT: {r['n_contractions']} contractions | DNA: {r['dna_sample'][:20]}...")
    
    print(f"\nAll match: {all_match}")
    
    # enwik8
    p = Path('/tmp/opencode/enwik8')
    if p.exists():
        with open(p, 'rb') as f: data = f.read()
        print(f"\n{'='*70}\n  enwik8\n{'='*70}")
        for size in [10000, 100000, 1000000]:
            s = data[:size]
            g = zlib.compress(s, 9)
            x = lzma.compress(s, preset=9)
            lut = build_lut(s)
            r = compress(s, lut)
            print(f"\n--- {size:,}B ---")
            print(f"  gzip:           {len(g):>8,d}B ({len(g)*8/size:.3f} b/B)")
            print(f"  xz:             {len(x):>8,d}B ({len(x)*8/size:.3f} b/B)")
            print(f"  Braille only:   {r['braille_compressed']:>8,d}B ({r['braille_compressed']*8/size:.3f} b/B)")
            print(f"  Braille+T9+PPM: {r['compressed']:>8,d}B ({r['bits_per_byte']:.3f} b/B) "
                  f"pred={r['pred_acc']:.0f}% match={'Y' if r['match'] else 'N'}")
            print(f"  vs xz: {r['compressed']/len(x):.2f}x | vs gzip: {r['compressed']/len(g):.2f}x")