mirror of
https://github.com/allaunthefox/SilverSight.git
synced 2026-07-31 01:25:21 +00:00
allaun
99c943dcd0
Records the full compression investigation as a repo doc plus the scripts that back every number (real coders, byte-exact lossless round-trips, no straw baselines). One law: recoverable <=> sparse/structured; no method beats K(data), schemes only relocate bits between model and residual columns. Findings (all measured): char-poly = integrity receipt not compressor; Braille/T9 = 4.167 b/B, loses to xz; "16D/583x" GW ringdown = zero-noise self-fit artifact, ~1.5x tying/losing to LPC on noisy strain; frozen-model conservation law (k=3 smallest tape, worst total); Semantic Mass Number = base conversion (1.00-1.10x, bijection); capstone superposition/compressed-sensing cliff (recoverable iff k <= ~d/log N). Honest home for all: receipts/addresses/ recoverability gates (GCCL/RRC), never the ratio column. docs/research/COMPRESSION_HONEST_FINDINGS.md + scripts/compression/ (7 scripts + README). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
78 lines
4 KiB
Python
78 lines
4 KiB
Python
"""
|
||
Test the strong claim: "encode data as a semantic mass number A(H), reconstruct
|
||
losslessly" as a COMPRESSOR. Faithful to the OTOM spec's own readings:
|
||
- nuclide address / "10-adic residue" reading -> the message IS one big number
|
||
- "observation = residue class mod ℓ" -> mixed-radix over the alphabet ℓ
|
||
Encoding a string as a single integer is BASE CONVERSION, which is a bijection:
|
||
information-preserving, therefore it cannot reduce bits below what the digit
|
||
radices already imply. Measured three ways, all lossless round-trip:
|
||
|
||
M1 base-256 positional : A(H) = int.from_bytes(data) (uniform radix 256)
|
||
M2 mixed-radix over Σ : digits = symbol ranks, radix |Σ| (residue-mod-ℓ reading)
|
||
M3 frequency-weighted : radix_i = 1/p(sym_i) == arithmetic coding (best case)
|
||
|
||
M1/M2 carry NO probability model -> ceil(n·log2 radix) bits, a pure repack.
|
||
M3 reaches entropy but the radices ARE the model, which must ship (conservation).
|
||
Baseline: xz -9 (order-N + long-range matching) on the same bytes.
|
||
"""
|
||
import os, sys, math, lzma, numpy as np
|
||
|
||
def load(root, cap):
|
||
buf = bytearray()
|
||
for dp, _, fs in os.walk(root):
|
||
for fn in fs:
|
||
if fn.endswith((".md", ".txt", ".lean", ".py")):
|
||
try:
|
||
with open(os.path.join(dp, fn), "rb") as f: buf += f.read()
|
||
except Exception: pass
|
||
if len(buf) >= cap: return bytes(buf[:cap])
|
||
return bytes(buf)
|
||
|
||
data = load(sys.argv[1] if len(sys.argv) > 1 else "/home/allaun/SilverSight", 100_000)
|
||
n = len(data)
|
||
print(f"test data: {n} bytes\n")
|
||
|
||
# ---- M1: base-256 positional big number (the literal "mass number of the doc") ----
|
||
A = int.from_bytes(data, "big")
|
||
m1_bytes = (A.bit_length() + 7) // 8
|
||
back = A.to_bytes(m1_bytes, "big").rjust(n, b"\x00") # leading-zero safe
|
||
m1_ok = back == data
|
||
# to be fully lossless you must also store n (leading zeros) -> +a few bytes; negligible
|
||
|
||
# ---- M2: mixed-radix over the observed alphabet Σ (residue-mod-ℓ reading) ----
|
||
alphabet = sorted(set(data)); L = len(alphabet)
|
||
rank = {b: i for i, b in enumerate(alphabet)}
|
||
# real bijection round-trip on a slice (big-int divmod is O(len^2) — prove on 3KB)
|
||
sl = data[:3000]
|
||
val = 0
|
||
for b in sl: val = val * L + rank[b] # encode slice as one mixed-radix integer
|
||
digits = []
|
||
v = val
|
||
for _ in range(len(sl)): v, r = divmod(v, L); digits.append(r)
|
||
dec = bytes(alphabet[r] for r in reversed(digits))
|
||
m2_ok = dec == sl
|
||
m2_bits_per_char = math.log2(L) # exact: ceil(n·log2 L)/n -> log2 L
|
||
m2_bytes_full = math.ceil(n * math.log2(L) / 8) + L # +L to ship the alphabet table
|
||
|
||
# ---- M3: frequency-weighted radix == order-0 arithmetic coding (ideal rate) ----
|
||
counts = np.bincount(np.frombuffer(data, np.uint8), minlength=256).astype(float)
|
||
p = counts[counts > 0] / n
|
||
H0 = float(-(p * np.log2(p)).sum()) # order-0 entropy = M3 ideal bits/char
|
||
m3_bytes = math.ceil(H0 * n / 8) + L * 2 # + shipped frequency model (~2B/symbol)
|
||
|
||
# ---- baseline ----
|
||
xz = len(lzma.compress(data, preset=9)); xz_bpc = xz * 8 / n
|
||
|
||
print(f"{'method':<34}{'bits/char':>10}{'bytes':>10}{'ratio':>8} lossless")
|
||
print("-" * 74)
|
||
print(f"{'M1 base-256 mass number':<34}{8.0:>10.3f}{m1_bytes:>10}{n/m1_bytes:>8.2f} "
|
||
f"{'PASS' if m1_ok else 'FAIL'} (= the data, no gain)")
|
||
print(f"{'M2 mixed-radix over Σ (|Σ|=%d)'%L:<34}{m2_bits_per_char:>10.3f}{m2_bytes_full:>10}"
|
||
f"{n/m2_bytes_full:>8.2f} {'PASS' if m2_ok else 'FAIL'} (flat order-0 only)")
|
||
print(f"{'M3 freq-weighted (=arith, order0)':<34}{H0:>10.3f}{m3_bytes:>10}{n/m3_bytes:>8.2f}"
|
||
f" n/a (needs shipped model)")
|
||
print(f"{'xz -9 (order-N + matching)':<34}{xz_bpc:>10.3f}{xz:>10}{n/xz:>8.2f} PASS")
|
||
print("\nM1/M2 carry no model: base conversion is a bijection, cannot beat the radix.")
|
||
print("M3 reaches entropy only by BEING the model (radices=1/p) -> conservation.")
|
||
print("xz wins because it exploits order-N structure + long-range repeats, which no")
|
||
print("single 'mass number' of the raw stream can see.")
|