"""
Test the strong claim: "encode data as a semantic mass number A(H), reconstruct
losslessly" as a COMPRESSOR. Faithful to the OTOM spec's own readings:
  - nuclide address / "10-adic residue" reading  -> the message IS one big number
  - "observation = residue class mod ℓ"           -> mixed-radix over the alphabet ℓ
Encoding a string as a single integer is BASE CONVERSION, which is a bijection:
information-preserving, therefore it cannot reduce bits below what the digit
radices already imply. Measured three ways, all lossless round-trip:

  M1  base-256 positional  : A(H) = int.from_bytes(data)          (uniform radix 256)
  M2  mixed-radix over Σ   : digits = symbol ranks, radix |Σ|     (residue-mod-ℓ reading)
  M3  frequency-weighted   : radix_i = 1/p(sym_i)  == arithmetic coding (best case)

M1/M2 carry NO probability model -> ceil(n·log2 radix) bits, a pure repack.
M3 reaches entropy but the radices ARE the model, which must ship (conservation).
Baseline: xz -9 (order-N + long-range matching) on the same bytes.
"""
import os, sys, math, lzma, numpy as np

def load(root, cap):
    buf = bytearray()
    for dp, _, fs in os.walk(root):
        for fn in fs:
            if fn.endswith((".md", ".txt", ".lean", ".py")):
                try:
                    with open(os.path.join(dp, fn), "rb") as f: buf += f.read()
                except Exception: pass
                if len(buf) >= cap: return bytes(buf[:cap])
    return bytes(buf)

data = load(sys.argv[1] if len(sys.argv) > 1 else "/home/allaun/SilverSight", 100_000)
n = len(data)
print(f"test data: {n} bytes\n")

# ---- M1: base-256 positional big number (the literal "mass number of the doc") ----
A = int.from_bytes(data, "big")
m1_bytes = (A.bit_length() + 7) // 8
back = A.to_bytes(m1_bytes, "big").rjust(n, b"\x00")  # leading-zero safe
m1_ok = back == data
# to be fully lossless you must also store n (leading zeros) -> +a few bytes; negligible

# ---- M2: mixed-radix over the observed alphabet Σ (residue-mod-ℓ reading) ----
alphabet = sorted(set(data)); L = len(alphabet)
rank = {b: i for i, b in enumerate(alphabet)}
# real bijection round-trip on a slice (big-int divmod is O(len^2) — prove on 3KB)
sl = data[:3000]
val = 0
for b in sl: val = val * L + rank[b]           # encode slice as one mixed-radix integer
digits = []
v = val
for _ in range(len(sl)): v, r = divmod(v, L); digits.append(r)
dec = bytes(alphabet[r] for r in reversed(digits))
m2_ok = dec == sl
m2_bits_per_char = math.log2(L)                # exact: ceil(n·log2 L)/n -> log2 L
m2_bytes_full = math.ceil(n * math.log2(L) / 8) + L  # +L to ship the alphabet table

# ---- M3: frequency-weighted radix == order-0 arithmetic coding (ideal rate) ----
counts = np.bincount(np.frombuffer(data, np.uint8), minlength=256).astype(float)
p = counts[counts > 0] / n
H0 = float(-(p * np.log2(p)).sum())            # order-0 entropy = M3 ideal bits/char
m3_bytes = math.ceil(H0 * n / 8) + L * 2       # + shipped frequency model (~2B/symbol)

# ---- baseline ----
xz = len(lzma.compress(data, preset=9)); xz_bpc = xz * 8 / n

print(f"{'method':<34}{'bits/char':>10}{'bytes':>10}{'ratio':>8}  lossless")
print("-" * 74)
print(f"{'M1 base-256 mass number':<34}{8.0:>10.3f}{m1_bytes:>10}{n/m1_bytes:>8.2f}  "
      f"{'PASS' if m1_ok else 'FAIL'}  (= the data, no gain)")
print(f"{'M2 mixed-radix over Σ (|Σ|=%d)'%L:<34}{m2_bits_per_char:>10.3f}{m2_bytes_full:>10}"
      f"{n/m2_bytes_full:>8.2f}  {'PASS' if m2_ok else 'FAIL'}  (flat order-0 only)")
print(f"{'M3 freq-weighted (=arith, order0)':<34}{H0:>10.3f}{m3_bytes:>10}{n/m3_bytes:>8.2f}"
      f"  n/a   (needs shipped model)")
print(f"{'xz -9 (order-N + matching)':<34}{xz_bpc:>10.3f}{xz:>10}{n/xz:>8.2f}  PASS")
print("\nM1/M2 carry no model: base conversion is a bijection, cannot beat the radix.")
print("M3 reaches entropy only by BEING the model (radices=1/p) -> conservation.")
print("xz wins because it exploits order-N structure + long-range repeats, which no")
print("single 'mass number' of the raw stream can see.")