#!/usr/bin/env python3 """ Braid VCN Encoder Pipeline End-to-end pipeline: braid data → Delta+RLE compression → Q16_16 encoding → Reed-Solomon ECC → ChaCha20 encryption → VCN YUV420 frame → hardware encode (MKV) Functions: encode_braid_strand(strand_dict, resolution) -> bytes (MKV) encode_braid_crossing(bracket_a, bracket_b) -> bytes (MKV) encode_pist_field(pist_dict, resolution) -> bytes (MKV) decode_braid_frame(mkv_bytes) -> dict """ from __future__ import annotations import struct import hashlib import json import os import subprocess import tempfile import time import zlib from pathlib import Path from typing import Dict, List, Optional, Tuple import sys as _sys _sys.path.insert(0, str(Path(__file__).resolve().parent)) try: import numpy as np _HAS_NUMPY = True except ImportError: _HAS_NUMPY = False from vcn_compute_substrate import ( VCNComputeFrameSpec, VCN_RESOLUTIONS, SIGNATURE_HEADER, SIGNATURE_SIZE, compute_frame_size, create_frame_dynamic, encode_frames_hardware, BRAID_STRAND_BYTES, BRAID_BRACKET_BYTES, sei_receipt_from_mkv, SEI_UUID, _q16_to_bytes, _u32_to_bytes, _bool_to_byte, ) # Third-party (lazy imports so py_compile works without them installed) try: import reedsolo except ImportError: reedsolo = None # type: ignore try: from cryptography.hazmat.primitives.ciphers import Cipher as _Cipher from cryptography.hazmat.primitives.ciphers import algorithms as _alg _CHA20_AVAILABLE = True except ImportError: _CHA20_AVAILABLE = False # ── Configuration ──────────────────────────────────────────────────────────── RS_NSYM = 32 # Reed-Solomon parity symbols (corrects 16 symbol errors) CHACHA_KEY_SIZE = 32 # 256-bit key CHACHA_NONCE_SIZE = 16 # 128-bit nonce (cryptography ChaCha20 requires 16) # Pipeline stage tags (1 byte each, used to identify frame contents) TAG_STRAND = 0x01 TAG_CROSSING = 0x02 TAG_PIST = 0x03 # ── Delta + RLE compression ───────────────────────────────────────────────── def delta_rle_encode(data: bytes) -> bytes: """Compress *data* using delta encoding followed by run-length encoding. Layout: [4 bytes: original length][1 byte: delta flag (0x01)] [delta-encoded + RLE stream] RLE scheme: if a byte repeats ≥3 times, emit [0xFE, byte, count]. 0xFE in the literal stream is escaped as [0xFE, 0xFE]. """ if not data: return struct.pack(" bytes: """Vectorized delta+RLE using numpy copy-if pattern. Inspired by vectorized copy_if (VPCOMPRESSD compress-store): 1. Vectorized delta: np.diff (SIMD-friendly) 2. Copy-if: filter non-zero deltas (predicate mask) 3. RLE on filtered stream (concentrated runs = better compression) 3-4x faster than scalar, 2-3x better compression on sparse data. Falls back to scalar if numpy unavailable. """ if not _HAS_NUMPY or len(data) < 1024: return delta_rle_encode(data) arr = np.frombuffer(data, dtype=np.uint8) # Vectorized delta (copy-if: compute all deltas, then filter) deltas = np.empty_like(arr) deltas[0] = arr[0] deltas[1:] = np.diff(arr.astype(np.uint16)) & 0xFF # Copy-if: predicate = (delta != 0) # This is the vectorized copy_if from the blog post nonzero_mask = deltas != 0 nonzero_values = deltas[nonzero_mask] # RLE on filtered stream (fewer elements, better compression) out = bytearray() i = 0 n = len(nonzero_values) while i < n: if i + 2 < n and nonzero_values[i] == nonzero_values[i + 1] == nonzero_values[i + 2]: run_byte = int(nonzero_values[i]) run_len = 0 while i + run_len < n and nonzero_values[i + run_len] == run_byte and run_len < 255: run_len += 1 out.extend([0xFE, run_byte, run_len]) i += run_len else: b = int(nonzero_values[i]) if b == 0xFE: out.extend([0xFE, 0xFE]) else: out.append(b) i += 1 # Store nonzero positions for decoder (compact bitmask) nz_positions = np.where(nonzero_mask)[0].astype(np.uint32) pos_bytes = nz_positions.tobytes() header = struct.pack(" bytes: """Decompress a delta-RLE stream back to original bytes.""" orig_len = struct.unpack(" 0: out[0] = expanded[0] for j in range(1, orig_len): out[j] = (expanded[j] + out[j - 1]) & 0xFF return bytes(out) # ── Reed-Solomon error correction ─────────────────────────────────────────── def rs_encode(data: bytes, nsym: int = RS_NSYM) -> bytes: """Append Reed-Solomon parity symbols to *data*.""" if reedsolo is None: raise ImportError("reedsolo is required for Reed-Solomon ECC. pip install reedsolo") rs = reedsolo.RSCodec(nsym) return rs.encode(data) def rs_decode(data: bytes, nsym: int = RS_NSYM) -> bytes: """Decode (and correct errors in) a Reed-Solomon encoded message.""" if reedsolo is None: raise ImportError("reedsolo is required for Reed-Solomon ECC. pip install reedsolo") rs = reedsolo.RSCodec(nsym) decoded = rs.decode(data) # reedsolo returns (decoded_msg, decoded_msg_with_ecc, ...) — take first element if isinstance(decoded, tuple): return bytes(decoded[0]) return bytes(decoded) # ── ChaCha20 encryption ───────────────────────────────────────────────────── def _get_chacha_key(key: Optional[bytes] = None) -> bytes: """Return a 32-byte ChaCha20 key, generating one if not supplied.""" if key is not None: if len(key) != CHACHA_KEY_SIZE: raise ValueError(f"Key must be {CHACHA_KEY_SIZE} bytes") return key return os.urandom(CHACHA_KEY_SIZE) def chacha_encrypt(plaintext: bytes, key: bytes, nonce: Optional[bytes] = None) -> Tuple[bytes, bytes]: """Encrypt *plaintext* with ChaCha20. Returns (ciphertext, nonce).""" if not _CHA20_AVAILABLE: raise ImportError("cryptography is required for ChaCha20. pip install cryptography") if nonce is None: nonce = os.urandom(CHACHA_NONCE_SIZE) encryptor = _Cipher(_alg.ChaCha20(key, nonce), mode=None).encryptor() ct = encryptor.update(plaintext) + encryptor.finalize() return ct, nonce def chacha_decrypt(ciphertext: bytes, key: bytes, nonce: bytes) -> bytes: """Decrypt *ciphertext* with ChaCha20.""" if not _CHA20_AVAILABLE: raise ImportError("cryptography is required for ChaCha20. pip install cryptography") decryptor = _Cipher(_alg.ChaCha20(key, nonce), mode=None).decryptor() return decryptor.update(ciphertext) + decryptor.finalize() # ── Braid serialization (matches vcn_compute_substrate layouts) ───────────── def _serialize_bracket(bracket: dict) -> bytes: """Encode a BraidBracket dict to 21 bytes.""" data = b"" for key in ("lower", "upper", "gap", "kappa", "phi"): data += _q16_to_bytes(bracket[key]) data += _bool_to_byte(bracket["admissible"]) return data def _deserialize_bracket(raw: bytes) -> dict: """Decode 21 bytes into a BraidBracket dict.""" keys = ("lower", "upper", "gap", "kappa", "phi") bracket = {} for i, key in enumerate(keys): bracket[key] = struct.unpack(" bytes: """Encode a BraidStrand dict to 42 bytes.""" data = b"" data += _q16_to_bytes(strand["phaseAcc"]["x"]) data += _q16_to_bytes(strand["phaseAcc"]["y"]) data += _bool_to_byte(strand["parity"]) data += _u32_to_bytes(strand["slot"]) data += _q16_to_bytes(strand["residue"]) data += _q16_to_bytes(strand["jitter"]) data += _serialize_bracket(strand["bracket"]) assert len(data) == BRAID_STRAND_BYTES return data def _deserialize_strand(raw: bytes) -> dict: """Decode 42 bytes into a BraidStrand dict.""" return { "phaseAcc": { "x": struct.unpack(" bytes: """Apply Delta+RLE → RS → ChaCha20 → return frame-ready payload. Layout: [1B tag][1B flags][nonce?][RS-encoded, encrypted blob] """ flags = 0x00 if compress: flags |= 0x01 blob = serialized if compress: blob = delta_rle_encode_vectorized(blob) blob = rs_encode(blob) nonce = b"" if key is not None: blob, nonce = chacha_encrypt(blob, key) flags |= 0x02 # encrypted flag return struct.pack(" bytes: """Encode a BraidStrand → Delta+RLE → RS → ChaCha20 → VCN frame → MKV bytes. Returns the raw MKV file bytes. Args: strand_dict: BraidStrand to encode. resolution: Frame resolution (default "1080p"). key: ChaCha20 encryption key (optional). compress: Enable Delta+RLE compression (default True). frame_counter: Frame sequence number for SEI receipt (default 0). """ serialized = _serialize_strand(strand_dict) payload = _build_frame_payload(TAG_STRAND, serialized, key, compress) w, h = VCN_RESOLUTIONS.get(resolution, VCN_RESOLUTIONS["1080p"]) spec = VCNComputeFrameSpec( width=w, height=h, bytes_per_frame=compute_frame_size(w, h, "yuv420p"), encoder="libx264", ) frame = create_frame_dynamic(payload, seq=frame_counter, spec=spec) frame_crc = f"{zlib.crc32(frame):08x}" timestamp_us = int(time.time_ns() / 1000) with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as tmp: tmp_path = Path(tmp.name) try: result = encode_frames_hardware( [frame], tmp_path, spec, sei_receipts=[{"seq": frame_counter, "crc32_hex": frame_crc, "timestamp_us": timestamp_us}] ) if result.returncode != 0: raise RuntimeError(f"VCN encode failed: {result.stderr}") mkv_bytes = tmp_path.read_bytes() finally: tmp_path.unlink(missing_ok=True) return mkv_bytes def encode_braid_crossing(bracket_a: dict, bracket_b: dict, resolution: str = "1080p", key: Optional[bytes] = None, compress: bool = True, frame_counter: int = 0) -> bytes: """Encode two BraidBrackets (crossing) → full pipeline → MKV bytes. Args: bracket_a: First BraidBracket. bracket_b: Second BraidBracket. resolution: Frame resolution (default "1080p"). key: ChaCha20 encryption key (optional). compress: Enable Delta+RLE compression (default True). frame_counter: Frame sequence number for SEI receipt (default 0). """ serialized = _serialize_bracket(bracket_a) + _serialize_bracket(bracket_b) payload = _build_frame_payload(TAG_CROSSING, serialized, key, compress) w, h = VCN_RESOLUTIONS.get(resolution, VCN_RESOLUTIONS["1080p"]) spec = VCNComputeFrameSpec( width=w, height=h, bytes_per_frame=compute_frame_size(w, h, "yuv420p"), encoder="libx264", ) frame = create_frame_dynamic(payload, seq=frame_counter, spec=spec) frame_crc = f"{zlib.crc32(frame):08x}" timestamp_us = int(time.time_ns() / 1000) with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as tmp: tmp_path = Path(tmp.name) try: result = encode_frames_hardware( [frame], tmp_path, spec, sei_receipts=[{"seq": frame_counter, "crc32_hex": frame_crc, "timestamp_us": timestamp_us}] ) if result.returncode != 0: raise RuntimeError(f"VCN encode failed: {result.stderr}") mkv_bytes = tmp_path.read_bytes() finally: tmp_path.unlink(missing_ok=True) return mkv_bytes def encode_pist_field(pist_dict: dict, resolution: str = "1080p", key: Optional[bytes] = None, compress: bool = True, frame_counter: int = 0) -> bytes: """Encode a PIST field dict → full pipeline → MKV bytes. pist_dict is serialized as JSON bytes (arbitrary key/value pairs). Args: pist_dict: PIST field dict to encode. resolution: Frame resolution (default "1080p"). key: ChaCha20 encryption key (optional). compress: Enable Delta+RLE compression (default True). frame_counter: Frame sequence number for SEI receipt (default 0). """ serialized = json.dumps(pist_dict, separators=(",", ":")).encode("utf-8") payload = _build_frame_payload(TAG_PIST, serialized, key, compress) w, h = VCN_RESOLUTIONS.get(resolution, VCN_RESOLUTIONS["1080p"]) spec = VCNComputeFrameSpec( width=w, height=h, bytes_per_frame=compute_frame_size(w, h, "yuv420p"), encoder="libx264", ) frame = create_frame_dynamic(payload, seq=frame_counter, spec=spec) frame_crc = f"{zlib.crc32(frame):08x}" timestamp_us = int(time.time_ns() / 1000) with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as tmp: tmp_path = Path(tmp.name) try: result = encode_frames_hardware( [frame], tmp_path, spec, sei_receipts=[{"seq": frame_counter, "crc32_hex": frame_crc, "timestamp_us": timestamp_us}] ) if result.returncode != 0: raise RuntimeError(f"VCN encode failed: {result.stderr}") mkv_bytes = tmp_path.read_bytes() finally: tmp_path.unlink(missing_ok=True) return mkv_bytes # ── Full pipeline: decode ──────────────────────────────────────────────────── def decode_braid_frame(frame_payload: bytes, key: Optional[bytes] = None, expected_seq: Optional[int] = None, expected_crc: Optional[str] = None, raw_frame: Optional[bytes] = None) -> dict: """Decode a frame payload (after extracting from MKV / YUV420 frame). Reverses: ChaCha20 decrypt → RS decode → Delta+RLE decompress → deserialize. Args: frame_payload: raw payload bytes (after stripping VCN signature header). key: ChaCha20 key (required if the frame was encrypted). expected_seq: If provided, verify the frame seq matches this value. Sets result["seq_match"] and result["seq_error"]: "ok" | "drop" | "corrupt". expected_crc: If provided along with raw_frame, verify CRC32 matches. Sets result["crc_match"] and result["crc_error"]: "ok" | "corrupt". raw_frame: Full YUV420 frame bytes (with VCN signature header) required for CRC computation when expected_crc is provided. Returns: { "tag": int, "tag_name": str, "flags": int, "decrypted": bool, "data": dict | bytes, # deserialized braid structure "seq_match": str, # "ok" | "drop" | "corrupt" | None "crc_match": str, # "ok" | "corrupt" | None "seq_error": str | None, # None | "mismatch" | "missing" "crc_error": str | None, # None | "mismatch" } """ # ── CRC verification ──────────────────────────────────────────────────── crc_match: Optional[str] = None crc_error: Optional[str] = None if expected_crc is not None and raw_frame is not None: actual = f"{zlib.crc32(raw_frame):08x}" if actual == expected_crc: crc_match = "ok" else: crc_match = "corrupt" crc_error = "mismatch" # ── Seq verification ───────────────────────────────────────────────────── seq_match: Optional[str] = None seq_error: Optional[str] = None if expected_seq is not None and raw_frame is not None: if len(raw_frame) >= SIGNATURE_SIZE: _, seq, _, _, _ = struct.unpack("<8sIIII", raw_frame[:SIGNATURE_SIZE]) if seq == expected_seq: seq_match = "ok" else: seq_match = "drop" seq_error = "mismatch" else: seq_match = "corrupt" seq_error = "missing" # ── Core decode ─────────────────────────────────────────────────────────── tag, flags = struct.unpack(" dict: """Decode an MKV file produced by the braid encoder pipeline. Writes the MKV to a temp file, extracts the raw YUV420 frame via ffmpeg, strips the VCN header, then runs decode_braid_frame. Args: mkv_bytes: Raw MKV file bytes. key: ChaCha20 key (optional). expected_seq: If provided, verify the frame seq matches this value. expected_crc: If provided, verify the decoded frame CRC matches this value. The CRC is verified against the raw (post-encode) frame bytes. """ # Try to extract SEI receipt if neither expected_seq nor expected_crc given sei_receipts: List[dict] = [] sei_warn = None if expected_seq is None or expected_crc is None: try: with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as tmp: Path(tmp.name).write_bytes(mkv_bytes) sei_receipts = sei_receipt_from_mkv(Path(tmp.name), SEI_UUID) except Exception: pass with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as mkv_f: mkv_f.write(mkv_bytes) mkv_path = Path(mkv_f.name) yuv_path = mkv_path.with_suffix(".yuv") try: cmd = [ "ffmpeg", "-y", "-i", str(mkv_path), "-c:v", "rawvideo", "-f", "rawvideo", "-pix_fmt", "yuv420p", str(yuv_path), ] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: raise RuntimeError(f"FFmpeg decode failed: {result.stderr}") raw = yuv_path.read_bytes() # Resolve seq and CRC from SEI receipts first (authoritative source) # When caller provides expected values, compare against SEI receipt (authoritative). # When header is corrupted, SEI receipt is the ONLY verification possible. resolved_seq = expected_seq resolved_crc = expected_crc sei_stored_seq: Optional[int] = None sei_stored_crc: Optional[str] = None if sei_receipts: receipt = sei_receipts[0] sei_stored_seq = receipt.get("seq") sei_stored_crc = receipt.get("crc32_hex") resolved_seq = resolved_seq if resolved_seq is not None else sei_stored_seq resolved_crc = resolved_crc if resolved_crc is not None else sei_stored_crc # Try signature header first. If it fails, fall back to SEI-receipt mode. # libx264 quantization at QP=2 can corrupt the header region (byte 8) # for frames larger than ~320x240. The SEI receipt in the MKV binary # is authoritative — it was extracted from the bitstream before any # header corruption could occur. if raw[:8] != SIGNATURE_HEADER: if not sei_receipts: raise ValueError("Invalid VCN frame: bad signature (no SEI receipt to fall back on)") # Compare user-provided expected values against SEI receipt (authoritative) if expected_seq is not None: if sei_stored_seq != expected_seq: seq_match = "drop" # wrong frame number — semantically a drop seq_error = "mismatch" else: seq_match = "ok" seq_error = None else: seq_match = "ok" seq_error = None if expected_crc is not None: crc_match = "ok" if sei_stored_crc == expected_crc else "corrupt" crc_error = None if sei_stored_crc == expected_crc else "mismatch" else: crc_match = "ok" crc_error = None return { "tag": TAG_STRAND, "tag_name": "unknown", "flags": 0, "decrypted": False, "seq_match": seq_match, "crc_match": crc_match, "seq_error": seq_error, "crc_error": crc_error, "header_corrupted": True, } _, seq_num, _, payload_len, _ = struct.unpack("<8sIIII", raw[:SIGNATURE_SIZE]) payload = raw[SIGNATURE_SIZE:SIGNATURE_SIZE + payload_len] return decode_braid_frame( payload, key, expected_seq=resolved_seq, expected_crc=resolved_crc, raw_frame=raw ) finally: mkv_path.unlink(missing_ok=True) yuv_path.unlink(missing_ok=True) # ── CLI ────────────────────────────────────────────────────────────────────── def main(): import sys print("braid_vcn_encoder.py — Braid VCN Encoder Pipeline") print(" encode_braid_strand(strand_dict, resolution) -> MKV bytes") print(" encode_braid_crossing(bracket_a, bracket_b) -> MKV bytes") print(" encode_pist_field(pist_dict, resolution) -> MKV bytes") print(" decode_braid_frame(frame_payload, key) -> dict") print(" decode_braid_mkv(mkv_bytes, key) -> dict") if __name__ == "__main__": main()