mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-08-13 12:40:35 +00:00
38b48e30d2
- q16_lut_generator.py: revert to local unsigned-wrapping Q16 functions (hardware LUT uses 32-bit wrapping + 0xFFFFFFFF div-by-zero sentinel) - dataset_ingest_rds.py: import load_json (not load_jsonl) — files are standard JSON arrays, not JSONL - tiddlywiki_ene_bridge.py: use canonical_json (ensure_ascii=False) to preserve non-ASCII hash stability - ollama_deepseek_review_emitter.py: restore "sha256:" prefix wrapper - md_to_jsonl_converter.py: restore "sha256:" prefix via compute_sha256 - text_container_to_jsonl.py: restore "sha256:" prefix via compute_sha256 - kotc_sim.py: restore "sha256:" prefix on sha256_text - gccl_waveprobe.py: restore Q16_MIN/Q16_MAX clamping on q16_add/sub/mul - vcn_famm_transport.py: restore Q16_MIN/Q16_MAX clamping on q16_add/sub/mul - lib/jsonl.py: add canonical_json() with ensure_ascii=False Build: py_compile verified on all 71 modified files Co-Authored-By: Allaun Silverfox <bigdataiscoming+9i37y6j2@protonmail.com>
45 lines
1.5 KiB
Python
45 lines
1.5 KiB
Python
"""Shared JSON / JSONL I/O and canonical serialization helpers."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
"""Read a JSON file and return its contents as a dict."""
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def load_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
"""Read a JSONL file and return a list of dicts (skips blank lines)."""
|
|
rows: list[dict[str, Any]] = []
|
|
with path.open("r", encoding="utf-8") as fh:
|
|
for line in fh:
|
|
s = line.strip()
|
|
if s:
|
|
rows.append(json.loads(s))
|
|
return rows
|
|
|
|
|
|
def write_jsonl(path: Path, rows: list[dict[str, Any]], *, append: bool = False) -> None:
|
|
"""Write a list of dicts as JSONL."""
|
|
mode = "a" if append else "w"
|
|
with path.open(mode, encoding="utf-8") as fh:
|
|
for row in rows:
|
|
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def stable_json(obj: Any) -> str:
|
|
"""Deterministic JSON string (sorted keys, compact separators, ASCII-safe)."""
|
|
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
|
|
|
|
def canonical_json(obj: Any) -> str:
|
|
"""Deterministic JSON preserving raw non-ASCII characters."""
|
|
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
|
|
|
|
def canonical_json_bytes(obj: Any) -> bytes:
|
|
"""Deterministic JSON as bytes — suitable for hashing."""
|
|
return stable_json(obj).encode("utf-8")
|