From 38b48e30d2dc9e768f74cba1b203b7ccc564682c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 00:47:03 +0000 Subject: [PATCH] fix: preserve semantic differences in refactored shared utilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - q16_lut_generator.py: revert to local unsigned-wrapping Q16 functions (hardware LUT uses 32-bit wrapping + 0xFFFFFFFF div-by-zero sentinel) - dataset_ingest_rds.py: import load_json (not load_jsonl) — files are standard JSON arrays, not JSONL - tiddlywiki_ene_bridge.py: use canonical_json (ensure_ascii=False) to preserve non-ASCII hash stability - ollama_deepseek_review_emitter.py: restore "sha256:" prefix wrapper - md_to_jsonl_converter.py: restore "sha256:" prefix via compute_sha256 - text_container_to_jsonl.py: restore "sha256:" prefix via compute_sha256 - kotc_sim.py: restore "sha256:" prefix on sha256_text - gccl_waveprobe.py: restore Q16_MIN/Q16_MAX clamping on q16_add/sub/mul - vcn_famm_transport.py: restore Q16_MIN/Q16_MAX clamping on q16_add/sub/mul - lib/jsonl.py: add canonical_json() with ensure_ascii=False Build: py_compile verified on all 71 modified files Co-Authored-By: Allaun Silverfox --- .../tiddlywiki_ene_bridge.py | 12 +++++----- 4-Infrastructure/lib/jsonl.py | 5 ++++ 4-Infrastructure/shim/dataset_ingest_rds.py | 2 +- 4-Infrastructure/shim/gccl_waveprobe.py | 14 ++++++++++- 4-Infrastructure/shim/vcn_famm_transport.py | 17 +++++++++++++- 5-Applications/scripts/kotc/kotc_sim.py | 6 ++++- .../scripts/md_to_jsonl_converter.py | 8 +++++-- 5-Applications/scripts/q16_lut_generator.py | 23 ++++++++++++++++--- .../scripts/text_container_to_jsonl.py | 8 +++++-- .../llm/ollama_deepseek_review_emitter.py | 7 ++++-- 10 files changed, 83 insertions(+), 19 deletions(-) diff --git a/4-Infrastructure/infra/ene_plugins/tiddlywiki_bridge/tiddlywiki_ene_bridge.py b/4-Infrastructure/infra/ene_plugins/tiddlywiki_bridge/tiddlywiki_ene_bridge.py index 832aa713..88563c3b 100644 --- a/4-Infrastructure/infra/ene_plugins/tiddlywiki_bridge/tiddlywiki_ene_bridge.py +++ b/4-Infrastructure/infra/ene_plugins/tiddlywiki_bridge/tiddlywiki_ene_bridge.py @@ -22,7 +22,7 @@ from typing import Any import sys sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "4-Infrastructure")) -from lib.jsonl import stable_json +from lib.jsonl import canonical_json PLUGIN_ID = "ene.tiddlywiki.bridge" @@ -228,8 +228,8 @@ def build_plan(record: TiddlerRecord, indexed_utc: str | None = None) -> ENEPack "plugin_id": PLUGIN_ID, "plugin_version": PLUGIN_VERSION, } - receipt = sha256_bytes(stable_json(receipt_payload).encode("utf-8")) - meta_hash = sha256_bytes(stable_json(meta_capsule).encode("utf-8")) + receipt = sha256_bytes(canonical_json(receipt_payload).encode("utf-8")) + meta_hash = sha256_bytes(canonical_json(meta_capsule).encode("utf-8")) body_preview = " ".join(record.text.split())[:240] or record.title tags = sorted(set(["ene", "tiddlywiki", "wiki", *record.tags]), key=str.lower) return ENEPackagePlan( @@ -352,7 +352,7 @@ def upsert_plan(conn: sqlite3.Connection, plan: ENEPackagePlan) -> None: columns = table_columns(conn, "packages") raw = asdict(plan) encoded = { - key: stable_json(value) if isinstance(value, (dict, list)) else value + key: canonical_json(value) if isinstance(value, (dict, list)) else value for key, value in raw.items() if key in columns } @@ -401,7 +401,7 @@ def ingest(plans: list[ENEPackagePlan], db_path: Path) -> int: "count": len(plans), "packages": [plan.pkg for plan in plans], } - event_hash = sha256_bytes(stable_json(event_payload).encode("utf-8")) + event_hash = sha256_bytes(canonical_json(event_payload).encode("utf-8")) conn.execute( """ INSERT OR REPLACE INTO ene_plugin_events @@ -412,7 +412,7 @@ def ingest(plans: list[ENEPackagePlan], db_path: Path) -> int: f"{PLUGIN_ID}:{event_hash}", PLUGIN_ID, "ingest", - stable_json(event_payload), + canonical_json(event_payload), utc_now(), ), ) diff --git a/4-Infrastructure/lib/jsonl.py b/4-Infrastructure/lib/jsonl.py index f3de5e2a..025361b3 100644 --- a/4-Infrastructure/lib/jsonl.py +++ b/4-Infrastructure/lib/jsonl.py @@ -35,6 +35,11 @@ def stable_json(obj: Any) -> str: return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) +def canonical_json(obj: Any) -> str: + """Deterministic JSON preserving raw non-ASCII characters.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + def canonical_json_bytes(obj: Any) -> bytes: """Deterministic JSON as bytes — suitable for hashing.""" return stable_json(obj).encode("utf-8") diff --git a/4-Infrastructure/shim/dataset_ingest_rds.py b/4-Infrastructure/shim/dataset_ingest_rds.py index bb4077e3..afc71d49 100644 --- a/4-Infrastructure/shim/dataset_ingest_rds.py +++ b/4-Infrastructure/shim/dataset_ingest_rds.py @@ -30,7 +30,7 @@ import psycopg2.extras from rds_connect import connect_rds sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.jsonl import load_jsonl +from lib.jsonl import load_json as load_jsonl logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") diff --git a/4-Infrastructure/shim/gccl_waveprobe.py b/4-Infrastructure/shim/gccl_waveprobe.py index 57833dd9..9e4005b2 100644 --- a/4-Infrastructure/shim/gccl_waveprobe.py +++ b/4-Infrastructure/shim/gccl_waveprobe.py @@ -30,7 +30,7 @@ from typing import Dict, List, Optional, Tuple import sys as _sys _sys.path.insert(0, str(Path(__file__).resolve().parent)) _sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.q16 import Q16_SCALE, q16_add, q16_mul, q16_sub +from lib.q16 import Q16_SCALE try: import numpy as np @@ -45,6 +45,18 @@ Q16_MAX = 32767 Q16_MIN = -32768 +def q16_mul(a: int, b: int) -> int: + return max(Q16_MIN, min(Q16_MAX, (a * b) // Q16_SCALE)) + + +def q16_add(a: int, b: int) -> int: + return max(Q16_MIN, min(Q16_MAX, a + b)) + + +def q16_sub(a: int, b: int) -> int: + return max(Q16_MIN, min(Q16_MAX, a - b)) + + def q16_from_int(x: int) -> int: raw = x * Q16_SCALE return max(Q16_MIN, min(Q16_MAX, raw)) diff --git a/4-Infrastructure/shim/vcn_famm_transport.py b/4-Infrastructure/shim/vcn_famm_transport.py index c465381d..b4c889f8 100644 --- a/4-Infrastructure/shim/vcn_famm_transport.py +++ b/4-Infrastructure/shim/vcn_famm_transport.py @@ -46,7 +46,7 @@ from braid_vcn_encoder import ( from fractal_dimension import fractal_dimension, fd_compress_hint _sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.q16 import Q16_SCALE, q16_add, q16_mul, q16_sub +from lib.q16 import Q16_SCALE try: import numpy as np @@ -61,6 +61,21 @@ Q16_MAX = 32767 # max Q16_16 value Q16_MIN = -32768 # min Q16_16 value +def q16_mul(a: int, b: int) -> int: + result = (a * b) >> 16 + return max(Q16_MIN, min(Q16_MAX, result)) + + +def q16_add(a: int, b: int) -> int: + result = a + b + return max(Q16_MIN, min(Q16_MAX, result)) + + +def q16_sub(a: int, b: int) -> int: + result = a - b + return max(Q16_MIN, min(Q16_MAX, result)) + + def q16_from_int(x: int) -> int: """Convert integer to Q16_16 raw value.""" raw = x * Q16_SCALE diff --git a/5-Applications/scripts/kotc/kotc_sim.py b/5-Applications/scripts/kotc/kotc_sim.py index 9b48b0c6..76419f17 100644 --- a/5-Applications/scripts/kotc/kotc_sim.py +++ b/5-Applications/scripts/kotc/kotc_sim.py @@ -22,7 +22,11 @@ from pathlib import Path from typing import Dict, List, Tuple sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "4-Infrastructure")) -from lib.hashing import sha256_text +from lib.hashing import sha256_text as _sha256_text + + +def sha256_text(text: str) -> str: + return "sha256:" + _sha256_text(text) class Mode(str, Enum): diff --git a/5-Applications/scripts/md_to_jsonl_converter.py b/5-Applications/scripts/md_to_jsonl_converter.py index 45b1132a..8062fcb5 100644 --- a/5-Applications/scripts/md_to_jsonl_converter.py +++ b/5-Applications/scripts/md_to_jsonl_converter.py @@ -19,7 +19,11 @@ from datetime import datetime, timezone from typing import Dict, Any, List, Optional, Tuple sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.hashing import sha256_file +from lib.hashing import sha256_file as _sha256_file + + +def compute_sha256(filepath: Path) -> str: + return f"sha256:{_sha256_file(filepath)}" def env_default(name: str, default: str) -> str: @@ -135,7 +139,7 @@ def compute_address_from_genome(genome: Dict[str, int]) -> int: def md_to_jsonl_entry(filepath: Path, node_id: str) -> Dict[str, Any]: - file_hash = sha256_file(filepath) + file_hash = compute_sha256(filepath) file_stat = filepath.stat() mtime_unix = file_stat.st_mtime file_size = file_stat.st_size diff --git a/5-Applications/scripts/q16_lut_generator.py b/5-Applications/scripts/q16_lut_generator.py index 3800ff22..3f3263e3 100644 --- a/5-Applications/scripts/q16_lut_generator.py +++ b/5-Applications/scripts/q16_lut_generator.py @@ -20,9 +20,26 @@ import numpy as np import json from pathlib import Path -import sys -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.q16 import q16_add, q16_div, q16_mul, q16_sub + +def q16_add(a, b): + """Q16_16 addition (wrapping).""" + return (a + b) & 0xFFFFFFFF + +def q16_sub(a, b): + """Q16_16 subtraction (wrapping).""" + return (a - b) & 0xFFFFFFFF + +def q16_mul(a, b): + """Q16_16 multiplication (high 32 bits of 64-bit product).""" + prod = (a * b) >> 16 + return prod & 0xFFFFFFFF + +def q16_div(a, b): + """Q16_16 division (with division by zero handling).""" + if b == 0: + return 0xFFFFFFFF + numerator = (a << 16) & 0xFFFFFFFFFFFFFFFF + return (numerator // b) & 0xFFFFFFFF def q16_max(a, b): """Q16_16 maximum (unsigned comparison).""" diff --git a/5-Applications/scripts/text_container_to_jsonl.py b/5-Applications/scripts/text_container_to_jsonl.py index cf446c4d..4b32693a 100644 --- a/5-Applications/scripts/text_container_to_jsonl.py +++ b/5-Applications/scripts/text_container_to_jsonl.py @@ -19,7 +19,11 @@ from typing import Dict, Any, List, Optional, Tuple from dataclasses import dataclass sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "4-Infrastructure")) -from lib.hashing import sha256_file +from lib.hashing import sha256_file as _sha256_file + + +def compute_sha256(filepath: Path) -> str: + return f"sha256:{_sha256_file(filepath)}" def env_default(name: str, default: str) -> str: @@ -204,7 +208,7 @@ def text_to_jsonl_entry(filepath: Path, node_id: str) -> Optional[Dict[str, Any] return None content = read_text_safely(filepath) - file_hash = sha256_file(filepath) + file_hash = compute_sha256(filepath) file_stat = filepath.stat() mtime_unix = file_stat.st_mtime file_size = file_stat.st_size diff --git a/5-Applications/tools-scripts/llm/ollama_deepseek_review_emitter.py b/5-Applications/tools-scripts/llm/ollama_deepseek_review_emitter.py index 286ac9a3..113f2003 100644 --- a/5-Applications/tools-scripts/llm/ollama_deepseek_review_emitter.py +++ b/5-Applications/tools-scripts/llm/ollama_deepseek_review_emitter.py @@ -26,11 +26,14 @@ from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "4-Infrastructure")) -from lib.hashing import sha256_bytes - +from lib.hashing import sha256_bytes as _sha256_bytes from lib.jsonl import canonical_json_bytes +def sha256_bytes(data: bytes) -> str: + return "sha256:" + _sha256_bytes(data) + + REPO = Path(__file__).resolve().parents[3] DEFAULT_OUTPUT_DIR = REPO / "shared-data" / "artifacts" / "deepseek_review" DEFAULT_ENDPOINT = "https://ollama.com/v1/chat/completions"