Research-Stack/4-Infrastructure/shim/wiki_tool_maturation_apply.py

#!/usr/bin/env python3
"""Apply a standard maturation block to tool-like TiddlyWiki entries.

This is an idempotent bulk pass. It recomputes the same local heuristic used by
wiki_tool_tuning_review_probe.py, then inserts or replaces a generated
Maturation Status section in every tool-like non-system tiddler.
"""

from __future__ import annotations

import hashlib
import importlib.util
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


REPO = Path(__file__).resolve().parents[2]
TIDDLERS = REPO / "6-Documentation" / "tiddlywiki-local" / "wiki" / "tiddlers"
REVIEW_SCRIPT = REPO / "4-Infrastructure" / "shim" / "wiki_tool_tuning_review_probe.py"
REVIEW_RECEIPT = REPO / "shared-data" / "data" / "wiki_tool_tuning_review" / "wiki_tool_tuning_review_receipt.json"
OUT_DIR = REPO / "shared-data" / "data" / "wiki_tool_maturation_pass"
PAYLOAD_JSON = OUT_DIR / "wiki_tool_maturation_pass.json"
SUMMARY = OUT_DIR / "wiki_tool_maturation_pass.md"
RECEIPT = OUT_DIR / "wiki_tool_maturation_pass_receipt.json"
PASS_TIDDLER = TIDDLERS / "Wiki Tool Maturation Pass.tid"
GENERATED_TIDDLERS = {
    "Wiki Tool Tuning Review.tid",
    "Wiki Tool Maturation Pass.tid",
}

SECTION_HEADER = "!! Maturation Status"
BEGIN_MARKER = "<!-- WIKI_TOOL_MATURATION_BEGIN -->"
END_MARKER = "<!-- WIKI_TOOL_MATURATION_END -->"


def load_review_module() -> Any:
    spec = importlib.util.spec_from_file_location("wiki_tool_tuning_review_probe", REVIEW_SCRIPT)
    if spec is None or spec.loader is None:
        raise RuntimeError(f"cannot load review script: {REVIEW_SCRIPT}")
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


REVIEW = load_review_module()


def stable_json(obj: Any) -> str:
    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)


def sha256_bytes(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def hash_obj(obj: Any) -> str:
    return sha256_bytes(stable_json(obj).encode("utf-8"))


def rel(path: Path) -> str:
    try:
        return str(path.relative_to(REPO))
    except ValueError:
        return str(path)


def review_receipt_hash() -> str | None:
    if not REVIEW_RECEIPT.exists():
        return None
    return json.loads(REVIEW_RECEIPT.read_text(encoding="utf-8")).get("receipt_hash")


def remove_existing_block(text: str) -> str:
    pattern = re.compile(
        rf"\n?{re.escape(SECTION_HEADER)}\n\n{re.escape(BEGIN_MARKER)}.*?{re.escape(END_MARKER)}\n?",
        flags=re.DOTALL,
    )
    return pattern.sub("\n", text).rstrip() + "\n"


def maturity_block(entry: dict[str, Any], receipt_hash: str | None) -> str:
    categories = ", ".join(f"`{category}`" for category in entry["categories"]) or "`uncategorized`"
    next_actions = entry.get("recommended_next") or ["add local fixture and receipt before promotion"]
    lines = [
        SECTION_HEADER,
        "",
        BEGIN_MARKER,
        "",
        "This section is generated by the wiki tool maturation pass. It is a",
        "review aid, not a validation claim.",
        "",
        f"* Maturity: `{entry['maturity']}`",
        f"* Tuning priority: `{entry['tuning_priority']}`",
        f"* Tool categories: {categories}",
        f"* Runner references: `{len(entry['runner_paths'])}`",
        f"* Receipt references: `{len(entry['receipt_paths'])}`",
        f"* Baseline signal: `{entry['baseline_signal']}`",
        f"* Exact replay signal: `{entry['exact_replay_signal']}`",
        f"* Review receipt: `{receipt_hash or 'missing'}`",
        "",
        "Next maturation actions:",
        "",
    ]
    for action in next_actions:
        lines.append(f"* {action}")
    lines.extend(
        [
            "",
            "Promotion remains HOLD until executable fixtures, exact replay or declared",
            "loss policy, baseline/negative-control evidence, and receipt hashes close.",
            "",
            END_MARKER,
        ]
    )
    return "\n".join(lines)


def apply_block(path: Path, entry: dict[str, Any], receipt_hash: str | None) -> dict[str, Any]:
    before = path.read_text(encoding="utf-8", errors="replace")
    cleaned = remove_existing_block(before)
    after = cleaned.rstrip() + "\n\n" + maturity_block(entry, receipt_hash) + "\n"
    changed = before != after
    if changed:
        path.write_text(after, encoding="utf-8")
    return {
        "path": rel(path),
        "title": entry["title"],
        "changed": changed,
        "before_sha256": sha256_bytes(before.encode("utf-8")),
        "after_sha256": sha256_bytes(after.encode("utf-8")),
        "maturity": entry["maturity"],
        "tuning_priority": entry["tuning_priority"],
        "categories": entry["categories"],
        "recommended_next": entry.get("recommended_next", []),
    }


def compute_entries() -> list[dict[str, Any]]:
    raw_entries = [
        REVIEW.parse_tiddler(path)
        for path in sorted(TIDDLERS.glob("*.tid"))
        if not path.name.startswith("$__") and path.name not in GENERATED_TIDDLERS
    ]
    tool_entries = [entry for entry in raw_entries if entry["tool_score"] >= 6]
    for entry in tool_entries:
        entry["tuning_priority"] = REVIEW.tuning_priority(entry)
        entry["recommended_next"] = REVIEW.recommended_next(entry)
    return sorted(tool_entries, key=lambda entry: entry["tuning_priority"], reverse=True)


def build_payload() -> dict[str, Any]:
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    receipt_hash = review_receipt_hash()
    if receipt_hash is None:
        raise RuntimeError(f"missing source review receipt: {REVIEW_RECEIPT}")
    entries = compute_entries()
    changes = [apply_block(Path(REPO / entry["path"]), entry, receipt_hash) for entry in entries]
    changed = [item for item in changes if item["changed"]]
    matured_block_count = 0
    for item in changes:
        text = (REPO / item["path"]).read_text(encoding="utf-8", errors="replace")
        if BEGIN_MARKER in text and END_MARKER in text:
            matured_block_count += 1
    maturity_rollup = {
        maturity: sum(1 for item in changes if item["maturity"] == maturity)
        for maturity in sorted({item["maturity"] for item in changes})
    }
    payload = {
        "schema": "wiki_tool_maturation_pass_v1",
        "claim_boundary": (
            "Bulk wiki maturation only. Generated sections annotate local tuning "
            "status and next actions; they do not validate any theory, result, or tool."
        ),
        "source_review_receipt": rel(REVIEW_RECEIPT),
        "source_review_receipt_hash": receipt_hash,
        "marker": {
            "section_header": SECTION_HEADER,
            "begin": BEGIN_MARKER,
            "end": END_MARKER,
        },
        "aggregates": {
            "tool_entries_seen": len(entries),
            "tiddlers_changed": len(changed),
            "tiddlers_unchanged": len(changes) - len(changed),
            "matured_block_count": matured_block_count,
            "all_tool_entries_have_maturation_block": matured_block_count == len(entries),
            "maturity_rollup": maturity_rollup,
        },
        "changed": changed,
        "all_entries": changes,
        "decision": "ADMIT_WIKI_TOOL_MATURATION_PASS_AS_HOLD_ANNOTATION",
    }
    payload["payload_hash"] = hash_obj({k: v for k, v in payload.items() if k != "payload_hash"})
    return payload


def build_receipt(payload: dict[str, Any]) -> dict[str, Any]:
    receipt = {
        "schema": "wiki_tool_maturation_pass_receipt_v1",
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "timestamp_role": "metadata_only",
        "generated_at_utc_included_in_receipt_hash": False,
        "payload_hash": payload["payload_hash"],
        "source_review_receipt_hash": payload["source_review_receipt_hash"],
        "aggregates": payload["aggregates"],
        "decision": payload["decision"],
        "claim_boundary": payload["claim_boundary"],
    }
    receipt["receipt_hash"] = sha256_bytes(
        stable_json({k: v for k, v in receipt.items() if k not in {"receipt_hash", "generated_at_utc"}}).encode("utf-8")
    )
    return receipt


def write_summary(payload: dict[str, Any], receipt: dict[str, Any]) -> None:
    agg = payload["aggregates"]
    lines = [
        "# Wiki Tool Maturation Pass",
        "",
        f"Decision: `{payload['decision']}`  ",
        f"Receipt hash: `{receipt['receipt_hash']}`",
        "",
        payload["claim_boundary"],
        "",
        "## Aggregate",
        "",
        f"- Tool entries seen: `{agg['tool_entries_seen']}`",
        f"- Tiddlers changed: `{agg['tiddlers_changed']}`",
        f"- Tiddlers unchanged: `{agg['tiddlers_unchanged']}`",
        f"- Maturation blocks present: `{agg['matured_block_count']}`",
        f"- All tool entries have maturation block: `{agg['all_tool_entries_have_maturation_block']}`",
        f"- Maturity rollup: `{agg['maturity_rollup']}`",
        "",
        "## Top Changed Entries",
        "",
        "| Tiddler | Maturity | Priority | Categories |",
        "|---|---|---:|---|",
    ]
    for item in payload["changed"][:30]:
        lines.append(f"| [[{item['title']}]] | {item['maturity']} | {item['tuning_priority']} | {', '.join(item['categories'][:4])} |")
    lines.extend(["", "## Receipt", "", f"`{rel(RECEIPT)}`"])
    SUMMARY.write_text("\n".join(lines) + "\n", encoding="utf-8")


def write_pass_tiddler(payload: dict[str, Any], receipt: dict[str, Any]) -> None:
    agg = payload["aggregates"]
    lines = [
        "title: Wiki Tool Maturation Pass",
        "tags: ResearchStack TiddlyWiki ToolTuning Maturation HOLD Receipt",
        "type: text/vnd.tiddlywiki",
        "",
        "! Wiki Tool Maturation Pass",
        "",
        f"Decision: `{payload['decision']}`",
        "",
        f"Receipt hash: `{receipt['receipt_hash']}`",
        "",
        "!! Aggregate",
        "",
        f"* Tool entries seen: `{agg['tool_entries_seen']}`",
        f"* Tiddlers changed: `{agg['tiddlers_changed']}`",
        f"* Tiddlers unchanged: `{agg['tiddlers_unchanged']}`",
        f"* Maturation blocks present: `{agg['matured_block_count']}`",
        f"* All tool entries have maturation block: `{agg['all_tool_entries_have_maturation_block']}`",
        f"* Maturity rollup: `{agg['maturity_rollup']}`",
        "",
        "!! What Changed",
        "",
        "Each tool-like tiddler now has a generated `Maturation Status` section",
        "with maturity class, tuning priority, categories, next actions, and the",
        "source review receipt.",
        "",
        "!! Boundary",
        "",
        payload["claim_boundary"],
        "",
        "!! Links",
        "",
        "* [[Wiki Tool Tuning Review]]",
        "* [[Combined Approach Equation Surface]]",
        "",
        f"Receipt: `{rel(RECEIPT)}`",
    ]
    PASS_TIDDLER.write_text("\n".join(lines) + "\n", encoding="utf-8")


def main() -> None:
    payload = build_payload()
    receipt = build_receipt(payload)
    PAYLOAD_JSON.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    RECEIPT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_summary(payload, receipt)
    write_pass_tiddler(payload, receipt)
    print(json.dumps(receipt, indent=2, sort_keys=True))


if __name__ == "__main__":
    main()