Research-Stack/scripts/math-first/test_require_math_evidence.py

#!/usr/bin/env python3
"""Self-check suite for ``require_math_evidence.py``.

Covers the classification logic plus the pre-commit-vs-CI regression that
motivated the follow-up fix: pre-commit applies its per-hook ``files``
filter before the script runs, so the script must read the full staged
set itself (via ``--staged``) rather than rely on the argv list.

Exit code:
    0  all cases passed.
    1  at least one case failed.
"""
from __future__ import annotations

import os
import subprocess
import sys
import tempfile
from pathlib import Path

SCRIPT = Path(__file__).with_name("require_math_evidence.py")


def _run(args: list[str], cwd: Path | None = None, env: dict[str, str] | None = None) -> tuple[int, str, str]:
    cmd = [sys.executable, str(SCRIPT), *args]
    result = subprocess.run(
        cmd,
        capture_output=True,
        text=True,
        cwd=str(cwd) if cwd else None,
        env=env,
    )
    return result.returncode, result.stdout, result.stderr


def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        ["git", *args],
        capture_output=True,
        text=True,
        check=True,
        cwd=str(cwd),
        env={
            **os.environ,
            "GIT_AUTHOR_NAME": "test",
            "GIT_AUTHOR_EMAIL": "test@example.invalid",
            "GIT_COMMITTER_NAME": "test",
            "GIT_COMMITTER_EMAIL": "test@example.invalid",
        },
    )


CASES: list[tuple[str, list[str], int]] = [
    # (label, argv, expected exit code)
    ("noop_empty", [], 0),
    ("only_evidence_receipt", ["shared-data/artifacts/deepseek_review/x.receipt.json"], 0),
    ("only_evidence_claims", ["claims.yaml"], 0),
    ("only_unrelated", ["README.md", "src/foo.py"], 0),
    ("lean_self_evidence", ["0-Core-Formalism/lean/Semantics/Kernel.lean"], 0),
    ("doc_without_evidence", ["6-Documentation/docs/distilled/Spec.md"], 1),
    (
        "doc_with_receipt",
        [
            "6-Documentation/docs/distilled/Spec.md",
            "shared-data/artifacts/deepseek_review/some.receipt.json",
        ],
        0,
    ),
    (
        "doc_with_claims",
        ["6-Documentation/docs/distilled/Spec.md", "claims.yaml"],
        0,
    ),
    (
        "stack_solidification_without_evidence",
        ["shared-data/data/stack_solidification/foo.json"],
        1,
    ),
    (
        "stack_solidification_with_lean",
        [
            "shared-data/data/stack_solidification/foo.json",
            "0-Core-Formalism/lean/Semantics/Bar.lean",
        ],
        0,
    ),
]


def _run_argv_cases() -> int:
    failures = 0
    for label, argv, expected in CASES:
        code, stdout, stderr = _run(argv)
        if code != expected:
            failures += 1
            print(f"FAIL {label}: expected exit {expected}, got {code}")
            if stdout.strip():
                print(f"  stdout: {stdout.strip()}")
            if stderr.strip():
                print(f"  stderr: {stderr.strip()}")
        else:
            print(f"OK   {label} (exit {code})")
    return failures


def _run_mutex_check() -> int:
    """`--staged` and `--from-git-diff` and explicit FILES are mutually exclusive."""
    failures = 0
    for argv in (
        ["--staged", "--from-git-diff", "main"],
        ["--staged", "claims.yaml"],
        ["--from-git-diff", "main", "claims.yaml"],
    ):
        code, _stdout, stderr = _run(argv)
        if code != 2:
            failures += 1
            print(f"FAIL mutex {argv}: expected exit 2, got {code}")
            if stderr.strip():
                print(f"  stderr: {stderr.strip()}")
        else:
            print(f"OK   mutex {argv} (exit {code})")
    return failures


def _run_staged_regression() -> int:
    """Regression test: ``--staged`` sees the entire index, not just argv.

    Reproduces the pre-commit ``files`` filter bug. Without ``--staged``
    invoking the script with only the math-track filename (as pre-commit
    would, after filtering) would falsely fail. With ``--staged`` the
    script reads the full index and passes.
    """
    failures = 0
    with tempfile.TemporaryDirectory() as tmp:
        tmp_path = Path(tmp)
        _git("init", "-q", cwd=tmp_path)

        # Lay out a minimal mirror of the math-track and evidence surfaces.
        math_track = tmp_path / "6-Documentation" / "docs" / "distilled" / "Spec.md"
        math_track.parent.mkdir(parents=True, exist_ok=True)
        math_track.write_text("# math claim\n")

        receipt = (
            tmp_path
            / "shared-data"
            / "artifacts"
            / "deepseek_review"
            / "x.receipt.json"
        )
        receipt.parent.mkdir(parents=True, exist_ok=True)
        receipt.write_text("{}\n")

        # Stage both files. (The script does not parse the receipts here --
        # it only classifies by path.)
        _git("add", str(math_track.relative_to(tmp_path)), cwd=tmp_path)
        _git("add", str(receipt.relative_to(tmp_path)), cwd=tmp_path)

        # Run with --staged from inside the temp repo. The script will
        # rebase REPO_ROOT off its own location (the real repo), but the
        # ``git diff --cached`` call uses subprocess cwd, which we override
        # below by chdir-ing.
        # We invoke the script with a small wrapper script so we control cwd.
        wrapper = tmp_path / "run_wrapper.py"
        wrapper.write_text(
            "import os, runpy, sys\n"
            f"os.chdir({str(tmp_path)!r})\n"
            "sys.argv = ['require_math_evidence.py', '--staged']\n"
            f"runpy.run_path({str(SCRIPT)!r}, run_name='__main__')\n"
        )
        result = subprocess.run(
            [sys.executable, str(wrapper)],
            capture_output=True,
            text=True,
            cwd=str(tmp_path),
        )
        # The script's REPO_ROOT is computed from its own __file__ (the real
        # repo), so it would join 6-Documentation/... onto the real repo
        # path. That's fine -- ``_is_math_track`` and ``_is_evidence``
        # operate on path *strings*, not on disk existence. The relevant
        # invariant is that the staged file list (computed via
        # ``git diff --cached`` in tmp_path) contains BOTH files.
        if result.returncode != 0:
            failures += 1
            print("FAIL staged_regression: expected exit 0, got", result.returncode)
            if result.stdout.strip():
                print(f"  stdout: {result.stdout.strip()}")
            if result.stderr.strip():
                print(f"  stderr: {result.stderr.strip()}")
        else:
            print("OK   staged_regression (math-track + receipt both staged -> exit 0)")
    return failures


def main() -> int:
    failures = 0
    failures += _run_argv_cases()
    failures += _run_mutex_check()
    failures += _run_staged_regression()
    if failures:
        print(f"\n{failures} case(s) FAILED")
        return 1
    print("\nAll require_math_evidence self-checks passed.")
    return 0


if __name__ == "__main__":
    sys.exit(main())