Research-Stack/scripts/math-first/test_require_math_evidence.py

#!/usr/bin/env python3
"""Self-check suite for ``require_math_evidence.py``.

Covers the classification logic plus the pre-commit-vs-CI regression that
motivated the follow-up fix: pre-commit applies its per-hook ``files``
filter before the script runs, so the script must read the full staged
set itself (via ``--staged``) rather than rely on the argv list.

Exit code:
    0  all cases passed.
    1  at least one case failed.
"""
from __future__ import annotations

import os
import subprocess
import sys
import tempfile
from pathlib import Path

SCRIPT = Path(__file__).with_name("require_math_evidence.py")


def _run(args: list[str], cwd: Path | None = None, env: dict[str, str] | None = None) -> tuple[int, str, str]:
    cmd = [sys.executable, str(SCRIPT), *args]
    result = subprocess.run(
        cmd,
        capture_output=True,
        text=True,
        cwd=str(cwd) if cwd else None,
        env=env,
    )
    return result.returncode, result.stdout, result.stderr


def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        ["git", *args],
        capture_output=True,
        text=True,
        check=True,
        cwd=str(cwd),
        env={
            **os.environ,
            "GIT_AUTHOR_NAME": "test",
            "GIT_AUTHOR_EMAIL": "test@example.invalid",
            "GIT_COMMITTER_NAME": "test",
            "GIT_COMMITTER_EMAIL": "test@example.invalid",
        },
    )


CASES: list[tuple[str, list[str], int]] = [
    # (label, argv, expected exit code)
    ("noop_empty", [], 0),
    ("only_evidence_receipt", ["shared-data/artifacts/deepseek_review/x.receipt.json"], 0),
    ("only_evidence_claims", ["claims.yaml"], 0),
    ("only_unrelated", ["README.md", "src/foo.py"], 0),
    ("lean_self_evidence", ["0-Core-Formalism/lean/Semantics/Kernel.lean"], 0),
    ("doc_without_evidence", ["6-Documentation/docs/distilled/Spec.md"], 1),
    (
        "doc_with_receipt",
        [
            "6-Documentation/docs/distilled/Spec.md",
            "shared-data/artifacts/deepseek_review/some.receipt.json",
        ],
        0,
    ),
    (
        "doc_with_claims",
        ["6-Documentation/docs/distilled/Spec.md", "claims.yaml"],
        0,
    ),
    (
        "stack_solidification_without_evidence",
        ["shared-data/data/stack_solidification/foo.json"],
        1,
    ),
    (
        "stack_solidification_with_lean",
        [
            "shared-data/data/stack_solidification/foo.json",
            "0-Core-Formalism/lean/Semantics/Bar.lean",
        ],
        0,
    ),
]


def _run_argv_cases() -> int:
    failures = 0
    for label, argv, expected in CASES:
        code, stdout, stderr = _run(argv)
        if code != expected:
            failures += 1
            print(f"FAIL {label}: expected exit {expected}, got {code}")
            if stdout.strip():
                print(f"  stdout: {stdout.strip()}")
            if stderr.strip():
                print(f"  stderr: {stderr.strip()}")
        else:
            print(f"OK   {label} (exit {code})")
    return failures


def _run_mutex_check() -> int:
    """`--staged` and `--from-git-diff` and explicit FILES are mutually exclusive."""
    failures = 0
    for argv in (
        ["--staged", "--from-git-diff", "main"],
        ["--staged", "claims.yaml"],
        ["--from-git-diff", "main", "claims.yaml"],
    ):
        code, _stdout, stderr = _run(argv)
        if code != 2:
            failures += 1
            print(f"FAIL mutex {argv}: expected exit 2, got {code}")
            if stderr.strip():
                print(f"  stderr: {stderr.strip()}")
        else:
            print(f"OK   mutex {argv} (exit {code})")
    return failures


def _run_staged_regression() -> int:
    """Regression for the pre-commit ``files``-filter bug.

    Builds a throwaway git repo so this test is self-contained -- the
    test does not depend on the state of the real repo's index. The
    script is run with ``cwd=tmp_path`` (NOT via ``runpy``) so that
    ``git rev-parse --show-toplevel`` inside the script resolves to the
    temp repo, and ``git diff --cached`` queries the temp repo's index.

    Three sub-cases:
        (a) math-track-only staged                -> expect exit 1
            (proves the script actually evaluates classification logic;
            without (a) the next sub-case could pass vacuously by
            short-circuiting on an empty diff)
        (b) math-track + receipt staged           -> expect exit 0
            (the original pre-commit ``files``-filter bug)
        (c) math-track + claims.yaml staged       -> expect exit 0
            (verifies the registry-update path)
    """
    failures = 0
    with tempfile.TemporaryDirectory() as tmp:
        tmp_path = Path(tmp).resolve()
        _git("init", "-q", cwd=tmp_path)

        math_track = tmp_path / "6-Documentation" / "docs" / "distilled" / "Spec.md"
        math_track.parent.mkdir(parents=True, exist_ok=True)
        math_track.write_text("# math claim\n")

        receipt = (
            tmp_path / "shared-data" / "artifacts" / "deepseek_review" / "x.receipt.json"
        )
        receipt.parent.mkdir(parents=True, exist_ok=True)
        receipt.write_text("{}\n")

        claims = tmp_path / "claims.yaml"
        claims.write_text("claims: []\n")

        def _stage_only(*relpaths: str) -> None:
            # Reset the index to a clean state, then stage exactly the
            # supplied paths. ``git reset`` is safe here -- the temp repo
            # has no commits, so there is no "HEAD" to reset against. We
            # instead remove everything currently in the index.
            _git("rm", "--cached", "-rf", "--ignore-unmatch", ".", cwd=tmp_path)
            for relpath in relpaths:
                _git("add", "--", relpath, cwd=tmp_path)

        def _invoke() -> subprocess.CompletedProcess[str]:
            return subprocess.run(
                [sys.executable, str(SCRIPT), "--staged"],
                capture_output=True,
                text=True,
                cwd=str(tmp_path),
            )

        def _assert(label: str, expected_exit: int, *staged: str) -> int:
            _stage_only(*staged)
            indexed = _git("diff", "--cached", "--name-only", cwd=tmp_path).stdout.splitlines()
            indexed = [line for line in indexed if line.strip()]
            if sorted(indexed) != sorted(staged):
                print(f"FAIL {label}: index does not match expected staging")
                print(f"  expected: {sorted(staged)}")
                print(f"  actual:   {sorted(indexed)}")
                return 1
            result = _invoke()
            if result.returncode != expected_exit:
                print(
                    f"FAIL {label}: expected exit {expected_exit}, "
                    f"got {result.returncode}"
                )
                if result.stdout.strip():
                    print(f"  stdout: {result.stdout.strip()}")
                if result.stderr.strip():
                    print(f"  stderr: {result.stderr.strip()}")
                return 1
            print(f"OK   {label} (exit {expected_exit})")
            return 0

        failures += _assert(
            "staged_regression_negative (math-only -> FAIL)",
            1,
            "6-Documentation/docs/distilled/Spec.md",
        )
        failures += _assert(
            "staged_regression_positive_receipt (math + receipt -> OK)",
            0,
            "6-Documentation/docs/distilled/Spec.md",
            "shared-data/artifacts/deepseek_review/x.receipt.json",
        )
        failures += _assert(
            "staged_regression_positive_claims (math + claims.yaml -> OK)",
            0,
            "6-Documentation/docs/distilled/Spec.md",
            "claims.yaml",
        )
    return failures


def main() -> int:
    failures = 0
    failures += _run_argv_cases()
    failures += _run_mutex_check()
    failures += _run_staged_regression()
    if failures:
        print(f"\n{failures} case(s) FAILED")
        return 1
    print("\nAll require_math_evidence self-checks passed.")
    return 0


if __name__ == "__main__":
    sys.exit(main())