mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-08-13 18:10:36 +00:00
1b4f717d13
Adds the 5 scripts required by .github/workflows/math-check.yml and .pre-commit-config.yaml: - validate_deepseek_receipts.py: validates receipt JSON against schemas - validate_claims_registry.py: validates claims.yaml against schema - require_math_evidence.py: enforces evidence requirement for math-track PRs - test_require_math_evidence.py: self-tests for the evidence gate - test_validate_deepseek_receipts.py: self-tests for the receipt validator All 5 scripts pass locally (2 receipts OK, 1 claim OK, 3/3 + 4/4 tests). Co-Authored-By: Allaun Silverfox <bigdataiscoming+9i37y6j2@protonmail.com>
126 lines
4 KiB
Python
Executable file
126 lines
4 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Self-tests for validate_deepseek_receipts.py.
|
|
|
|
Runs minimal smoke tests to verify the validator works against the live
|
|
schema and any existing receipts. Exits 0 on success, 1 on failure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# Ensure we can import the validator's logic
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
REPO_ROOT = SCRIPT_DIR.parents[1]
|
|
SCHEMA_PATH = REPO_ROOT / "shared-data" / "schemas" / "deepseek-review-receipt.schema.json"
|
|
|
|
try:
|
|
from jsonschema import Draft202012Validator, ValidationError
|
|
except ImportError:
|
|
print("SKIP: jsonschema not installed")
|
|
sys.exit(0)
|
|
|
|
|
|
def test_schema_compiles() -> bool:
|
|
"""The schema itself must be valid JSON Schema."""
|
|
if not SCHEMA_PATH.exists():
|
|
print("SKIP: schema file not found")
|
|
return True
|
|
schema = json.loads(SCHEMA_PATH.read_text())
|
|
Draft202012Validator.check_schema(schema)
|
|
print("PASS: schema compiles")
|
|
return True
|
|
|
|
|
|
def test_valid_receipt_accepted() -> bool:
|
|
"""A minimal valid primary receipt must pass validation."""
|
|
if not SCHEMA_PATH.exists():
|
|
print("SKIP: schema file not found")
|
|
return True
|
|
schema = json.loads(SCHEMA_PATH.read_text())
|
|
validator = Draft202012Validator(schema)
|
|
|
|
valid_receipt = {
|
|
"schema": "ollama_deepseek_review_receipt_v1",
|
|
"created_at": "2026-01-01T00:00:00+00:00",
|
|
"model": "deepseek-v3.2",
|
|
"endpoint": "https://ollama.com/v1/chat/completions",
|
|
"prompt_sha256": "sha256:" + "a" * 64,
|
|
"answer_sha256": "sha256:" + "b" * 64,
|
|
"usage": {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300},
|
|
"context_files": ["some/file.lean"],
|
|
"answer_path": "shared-data/artifacts/deepseek_review/test.md",
|
|
}
|
|
|
|
errors = list(validator.iter_errors(valid_receipt))
|
|
if errors:
|
|
print("FAIL: valid receipt rejected:")
|
|
for err in errors:
|
|
print(f" {err.message}")
|
|
return False
|
|
print("PASS: valid receipt accepted")
|
|
return True
|
|
|
|
|
|
def test_invalid_receipt_rejected() -> bool:
|
|
"""A receipt missing required fields must be rejected."""
|
|
if not SCHEMA_PATH.exists():
|
|
print("SKIP: schema file not found")
|
|
return True
|
|
schema = json.loads(SCHEMA_PATH.read_text())
|
|
validator = Draft202012Validator(schema)
|
|
|
|
invalid_receipt = {"schema": "ollama_deepseek_review_receipt_v1"}
|
|
errors = list(validator.iter_errors(invalid_receipt))
|
|
if not errors:
|
|
print("FAIL: invalid receipt was accepted")
|
|
return False
|
|
print("PASS: invalid receipt rejected")
|
|
return True
|
|
|
|
|
|
def test_bad_sha256_rejected() -> bool:
|
|
"""A receipt with malformed SHA-256 must be rejected."""
|
|
if not SCHEMA_PATH.exists():
|
|
print("SKIP: schema file not found")
|
|
return True
|
|
schema = json.loads(SCHEMA_PATH.read_text())
|
|
validator = Draft202012Validator(schema)
|
|
|
|
receipt = {
|
|
"schema": "ollama_deepseek_review_receipt_v1",
|
|
"created_at": "2026-01-01T00:00:00+00:00",
|
|
"model": "deepseek-v3.2",
|
|
"endpoint": "https://ollama.com/v1/chat/completions",
|
|
"prompt_sha256": "not-a-hash",
|
|
"answer_sha256": "sha256:" + "b" * 64,
|
|
"usage": {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300},
|
|
"context_files": ["some/file.lean"],
|
|
"answer_path": "shared-data/artifacts/deepseek_review/test.md",
|
|
}
|
|
errors = list(validator.iter_errors(receipt))
|
|
if not errors:
|
|
print("FAIL: bad SHA-256 was accepted")
|
|
return False
|
|
print("PASS: bad SHA-256 rejected")
|
|
return True
|
|
|
|
|
|
def main() -> int:
|
|
tests = [
|
|
test_schema_compiles,
|
|
test_valid_receipt_accepted,
|
|
test_invalid_receipt_rejected,
|
|
test_bad_sha256_rejected,
|
|
]
|
|
passed = sum(1 for t in tests if t())
|
|
total = len(tests)
|
|
print(f"\n{passed}/{total} tests passed")
|
|
return 0 if passed == total else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|