SilverSight/.github/scripts/glossary_lint.py

#!/usr/bin/env python3
"""Glossary lint: warn when domain terms are used but not defined.

This script scans SilverSight source, docs, and CI files for terms that look
like project-specific concepts.  If a term appears multiple times but is not
listed in docs/GLOSSARY.md (or docs/GLOSSARY_ALLOWLIST.md), it prints a
warning.  It is intentionally a warning, not a build failure, to avoid
blocking work-in-progress.

Run from repo root:
    python3 .github/scripts/glossary_lint.py
"""

import re
import sys
from collections import Counter
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent.parent
GLOSSARY = ROOT / "docs" / "GLOSSARY.md"
ALLOWLIST = ROOT / "docs" / "GLOSSARY_ALLOWLIST.md"

# Files and globs to scan.
# v1 scans Markdown documentation only; this is where terminology is introduced
# and where LLMs (and humans) most need glossary support.  Code-only identifiers
# that never surface in docs are not flagged.
SCAN_GLOBS = [
    "*.md",
    "docs/**/*.md",
]

# Auto-generated reports from Research Stack are not SilverSight terminology.
SKIP_PATHS = {
    "docs/research_stack_usage_graph.md",
    "docs/research_stack_porting_candidates.md",
}

# Minimum number of occurrences before a term is reported.
MIN_OCCURRENCES = 2

# Always-allowed words: project names, common acronyms, file formats, etc.
ALWAYS_ALLOWED = {
    # Project / repo names
    "SilverSight",
    "Research Stack",
    "Research-Stack",
    "GitHub",
    # Licenses / formats
    "MIT",
    "Apache-2.0",
    "Apache",
    "YAML",
    "JSON",
    "JSONL",
    "SVG",
    "PNG",
    "CFF",
    # Common acronyms
    "CI",
    "LLM",
    "AI",
    "DOI",
    "arXiv",
    "URL",
    "API",
    "GPU",
    "CPU",
    "RAM",
    "OS",
    "UI",
    "CLI",
    "HTTP",
    "HTTPS",
    # Meta / workflow
    "TODO",
    "FIXME",
    "README",
    "AGENTS.md",
    "PORTING_MAP.md",
    "CITATION.cff",
    "GLOSSARY.md",
    "REBASE_RULES.md",
    # File extensions / toolchains
    ".lean",
    ".py",
    ".md",
    ".yml",
    ".yaml",
    ".json",
    ".svg",
    ".png",
    # Lean / math common
    "Mathlib",
    "Prop",
    "Type",
    "Sort",
    " theorem",
    " lemma",
    " definition",
    " structure",
    " inductive",
    # Git / GitHub
    "push",
    "pull request",
    "pull_request",
    "workflow",
    "action",
}


def extract_glossary_terms(path: Path) -> set[str]:
    """Return the set of SilverSight terms defined in the glossary."""
    text = path.read_text(encoding="utf-8")
    terms: set[str] = set()
    for line in text.splitlines():
        line = line.strip()
        # Table rows of the form | **Term** | ... |
        m = re.match(r"^\|\s*\*\*([^*|]+?)\*\*\s*\|", line)
        if m:
            terms.add(m.group(1).strip())
        # Crosswalk first column: | Term | standard | note |
        m = re.match(r"^\|\s*\*\*([^*|]+?)\*\*\s*\|", line)
        if m:
            terms.add(m.group(1).strip())
    return terms


def extract_allowlist(path: Path) -> set[str]:
    """Return the set of intentionally unglossary'd terms."""
    terms: set[str] = set()
    if not path.exists():
        return terms
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        if line.startswith("- "):
            line = line[2:]
        terms.add(line.strip())
    return terms


TERM_RE = re.compile(
    r"""
    (?:\*\*([^*|\n]{2,40})\*\*)      # **Term** (do not span table cells)
    |
    (?:`([^`|\n]{2,40})`)            # `Term`
    """,
    re.VERBOSE,
)

# Terms that are clearly code snippets / paths rather than domain terms.
SKIP_RE = re.compile(
    r"""
    ^https?://|
    [/.\\]|          # file paths or dotted identifiers
    ^\.|            # file extensions or dotted paths
    ^/|\$|
    [=<>(){}\[\]]|
    \b\d+\b
    """,
    re.VERBOSE,
)

# Common words / UI labels / sentence fragments that are not domain terms.
COMMON_WORDS = {
    "Input",
    "Output",
    "Status",
    "Maps to",
    "Delta",
    "Note",
    "Notes",
    "Principle",
    "Principles",
    "Rule",
    "Rules",
    "Boundary",
    "Boundaries",
    "Verification",
    "Grounding",
    "Atoms",
    "Append",
    "Only",
    "Not",
    "Stop",
    "Why the original failed",
    "The Complete Variety Isomorphism",
    "Python shim OK",
    "PORT ONLY IF",
    "No `Float` in Lean compute paths",
    "Theorem 1a",
    "Theorem 1b",
    "Theorem 1d",
}


def scan_file(path: Path) -> Counter[str]:
    """Return a Counter of candidate terms found in one file."""
    counts: Counter[str] = Counter()
    try:
        text = path.read_text(encoding="utf-8")
    except UnicodeDecodeError:
        return counts
    for bold, code in TERM_RE.findall(text):
        term = (bold or code).strip()
        if len(term) < 2 or len(term) > 40:
            continue
        if SKIP_RE.search(term):
            continue
        # Strip leading/trailing punctuation
        term = term.strip(".,;:!?')\"")
        if not term:
            continue
        if term in COMMON_WORDS:
            continue
        # Skip all-lowercase phrases (not domain terms) unless they are
        # known acronyms already in ALWAYS_ALLOWED.
        if term.islower():
            continue
        counts[term] += 1
    return counts


def normalize(term: str) -> str:
    """Normalize for case-insensitive comparison."""
    return re.sub(r"\s+", " ", term).lower()


def main() -> int:
    glossary = extract_glossary_terms(GLOSSARY)
    allowlist = extract_allowlist(ALLOWLIST)
    allowed = ALWAYS_ALLOWED | allowlist
    allowed_norm = {normalize(t) for t in allowed}

    total_counts: Counter[str] = Counter()
    for glob in SCAN_GLOBS:
        for path in ROOT.glob(glob):
            if not path.is_file():
                continue
            rel = path.relative_to(ROOT).as_posix()
            if rel in SKIP_PATHS:
                continue
            total_counts.update(scan_file(path))

    warnings: list[tuple[str, int]] = []
    for term, count in total_counts.items():
        if count < MIN_OCCURRENCES:
            continue
        if term in allowed:
            continue
        if term in glossary:
            continue
        term_norm = normalize(term)
        if term_norm in allowed_norm:
            continue
        # Also accept if any glossary term normalizes the same way.
        if term_norm in {normalize(t) for t in glossary}:
            continue
        warnings.append((term, count))

    warnings.sort(key=lambda x: (-x[1], x[0]))

    if warnings:
        print(f"⚠️  Glossary lint: {len(warnings)} terms used {MIN_OCCURRENCES}+ times but not defined")
        for term, count in warnings:
            print(f"   '{term}' — {count} occurrence(s)")
        print()
        print("Add genuine terms to docs/GLOSSARY.md or intentional omissions to docs/GLOSSARY_ALLOWLIST.md.")
        return 0  # warning only
    else:
        print("✅ Glossary lint: no undefined repeated terms.")
        return 0


if __name__ == "__main__":
    sys.exit(main())