#!/usr/bin/env python3 """Glossary lint: warn when domain terms are used but not defined. This script scans SilverSight source, docs, and CI files for terms that look like project-specific concepts. If a term appears multiple times but is not listed in docs/GLOSSARY.md (or docs/GLOSSARY_ALLOWLIST.md), it prints a warning. It is intentionally a warning, not a build failure, to avoid blocking work-in-progress. Run from repo root: python3 .github/scripts/glossary_lint.py """ import re import sys from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent GLOSSARY = ROOT / "docs" / "GLOSSARY.md" ALLOWLIST = ROOT / "docs" / "GLOSSARY_ALLOWLIST.md" # Files and globs to scan. # v1 scans Markdown documentation only; this is where terminology is introduced # and where LLMs (and humans) most need glossary support. Code-only identifiers # that never surface in docs are not flagged. SCAN_GLOBS = [ "*.md", "docs/**/*.md", ] # Auto-generated reports from Research Stack are not SilverSight terminology. SKIP_PATHS = { "docs/research_stack_usage_graph.md", "docs/research_stack_porting_candidates.md", } # Minimum number of occurrences before a term is reported. MIN_OCCURRENCES = 2 # Always-allowed words: project names, common acronyms, file formats, etc. ALWAYS_ALLOWED = { # Project / repo names "SilverSight", "Research Stack", "Research-Stack", "GitHub", # Licenses / formats "MIT", "Apache-2.0", "Apache", "YAML", "JSON", "JSONL", "SVG", "PNG", "CFF", # Common acronyms "CI", "LLM", "AI", "DOI", "arXiv", "URL", "API", "GPU", "CPU", "RAM", "OS", "UI", "CLI", "HTTP", "HTTPS", # Meta / workflow "TODO", "FIXME", "README", "AGENTS.md", "PORTING_MAP.md", "CITATION.cff", "GLOSSARY.md", "REBASE_RULES.md", # File extensions / toolchains ".lean", ".py", ".md", ".yml", ".yaml", ".json", ".svg", ".png", # Lean / math common "Mathlib", "Prop", "Type", "Sort", " theorem", " lemma", " definition", " structure", " inductive", # Git / GitHub "push", "pull request", "pull_request", "workflow", "action", } def extract_glossary_terms(path: Path) -> set[str]: """Return the set of SilverSight terms defined in the glossary.""" text = path.read_text(encoding="utf-8") terms: set[str] = set() for line in text.splitlines(): line = line.strip() # Table rows of the form | **Term** | ... | m = re.match(r"^\|\s*\*\*([^*|]+?)\*\*\s*\|", line) if m: terms.add(m.group(1).strip()) # Crosswalk first column: | Term | standard | note | m = re.match(r"^\|\s*\*\*([^*|]+?)\*\*\s*\|", line) if m: terms.add(m.group(1).strip()) return terms def extract_allowlist(path: Path) -> set[str]: """Return the set of intentionally unglossary'd terms.""" terms: set[str] = set() if not path.exists(): return terms for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line or line.startswith("#"): continue if line.startswith("- "): line = line[2:] terms.add(line.strip()) return terms TERM_RE = re.compile( r""" (?:\*\*([^*|\n]{2,40})\*\*) # **Term** (do not span table cells) | (?:`([^`|\n]{2,40})`) # `Term` """, re.VERBOSE, ) # Terms that are clearly code snippets / paths rather than domain terms. SKIP_RE = re.compile( r""" ^https?://| [/.\\]| # file paths or dotted identifiers ^\.| # file extensions or dotted paths ^/|\$| [=<>(){}\[\]]| \b\d+\b """, re.VERBOSE, ) # Common words / UI labels / sentence fragments that are not domain terms. COMMON_WORDS = { "Input", "Output", "Status", "Maps to", "Delta", "Note", "Notes", "Principle", "Principles", "Rule", "Rules", "Boundary", "Boundaries", "Verification", "Grounding", "Atoms", "Append", "Only", "Not", "Stop", "Why the original failed", "The Complete Variety Isomorphism", "Python shim OK", "PORT ONLY IF", "No `Float` in Lean compute paths", "Theorem 1a", "Theorem 1b", "Theorem 1d", } def scan_file(path: Path) -> Counter[str]: """Return a Counter of candidate terms found in one file.""" counts: Counter[str] = Counter() try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: return counts for bold, code in TERM_RE.findall(text): term = (bold or code).strip() if len(term) < 2 or len(term) > 40: continue if SKIP_RE.search(term): continue # Strip leading/trailing punctuation term = term.strip(".,;:!?')\"") if not term: continue if term in COMMON_WORDS: continue # Skip all-lowercase phrases (not domain terms) unless they are # known acronyms already in ALWAYS_ALLOWED. if term.islower(): continue counts[term] += 1 return counts def normalize(term: str) -> str: """Normalize for case-insensitive comparison.""" return re.sub(r"\s+", " ", term).lower() def main() -> int: glossary = extract_glossary_terms(GLOSSARY) allowlist = extract_allowlist(ALLOWLIST) allowed = ALWAYS_ALLOWED | allowlist allowed_norm = {normalize(t) for t in allowed} total_counts: Counter[str] = Counter() for glob in SCAN_GLOBS: for path in ROOT.glob(glob): if not path.is_file(): continue rel = path.relative_to(ROOT).as_posix() if rel in SKIP_PATHS: continue total_counts.update(scan_file(path)) warnings: list[tuple[str, int]] = [] for term, count in total_counts.items(): if count < MIN_OCCURRENCES: continue if term in allowed: continue if term in glossary: continue term_norm = normalize(term) if term_norm in allowed_norm: continue # Also accept if any glossary term normalizes the same way. if term_norm in {normalize(t) for t in glossary}: continue warnings.append((term, count)) warnings.sort(key=lambda x: (-x[1], x[0])) if warnings: print(f"⚠️ Glossary lint: {len(warnings)} terms used {MIN_OCCURRENCES}+ times but not defined") for term, count in warnings: print(f" '{term}' — {count} occurrence(s)") print() print("Add genuine terms to docs/GLOSSARY.md or intentional omissions to docs/GLOSSARY_ALLOWLIST.md.") return 0 # warning only else: print("✅ Glossary lint: no undefined repeated terms.") return 0 if __name__ == "__main__": sys.exit(main())