#!/usr/bin/env python3 """ rrc_arxiv_kernel_refine.py — Refine RRC classification using arxiv DB. Queries the arxiv_papers table (titles + abstracts) for each RRC equation using extracted keywords. Adds matches to the classifier receipt. Stages: kernel_refine_v1 — generic keyword-based matching (abstract + title) kernel_refine_v2 — number theory kernel (structural pattern matching against Diophantine equation taxonomy) kernel_refine_v3 — combinatorics kernel (structural pattern matching against combinatorics subfield taxonomy) kernel_refine_v4 — dataset kernel (domain taxonomy from Big-Math-RL-Verified, web patterns from AutoMathText, theorem patterns from TheoremQA) kernel_refine_v5 — obscure math kernel (28 niche subfields: tropical geometry, fusion categories, operads, cluster algebras, quandles, magmas, etc.) kernel_refine_v6 — Sidon generation kernel (359 entries across 9 topics: Sidon sets, sumsets, Freiman, additive energy, difference sets, cap sets, etc.) kernel_refine_v7 — Wannier Hamiltonian kernel (21 materials, 30 files: tight-binding band structures from BN, graphene, Si, GaAs, Fe, Cu, CrI3, etc.) kernel_refine_v7 — geometry/topology kernel (pure-local, 9 subfields: Riemannian geometry, curvature invariants, geometric flows, characteristic classes, symplectic/contact, Kähler, general relativity, differential & low-dim topology). No DB dependency. Usage: python3 4-Infrastructure/shim/rrc_arxiv_kernel_refine.py """ from __future__ import annotations import json import re import subprocess import sys from collections import defaultdict from pathlib import Path NEON_HOST = "neon-64gb" CONTAINER = "arxiv-pg" DB = "arxiv" RECEIPT_PATH = Path("archive/experimental-shim-probes/rrc_equation_classifier_receipt.json") # Math-symbol normalizer (LaTeX/Unicode → canonical) backing the geometry kernel. # Same-dir import; degrade to identity if the module/DB is unavailable. try: import sys as _sys _sys.path.insert(0, str(Path(__file__).resolve().parent)) from math_symbols import normalize_math except Exception: def normalize_math(text: str) -> str: return text or "" STOPWORDS = { "the", "and", "for", "where", "with", "this", "from", "that", "are", "but", "not", "have", "has", "been", "was", "were", "will", "would", "could", "should", "their", "them", "they", "its", "also", "can", "may", "however", "thus", "proof", "theorem", "lemma", "corollary", "proposition", "function", "functions", "using", "used", "use", "given", "show", "shows", "paper", "result", "results", "method", "methods", "well", "first", "new", "one", "two", "three", "equation", "equations", } NT_KEYWORDS = { "goormaghtigh": 5, "repunit": 5, "baker theorem": 5, "linear form": 5, "exponential diophantine": 5, "logarithm": 4, "lower bound": 4, "upper bound": 3, "diophantine": 4, "number theory": 2, "algebraic number": 3, "transcendence": 3, "integer solution": 4, "laurent mignotte": 5, "nesterenko": 5, "prime": 2, "congruence": 2, "modular": 2, } # Structured number theory kernel: equation-type patterns → relevant arxiv papers # Each entry: pattern regex → (type_label, [paper_id, ...]) DIOPHANTINE_KERNEL = [ # Goormaghtigh / repunit: (x^m - 1)/(x - 1) = (y^n - 1)/(y - 1) (r"repunit|goormaghtigh|(x\^[a-z]+\s*-\s*1)\s*(/|\\over)\s*(x\s*-\s*1)|(y\^[a-z]+\s*-\s*1)\s*(/|\\over)\s*(y\s*-\s*1)|(a\^\s*[a-z]+\s*-\s*1).*(a\s*-\s*1)", "goormaghtigh", ["2410.03677", "2505.08160", "2510.11252"]), # Exponential Diophantine: a^x + b^y = c^z or similar (r"[a-z]\^\{?[a-z]\}?\s*\+\s*[a-z]\^\{?[a-z]\}?\s*=", "exponential_diophantine", ["1808.06557", "1702.03424", "1808.06272", "2508.17601", "2503.00843"]), # Baker bounds: linear forms in logarithms (r"baker.*(theorem|bound|lower)|linear form.*logarithm|logarithm.*linear form", "baker_bounds", ["1309.5987", "2205.08899", "2303.02037", "1906.00419", "2107.00971"]), # S-unit equations: a_1 x_1 + ... + a_k x_k = 1 with S-units (r"\bs-unit\b.*\bequation\b|unit equation.*diophantine|S-unit.*module|S-unit.*linear", "s_unit", ["2505.19141", "2604.26497", "1911.11963"]), # Pell-type: x^2 - D y^2 = N (r"x\^\{?2\}?\s*[−\-]\s*[a-z]\s*y\^\{?2\}?\s*=", "pell", ["2509.17882", "2411.11103", "2403.18924"]), # Thue: F(x,y) = m (homogeneous degree >= 3) (r"thue|F\(x,\s*y\)\s*=\s*\d|homogeneous.*degree.*integer|binary.*form.*integer", "thue", ["2210.09631", "2406.01111", "2208.03830"]), # Ramanujan-Nagell: x^2 + D = 2^n (r"ramanujan.*nagell|nagell|2\^.*=.*x\^.*\+|x\^.*\+.*=.*2\^", "ramanujan_nagell", ["2602.06073"]), # Catalan / Mihailescu: a^x - b^y = 1 (r"catalan|mihailescu|a\^.*−\s*b\^.*=\s*1|a\^.*-\s*b\^.*=\s*1|x\^.*−\s*y\^.*=\s*1", "catalan", []), # abc conjecture (r"abc\s*conjecture|masser.*oesterle|a\s*\+\s*b\s*=\s*c|quality.*abc", "abc_conjecture", []), # Generalized Fermat: x^p + y^q = z^r (r"fermat.*last|generalized.*fermat|x\^.*\+\s*y\^.*=\s*z\^|x\^4\s*\+\s*y\^4\s*=\s*z\^", "fermat_generalized", ["math/0403046", "2311.12044"]), # Modular approach to Diophantine equations (r"modular.*diophantine|classical.*modular.*exponential|frey.*curve|galois.*representation.*diophantine", "modular_diophantine", ["math/0403046", "math/0405220", "2311.12044"]), # Skolem / p-adic method (r"skolem|p-adic.*diophantine|p-adic.*method|chabauty|coleman.*chabauty", "p_adic_diophantine", ["math/0005186", "2208.03830", "2508.17601"]), ] # ───────────────────────────────────────────────────────────────────────────── # §2b COMBINATORICS KERNEL — structural patterns bridging to stack concepts # ───────────────────────────────────────────────────────────────────────────── # Each entry: (pattern_regex, subfield_label, [paper_ids]) COMBINATORICS_KERNEL = [ # Graph theory / extremal (r"regularity lemma|szemerédi regular|szemeredi regular|regularisation lemma", "extremal_combinatorics", ["math/0504472", "2606.06192", "math/0310476"]), (r"flag algebra|razborov|turán density|turán number|turán problem", "extremal_combinatorics", ["2601.12741", "1607.04741", "2601.06590"]), (r"hypergraph container|container method|container theorem", "extremal_combinatorics", ["1204.6595", "1801.04584", "2408.08514", "2408.06617"]), # Combinatorial Nullstellensatz / polynomial method (r"combinatorial nullstellensatz|alom nullstellensatz|chevalley-warning", "combinatorial_nullstellensatz", ["2404.10778", "2605.18323", "2508.07257", "2408.03443"]), (r"polynomial method.*(combinatoric|additive|cap|bound)|croot-lev-pach|ellenberg.gijswijt", "polynomial_method", ["2311.08873", "1912.07679", "1811.07865"]), # Graph minors (r"graph minor|robertson.seymour|minor.theorem|graph structure theorem|excluded minor", "graph_minor_theory", ["2504.02532", "2507.02769", "2510.19285", "2409.18902", "2212.07670"]), # Matroid theory (r"matroid|oriented matroid|tutte polynomial|matroid represent", "matroid_theory", ["math/9804004", "math/9702219", "math/0612073", "math/0609840", "2606.16832"]), # Design theory (r"(block|combinatorial) design|finite projective plane|difference set|latin square|mutually orthogonal", "design_theory", ["math/0611492", "2509.06247", "math/0609244", "math/0609586", "2606.13536"]), # Ramsey theory (r"ramsey (number|theory|theorem)|van der waerden|hales.jewett|gallai-ramsey|anti-ramsey", "ramsey_theory", ["2601.05442", "2309.08370", "2212.07180", "2110.07144"]), # Probabilistic method (r"probabilistic method|lovász local lemma|lovasz local lemma", "probabilistic_method", ["2310.00513", "1909.11078", "1402.6817", "1511.04739"]), # Additive combinatorics (r"additive combinatoric|sumset|sum.set|freiman|balog.szemerédi|cauchy.davenport|erdős.szemerédi|erdős.szemerédi", "additive_combinatorics", ["math/0608105", "2211.01893", "math/0402285", "math/0507539", "math/0703668"]), # Algebraic combinatorics / spectral graph (r"algebraic (combinatoric|graph theory)|spectral graph|association scheme|strongly regular", "algebraic_combinatorics", ["2504.10624", "2504.03566", "2605.19542", "2308.14137"]), # Sidon / difference sets bridge (r"sidon set|sidon sequence|perfect difference set", "design_theory", ["math/0609244", "math/0504226"]), # Enumerative combinatorics (r"catalan number|dyck path|non.crossing partition|enumerative combinatoric", "enumerative_combinatorics", ["math/9904107", "2605.19979", "1910.00299"]), # Combinatorial geometry (r"combinatorial geometr|discrete geometr|zonotope|erdős.szekeres|erdős.szekeres|incidence geometry", "combinatorial_geometry", ["math/0609053", "2606.16832", "2605.23866", "2605.04183"]), # Random graphs (r"random graph|erdős.rényi|erdos.renyi|percolation.*graph|graph limit|graphon", "random_graphs", ["math/0703269", "math/0701316", "math/0612827"]), # Combinatorial topology (r"combinatorial topolog|topological combinatoric|discrete topolog", "combinatorial_topology", ["2112.14700", "2305.06288", "2402.06024"]), ] # ───────────────────────────────────────────────────────────────────────────── # §2c DATASET KERNELS — loaded from shared-data/data/ (domain, webmath, theorem) # ───────────────────────────────────────────────────────────────────────────── DATASET_KERNELS: dict | None = None def load_reconstruction_kernel() -> dict: base = Path("shared-data/data") p = base / "reconstruction_kernel_v1.json" if p.exists(): return json.loads(p.read_text()) return {"papers": [], "lean_proofs": []} def detect_reconstruction_type(name: str, eq_text: str) -> list[dict]: """Match against the graph reconstruction conjecture kernel.""" combined = (name + " " + eq_text).lower() kernel = load_reconstruction_kernel() papers = kernel.get("papers", []) # Keywords for reconstruction conjecture recon_kws = ["reconstruction", "reconstructible", "graph reconstruction", "deck of graph", "hypomorphic", "vertex deleted", "kelly lemma", "ulam conjecture"] score = sum(3 for kw in recon_kws if kw in combined) if score >= 3: matches = [] for p in papers[:3]: matches.append({"paper_id": p["paper_id"], "title": p["title"], "score": score + 3, "match_type": "reconstruction_conjecture", "abstract_snippet": p.get("abstract_snippet", "")}) return matches # Also check for graph theory keywords + combinatorics route graph_kws = ["graph", "subgraph", "tree", "vertex", "edge", "deck", "spanning"] graph_score = sum(1 for kw in graph_kws if kw in combined) if graph_score >= 3 and any(r in combined for r in ["conjecture", "reconstruct"]): matches = [] for p in papers[:2]: matches.append({"paper_id": p["paper_id"], "title": p["title"], "score": graph_score, "match_type": "reconstruction_conjecture", "abstract_snippet": p.get("abstract_snippet", "")}) return matches return [] def load_wannier_kernel() -> dict: base = Path("shared-data/data") p = base / "wannier_hamiltonian_kernel_v1.json" if p.exists(): return json.loads(p.read_text()) return {"materials": []} def detect_wannier_type(name: str, eq_text: str) -> list[dict]: """Match against Wannier tight-binding Hamiltonian materials.""" combined = (name + " " + eq_text).lower() kernel = load_wannier_kernel() materials_list = kernel.get("materials", []) matches = [] for mat in materials_list: mname = mat["name"].lower() if mname in combined: matches.append({ "paper_id": f"wannier:{mat['name']}", "title": f"Wannier tight-binding Hamiltonian: {mat['name']}", "abstract_snippet": f"bands={mat['bands']}, kpoints={mat['kpoints']}, degree={mat['degree']}, graph={mat['graph_type']}", "score": 5, "match_type": "condensed_matter_hamiltonian", }) return matches def load_sidon_kernel() -> dict: base = Path("shared-data/data") p = base / "sidon_generation_kernel_v1.json" if p.exists(): return json.loads(p.read_text()) return {"papers": [], "rrc_equations": [], "sidon_types": {}} # Partial-correlation hub weights for the operator grammar (2418-paper bootstrap). # The operator roles form an AFFINE Ã₂ extended-Dynkin diagram: a 3-cycle # relation–binary_op–arrow, with greek_letter & nary_operator as A₁ pendants on # the `relation` hub, and `operator` conditionally independent (its correlations # are fully mediated through relation). A cyclic diagram ⇒ affine, not finite ADE # (rooted in the extended-Dynkin / affine Kac–Moody classification). Weights = # partial-correlation degree centrality: relation (deg-4 hub) highest, operator # (deg-0, isolated) lowest. Sidon notation (greek_letter+binary_op+relation) sits # at this hub, so structurally-central equations are up-weighted among candidates. SIDON_ROLE_WEIGHTS = { "relation": 4.0, # hub of the grammar (degree-4 center) "binary_op": 2.0, # in the Ã₂ cycle "arrow": 2.0, # in the Ã₂ cycle "greek_letter": 1.5, # A₁ pendant on the hub "nary_operator": 1.5, # A₁ pendant on the hub "delimiter": 1.0, "delimiter_open": 1.0, "delimiter_close": 1.0, "operator": 0.5, # conditionally isolated (lowest) } def sidon_structure_score(text: str) -> float: """Hub-weighted operator-role density (the partial-correlation structure). Uses the full role vector (role_histogram on normalized text) weighted by SIDON_ROLE_WEIGHTS, normalized by total symbol count → a scale-free measure of how strongly an equation sits at the grammar hub. """ hist = role_histogram(normalize_math(text)) total = sum(hist.values()) if total == 0: return 0.0 wsum = sum(SIDON_ROLE_WEIGHTS.get(r, 0.0) * c for r, c in hist.items()) return wsum / total def detect_sidon_type(name: str, eq_text: str) -> list[dict]: """Match against the Sidon generation kernel, ranked by hub-weighted role structure (relation-hub partial correlations) among keyword candidates.""" combined = (name + " " + eq_text).lower() struct = sidon_structure_score(name + " " + eq_text) kernel = load_sidon_kernel() sidon_papers = kernel.get("papers", []) sidon_types = kernel.get("sidon_types", {}) matches = [] # Keyword patterns for each Sidon type sidon_patterns = { "sidon_set": ["sidon set", "sidon sequence", "b_h set", "bh set"], "sumset": ["sumset", "sum set", "minkowski sum"], "freiman": ["freiman", "freiman theorem"], "additive_energy": ["additive energy", "additive combinatoric"], "difference_set": ["difference set", "perfect difference"], "projective_plane": ["projective plane", "finite projective"], "cap_set": ["cap set", "caps set"], "cauchy_davenport": ["cauchy-davenport"], "singer": ["singer construction", "singer difference"], "szemeredi": ["szemerédi", "szemeredi", "arithmetic progression"], } for stype, kws in sidon_patterns.items(): score = sum(4 for kw in kws if kw in combined) if score >= 4: # Find a matching paper from this Sidon type for p in sidon_papers: if p.get("topic") == stype: matches.append({ "paper_id": p["paper_id"], "title": p["title"], "abstract_snippet": p.get("abstract_snippet", ""), # boost keyword score by hub-weighted role structure "score": score + round(struct), "match_type": f"sidon_{stype}", "role_structure": round(struct, 3), }) break # rank by structure-boosted score so hub-central candidates surface first return sorted(matches, key=lambda m: -m["score"])[:3] def load_obscure_kernel() -> dict: base = Path("shared-data/data") p = base / "obscure_math_kernel_v1.json" if p.exists(): return json.loads(p.read_text()) return {"domains": [], "papers_index": {}} def detect_obscure_type(name: str, eq_text: str) -> list[dict]: """Match against 28 obscure math subfields by keyword detection.""" combined = (name + " " + eq_text).lower() kernel = load_obscure_kernel() index = kernel.get("papers_index", {}) matches = [] # Keyword patterns for each obscure domain obscure_patterns = { "tropical_geometry": ["tropical", "tropical geometry", "max-plus", "tropicalization"], "quantum_topology": ["quantum invariant", "jones polynomial", "quantum knot"], "noncommutative_geometry": ["noncommutative geometry", "noncommutative space"], "fusion_categories": ["fusion category", "fusion categor", "modular tensor"], "planar_algebras": ["planar algebra", "subfactor planar"], "operads": ["operad", "multicategory"], "cluster_algebras": ["cluster algebra", "cluster variable", "quiver mutat"], "higher_category_theory": ["higher category", "infinity categor", "∞-category", "∞-categor"], "nimber_combinatorial_games": ["nimber", "combinatorial game", "surreal number", "partizan"], "mock_modular_forms": ["mock theta", "mock modular", "harmonic maass"], "sandpile_chip_firing": ["sandpile", "chip firing", "chip-firing"], "dessins_enfants": ["dessin", "belyi", "dessins d'enfants"], "pisot_salem_numbers": ["pisot", "salem number", "beta-expansion"], "subshifts_symbolic_dynamics": ["subshift", "sturmian word", "substitution sequence"], "quandles": ["quandle", "biquandle"], "magmas": ["magma", "magmatic"], "rigid_analytic_geometry": ["rigid anal", "rigid geometr", "rigid space"], "synthetic_differential_geometry": ["synthetic differ", "synthetic geometry"], "bornological_vector_spaces": ["bornolog", "bornivor"], "automatic_groups": ["automatic group", "knuth-bendix"], "affine_hecke_algebras": ["affine hecke", "double affine hecke"], "koszul_duality": ["koszul dual"], "combinatorial_species": ["combinatorial species", "combinatorial spec"], "deligne_mumford_stacks": ["deligne-mumford", "dm stack"], "postcritically_finite_maps": ["postcritic", "post-critic"], "golay_codes": ["golay", "hexacode"], "profinite_groups": ["profinite complet", "profinite group"], "mahler_measure": ["mahler measure", "mahler's measure"], } for domain, kws in obscure_patterns.items(): score = sum(3 for kw in kws if kw in combined) if score >= 3: # Find the best paper for this domain for d in kernel.get("domains", []): if d["name"] == domain: for pid in d.get("paper_ids", []): if pid in index: matches.append({ "paper_id": pid, "title": index[pid]["title"], "abstract_snippet": f"obscure domain: {domain}", "score": score, "match_type": f"obscure_{domain}", }) break return matches[:3] # ───────────────────────────────────────────────────────────────────────────── # §2d GEOMETRY / TOPOLOGY KERNEL — Riemannian & differential geometry taxonomy # ───────────────────────────────────────────────────────────────────────────── # Pure-local detector (NO arxiv-DB dependency): classifies differential-geometry # and topology equations that fall outside the number-theory and combinatorics # kernels (e.g. the geodesic equation, which previously fell through to # "unmatched"). Each subfield is rooted in a named invariant / theorem, per the # OTM provability doctrine. paper_id uses the synthetic ``geom:`` # convention, mirroring the ``domain:`` / ``theorem:`` ids of detect_dataset_type. GEOMETRY_PATTERNS: dict[str, list[str]] = { "riemannian_geometry": [ "geodesic", "christoffel", "metric tensor", "covariant derivative", "levi-civita", "levi civita", "parallel transport", "riemannian manifold", "tangent bundle", "first fundamental form", ], "curvature_invariants": [ "riemann curvature", "curvature tensor", "ricci tensor", "ricci curvature", "scalar curvature", "sectional curvature", "gaussian curvature", "gauss curvature", "mean curvature", "second fundamental form", ], "geometric_flows": [ "ricci flow", "perelman", "mean curvature flow", "geometric flow", "yamabe flow", "willmore flow", ], "characteristic_classes": [ "gauss-bonnet", "gauss bonnet", "chern class", "chern-weil", "euler characteristic", "pontryagin", "characteristic class", "atiyah-singer", "index theorem", ], "symplectic_contact": [ "symplectic", "contact structure", "hamiltonian vector field", "darboux", "moment map", "poisson manifold", "lagrangian submanifold", ], "complex_kahler": [ "kähler", "kahler", "calabi-yau", "calabi yau", "hermitian metric", "hodge theory", "hodge decomposition", "dolbeault", "holomorphic bundle", ], "general_relativity": [ "einstein field", "einstein tensor", "stress-energy", "stress energy", "lorentzian", "pseudo-riemannian", "schwarzschild", "spacetime", ], "differential_topology": [ "de rham", "morse theory", "fiber bundle", "vector bundle", "principal bundle", "connection form", "curvature form", "exterior derivative", ], "low_dimensional_topology": [ "3-manifold", "three-manifold", "knot invariant", "heegaard", "floer homology", "dehn surgery", "mapping class group", "seifert fiber", ], } GEOMETRY_TITLES: dict[str, str] = { "riemannian_geometry": "Riemannian geometry (geodesics, Levi-Civita connection)", "curvature_invariants": "Curvature invariants (Riemann/Ricci/scalar curvature)", "geometric_flows": "Geometric flows (Ricci / mean-curvature flow)", "characteristic_classes": "Characteristic classes & index theory (Gauss-Bonnet, Chern)", "symplectic_contact": "Symplectic & contact geometry", "complex_kahler": "Complex & Kähler geometry (Hodge theory)", "general_relativity": "Pseudo-Riemannian / general relativity", "differential_topology": "Differential topology (de Rham, Morse, bundles)", "low_dimensional_topology": "Low-dimensional topology (knots, 3-manifolds, Floer)", } # ── Notation-level signatures (case-SENSITIVE, run on RAW text) ────────────── # The keyword layer lowercases everything, which collapses the R (Riemann/Ricci) # vs g (metric) vs G (Einstein) case distinction that IS the signal. These regex # signatures run on the raw, case-preserving text so an equation in pure tensor- # index notation — with zero English words — still classifies. Each entry: # (compiled_regex, subfield, weight, label) # Weights: 5 = unambiguous anchor (Christoffel/Riemann), 4 = strong, 3 = solid, # 2 = corroborating-only (won't clear the score>=3 threshold alone). _NOTATION_SPECS: list[tuple[str, str, int, str]] = [ # Christoffel symbol Γ^i_{jk} / \Gamma^i_jk (the connection coefficients) (r"(?:Γ|\\Gamma)\s*[\^_]", "riemannian_geometry", 5, "christoffel_symbol"), # Geodesic structure d²x^i/ds² (2nd derivative of a coordinate wrt arclength; # accepts Unicode ² and LaTeX ^2 / ^{2}) (r"d\s*\^?\{?\s*[²2]\}?\s*x.{0,20}?d\s*s\s*\^?\{?\s*[²2]", "riemannian_geometry", 4, "geodesic_2nd_deriv"), # Covariant derivative ∇_μ / \nabla_i (indexed — distinguishes from a bare gradient) (r"(?:∇|\\nabla)\s*[_^]", "riemannian_geometry", 4, "covariant_derivative"), # Riemann curvature tensor R^ρ_{σμν} / R^i_{jkl} (R + ≥2 index groups) — # the most specific curvature signal, so it out-anchors a bare Christoffel. (r"R(?:\s*[\^_]\s*\{?[A-Za-zα-ω]+\}?){2,}", "curvature_invariants", 6, "riemann_tensor"), # Riemann, fully lowered R_{ρσμν} (4 indices in a single brace group) (r"R\s*_\s*\{[A-Za-zα-ω]{4}\}", "curvature_invariants", 6, "riemann_tensor_lowered"), # Ricci tensor R_{μν} / R_{ij} (exactly two indices, not part of a longer group) (r"R\s*_\s*\{?[A-Za-zα-ω]{2}\}?(?![A-Za-zα-ω])", "curvature_invariants", 4, "ricci_tensor"), # Einstein tensor G_{μν} (capital G, Greek pair — GR convention) (r"G\s*_\s*\{?[α-ω]{2}\}?", "general_relativity", 4, "einstein_tensor"), # Stress–energy tensor T_{μν} (r"T\s*_\s*\{?[α-ω]{2}\}?", "general_relativity", 4, "stress_energy_tensor"), # Metric tensor, Greek indices g_{μν} / g^{μν} (r"g\s*[\^_]\s*\{?[α-ω]{2}\}?", "riemannian_geometry", 3, "metric_tensor_greek"), # Metric tensor, Latin indices g_{ij} (weaker — could be a generic Gram matrix) (r"g\s*[\^_]\s*\{?[ijklmn]{2}\}?", "riemannian_geometry", 2, "metric_tensor_latin"), # Line element ds² = … (Unicode ² or LaTeX ^2 / ^{2}) (r"d\s*s\s*\^?\{?\s*[²2]\}?\s*=", "riemannian_geometry", 3, "line_element"), # Exterior-derivative nilpotency d²=0 / d^2=0 / dd=0 (r"d\s*\^?\{?\s*[²2]\}?\s*=\s*0|d\s*d\s*=\s*0", "differential_topology",3, "exterior_deriv_nilpotent"), # Hodge–de Rham Laplacian Δ = dδ + δd (r"d\s*δ\s*\+\s*δ\s*d|δ\s*d\s*\+\s*d\s*δ", "differential_topology",3, "hodge_laplacian"), # Curvature / connection 2-form Ω = dω + ω∧ω (r"(?:Ω|\\Omega)\s*=\s*d|d\s*ω|ω\s*∧\s*ω", "differential_topology",3, "curvature_2form"), # Wedge product of differential forms α ∧ β (corroborating) (r"∧|\\wedge", "differential_topology",2, "wedge_product"), # Second fundamental form II(·,·) (r"\bII\b\s*[=(]", "curvature_invariants", 3, "second_fundamental_form"), # d'Alembertian / wave operator on a Lorentzian manifold □φ (r"□|\\Box", "general_relativity", 2, "dalembertian"), # Indexed partial derivative ∂_μ (corroborates a covariant-derivative ctx) (r"∂\s*[_^]", "riemannian_geometry", 2, "partial_indexed"), ] NOTATION_SIGNATURES = [(re.compile(rx), sf, w, lbl) for rx, sf, w, lbl in _NOTATION_SPECS] def detect_geometry_type(name: str, eq_text: str) -> list[dict]: """Pure-local detector for differential-geometry / topology equations. Two fused layers, no arxiv-DB dependency: 1. keyword layer — English term names (lowercased), weight 3 each. 2. notation layer — case-sensitive tensor-index signatures on the RAW text (Christoffel Γ^i_jk, Riemann R^ρ_σμν, Ricci R_μν, Einstein G_μν, covariant derivative ∇_μ, metric g_ij, line element ds², …). The notation layer means an equation in pure index notation — e.g. the geodesic equation ``d²x^i/ds² + Γ^i_jk dx^j/ds dx^k/ds = 0`` — classifies even with zero English words. Returns ``geom:`` taxonomy matches, each carrying a ``signals`` list of the exact cues that fired (for drill-down). """ # Normalize LaTeX/Unicode first so \Gamma ≡ Γ, \rho\sigma ≡ ρσ, and Penrose # R^{a}{}_{bcd} collapses — signatures then match regardless of encoding. raw = normalize_math(name + " " + eq_text) # case-preserved for notation combined = raw.lower() scores: dict[str, int] = defaultdict(int) signals: dict[str, list[str]] = defaultdict(list) # Layer 1 — English term names (case-insensitive) for subfield, kws in GEOMETRY_PATTERNS.items(): for kw in kws: if kw in combined: scores[subfield] += 3 signals[subfield].append(f"kw:{kw}") # Layer 2 — tensor-index notation (case-sensitive, raw text) for rx, subfield, weight, label in NOTATION_SIGNATURES: if rx.search(raw): scores[subfield] += weight signals[subfield].append(f"sig:{label}(+{weight})") matches = [] for subfield, score in scores.items(): if score >= 3: matches.append({ "paper_id": f"geom:{subfield}", "title": GEOMETRY_TITLES.get(subfield, subfield), "abstract_snippet": f"geometry kernel subfield: {subfield}", "score": score, "match_type": f"geometry_{subfield}", "signals": signals[subfield][:8], }) return sorted(matches, key=lambda x: -x["score"])[:3] def load_dataset_kernels() -> dict: global DATASET_KERNELS if DATASET_KERNELS is not None: return DATASET_KERNELS base = Path("shared-data/data") dk = json.loads((base / "domain_kernel_v1.json").read_text()) if (base / "domain_kernel_v1.json").exists() else {} wk = json.loads((base / "webmath_kernel_v1.json").read_text()) if (base / "webmath_kernel_v1.json").exists() else {} tk = json.loads((base / "theorem_kernel_v1.json").read_text()) if (base / "theorem_kernel_v1.json").exists() else {} DATASET_KERNELS = {"domain": dk, "webmath": wk, "theorem": tk} return DATASET_KERNELS def detect_dataset_type(name: str, eq_text: str) -> list[dict]: """Match against domain taxonomy, web patterns, and theorem QA.""" combined = (name + " " + eq_text).lower() kernels = load_dataset_kernels() matches = [] # 1. Domain kernel: match keywords against equation name + text dk = kernels.get("domain", {}) for dom in dk.get("domains", []): kws = dom.get("keywords", []) score = 0 for kw in kws: if kw in combined: score += 1 if score >= 3: # at least 3 keyword hits matches.append({ "paper_id": f"domain:{dom['path']}", "title": dom["path"], "abstract_snippet": f"count={dom['count']}, solve_rate={dom['avg_solve_rate']}", "score": score, "match_type": "dataset_domain", "domain_path": dom["path"], "domain_count": dom["count"], "domain_solve_rate": dom["avg_solve_rate"], }) # 2. Theorem kernel: match theorem kinds against equation tk = kernels.get("theorem", {}) kind_counts: dict[str, int] = {} for t in tk.get("theorems", []): kws = t.get("keywords", []) hit = sum(1 for kw in kws if kw in combined) if hit >= 2: kind = t.get("kind", "other") kind_counts[kind] = kind_counts.get(kind, 0) + 1 for kind, cnt in sorted(kind_counts.items(), key=lambda x: -x[1])[:3]: matches.append({ "paper_id": f"theorem:{kind}", "title": f"TheoremQA: {kind}", "abstract_snippet": f"{cnt} theorem matches", "score": cnt, "match_type": "dataset_theorem", }) return sorted(matches, key=lambda x: -x["score"])[:3] def ssh_query(sql: str, timeout: int = 60) -> list[list[str]]: result = subprocess.run([ "ssh", NEON_HOST, f"podman exec {CONTAINER} psql -U postgres -d {DB} -t -A -F '|' -c \"{sql}\"" ], capture_output=True, text=True, timeout=timeout) return [line.split("|") for line in result.stdout.strip().split("\n") if line] _SYMBOL_ROLE_MAP: dict[str, tuple[str, str]] | None = None def _load_symbol_role_map() -> dict[str, tuple[str, str]]: """Load CHAR (Unicode char → English name, role) from math_symbols_v1.json. This replaces the hand-maintained 28-entry LaTeX→English map with the 2953-entry matrix's Unicode character → role/normalized-name lookup. """ global _SYMBOL_ROLE_MAP if _SYMBOL_ROLE_MAP is not None: return _SYMBOL_ROLE_MAP path = Path("shared-data/data/math_symbols_v1.json") _SYMBOL_ROLE_MAP = {} if not path.exists(): return _SYMBOL_ROLE_MAP data = json.loads(path.read_text()) for s in data.get("symbols", []): ch = s.get("char", "") name = s.get("name", "") role = s.get("role", "") if ch and name and role: # Fix: ASCII letters A-Z, a-z are operands, not operators if ch.isalpha() and ord(ch) < 128: role = "math_letter" eng = re.sub(r"[^a-zA-Z0-9 ]", "", name.split(",")[0].strip()).lower() _SYMBOL_ROLE_MAP[ch] = (eng, role) return _SYMBOL_ROLE_MAP def extract_keywords(text: str) -> dict[str, float]: if not text: return {} t = text.lower() # 1. Hand-maintained LaTeX commands (28 Greeks + common constructs) for sym, eng in [ (r"\\sigma", " sigma "), (r"\\lambda", " lambda "), (r"\\alpha", " alpha "), (r"\\beta", " beta "), (r"\\gamma", " gamma "), (r"\\delta", " delta "), (r"\\theta", " theta "), (r"\\mu", " mu "), (r"\\pi", " pi "), (r"\\rho", " rho "), (r"\\omega", " omega "), (r"\\tau", " tau "), (r"\\phi", " phi "), (r"\\psi", " psi "), (r"\\zeta", " zeta "), (r"\\xi", " xi "), (r"\\eta", " eta "), (r"\\kappa", " kappa "), (r"\\nu", " nu "), (r"\\chi", " chi "), (r"\\epsilon", " epsilon "), (r"\\partial", " partial "), (r"\\nabla", " gradient "), (r"\\Delta", " delta "), (r"\\Gamma", " gamma "), (r"\\infty", " infinity "), (r"\\sum", " sum "), (r"\\prod", " product "), (r"\\int", " integral "), (r"\$", " "), ]: t = re.sub(sym, eng, t) t = re.sub(r"\\[A-Za-z]+", " ", t) # 2. Matrix-backed Unicode symbol → English substitution (non-ASCII only: # Greek, arrows, operators, relations — not ASCII letters or common math ops) role_map = _load_symbol_role_map() for ch, (eng, role) in role_map.items(): if ch in t and ord(ch) > 127: t = t.replace(ch, f" {eng} ") t = re.sub(r"[{}()\[\]^_=+\-*/|<>~]", " ", t) t = re.sub(r"[0-9]+", " ", t) tokens = re.findall(r"[a-z][a-z-]{2,}", t) freq: dict[str, float] = defaultdict(float) for tok in tokens: if tok not in STOPWORDS: freq[tok] += 1.0 t_lower = text.lower() for kw, boost in NT_KEYWORDS.items(): if kw in t_lower: freq[kw.replace(" ", "_")] += boost return dict(freq) def role_histogram(text: str) -> dict[str, int]: """Compute a 15-dim role-count vector from math_symbols_v1 CHAR_INFO taxonomy. Roles: greek_letter, nary_operator, relation, arrow, binary_op, accent, delimiter_open, delimiter_close, ordinary, symbol, math_letter, letter, operator, punctuation, delimiter. Replaces the all-zeros foundation_vector in build_unified_forest.py. """ if not text: return {} role_map = _load_symbol_role_map() hist: dict[str, int] = {} for ch, (eng, role) in role_map.items(): if ch in text: # Apply the same ASCII letter fix: A-Z, a-z are math_letter not symbol fixed_role = role if ch.isalpha() and ord(ch) < 128: fixed_role = "math_letter" hist[fixed_role] = hist.get(fixed_role, 0) + text.count(ch) return hist def search_papers(keywords: dict[str, float]) -> list[dict]: if not keywords: return [] sorted_kw = sorted(keywords.items(), key=lambda x: -x[1])[:6] terms = [kw.replace("_", " ") for kw, _ in sorted_kw] results = [] for t in terms: sql = ( f"SELECT paper_id, title, substring(abstract, 1, 200) " f"FROM arxiv_papers " f"WHERE title ILIKE '%{t}%' OR abstract ILIKE '%{t}%' " f"LIMIT 3" ) rows = ssh_query(sql) for r in rows: if len(r) >= 2: pid = r[0] score = 0 for kw, w in sorted_kw: kw_s = kw.replace("_", " ") if kw_s in r[1].lower(): score += w * 3 elif len(r) > 2 and kw_s in r[2].lower(): score += w results.append({ "paper_id": pid, "title": r[1], "abstract_snippet": r[2] if len(r) > 2 else "", "score": score, }) best: dict[str, dict] = {} for r in results: if r["paper_id"] not in best or r["score"] > best[r["paper_id"]]["score"]: best[r["paper_id"]] = r out = sorted(best.values(), key=lambda x: -x["score"]) return out[:3] def detect_combinatorics_type(name: str, eq_text: str) -> list[dict]: """Structural pattern matching against combinatorics taxonomy.""" combined = (name + " " + eq_text).lower() matches = [] for pattern, subfield, paper_ids in COMBINATORICS_KERNEL: if re.search(pattern, combined, re.IGNORECASE): if not paper_ids: continue verified = [] for pid in paper_ids: rows = ssh_query( f"SELECT paper_id, title, substring(abstract, 1, 200) " f"FROM arxiv_papers WHERE paper_id = '{pid}'" ) if rows and len(rows[0]) >= 2: r = rows[0] verified.append({ "paper_id": r[0], "title": r[1], "abstract_snippet": r[2] if len(r) > 2 and r[2] != "None" else "", "score": 5, "match_type": subfield, }) matches.extend(verified) return matches def detect_diophantine_type(name: str, eq_text: str) -> list[dict]: """Structural pattern matching against Diophantine equation taxonomy.""" combined = (name + " " + eq_text).lower() matches = [] for pattern, eq_type, paper_ids in DIOPHANTINE_KERNEL: if re.search(pattern, combined, re.IGNORECASE): if not paper_ids: continue # Verify the papers actually exist in our DB verified = [] for pid in paper_ids: rows = ssh_query( f"SELECT paper_id, title, substring(abstract, 1, 200) " f"FROM arxiv_papers WHERE paper_id = '{pid}'" ) if rows and len(rows[0]) >= 2: r = rows[0] verified.append({ "paper_id": r[0], "title": r[1], "abstract_snippet": r[2] if len(r) > 2 and r[2] != "None" else "", "score": 5, # structural match = high confidence "match_type": eq_type, }) matches.extend(verified) return matches def main(): print("RRC arXiv Kernel Refinement", file=sys.stderr) d = json.loads(RECEIPT_PATH.read_text()) eqs = d["compiled_equations"] total = len(eqs) n_matched_before = sum(1 for e in eqs if e["equation_record"].get("arxiv_paper_id")) new_reconstruction = 0 new_wannier = 0 new_generic = 0 new_diophantine = 0 new_combinatorics = 0 new_dataset = 0 new_obscure = 0 new_sidon = 0 new_geometry = 0 for e in eqs: rec = e["equation_record"] name = rec.get("name", "") eq_text = rec.get("equation", "") route_hint = rec.get("route_hint", "") # --- Stage 0: Reconstruction kernel (graph theory sector) --- recon_matches = detect_reconstruction_type(name, eq_text) if recon_matches: existing = rec.get("arxiv_paper_id") best = recon_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_type"] = "reconstruction_conjecture" rec["arxiv_match_stage"] = "kernel_refine_v0" new_reconstruction += 1 print(f" RECON: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{' (was ' + existing + ')' if existing else ''}", file=sys.stderr) continue # --- Stage 1a: Combinatorics kernel (structural matching) --- comb_matches = detect_combinatorics_type(name, eq_text) if comb_matches: existing = rec.get("arxiv_paper_id") best = comb_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_comb_paper_ids"] = [m["paper_id"] for m in comb_matches] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_abstract"] = best["abstract_snippet"] rec["arxiv_match_type"] = best["match_type"] rec["arxiv_match_stage"] = "kernel_refine_v3" new_combinatorics += 1 note = f" (was {existing})" if existing else "" print(f" COMB: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1b: Dataset kernel (domain/theorem/webmath matching) --- ds_matches = detect_dataset_type(name, eq_text) if ds_matches: existing = rec.get("arxiv_paper_id") best = ds_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_abstract"] = best.get("abstract_snippet", "") rec["arxiv_match_type"] = best["match_type"] if "domain_path" in best: rec["arxiv_domain_path"] = best["domain_path"] rec["arxiv_domain_count"] = best["domain_count"] rec["arxiv_domain_solve_rate"] = best["domain_solve_rate"] rec["arxiv_match_stage"] = "kernel_refine_v4" new_dataset += 1 note = f" (was {existing})" if existing else "" print(f" DS: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1c: Obscure math kernel (28 niche subfields) --- ob_matches = detect_obscure_type(name, eq_text) if ob_matches: existing = rec.get("arxiv_paper_id") best = ob_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_obscure_type"] = best["match_type"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_stage"] = "kernel_refine_v5" new_obscure += 1 note = f" (was {existing})" if existing else "" print(f" OB: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1d: Sidon generation kernel (359 entries) --- sd_matches = detect_sidon_type(name, eq_text) if sd_matches: existing = rec.get("arxiv_paper_id") best = sd_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_type"] = best["match_type"] rec["arxiv_match_stage"] = "kernel_refine_v6" new_sidon += 1 note = f" (was {existing})" if existing else "" print(f" SD: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1e: Wannier Hamiltonian kernel (21 materials) --- wn_matches = detect_wannier_type(name, eq_text) if wn_matches: existing = rec.get("arxiv_paper_id") best = wn_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_type"] = best["match_type"] rec["arxiv_match_stage"] = "kernel_refine_v7" new_wannier += 1 note = f" (was {existing})" if existing else "" print(f" WN: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1f: Number theory kernel (structural matching) --- nt_matches = detect_diophantine_type(name, eq_text) if nt_matches: # Always record NT matches; they are high-confidence structural links existing = rec.get("arxiv_paper_id") best = nt_matches[0] # Store supplemental even if already matched rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_nt_paper_ids"] = [m["paper_id"] for m in nt_matches] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_abstract"] = best["abstract_snippet"] rec["arxiv_match_type"] = best["match_type"] rec["arxiv_match_stage"] = "kernel_refine_v2" new_diophantine += 1 note = f" (was {existing})" if existing else "" print(f" NT: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 1f: Geometry/topology kernel (pure-local, no DB) --- geo_matches = detect_geometry_type(name, eq_text) if geo_matches: existing = rec.get("arxiv_paper_id") best = geo_matches[0] rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"]) rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_abstract"] = best.get("abstract_snippet", "") rec["arxiv_match_type"] = best["match_type"] rec["arxiv_match_stage"] = "kernel_refine_v7" new_geometry += 1 note = f" (was {existing})" if existing else "" print(f" GEO: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr) continue # --- Stage 2: Generic keyword matching (only if unmatched) --- if not rec.get("arxiv_paper_id"): kw = extract_keywords(name + " " + eq_text + " " + route_hint) if route_hint == "compression_route": kw["compression"] = 3.0; kw["entropy"] = 2.0; kw["encoding"] = 2.0 elif route_hint in ("thermodynamic_energy", "magnetic_signal"): kw["energy"] = 3.0; kw["thermodynamic"] = 2.0 elif route_hint == "geometry_topology": kw["geometry"] = 3.0; kw["manifold"] = 2.0; kw["topology"] = 2.0 elif route_hint in ("cognitive_load", "control_signal"): kw["signal"] = 2.0; kw["cognitive"] = 2.0 results = search_papers(kw) if results and results[0]["score"] >= 3: best = results[0] rec["arxiv_paper_id"] = best["paper_id"] rec["arxiv_match_count"] = best["score"] rec["arxiv_match_title"] = best["title"][:200] rec["arxiv_match_abstract"] = best["abstract_snippet"] rec["arxiv_match_stage"] = "kernel_refine_v1" new_generic += 1 print(f" KW: {name:35s} → {best['paper_id']} ({best['score']})", file=sys.stderr) RECEIPT_PATH.write_text(json.dumps(d, indent=2, ensure_ascii=False)) n_matched_after = sum(1 for e in eqs if e["equation_record"].get("arxiv_paper_id")) print(f"\nReconstruction kernel matches: {new_reconstruction}", file=sys.stderr) print(f"Wannier Hamiltonian kernel matches: {new_wannier}", file=sys.stderr) print(f"Geometry/topology kernel matches: {new_geometry}", file=sys.stderr) print(f"Sidon generation kernel matches: {new_sidon}", file=sys.stderr) print(f"Obscure math kernel matches: {new_obscure}", file=sys.stderr) print(f"Dataset kernel matches: {new_dataset}", file=sys.stderr) print(f"Combinatorics kernel matches: {new_combinatorics}", file=sys.stderr) print(f"NT kernel matches: {new_diophantine}", file=sys.stderr) print(f"Generic keyword matches: {new_generic}", file=sys.stderr) print(f"Before: {n_matched_before}/{total}", file=sys.stderr) print(f"After: {n_matched_after}/{total}", file=sys.stderr) if __name__ == "__main__": main()