mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-30 18:56:16 +00:00
allaun
77488ac0ae
Lean proof fixes: - N3L_Energy.lean: fully close gaussian_line_integral_unit_dir (nlinarith+hab for unit-circle quadratic, sqrt_mul+neg_div for integral_gaussian_1d match, exp_sum_of_sq order fix, add_assoc for h_gauss_shift, sq_sqrt for field_simp, sq_abs for perpDistance hd) - Add Adapters/AlphaProofNexus: 12 Erdos/graph adapter stubs (AlphaProof nexus) - Add Adapters/ErgodicAdditive.lean, SidonMatroid.lean - Add AntiDiophantine.lean, EffectiveBoundDQ.lean, PVGS_DQ_Bridge.lean - Add FormalConjectures/Util/ProblemImports.lean - Add RRC/EntropyCandidates/Candidates.lean - Add OTOM external project (lakefile.toml, lake-manifest.json, lean-toolchain) Infrastructure: - Add 4-Infrastructure/shim/: 17 Python probes (RRC manifold, Sidon kernel, Wannier, arxiv harvest, math_symbols DB, coverage density, geometric entropy) - Add 4-Infrastructure/NoDupeLabs/: Node server + package files - Add 6-Documentation/docs/specs/DP_RRC_RECEIPT_ENCODING_SPEC.md - Add fix_offloat.py Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1066 lines
50 KiB
Python
1066 lines
50 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
rrc_arxiv_kernel_refine.py — Refine RRC classification using arxiv DB.
|
||
|
||
Queries the arxiv_papers table (titles + abstracts) for each RRC equation
|
||
using extracted keywords. Adds matches to the classifier receipt.
|
||
|
||
Stages:
|
||
kernel_refine_v1 — generic keyword-based matching (abstract + title)
|
||
kernel_refine_v2 — number theory kernel (structural pattern matching
|
||
against Diophantine equation taxonomy)
|
||
kernel_refine_v3 — combinatorics kernel (structural pattern matching
|
||
against combinatorics subfield taxonomy)
|
||
kernel_refine_v4 — dataset kernel (domain taxonomy from
|
||
Big-Math-RL-Verified, web patterns from
|
||
AutoMathText, theorem patterns from TheoremQA)
|
||
kernel_refine_v5 — obscure math kernel (28 niche subfields:
|
||
tropical geometry, fusion categories, operads,
|
||
cluster algebras, quandles, magmas, etc.)
|
||
kernel_refine_v6 — Sidon generation kernel (359 entries across
|
||
9 topics: Sidon sets, sumsets, Freiman, additive
|
||
energy, difference sets, cap sets, etc.)
|
||
kernel_refine_v7 — Wannier Hamiltonian kernel (21 materials,
|
||
30 files: tight-binding band structures from
|
||
BN, graphene, Si, GaAs, Fe, Cu, CrI3, etc.)
|
||
kernel_refine_v7 — geometry/topology kernel (pure-local, 9 subfields:
|
||
Riemannian geometry, curvature invariants, geometric
|
||
flows, characteristic classes, symplectic/contact,
|
||
Kähler, general relativity, differential & low-dim
|
||
topology). No DB dependency.
|
||
|
||
Usage:
|
||
python3 4-Infrastructure/shim/rrc_arxiv_kernel_refine.py
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
NEON_HOST = "neon-64gb"
|
||
CONTAINER = "arxiv-pg"
|
||
DB = "arxiv"
|
||
RECEIPT_PATH = Path("archive/experimental-shim-probes/rrc_equation_classifier_receipt.json")
|
||
|
||
# Math-symbol normalizer (LaTeX/Unicode → canonical) backing the geometry kernel.
|
||
# Same-dir import; degrade to identity if the module/DB is unavailable.
|
||
try:
|
||
import sys as _sys
|
||
_sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
from math_symbols import normalize_math
|
||
except Exception:
|
||
def normalize_math(text: str) -> str:
|
||
return text or ""
|
||
|
||
STOPWORDS = {
|
||
"the", "and", "for", "where", "with", "this", "from", "that", "are", "but",
|
||
"not", "have", "has", "been", "was", "were", "will", "would", "could",
|
||
"should", "their", "them", "they", "its", "also", "can", "may", "however",
|
||
"thus", "proof", "theorem", "lemma", "corollary", "proposition",
|
||
"function", "functions", "using", "used", "use", "given", "show", "shows",
|
||
"paper", "result", "results", "method", "methods", "well", "first", "new",
|
||
"one", "two", "three", "equation", "equations",
|
||
}
|
||
|
||
NT_KEYWORDS = {
|
||
"goormaghtigh": 5, "repunit": 5, "baker theorem": 5,
|
||
"linear form": 5, "exponential diophantine": 5,
|
||
"logarithm": 4, "lower bound": 4, "upper bound": 3,
|
||
"diophantine": 4, "number theory": 2, "algebraic number": 3,
|
||
"transcendence": 3, "integer solution": 4,
|
||
"laurent mignotte": 5, "nesterenko": 5,
|
||
"prime": 2, "congruence": 2, "modular": 2,
|
||
}
|
||
|
||
# Structured number theory kernel: equation-type patterns → relevant arxiv papers
|
||
# Each entry: pattern regex → (type_label, [paper_id, ...])
|
||
DIOPHANTINE_KERNEL = [
|
||
# Goormaghtigh / repunit: (x^m - 1)/(x - 1) = (y^n - 1)/(y - 1)
|
||
(r"repunit|goormaghtigh|(x\^[a-z]+\s*-\s*1)\s*(/|\\over)\s*(x\s*-\s*1)|(y\^[a-z]+\s*-\s*1)\s*(/|\\over)\s*(y\s*-\s*1)|(a\^\s*[a-z]+\s*-\s*1).*(a\s*-\s*1)",
|
||
"goormaghtigh",
|
||
["2410.03677", "2505.08160", "2510.11252"]),
|
||
|
||
# Exponential Diophantine: a^x + b^y = c^z or similar
|
||
(r"[a-z]\^\{?[a-z]\}?\s*\+\s*[a-z]\^\{?[a-z]\}?\s*=",
|
||
"exponential_diophantine",
|
||
["1808.06557", "1702.03424", "1808.06272", "2508.17601", "2503.00843"]),
|
||
|
||
# Baker bounds: linear forms in logarithms
|
||
(r"baker.*(theorem|bound|lower)|linear form.*logarithm|logarithm.*linear form",
|
||
"baker_bounds",
|
||
["1309.5987", "2205.08899", "2303.02037", "1906.00419", "2107.00971"]),
|
||
|
||
# S-unit equations: a_1 x_1 + ... + a_k x_k = 1 with S-units
|
||
(r"\bs-unit\b.*\bequation\b|unit equation.*diophantine|S-unit.*module|S-unit.*linear",
|
||
"s_unit",
|
||
["2505.19141", "2604.26497", "1911.11963"]),
|
||
|
||
# Pell-type: x^2 - D y^2 = N
|
||
(r"x\^\{?2\}?\s*[−\-]\s*[a-z]\s*y\^\{?2\}?\s*=",
|
||
"pell",
|
||
["2509.17882", "2411.11103", "2403.18924"]),
|
||
|
||
# Thue: F(x,y) = m (homogeneous degree >= 3)
|
||
(r"thue|F\(x,\s*y\)\s*=\s*\d|homogeneous.*degree.*integer|binary.*form.*integer",
|
||
"thue",
|
||
["2210.09631", "2406.01111", "2208.03830"]),
|
||
|
||
# Ramanujan-Nagell: x^2 + D = 2^n
|
||
(r"ramanujan.*nagell|nagell|2\^.*=.*x\^.*\+|x\^.*\+.*=.*2\^",
|
||
"ramanujan_nagell",
|
||
["2602.06073"]),
|
||
|
||
# Catalan / Mihailescu: a^x - b^y = 1
|
||
(r"catalan|mihailescu|a\^.*−\s*b\^.*=\s*1|a\^.*-\s*b\^.*=\s*1|x\^.*−\s*y\^.*=\s*1",
|
||
"catalan",
|
||
[]),
|
||
|
||
# abc conjecture
|
||
(r"abc\s*conjecture|masser.*oesterle|a\s*\+\s*b\s*=\s*c|quality.*abc",
|
||
"abc_conjecture",
|
||
[]),
|
||
|
||
# Generalized Fermat: x^p + y^q = z^r
|
||
(r"fermat.*last|generalized.*fermat|x\^.*\+\s*y\^.*=\s*z\^|x\^4\s*\+\s*y\^4\s*=\s*z\^",
|
||
"fermat_generalized",
|
||
["math/0403046", "2311.12044"]),
|
||
|
||
# Modular approach to Diophantine equations
|
||
(r"modular.*diophantine|classical.*modular.*exponential|frey.*curve|galois.*representation.*diophantine",
|
||
"modular_diophantine",
|
||
["math/0403046", "math/0405220", "2311.12044"]),
|
||
|
||
# Skolem / p-adic method
|
||
(r"skolem|p-adic.*diophantine|p-adic.*method|chabauty|coleman.*chabauty",
|
||
"p_adic_diophantine",
|
||
["math/0005186", "2208.03830", "2508.17601"]),
|
||
]
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# §2b COMBINATORICS KERNEL — structural patterns bridging to stack concepts
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Each entry: (pattern_regex, subfield_label, [paper_ids])
|
||
COMBINATORICS_KERNEL = [
|
||
# Graph theory / extremal
|
||
(r"regularity lemma|szemerédi regular|szemeredi regular|regularisation lemma",
|
||
"extremal_combinatorics",
|
||
["math/0504472", "2606.06192", "math/0310476"]),
|
||
|
||
(r"flag algebra|razborov|turán density|turán number|turán problem",
|
||
"extremal_combinatorics",
|
||
["2601.12741", "1607.04741", "2601.06590"]),
|
||
|
||
(r"hypergraph container|container method|container theorem",
|
||
"extremal_combinatorics",
|
||
["1204.6595", "1801.04584", "2408.08514", "2408.06617"]),
|
||
|
||
# Combinatorial Nullstellensatz / polynomial method
|
||
(r"combinatorial nullstellensatz|alom nullstellensatz|chevalley-warning",
|
||
"combinatorial_nullstellensatz",
|
||
["2404.10778", "2605.18323", "2508.07257", "2408.03443"]),
|
||
|
||
(r"polynomial method.*(combinatoric|additive|cap|bound)|croot-lev-pach|ellenberg.gijswijt",
|
||
"polynomial_method",
|
||
["2311.08873", "1912.07679", "1811.07865"]),
|
||
|
||
# Graph minors
|
||
(r"graph minor|robertson.seymour|minor.theorem|graph structure theorem|excluded minor",
|
||
"graph_minor_theory",
|
||
["2504.02532", "2507.02769", "2510.19285", "2409.18902", "2212.07670"]),
|
||
|
||
# Matroid theory
|
||
(r"matroid|oriented matroid|tutte polynomial|matroid represent",
|
||
"matroid_theory",
|
||
["math/9804004", "math/9702219", "math/0612073", "math/0609840", "2606.16832"]),
|
||
|
||
# Design theory
|
||
(r"(block|combinatorial) design|finite projective plane|difference set|latin square|mutually orthogonal",
|
||
"design_theory",
|
||
["math/0611492", "2509.06247", "math/0609244", "math/0609586", "2606.13536"]),
|
||
|
||
# Ramsey theory
|
||
(r"ramsey (number|theory|theorem)|van der waerden|hales.jewett|gallai-ramsey|anti-ramsey",
|
||
"ramsey_theory",
|
||
["2601.05442", "2309.08370", "2212.07180", "2110.07144"]),
|
||
|
||
# Probabilistic method
|
||
(r"probabilistic method|lovász local lemma|lovasz local lemma",
|
||
"probabilistic_method",
|
||
["2310.00513", "1909.11078", "1402.6817", "1511.04739"]),
|
||
|
||
# Additive combinatorics
|
||
(r"additive combinatoric|sumset|sum.set|freiman|balog.szemerédi|cauchy.davenport|erdős.szemerédi|erdős.szemerédi",
|
||
"additive_combinatorics",
|
||
["math/0608105", "2211.01893", "math/0402285", "math/0507539", "math/0703668"]),
|
||
|
||
# Algebraic combinatorics / spectral graph
|
||
(r"algebraic (combinatoric|graph theory)|spectral graph|association scheme|strongly regular",
|
||
"algebraic_combinatorics",
|
||
["2504.10624", "2504.03566", "2605.19542", "2308.14137"]),
|
||
|
||
# Sidon / difference sets bridge
|
||
(r"sidon set|sidon sequence|perfect difference set",
|
||
"design_theory",
|
||
["math/0609244", "math/0504226"]),
|
||
|
||
# Enumerative combinatorics
|
||
(r"catalan number|dyck path|non.crossing partition|enumerative combinatoric",
|
||
"enumerative_combinatorics",
|
||
["math/9904107", "2605.19979", "1910.00299"]),
|
||
|
||
# Combinatorial geometry
|
||
(r"combinatorial geometr|discrete geometr|zonotope|erdős.szekeres|erdős.szekeres|incidence geometry",
|
||
"combinatorial_geometry",
|
||
["math/0609053", "2606.16832", "2605.23866", "2605.04183"]),
|
||
|
||
# Random graphs
|
||
(r"random graph|erdős.rényi|erdos.renyi|percolation.*graph|graph limit|graphon",
|
||
"random_graphs",
|
||
["math/0703269", "math/0701316", "math/0612827"]),
|
||
|
||
# Combinatorial topology
|
||
(r"combinatorial topolog|topological combinatoric|discrete topolog",
|
||
"combinatorial_topology",
|
||
["2112.14700", "2305.06288", "2402.06024"]),
|
||
]
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# §2c DATASET KERNELS — loaded from shared-data/data/ (domain, webmath, theorem)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
DATASET_KERNELS: dict | None = None
|
||
|
||
def load_reconstruction_kernel() -> dict:
|
||
base = Path("shared-data/data")
|
||
p = base / "reconstruction_kernel_v1.json"
|
||
if p.exists():
|
||
return json.loads(p.read_text())
|
||
return {"papers": [], "lean_proofs": []}
|
||
|
||
|
||
def detect_reconstruction_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Match against the graph reconstruction conjecture kernel."""
|
||
combined = (name + " " + eq_text).lower()
|
||
kernel = load_reconstruction_kernel()
|
||
papers = kernel.get("papers", [])
|
||
# Keywords for reconstruction conjecture
|
||
recon_kws = ["reconstruction", "reconstructible", "graph reconstruction", "deck of graph",
|
||
"hypomorphic", "vertex deleted", "kelly lemma", "ulam conjecture"]
|
||
score = sum(3 for kw in recon_kws if kw in combined)
|
||
if score >= 3:
|
||
matches = []
|
||
for p in papers[:3]:
|
||
matches.append({"paper_id": p["paper_id"], "title": p["title"], "score": score + 3,
|
||
"match_type": "reconstruction_conjecture",
|
||
"abstract_snippet": p.get("abstract_snippet", "")})
|
||
return matches
|
||
# Also check for graph theory keywords + combinatorics route
|
||
graph_kws = ["graph", "subgraph", "tree", "vertex", "edge", "deck", "spanning"]
|
||
graph_score = sum(1 for kw in graph_kws if kw in combined)
|
||
if graph_score >= 3 and any(r in combined for r in ["conjecture", "reconstruct"]):
|
||
matches = []
|
||
for p in papers[:2]:
|
||
matches.append({"paper_id": p["paper_id"], "title": p["title"], "score": graph_score,
|
||
"match_type": "reconstruction_conjecture",
|
||
"abstract_snippet": p.get("abstract_snippet", "")})
|
||
return matches
|
||
return []
|
||
|
||
|
||
def load_wannier_kernel() -> dict:
|
||
base = Path("shared-data/data")
|
||
p = base / "wannier_hamiltonian_kernel_v1.json"
|
||
if p.exists():
|
||
return json.loads(p.read_text())
|
||
return {"materials": []}
|
||
|
||
|
||
def detect_wannier_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Match against Wannier tight-binding Hamiltonian materials."""
|
||
combined = (name + " " + eq_text).lower()
|
||
kernel = load_wannier_kernel()
|
||
materials_list = kernel.get("materials", [])
|
||
matches = []
|
||
for mat in materials_list:
|
||
mname = mat["name"].lower()
|
||
if mname in combined:
|
||
matches.append({
|
||
"paper_id": f"wannier:{mat['name']}",
|
||
"title": f"Wannier tight-binding Hamiltonian: {mat['name']}",
|
||
"abstract_snippet": f"bands={mat['bands']}, kpoints={mat['kpoints']}, degree={mat['degree']}, graph={mat['graph_type']}",
|
||
"score": 5,
|
||
"match_type": "condensed_matter_hamiltonian",
|
||
})
|
||
return matches
|
||
|
||
|
||
def load_sidon_kernel() -> dict:
|
||
base = Path("shared-data/data")
|
||
p = base / "sidon_generation_kernel_v1.json"
|
||
if p.exists():
|
||
return json.loads(p.read_text())
|
||
return {"papers": [], "rrc_equations": [], "sidon_types": {}}
|
||
|
||
|
||
# Partial-correlation hub weights for the operator grammar (2418-paper bootstrap).
|
||
# The operator roles form an AFFINE Ã₂ extended-Dynkin diagram: a 3-cycle
|
||
# relation–binary_op–arrow, with greek_letter & nary_operator as A₁ pendants on
|
||
# the `relation` hub, and `operator` conditionally independent (its correlations
|
||
# are fully mediated through relation). A cyclic diagram ⇒ affine, not finite ADE
|
||
# (rooted in the extended-Dynkin / affine Kac–Moody classification). Weights =
|
||
# partial-correlation degree centrality: relation (deg-4 hub) highest, operator
|
||
# (deg-0, isolated) lowest. Sidon notation (greek_letter+binary_op+relation) sits
|
||
# at this hub, so structurally-central equations are up-weighted among candidates.
|
||
SIDON_ROLE_WEIGHTS = {
|
||
"relation": 4.0, # hub of the grammar (degree-4 center)
|
||
"binary_op": 2.0, # in the Ã₂ cycle
|
||
"arrow": 2.0, # in the Ã₂ cycle
|
||
"greek_letter": 1.5, # A₁ pendant on the hub
|
||
"nary_operator": 1.5, # A₁ pendant on the hub
|
||
"delimiter": 1.0, "delimiter_open": 1.0, "delimiter_close": 1.0,
|
||
"operator": 0.5, # conditionally isolated (lowest)
|
||
}
|
||
|
||
|
||
def sidon_structure_score(text: str) -> float:
|
||
"""Hub-weighted operator-role density (the partial-correlation structure).
|
||
|
||
Uses the full role vector (role_histogram on normalized text) weighted by
|
||
SIDON_ROLE_WEIGHTS, normalized by total symbol count → a scale-free measure
|
||
of how strongly an equation sits at the grammar hub.
|
||
"""
|
||
hist = role_histogram(normalize_math(text))
|
||
total = sum(hist.values())
|
||
if total == 0:
|
||
return 0.0
|
||
wsum = sum(SIDON_ROLE_WEIGHTS.get(r, 0.0) * c for r, c in hist.items())
|
||
return wsum / total
|
||
|
||
|
||
def detect_sidon_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Match against the Sidon generation kernel, ranked by hub-weighted role
|
||
structure (relation-hub partial correlations) among keyword candidates."""
|
||
combined = (name + " " + eq_text).lower()
|
||
struct = sidon_structure_score(name + " " + eq_text)
|
||
kernel = load_sidon_kernel()
|
||
sidon_papers = kernel.get("papers", [])
|
||
sidon_types = kernel.get("sidon_types", {})
|
||
matches = []
|
||
|
||
# Keyword patterns for each Sidon type
|
||
sidon_patterns = {
|
||
"sidon_set": ["sidon set", "sidon sequence", "b_h set", "bh set"],
|
||
"sumset": ["sumset", "sum set", "minkowski sum"],
|
||
"freiman": ["freiman", "freiman theorem"],
|
||
"additive_energy": ["additive energy", "additive combinatoric"],
|
||
"difference_set": ["difference set", "perfect difference"],
|
||
"projective_plane": ["projective plane", "finite projective"],
|
||
"cap_set": ["cap set", "caps set"],
|
||
"cauchy_davenport": ["cauchy-davenport"],
|
||
"singer": ["singer construction", "singer difference"],
|
||
"szemeredi": ["szemerédi", "szemeredi", "arithmetic progression"],
|
||
}
|
||
|
||
for stype, kws in sidon_patterns.items():
|
||
score = sum(4 for kw in kws if kw in combined)
|
||
if score >= 4:
|
||
# Find a matching paper from this Sidon type
|
||
for p in sidon_papers:
|
||
if p.get("topic") == stype:
|
||
matches.append({
|
||
"paper_id": p["paper_id"],
|
||
"title": p["title"],
|
||
"abstract_snippet": p.get("abstract_snippet", ""),
|
||
# boost keyword score by hub-weighted role structure
|
||
"score": score + round(struct),
|
||
"match_type": f"sidon_{stype}",
|
||
"role_structure": round(struct, 3),
|
||
})
|
||
break
|
||
# rank by structure-boosted score so hub-central candidates surface first
|
||
return sorted(matches, key=lambda m: -m["score"])[:3]
|
||
|
||
|
||
def load_obscure_kernel() -> dict:
|
||
base = Path("shared-data/data")
|
||
p = base / "obscure_math_kernel_v1.json"
|
||
if p.exists():
|
||
return json.loads(p.read_text())
|
||
return {"domains": [], "papers_index": {}}
|
||
|
||
|
||
def detect_obscure_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Match against 28 obscure math subfields by keyword detection."""
|
||
combined = (name + " " + eq_text).lower()
|
||
kernel = load_obscure_kernel()
|
||
index = kernel.get("papers_index", {})
|
||
matches = []
|
||
|
||
# Keyword patterns for each obscure domain
|
||
obscure_patterns = {
|
||
"tropical_geometry": ["tropical", "tropical geometry", "max-plus", "tropicalization"],
|
||
"quantum_topology": ["quantum invariant", "jones polynomial", "quantum knot"],
|
||
"noncommutative_geometry": ["noncommutative geometry", "noncommutative space"],
|
||
"fusion_categories": ["fusion category", "fusion categor", "modular tensor"],
|
||
"planar_algebras": ["planar algebra", "subfactor planar"],
|
||
"operads": ["operad", "multicategory"],
|
||
"cluster_algebras": ["cluster algebra", "cluster variable", "quiver mutat"],
|
||
"higher_category_theory": ["higher category", "infinity categor", "∞-category", "∞-categor"],
|
||
"nimber_combinatorial_games": ["nimber", "combinatorial game", "surreal number", "partizan"],
|
||
"mock_modular_forms": ["mock theta", "mock modular", "harmonic maass"],
|
||
"sandpile_chip_firing": ["sandpile", "chip firing", "chip-firing"],
|
||
"dessins_enfants": ["dessin", "belyi", "dessins d'enfants"],
|
||
"pisot_salem_numbers": ["pisot", "salem number", "beta-expansion"],
|
||
"subshifts_symbolic_dynamics": ["subshift", "sturmian word", "substitution sequence"],
|
||
"quandles": ["quandle", "biquandle"],
|
||
"magmas": ["magma", "magmatic"],
|
||
"rigid_analytic_geometry": ["rigid anal", "rigid geometr", "rigid space"],
|
||
"synthetic_differential_geometry": ["synthetic differ", "synthetic geometry"],
|
||
"bornological_vector_spaces": ["bornolog", "bornivor"],
|
||
"automatic_groups": ["automatic group", "knuth-bendix"],
|
||
"affine_hecke_algebras": ["affine hecke", "double affine hecke"],
|
||
"koszul_duality": ["koszul dual"],
|
||
"combinatorial_species": ["combinatorial species", "combinatorial spec"],
|
||
"deligne_mumford_stacks": ["deligne-mumford", "dm stack"],
|
||
"postcritically_finite_maps": ["postcritic", "post-critic"],
|
||
"golay_codes": ["golay", "hexacode"],
|
||
"profinite_groups": ["profinite complet", "profinite group"],
|
||
"mahler_measure": ["mahler measure", "mahler's measure"],
|
||
}
|
||
|
||
for domain, kws in obscure_patterns.items():
|
||
score = sum(3 for kw in kws if kw in combined)
|
||
if score >= 3:
|
||
# Find the best paper for this domain
|
||
for d in kernel.get("domains", []):
|
||
if d["name"] == domain:
|
||
for pid in d.get("paper_ids", []):
|
||
if pid in index:
|
||
matches.append({
|
||
"paper_id": pid,
|
||
"title": index[pid]["title"],
|
||
"abstract_snippet": f"obscure domain: {domain}",
|
||
"score": score,
|
||
"match_type": f"obscure_{domain}",
|
||
})
|
||
break
|
||
return matches[:3]
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# §2d GEOMETRY / TOPOLOGY KERNEL — Riemannian & differential geometry taxonomy
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Pure-local detector (NO arxiv-DB dependency): classifies differential-geometry
|
||
# and topology equations that fall outside the number-theory and combinatorics
|
||
# kernels (e.g. the geodesic equation, which previously fell through to
|
||
# "unmatched"). Each subfield is rooted in a named invariant / theorem, per the
|
||
# OTM provability doctrine. paper_id uses the synthetic ``geom:<subfield>``
|
||
# convention, mirroring the ``domain:`` / ``theorem:`` ids of detect_dataset_type.
|
||
GEOMETRY_PATTERNS: dict[str, list[str]] = {
|
||
"riemannian_geometry": [
|
||
"geodesic", "christoffel", "metric tensor", "covariant derivative",
|
||
"levi-civita", "levi civita", "parallel transport", "riemannian manifold",
|
||
"tangent bundle", "first fundamental form",
|
||
],
|
||
"curvature_invariants": [
|
||
"riemann curvature", "curvature tensor", "ricci tensor", "ricci curvature",
|
||
"scalar curvature", "sectional curvature", "gaussian curvature",
|
||
"gauss curvature", "mean curvature", "second fundamental form",
|
||
],
|
||
"geometric_flows": [
|
||
"ricci flow", "perelman", "mean curvature flow", "geometric flow",
|
||
"yamabe flow", "willmore flow",
|
||
],
|
||
"characteristic_classes": [
|
||
"gauss-bonnet", "gauss bonnet", "chern class", "chern-weil",
|
||
"euler characteristic", "pontryagin", "characteristic class",
|
||
"atiyah-singer", "index theorem",
|
||
],
|
||
"symplectic_contact": [
|
||
"symplectic", "contact structure", "hamiltonian vector field",
|
||
"darboux", "moment map", "poisson manifold", "lagrangian submanifold",
|
||
],
|
||
"complex_kahler": [
|
||
"kähler", "kahler", "calabi-yau", "calabi yau", "hermitian metric",
|
||
"hodge theory", "hodge decomposition", "dolbeault", "holomorphic bundle",
|
||
],
|
||
"general_relativity": [
|
||
"einstein field", "einstein tensor", "stress-energy", "stress energy",
|
||
"lorentzian", "pseudo-riemannian", "schwarzschild", "spacetime",
|
||
],
|
||
"differential_topology": [
|
||
"de rham", "morse theory", "fiber bundle", "vector bundle",
|
||
"principal bundle", "connection form", "curvature form", "exterior derivative",
|
||
],
|
||
"low_dimensional_topology": [
|
||
"3-manifold", "three-manifold", "knot invariant", "heegaard",
|
||
"floer homology", "dehn surgery", "mapping class group", "seifert fiber",
|
||
],
|
||
}
|
||
|
||
GEOMETRY_TITLES: dict[str, str] = {
|
||
"riemannian_geometry": "Riemannian geometry (geodesics, Levi-Civita connection)",
|
||
"curvature_invariants": "Curvature invariants (Riemann/Ricci/scalar curvature)",
|
||
"geometric_flows": "Geometric flows (Ricci / mean-curvature flow)",
|
||
"characteristic_classes": "Characteristic classes & index theory (Gauss-Bonnet, Chern)",
|
||
"symplectic_contact": "Symplectic & contact geometry",
|
||
"complex_kahler": "Complex & Kähler geometry (Hodge theory)",
|
||
"general_relativity": "Pseudo-Riemannian / general relativity",
|
||
"differential_topology": "Differential topology (de Rham, Morse, bundles)",
|
||
"low_dimensional_topology": "Low-dimensional topology (knots, 3-manifolds, Floer)",
|
||
}
|
||
|
||
|
||
# ── Notation-level signatures (case-SENSITIVE, run on RAW text) ──────────────
|
||
# The keyword layer lowercases everything, which collapses the R (Riemann/Ricci)
|
||
# vs g (metric) vs G (Einstein) case distinction that IS the signal. These regex
|
||
# signatures run on the raw, case-preserving text so an equation in pure tensor-
|
||
# index notation — with zero English words — still classifies. Each entry:
|
||
# (compiled_regex, subfield, weight, label)
|
||
# Weights: 5 = unambiguous anchor (Christoffel/Riemann), 4 = strong, 3 = solid,
|
||
# 2 = corroborating-only (won't clear the score>=3 threshold alone).
|
||
_NOTATION_SPECS: list[tuple[str, str, int, str]] = [
|
||
# Christoffel symbol Γ^i_{jk} / \Gamma^i_jk (the connection coefficients)
|
||
(r"(?:Γ|\\Gamma)\s*[\^_]", "riemannian_geometry", 5, "christoffel_symbol"),
|
||
# Geodesic structure d²x^i/ds² (2nd derivative of a coordinate wrt arclength;
|
||
# accepts Unicode ² and LaTeX ^2 / ^{2})
|
||
(r"d\s*\^?\{?\s*[²2]\}?\s*x.{0,20}?d\s*s\s*\^?\{?\s*[²2]", "riemannian_geometry", 4, "geodesic_2nd_deriv"),
|
||
# Covariant derivative ∇_μ / \nabla_i (indexed — distinguishes from a bare gradient)
|
||
(r"(?:∇|\\nabla)\s*[_^]", "riemannian_geometry", 4, "covariant_derivative"),
|
||
# Riemann curvature tensor R^ρ_{σμν} / R^i_{jkl} (R + ≥2 index groups) —
|
||
# the most specific curvature signal, so it out-anchors a bare Christoffel.
|
||
(r"R(?:\s*[\^_]\s*\{?[A-Za-zα-ω]+\}?){2,}", "curvature_invariants", 6, "riemann_tensor"),
|
||
# Riemann, fully lowered R_{ρσμν} (4 indices in a single brace group)
|
||
(r"R\s*_\s*\{[A-Za-zα-ω]{4}\}", "curvature_invariants", 6, "riemann_tensor_lowered"),
|
||
# Ricci tensor R_{μν} / R_{ij} (exactly two indices, not part of a longer group)
|
||
(r"R\s*_\s*\{?[A-Za-zα-ω]{2}\}?(?![A-Za-zα-ω])", "curvature_invariants", 4, "ricci_tensor"),
|
||
# Einstein tensor G_{μν} (capital G, Greek pair — GR convention)
|
||
(r"G\s*_\s*\{?[α-ω]{2}\}?", "general_relativity", 4, "einstein_tensor"),
|
||
# Stress–energy tensor T_{μν}
|
||
(r"T\s*_\s*\{?[α-ω]{2}\}?", "general_relativity", 4, "stress_energy_tensor"),
|
||
# Metric tensor, Greek indices g_{μν} / g^{μν}
|
||
(r"g\s*[\^_]\s*\{?[α-ω]{2}\}?", "riemannian_geometry", 3, "metric_tensor_greek"),
|
||
# Metric tensor, Latin indices g_{ij} (weaker — could be a generic Gram matrix)
|
||
(r"g\s*[\^_]\s*\{?[ijklmn]{2}\}?", "riemannian_geometry", 2, "metric_tensor_latin"),
|
||
# Line element ds² = … (Unicode ² or LaTeX ^2 / ^{2})
|
||
(r"d\s*s\s*\^?\{?\s*[²2]\}?\s*=", "riemannian_geometry", 3, "line_element"),
|
||
# Exterior-derivative nilpotency d²=0 / d^2=0 / dd=0
|
||
(r"d\s*\^?\{?\s*[²2]\}?\s*=\s*0|d\s*d\s*=\s*0", "differential_topology",3, "exterior_deriv_nilpotent"),
|
||
# Hodge–de Rham Laplacian Δ = dδ + δd
|
||
(r"d\s*δ\s*\+\s*δ\s*d|δ\s*d\s*\+\s*d\s*δ", "differential_topology",3, "hodge_laplacian"),
|
||
# Curvature / connection 2-form Ω = dω + ω∧ω
|
||
(r"(?:Ω|\\Omega)\s*=\s*d|d\s*ω|ω\s*∧\s*ω", "differential_topology",3, "curvature_2form"),
|
||
# Wedge product of differential forms α ∧ β (corroborating)
|
||
(r"∧|\\wedge", "differential_topology",2, "wedge_product"),
|
||
# Second fundamental form II(·,·)
|
||
(r"\bII\b\s*[=(]", "curvature_invariants", 3, "second_fundamental_form"),
|
||
# d'Alembertian / wave operator on a Lorentzian manifold □φ
|
||
(r"□|\\Box", "general_relativity", 2, "dalembertian"),
|
||
# Indexed partial derivative ∂_μ (corroborates a covariant-derivative ctx)
|
||
(r"∂\s*[_^]", "riemannian_geometry", 2, "partial_indexed"),
|
||
]
|
||
NOTATION_SIGNATURES = [(re.compile(rx), sf, w, lbl) for rx, sf, w, lbl in _NOTATION_SPECS]
|
||
|
||
|
||
def detect_geometry_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Pure-local detector for differential-geometry / topology equations.
|
||
|
||
Two fused layers, no arxiv-DB dependency:
|
||
1. keyword layer — English term names (lowercased), weight 3 each.
|
||
2. notation layer — case-sensitive tensor-index signatures on the RAW
|
||
text (Christoffel Γ^i_jk, Riemann R^ρ_σμν, Ricci R_μν, Einstein G_μν,
|
||
covariant derivative ∇_μ, metric g_ij, line element ds², …).
|
||
|
||
The notation layer means an equation in pure index notation — e.g. the
|
||
geodesic equation ``d²x^i/ds² + Γ^i_jk dx^j/ds dx^k/ds = 0`` — classifies
|
||
even with zero English words. Returns ``geom:<subfield>`` taxonomy matches,
|
||
each carrying a ``signals`` list of the exact cues that fired (for drill-down).
|
||
"""
|
||
# Normalize LaTeX/Unicode first so \Gamma ≡ Γ, \rho\sigma ≡ ρσ, and Penrose
|
||
# R^{a}{}_{bcd} collapses — signatures then match regardless of encoding.
|
||
raw = normalize_math(name + " " + eq_text) # case-preserved for notation
|
||
combined = raw.lower()
|
||
scores: dict[str, int] = defaultdict(int)
|
||
signals: dict[str, list[str]] = defaultdict(list)
|
||
|
||
# Layer 1 — English term names (case-insensitive)
|
||
for subfield, kws in GEOMETRY_PATTERNS.items():
|
||
for kw in kws:
|
||
if kw in combined:
|
||
scores[subfield] += 3
|
||
signals[subfield].append(f"kw:{kw}")
|
||
|
||
# Layer 2 — tensor-index notation (case-sensitive, raw text)
|
||
for rx, subfield, weight, label in NOTATION_SIGNATURES:
|
||
if rx.search(raw):
|
||
scores[subfield] += weight
|
||
signals[subfield].append(f"sig:{label}(+{weight})")
|
||
|
||
matches = []
|
||
for subfield, score in scores.items():
|
||
if score >= 3:
|
||
matches.append({
|
||
"paper_id": f"geom:{subfield}",
|
||
"title": GEOMETRY_TITLES.get(subfield, subfield),
|
||
"abstract_snippet": f"geometry kernel subfield: {subfield}",
|
||
"score": score,
|
||
"match_type": f"geometry_{subfield}",
|
||
"signals": signals[subfield][:8],
|
||
})
|
||
return sorted(matches, key=lambda x: -x["score"])[:3]
|
||
|
||
|
||
def load_dataset_kernels() -> dict:
|
||
global DATASET_KERNELS
|
||
if DATASET_KERNELS is not None:
|
||
return DATASET_KERNELS
|
||
base = Path("shared-data/data")
|
||
dk = json.loads((base / "domain_kernel_v1.json").read_text()) if (base / "domain_kernel_v1.json").exists() else {}
|
||
wk = json.loads((base / "webmath_kernel_v1.json").read_text()) if (base / "webmath_kernel_v1.json").exists() else {}
|
||
tk = json.loads((base / "theorem_kernel_v1.json").read_text()) if (base / "theorem_kernel_v1.json").exists() else {}
|
||
DATASET_KERNELS = {"domain": dk, "webmath": wk, "theorem": tk}
|
||
return DATASET_KERNELS
|
||
|
||
|
||
def detect_dataset_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Match against domain taxonomy, web patterns, and theorem QA."""
|
||
combined = (name + " " + eq_text).lower()
|
||
kernels = load_dataset_kernels()
|
||
matches = []
|
||
|
||
# 1. Domain kernel: match keywords against equation name + text
|
||
dk = kernels.get("domain", {})
|
||
for dom in dk.get("domains", []):
|
||
kws = dom.get("keywords", [])
|
||
score = 0
|
||
for kw in kws:
|
||
if kw in combined:
|
||
score += 1
|
||
if score >= 3: # at least 3 keyword hits
|
||
matches.append({
|
||
"paper_id": f"domain:{dom['path']}",
|
||
"title": dom["path"],
|
||
"abstract_snippet": f"count={dom['count']}, solve_rate={dom['avg_solve_rate']}",
|
||
"score": score,
|
||
"match_type": "dataset_domain",
|
||
"domain_path": dom["path"],
|
||
"domain_count": dom["count"],
|
||
"domain_solve_rate": dom["avg_solve_rate"],
|
||
})
|
||
|
||
# 2. Theorem kernel: match theorem kinds against equation
|
||
tk = kernels.get("theorem", {})
|
||
kind_counts: dict[str, int] = {}
|
||
for t in tk.get("theorems", []):
|
||
kws = t.get("keywords", [])
|
||
hit = sum(1 for kw in kws if kw in combined)
|
||
if hit >= 2:
|
||
kind = t.get("kind", "other")
|
||
kind_counts[kind] = kind_counts.get(kind, 0) + 1
|
||
for kind, cnt in sorted(kind_counts.items(), key=lambda x: -x[1])[:3]:
|
||
matches.append({
|
||
"paper_id": f"theorem:{kind}",
|
||
"title": f"TheoremQA: {kind}",
|
||
"abstract_snippet": f"{cnt} theorem matches",
|
||
"score": cnt,
|
||
"match_type": "dataset_theorem",
|
||
})
|
||
|
||
return sorted(matches, key=lambda x: -x["score"])[:3]
|
||
|
||
|
||
def ssh_query(sql: str, timeout: int = 60) -> list[list[str]]:
|
||
result = subprocess.run([
|
||
"ssh", NEON_HOST,
|
||
f"podman exec {CONTAINER} psql -U postgres -d {DB} -t -A -F '|' -c \"{sql}\""
|
||
], capture_output=True, text=True, timeout=timeout)
|
||
return [line.split("|") for line in result.stdout.strip().split("\n") if line]
|
||
|
||
_SYMBOL_ROLE_MAP: dict[str, tuple[str, str]] | None = None
|
||
|
||
def _load_symbol_role_map() -> dict[str, tuple[str, str]]:
|
||
"""Load CHAR (Unicode char → English name, role) from math_symbols_v1.json.
|
||
This replaces the hand-maintained 28-entry LaTeX→English map with
|
||
the 2953-entry matrix's Unicode character → role/normalized-name lookup.
|
||
"""
|
||
global _SYMBOL_ROLE_MAP
|
||
if _SYMBOL_ROLE_MAP is not None:
|
||
return _SYMBOL_ROLE_MAP
|
||
path = Path("shared-data/data/math_symbols_v1.json")
|
||
_SYMBOL_ROLE_MAP = {}
|
||
if not path.exists():
|
||
return _SYMBOL_ROLE_MAP
|
||
data = json.loads(path.read_text())
|
||
for s in data.get("symbols", []):
|
||
ch = s.get("char", "")
|
||
name = s.get("name", "")
|
||
role = s.get("role", "")
|
||
if ch and name and role:
|
||
# Fix: ASCII letters A-Z, a-z are operands, not operators
|
||
if ch.isalpha() and ord(ch) < 128:
|
||
role = "math_letter"
|
||
eng = re.sub(r"[^a-zA-Z0-9 ]", "", name.split(",")[0].strip()).lower()
|
||
_SYMBOL_ROLE_MAP[ch] = (eng, role)
|
||
return _SYMBOL_ROLE_MAP
|
||
|
||
def extract_keywords(text: str) -> dict[str, float]:
|
||
if not text:
|
||
return {}
|
||
t = text.lower()
|
||
# 1. Hand-maintained LaTeX commands (28 Greeks + common constructs)
|
||
for sym, eng in [
|
||
(r"\\sigma", " sigma "), (r"\\lambda", " lambda "),
|
||
(r"\\alpha", " alpha "), (r"\\beta", " beta "),
|
||
(r"\\gamma", " gamma "), (r"\\delta", " delta "),
|
||
(r"\\theta", " theta "), (r"\\mu", " mu "),
|
||
(r"\\pi", " pi "), (r"\\rho", " rho "),
|
||
(r"\\omega", " omega "), (r"\\tau", " tau "),
|
||
(r"\\phi", " phi "), (r"\\psi", " psi "),
|
||
(r"\\zeta", " zeta "), (r"\\xi", " xi "),
|
||
(r"\\eta", " eta "), (r"\\kappa", " kappa "),
|
||
(r"\\nu", " nu "), (r"\\chi", " chi "),
|
||
(r"\\epsilon", " epsilon "), (r"\\partial", " partial "),
|
||
(r"\\nabla", " gradient "), (r"\\Delta", " delta "),
|
||
(r"\\Gamma", " gamma "), (r"\\infty", " infinity "),
|
||
(r"\\sum", " sum "), (r"\\prod", " product "),
|
||
(r"\\int", " integral "), (r"\$", " "),
|
||
]:
|
||
t = re.sub(sym, eng, t)
|
||
t = re.sub(r"\\[A-Za-z]+", " ", t)
|
||
# 2. Matrix-backed Unicode symbol → English substitution (non-ASCII only:
|
||
# Greek, arrows, operators, relations — not ASCII letters or common math ops)
|
||
role_map = _load_symbol_role_map()
|
||
for ch, (eng, role) in role_map.items():
|
||
if ch in t and ord(ch) > 127:
|
||
t = t.replace(ch, f" {eng} ")
|
||
t = re.sub(r"[{}()\[\]^_=+\-*/|<>~]", " ", t)
|
||
t = re.sub(r"[0-9]+", " ", t)
|
||
tokens = re.findall(r"[a-z][a-z-]{2,}", t)
|
||
freq: dict[str, float] = defaultdict(float)
|
||
for tok in tokens:
|
||
if tok not in STOPWORDS:
|
||
freq[tok] += 1.0
|
||
t_lower = text.lower()
|
||
for kw, boost in NT_KEYWORDS.items():
|
||
if kw in t_lower:
|
||
freq[kw.replace(" ", "_")] += boost
|
||
return dict(freq)
|
||
|
||
def role_histogram(text: str) -> dict[str, int]:
|
||
"""Compute a 15-dim role-count vector from math_symbols_v1 CHAR_INFO taxonomy.
|
||
Roles: greek_letter, nary_operator, relation, arrow, binary_op, accent,
|
||
delimiter_open, delimiter_close, ordinary, symbol, math_letter, letter,
|
||
operator, punctuation, delimiter.
|
||
Replaces the all-zeros foundation_vector in build_unified_forest.py.
|
||
"""
|
||
if not text:
|
||
return {}
|
||
role_map = _load_symbol_role_map()
|
||
hist: dict[str, int] = {}
|
||
for ch, (eng, role) in role_map.items():
|
||
if ch in text:
|
||
# Apply the same ASCII letter fix: A-Z, a-z are math_letter not symbol
|
||
fixed_role = role
|
||
if ch.isalpha() and ord(ch) < 128:
|
||
fixed_role = "math_letter"
|
||
hist[fixed_role] = hist.get(fixed_role, 0) + text.count(ch)
|
||
return hist
|
||
|
||
|
||
def search_papers(keywords: dict[str, float]) -> list[dict]:
|
||
if not keywords:
|
||
return []
|
||
sorted_kw = sorted(keywords.items(), key=lambda x: -x[1])[:6]
|
||
terms = [kw.replace("_", " ") for kw, _ in sorted_kw]
|
||
|
||
results = []
|
||
for t in terms:
|
||
sql = (
|
||
f"SELECT paper_id, title, substring(abstract, 1, 200) "
|
||
f"FROM arxiv_papers "
|
||
f"WHERE title ILIKE '%{t}%' OR abstract ILIKE '%{t}%' "
|
||
f"LIMIT 3"
|
||
)
|
||
rows = ssh_query(sql)
|
||
for r in rows:
|
||
if len(r) >= 2:
|
||
pid = r[0]
|
||
score = 0
|
||
for kw, w in sorted_kw:
|
||
kw_s = kw.replace("_", " ")
|
||
if kw_s in r[1].lower():
|
||
score += w * 3
|
||
elif len(r) > 2 and kw_s in r[2].lower():
|
||
score += w
|
||
results.append({
|
||
"paper_id": pid,
|
||
"title": r[1],
|
||
"abstract_snippet": r[2] if len(r) > 2 else "",
|
||
"score": score,
|
||
})
|
||
|
||
best: dict[str, dict] = {}
|
||
for r in results:
|
||
if r["paper_id"] not in best or r["score"] > best[r["paper_id"]]["score"]:
|
||
best[r["paper_id"]] = r
|
||
out = sorted(best.values(), key=lambda x: -x["score"])
|
||
return out[:3]
|
||
|
||
def detect_combinatorics_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Structural pattern matching against combinatorics taxonomy."""
|
||
combined = (name + " " + eq_text).lower()
|
||
matches = []
|
||
for pattern, subfield, paper_ids in COMBINATORICS_KERNEL:
|
||
if re.search(pattern, combined, re.IGNORECASE):
|
||
if not paper_ids:
|
||
continue
|
||
verified = []
|
||
for pid in paper_ids:
|
||
rows = ssh_query(
|
||
f"SELECT paper_id, title, substring(abstract, 1, 200) "
|
||
f"FROM arxiv_papers WHERE paper_id = '{pid}'"
|
||
)
|
||
if rows and len(rows[0]) >= 2:
|
||
r = rows[0]
|
||
verified.append({
|
||
"paper_id": r[0],
|
||
"title": r[1],
|
||
"abstract_snippet": r[2] if len(r) > 2 and r[2] != "None" else "",
|
||
"score": 5,
|
||
"match_type": subfield,
|
||
})
|
||
matches.extend(verified)
|
||
return matches
|
||
|
||
|
||
def detect_diophantine_type(name: str, eq_text: str) -> list[dict]:
|
||
"""Structural pattern matching against Diophantine equation taxonomy."""
|
||
combined = (name + " " + eq_text).lower()
|
||
matches = []
|
||
for pattern, eq_type, paper_ids in DIOPHANTINE_KERNEL:
|
||
if re.search(pattern, combined, re.IGNORECASE):
|
||
if not paper_ids:
|
||
continue
|
||
# Verify the papers actually exist in our DB
|
||
verified = []
|
||
for pid in paper_ids:
|
||
rows = ssh_query(
|
||
f"SELECT paper_id, title, substring(abstract, 1, 200) "
|
||
f"FROM arxiv_papers WHERE paper_id = '{pid}'"
|
||
)
|
||
if rows and len(rows[0]) >= 2:
|
||
r = rows[0]
|
||
verified.append({
|
||
"paper_id": r[0],
|
||
"title": r[1],
|
||
"abstract_snippet": r[2] if len(r) > 2 and r[2] != "None" else "",
|
||
"score": 5, # structural match = high confidence
|
||
"match_type": eq_type,
|
||
})
|
||
matches.extend(verified)
|
||
return matches
|
||
|
||
def main():
|
||
print("RRC arXiv Kernel Refinement", file=sys.stderr)
|
||
d = json.loads(RECEIPT_PATH.read_text())
|
||
eqs = d["compiled_equations"]
|
||
total = len(eqs)
|
||
n_matched_before = sum(1 for e in eqs if e["equation_record"].get("arxiv_paper_id"))
|
||
|
||
new_reconstruction = 0
|
||
new_wannier = 0
|
||
new_generic = 0
|
||
new_diophantine = 0
|
||
new_combinatorics = 0
|
||
new_dataset = 0
|
||
new_obscure = 0
|
||
new_sidon = 0
|
||
new_geometry = 0
|
||
|
||
for e in eqs:
|
||
rec = e["equation_record"]
|
||
name = rec.get("name", "")
|
||
eq_text = rec.get("equation", "")
|
||
route_hint = rec.get("route_hint", "")
|
||
|
||
# --- Stage 0: Reconstruction kernel (graph theory sector) ---
|
||
recon_matches = detect_reconstruction_type(name, eq_text)
|
||
if recon_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = recon_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_type"] = "reconstruction_conjecture"
|
||
rec["arxiv_match_stage"] = "kernel_refine_v0"
|
||
new_reconstruction += 1
|
||
print(f" RECON: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{' (was ' + existing + ')' if existing else ''}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1a: Combinatorics kernel (structural matching) ---
|
||
comb_matches = detect_combinatorics_type(name, eq_text)
|
||
if comb_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = comb_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_comb_paper_ids"] = [m["paper_id"] for m in comb_matches]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_abstract"] = best["abstract_snippet"]
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v3"
|
||
new_combinatorics += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" COMB: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1b: Dataset kernel (domain/theorem/webmath matching) ---
|
||
ds_matches = detect_dataset_type(name, eq_text)
|
||
if ds_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = ds_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_abstract"] = best.get("abstract_snippet", "")
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
if "domain_path" in best:
|
||
rec["arxiv_domain_path"] = best["domain_path"]
|
||
rec["arxiv_domain_count"] = best["domain_count"]
|
||
rec["arxiv_domain_solve_rate"] = best["domain_solve_rate"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v4"
|
||
new_dataset += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" DS: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1c: Obscure math kernel (28 niche subfields) ---
|
||
ob_matches = detect_obscure_type(name, eq_text)
|
||
if ob_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = ob_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_obscure_type"] = best["match_type"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v5"
|
||
new_obscure += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" OB: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1d: Sidon generation kernel (359 entries) ---
|
||
sd_matches = detect_sidon_type(name, eq_text)
|
||
if sd_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = sd_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v6"
|
||
new_sidon += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" SD: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1e: Wannier Hamiltonian kernel (21 materials) ---
|
||
wn_matches = detect_wannier_type(name, eq_text)
|
||
if wn_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = wn_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v7"
|
||
new_wannier += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" WN: {name:35s} [{best['match_type']:28s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1f: Number theory kernel (structural matching) ---
|
||
nt_matches = detect_diophantine_type(name, eq_text)
|
||
if nt_matches:
|
||
# Always record NT matches; they are high-confidence structural links
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = nt_matches[0]
|
||
# Store supplemental even if already matched
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_nt_paper_ids"] = [m["paper_id"] for m in nt_matches]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_abstract"] = best["abstract_snippet"]
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v2"
|
||
new_diophantine += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" NT: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 1f: Geometry/topology kernel (pure-local, no DB) ---
|
||
geo_matches = detect_geometry_type(name, eq_text)
|
||
if geo_matches:
|
||
existing = rec.get("arxiv_paper_id")
|
||
best = geo_matches[0]
|
||
rec["arxiv_paper_id_primary"] = rec.get("arxiv_paper_id", best["paper_id"])
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_abstract"] = best.get("abstract_snippet", "")
|
||
rec["arxiv_match_type"] = best["match_type"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v7"
|
||
new_geometry += 1
|
||
note = f" (was {existing})" if existing else ""
|
||
print(f" GEO: {name:35s} [{best['match_type']:25s}] → {best['paper_id']}{note}", file=sys.stderr)
|
||
continue
|
||
|
||
# --- Stage 2: Generic keyword matching (only if unmatched) ---
|
||
if not rec.get("arxiv_paper_id"):
|
||
kw = extract_keywords(name + " " + eq_text + " " + route_hint)
|
||
if route_hint == "compression_route":
|
||
kw["compression"] = 3.0; kw["entropy"] = 2.0; kw["encoding"] = 2.0
|
||
elif route_hint in ("thermodynamic_energy", "magnetic_signal"):
|
||
kw["energy"] = 3.0; kw["thermodynamic"] = 2.0
|
||
elif route_hint == "geometry_topology":
|
||
kw["geometry"] = 3.0; kw["manifold"] = 2.0; kw["topology"] = 2.0
|
||
elif route_hint in ("cognitive_load", "control_signal"):
|
||
kw["signal"] = 2.0; kw["cognitive"] = 2.0
|
||
|
||
results = search_papers(kw)
|
||
if results and results[0]["score"] >= 3:
|
||
best = results[0]
|
||
rec["arxiv_paper_id"] = best["paper_id"]
|
||
rec["arxiv_match_count"] = best["score"]
|
||
rec["arxiv_match_title"] = best["title"][:200]
|
||
rec["arxiv_match_abstract"] = best["abstract_snippet"]
|
||
rec["arxiv_match_stage"] = "kernel_refine_v1"
|
||
new_generic += 1
|
||
print(f" KW: {name:35s} → {best['paper_id']} ({best['score']})", file=sys.stderr)
|
||
|
||
RECEIPT_PATH.write_text(json.dumps(d, indent=2, ensure_ascii=False))
|
||
n_matched_after = sum(1 for e in eqs if e["equation_record"].get("arxiv_paper_id"))
|
||
print(f"\nReconstruction kernel matches: {new_reconstruction}", file=sys.stderr)
|
||
print(f"Wannier Hamiltonian kernel matches: {new_wannier}", file=sys.stderr)
|
||
print(f"Geometry/topology kernel matches: {new_geometry}", file=sys.stderr)
|
||
print(f"Sidon generation kernel matches: {new_sidon}", file=sys.stderr)
|
||
print(f"Obscure math kernel matches: {new_obscure}", file=sys.stderr)
|
||
print(f"Dataset kernel matches: {new_dataset}", file=sys.stderr)
|
||
print(f"Combinatorics kernel matches: {new_combinatorics}", file=sys.stderr)
|
||
print(f"NT kernel matches: {new_diophantine}", file=sys.stderr)
|
||
print(f"Generic keyword matches: {new_generic}", file=sys.stderr)
|
||
print(f"Before: {n_matched_before}/{total}", file=sys.stderr)
|
||
print(f"After: {n_matched_after}/{total}", file=sys.stderr)
|
||
|
||
if __name__ == "__main__":
|
||
main()
|