From 5a8179fe384acf5d191a4e93aa34b2f4bfcd61b8 Mon Sep 17 00:00:00 2001
From: allaun <allaun@researchstack.info>
Date: Sun, 21 Jun 2026 18:39:59 -0500
Subject: [PATCH] feat(infra): add Jaccard matcher for arxiv paper discovery

Finds related arxiv papers for cornfield concepts via keyword overlap.
Uses /shm/arxiv_texts.tsv (559 MB) for fast local matching.
pg_trgm GIN indexes on neon-64gb for Postgres queries.

Results: 240 candidate citations for 48 concepts.
Loaded into concept_citations table (relation_type='candidate').
---
 4-Infrastructure/shim/jaccard_matcher.py | 116 +++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 4-Infrastructure/shim/jaccard_matcher.py

diff --git a/4-Infrastructure/shim/jaccard_matcher.py b/4-Infrastructure/shim/jaccard_matcher.py
new file mode 100644
index 00000000..bed68b6f
--- /dev/null
+++ b/4-Infrastructure/shim/jaccard_matcher.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Jaccard matcher v6 — file-based with keyword filtering.
+Uses /shm/arxiv_texts.tsv for fast local matching.
+Only uses keywords >= 6 chars to reduce noise.
+"""
+
+import json
+import re
+from collections import defaultdict
+from pathlib import Path
+
+CONCEPTS_PATH = "/tmp/cornfield_concepts.json"
+ARXIV_TEXTS_PATH = "/shm/arxiv_texts.tsv"
+TOP_K = 5
+MIN_COVERAGE = 0.15
+MIN_KEYWORD_LEN = 6  # Only use keywords >= 6 chars
+
+
+def extract_keywords(concept: dict) -> set[str]:
+    words = set()
+    for tag in concept.get("tags", []):
+        for w in re.split(r'[_\s\-/]+', tag):
+            w = w.strip().lower()
+            if len(w) >= MIN_KEYWORD_LEN:
+                words.add(w)
+    name = concept.get("name", "")
+    for w in re.sub(r'([a-z])([A-Z])', r'\1 \2', name).split():
+        w = w.strip().lower()
+        if len(w) >= MIN_KEYWORD_LEN:
+            words.add(w)
+    desc = concept.get("description", "")
+    for w in re.findall(r'[a-zA-Z]{6,}', desc):
+        words.add(w.lower())
+    novelty = concept.get("novelty_statement", "")
+    for w in re.findall(r'[a-zA-Z]{6,}', novelty):
+        words.add(w.lower())
+    return words
+
+
+def main():
+    data = json.loads(Path(CONCEPTS_PATH).read_text())
+    concepts = [c for c in data["concepts"] if c.get("id", "").startswith("cf_")]
+    print(f"Processing {len(concepts)} cornfield concepts")
+
+    # Extract keywords for all concepts
+    concept_keywords = {}
+    for c in concepts:
+        kw = extract_keywords(c)
+        if len(kw) >= 2:
+            concept_keywords[c["id"]] = kw
+    print(f"Keywords extracted for {len(concept_keywords)} concepts (min len {MIN_KEYWORD_LEN})")
+
+    # Collect all unique keywords
+    all_keywords = set()
+    for kw_set in concept_keywords.values():
+        all_keywords.update(kw_set)
+    print(f"Total unique keywords: {len(all_keywords)}")
+
+    # Phase 1: Build inverted index
+    print("Phase 1: Building inverted index...")
+    inverted = defaultdict(set)
+    count = 0
+    with open(ARXIV_TEXTS_PATH) as f:
+        for line in f:
+            parts = line.split('\t', 1)
+            if len(parts) < 2:
+                continue
+            pid, text = parts[0], parts[1].lower()
+            count += 1
+            if count % 200000 == 0:
+                print(f"  {count} papers scanned...")
+            for kw in all_keywords:
+                if kw in text:
+                    inverted[kw].add(pid)
+    print(f"  Done: {count} papers, {len(inverted)} keywords with hits")
+
+    # Phase 2: Match concepts
+    print("Phase 2: Matching concepts...")
+    results = []
+    for cid, kw_set in concept_keywords.items():
+        candidates = defaultdict(set)
+        for kw in kw_set:
+            for pid in inverted.get(kw, set()):
+                candidates[pid].add(kw)
+
+        scored = []
+        for pid, matched in candidates.items():
+            cov = len(matched) / len(kw_set)
+            if cov >= MIN_COVERAGE:
+                scored.append((pid, cov, matched))
+        scored.sort(key=lambda x: x[1], reverse=True)
+        top = scored[:TOP_K]
+
+        if top:
+            print(f"  {cid}: {len(candidates)} candidates, top={top[0][1]:.3f}")
+            results.append({
+                "concept_id": cid,
+                "keywords": sorted(kw_set),
+                "candidates_found": len(candidates),
+                "top_matches": [
+                    {"paper_id": pid, "coverage": round(cov, 4), "matched_keywords": sorted(mk)}
+                    for pid, cov, mk in top
+                ]
+            })
+        else:
+            print(f"  {cid}: no matches above {MIN_COVERAGE}")
+
+    out = Path("/tmp/jaccard_matches.json")
+    out.write_text(json.dumps(results, indent=2))
+    total = sum(len(r["top_matches"]) for r in results)
+    print(f"\nDone: {len(results)} concepts, {total} citations → {out}")
+
+
+if __name__ == "__main__":
+    main()