From 2b3d255a4025f9ac5ff694192e11504c20967faf Mon Sep 17 00:00:00 2001
From: allaun <allaun@researchstack.info>
Date: Sun, 21 Jun 2026 18:56:30 -0500
Subject: [PATCH] feat(infra): add fast Jaccard matcher using /shm keyword
 index
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pre-built keyword index on /shm (272 MB, 111k keywords, 700k papers).
Runs in 39 seconds vs 10+ minutes timeout for SQL-based approach.

Usage:
  # Build index (one-time, ~2 min)
  ssh neon-64gb 'python3 /tmp/build_keyword_index.py'

  # Run matcher (39 seconds)
  ssh neon-64gb 'python3 /tmp/jaccard_fast.py'

/shm setup:
  - tmpfs mount (32 GB), persistent in fstab
  - /shm/arxiv_texts.tsv (559 MB) — raw paper texts
  - /shm/keyword_index.json (272 MB) — inverted index
---
 4-Infrastructure/shim/jaccard_fast.py | 83 +++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 4-Infrastructure/shim/jaccard_fast.py

diff --git a/4-Infrastructure/shim/jaccard_fast.py b/4-Infrastructure/shim/jaccard_fast.py
new file mode 100644
index 00000000..fdd31c74
--- /dev/null
+++ b/4-Infrastructure/shim/jaccard_fast.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+Fast Jaccard matcher using pre-built keyword index on /shm.
+Loads the index once, then does instant lookups per concept.
+"""
+
+import json
+import re
+from collections import defaultdict
+from pathlib import Path
+
+INDEX_PATH = "/shm/keyword_index.json"
+CONCEPTS_PATH = "/tmp/cornfield_concepts.json"
+TOP_K = 5
+MIN_COVERAGE = 0.15
+MIN_KEYWORD_LEN = 6
+
+def extract_keywords(concept):
+    words = set()
+    for tag in concept.get("tags", []):
+        for w in re.split(r'[_\s\-/]+', tag):
+            w = w.strip().lower()
+            if len(w) >= MIN_KEYWORD_LEN:
+                words.add(w)
+    name = concept.get("name", "")
+    for w in re.sub(r'([a-z])([A-Z])', r'\1 \2', name).split():
+        w = w.strip().lower()
+        if len(w) >= MIN_KEYWORD_LEN:
+            words.add(w)
+    for field in ["description", "novelty_statement"]:
+        for w in re.findall(r'[a-zA-Z]{6,}', concept.get(field, "")):
+            words.add(w.lower())
+    return words
+
+def main():
+    print("Loading keyword index...")
+    index = json.loads(Path(INDEX_PATH).read_text())
+    print(f"Loaded {len(index)} keywords")
+
+    data = json.loads(Path(CONCEPTS_PATH).read_text())
+    concepts = [c for c in data["concepts"] if c.get("id", "").startswith("cf_")]
+    
+    results = []
+    for concept in concepts:
+        cid = concept["id"]
+        kw = extract_keywords(concept)
+        if len(kw) < 2:
+            continue
+
+        # Find papers matching any keyword
+        candidates = defaultdict(set)
+        for word in kw:
+            for pid in index.get(word, []):
+                candidates[pid].add(word)
+
+        scored = []
+        for pid, matched in candidates.items():
+            cov = len(matched) / len(kw)
+            if cov >= MIN_COVERAGE:
+                scored.append((pid, cov, matched))
+        scored.sort(key=lambda x: x[1], reverse=True)
+        top = scored[:TOP_K]
+
+        if top:
+            print(f"  {cid}: {len(candidates)} candidates, top={top[0][1]:.3f}")
+            results.append({
+                "concept_id": cid,
+                "keywords": sorted(kw),
+                "top_matches": [
+                    {"paper_id": pid, "coverage": round(cov, 4), "matched_keywords": sorted(mk)}
+                    for pid, cov, mk in top
+                ]
+            })
+        else:
+            print(f"  {cid}: no matches")
+
+    out = Path("/tmp/jaccard_matches_fast.json")
+    out.write_text(json.dumps(results, indent=2))
+    total = sum(len(r["top_matches"]) for r in results)
+    print(f"\nDone: {len(results)} concepts, {total} citations → {out}")
+
+if __name__ == "__main__":
+    main()