mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
allaun
2b3d255a40
Pre-built keyword index on /shm (272 MB, 111k keywords, 700k papers). Runs in 39 seconds vs 10+ minutes timeout for SQL-based approach. Usage: # Build index (one-time, ~2 min) ssh neon-64gb 'python3 /tmp/build_keyword_index.py' # Run matcher (39 seconds) ssh neon-64gb 'python3 /tmp/jaccard_fast.py' /shm setup: - tmpfs mount (32 GB), persistent in fstab - /shm/arxiv_texts.tsv (559 MB) — raw paper texts - /shm/keyword_index.json (272 MB) — inverted index
83 lines
2.6 KiB
Python
83 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Fast Jaccard matcher using pre-built keyword index on /shm.
|
|
Loads the index once, then does instant lookups per concept.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
INDEX_PATH = "/shm/keyword_index.json"
|
|
CONCEPTS_PATH = "/tmp/cornfield_concepts.json"
|
|
TOP_K = 5
|
|
MIN_COVERAGE = 0.15
|
|
MIN_KEYWORD_LEN = 6
|
|
|
|
def extract_keywords(concept):
|
|
words = set()
|
|
for tag in concept.get("tags", []):
|
|
for w in re.split(r'[_\s\-/]+', tag):
|
|
w = w.strip().lower()
|
|
if len(w) >= MIN_KEYWORD_LEN:
|
|
words.add(w)
|
|
name = concept.get("name", "")
|
|
for w in re.sub(r'([a-z])([A-Z])', r'\1 \2', name).split():
|
|
w = w.strip().lower()
|
|
if len(w) >= MIN_KEYWORD_LEN:
|
|
words.add(w)
|
|
for field in ["description", "novelty_statement"]:
|
|
for w in re.findall(r'[a-zA-Z]{6,}', concept.get(field, "")):
|
|
words.add(w.lower())
|
|
return words
|
|
|
|
def main():
|
|
print("Loading keyword index...")
|
|
index = json.loads(Path(INDEX_PATH).read_text())
|
|
print(f"Loaded {len(index)} keywords")
|
|
|
|
data = json.loads(Path(CONCEPTS_PATH).read_text())
|
|
concepts = [c for c in data["concepts"] if c.get("id", "").startswith("cf_")]
|
|
|
|
results = []
|
|
for concept in concepts:
|
|
cid = concept["id"]
|
|
kw = extract_keywords(concept)
|
|
if len(kw) < 2:
|
|
continue
|
|
|
|
# Find papers matching any keyword
|
|
candidates = defaultdict(set)
|
|
for word in kw:
|
|
for pid in index.get(word, []):
|
|
candidates[pid].add(word)
|
|
|
|
scored = []
|
|
for pid, matched in candidates.items():
|
|
cov = len(matched) / len(kw)
|
|
if cov >= MIN_COVERAGE:
|
|
scored.append((pid, cov, matched))
|
|
scored.sort(key=lambda x: x[1], reverse=True)
|
|
top = scored[:TOP_K]
|
|
|
|
if top:
|
|
print(f" {cid}: {len(candidates)} candidates, top={top[0][1]:.3f}")
|
|
results.append({
|
|
"concept_id": cid,
|
|
"keywords": sorted(kw),
|
|
"top_matches": [
|
|
{"paper_id": pid, "coverage": round(cov, 4), "matched_keywords": sorted(mk)}
|
|
for pid, cov, mk in top
|
|
]
|
|
})
|
|
else:
|
|
print(f" {cid}: no matches")
|
|
|
|
out = Path("/tmp/jaccard_matches_fast.json")
|
|
out.write_text(json.dumps(results, indent=2))
|
|
total = sum(len(r["top_matches"]) for r in results)
|
|
print(f"\nDone: {len(results)} concepts, {total} citations → {out}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|