#!/usr/bin/env python3
"""
Fast Jaccard matcher using pre-built keyword index on /shm.
Loads the index once, then does instant lookups per concept.
"""

import json
import re
from collections import defaultdict
from pathlib import Path

INDEX_PATH = "/shm/keyword_index.json"
CONCEPTS_PATH = "/tmp/cornfield_concepts.json"
TOP_K = 5
MIN_COVERAGE = 0.15
MIN_KEYWORD_LEN = 6

def extract_keywords(concept):
    words = set()
    for tag in concept.get("tags", []):
        for w in re.split(r'[_\s\-/]+', tag):
            w = w.strip().lower()
            if len(w) >= MIN_KEYWORD_LEN:
                words.add(w)
    name = concept.get("name", "")
    for w in re.sub(r'([a-z])([A-Z])', r'\1 \2', name).split():
        w = w.strip().lower()
        if len(w) >= MIN_KEYWORD_LEN:
            words.add(w)
    for field in ["description", "novelty_statement"]:
        for w in re.findall(r'[a-zA-Z]{6,}', concept.get(field, "")):
            words.add(w.lower())
    return words

def main():
    print("Loading keyword index...")
    index = json.loads(Path(INDEX_PATH).read_text())
    print(f"Loaded {len(index)} keywords")

    data = json.loads(Path(CONCEPTS_PATH).read_text())
    concepts = [c for c in data["concepts"] if c.get("id", "").startswith("cf_")]
    
    results = []
    for concept in concepts:
        cid = concept["id"]
        kw = extract_keywords(concept)
        if len(kw) < 2:
            continue

        # Find papers matching any keyword
        candidates = defaultdict(set)
        for word in kw:
            for pid in index.get(word, []):
                candidates[pid].add(word)

        scored = []
        for pid, matched in candidates.items():
            cov = len(matched) / len(kw)
            if cov >= MIN_COVERAGE:
                scored.append((pid, cov, matched))
        scored.sort(key=lambda x: x[1], reverse=True)
        top = scored[:TOP_K]

        if top:
            print(f"  {cid}: {len(candidates)} candidates, top={top[0][1]:.3f}")
            results.append({
                "concept_id": cid,
                "keywords": sorted(kw),
                "top_matches": [
                    {"paper_id": pid, "coverage": round(cov, 4), "matched_keywords": sorted(mk)}
                    for pid, cov, mk in top
                ]
            })
        else:
            print(f"  {cid}: no matches")

    out = Path("/tmp/jaccard_matches_fast.json")
    out.write_text(json.dumps(results, indent=2))
    total = sum(len(r["top_matches"]) for r in results)
    print(f"\nDone: {len(results)} concepts, {total} citations → {out}")

if __name__ == "__main__":
    main()