mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
allaun
7b0c0a363b
Add exploratory Python shims for: - OpenAlex citation crawling - Semantic Scholar citation crawling - PageRank eigenvalue survey - SNAP PIST spectral cross-validation - Fiedler vector validation
444 lines
20 KiB
Python
444 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
snap_pist_spectral_crossval — Cross-validate SNAP citation graph through PIST
|
||
matrix projection vs spectral survey.
|
||
|
||
Pipeline:
|
||
1. Load SNAP edge list → NetworkX DiGraph
|
||
2. (Optionally) sample subgraph for spectral analysis
|
||
3. Serialize each paper's citation neighborhood as text
|
||
4. Run PIST token→strand 8×8 matrix projection
|
||
5. Run spectral survey (PageRank, HITS, Fiedler)
|
||
6. Cross-tabulate PIST matrix hashes vs spectral bin assignments
|
||
7. Emit comparison receipt JSON
|
||
|
||
Claim boundary: cross-validation comparison only; no classifier output.
|
||
"""
|
||
|
||
import gzip
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
import random
|
||
import re
|
||
import sys
|
||
from collections import defaultdict, Counter
|
||
|
||
import networkx as nx
|
||
import numpy as np
|
||
|
||
N_STRANDS = 8
|
||
random.seed(42)
|
||
np.random.seed(42)
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §1 SNAP PARSING
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def load_snap_edgelist(path: str) -> nx.DiGraph:
|
||
g = nx.DiGraph()
|
||
with gzip.open(path, "rt") if path.endswith(".gz") else open(path) as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 2:
|
||
continue
|
||
g.add_edge(int(parts[0]), int(parts[1]))
|
||
return g
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §2 CITATION CONTEXT TEXT + PIST TOKENIZER
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def tokenize(text: str) -> list[str]:
|
||
"""PIST-compatible tokenizer: split on space, _, and : (added space for citation IDs)."""
|
||
normalized = text.replace("-", "_")
|
||
return [p for p in re.split(r"[ _:]", normalized) if p]
|
||
|
||
|
||
def node_context_text(g: nx.DiGraph, node: int) -> str:
|
||
"""Serialize a paper's citation neighborhood as space-separated text.
|
||
|
||
Format: paper_id cited_id1 cited_id2 ... cited_by_id1 cited_by_id2 ...
|
||
|
||
After space-splitting tokenization, adjacent paper IDs become bigram
|
||
pairs → mapped to 8×8 strand adjacency in PIST space.
|
||
"""
|
||
parts = [str(node)]
|
||
parts.extend(str(v) for v in g.successors(node))
|
||
parts.extend(str(v) for v in g.predecessors(node))
|
||
return " ".join(parts)
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §3 PIST 8×8 ADJACENCY MATRIX
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def build_global_vocabulary(all_texts: list[str]) -> dict[str, int]:
|
||
all_tokens = set()
|
||
for text in all_texts:
|
||
for t in tokenize(text):
|
||
all_tokens.add(t)
|
||
return {t: i for i, t in enumerate(sorted(all_tokens))}
|
||
|
||
|
||
def strand_for_token(token: str, vocab: dict[str, int]) -> int:
|
||
return vocab[token] % N_STRANDS
|
||
|
||
|
||
def build_adjacency_matrix(tokens: list[str], vocab: dict[str, int]) -> list[list[int]]:
|
||
M = [[0] * N_STRANDS for _ in range(N_STRANDS)]
|
||
strands = [strand_for_token(t, vocab) for t in tokens]
|
||
for i in range(len(strands) - 1):
|
||
s1, s2 = strands[i], strands[i + 1]
|
||
M[s1][s2] += 1
|
||
return M
|
||
|
||
|
||
def matrix_hash(M: list[list[int]]) -> str:
|
||
return hashlib.sha256(
|
||
json.dumps(M, separators=(",", ":")).encode("utf-8")
|
||
).hexdigest()
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §4 SPECTRAL BIN ASSIGNMENT
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def classify_pagerank_bin(score: float, max_score: float) -> str:
|
||
if score >= 0.1 * max_score: return "authority"
|
||
elif score >= 0.01 * max_score: return "secondary"
|
||
return "noise"
|
||
|
||
|
||
def classify_hits_bin(auth: float, max_auth: float, hub: float, max_hub: float) -> str:
|
||
auth_norm = auth / max_auth if max_auth else 0
|
||
hub_norm = hub / max_hub if max_hub else 0
|
||
if auth_norm > 0.1 and auth_norm > hub_norm: return "authority"
|
||
elif hub_norm > 0.1: return "hub"
|
||
elif auth_norm > 0.01 or hub_norm > 0.01: return "secondary"
|
||
return "noise"
|
||
|
||
|
||
def classify_fiedler_bin(val: float) -> str:
|
||
return "community_A" if val < 0 else "community_B"
|
||
|
||
|
||
def spectral_bin_summary(pr_bin: str, hits_bin: str) -> str:
|
||
return f"{pr_bin}|{hits_bin}"
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §5 FIEDLER ON DEGREE-WEIGHTED SUBSAMPLE + PAGERANK-KNN EXTRAPOLATION
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def compute_fiedler_full(g: nx.DiGraph) -> tuple[dict[int, str], dict[int, float], dict]:
|
||
"""Compute Fiedler vector on a random 2-hop sampled subgraph,
|
||
then extend to all nodes via PageRank-kNN extrapolation.
|
||
|
||
Strategy:
|
||
1. Pick 500 random seeds, expand 2 hops for connectivity
|
||
2. Compute Fiedler via small eigsh (which='SM')
|
||
3. Extrapolate to all 34K nodes via kNN in PageRank space
|
||
(PageRank is an eigenvector centrality that preserves spectral
|
||
ordering; random sampling avoids degree bias in the computed set.)
|
||
"""
|
||
from scipy.sparse.linalg import eigsh
|
||
|
||
nodes = sorted(g.nodes())
|
||
ug = g.to_undirected()
|
||
n_total = len(nodes)
|
||
|
||
# 1. Random 2-hop sampling (representative, no degree bias)
|
||
seeds = random.sample(nodes, min(500, n_total))
|
||
sample_set: set[int] = set(seeds)
|
||
for s in seeds:
|
||
sample_set.update(g.successors(s))
|
||
sample_set.update(g.predecessors(s))
|
||
for s2 in list(g.successors(s)) + list(g.predecessors(s)):
|
||
sample_set.update(g.successors(s2))
|
||
sample_set.update(g.predecessors(s2))
|
||
sample = sorted(sample_set & set(nodes))
|
||
print(f" Fiedler: {len(seeds)} seeds, 2-hop → {len(sample)} nodes...", flush=True)
|
||
|
||
# Build undirected subgraph, take largest connected component
|
||
ug_s = g.subgraph(sample).to_undirected()
|
||
cc = max(nx.connected_components(ug_s), key=len)
|
||
ug_s = ug_s.subgraph(cc)
|
||
cc_nodes = sorted(cc)
|
||
print(f" Fiedler component: {len(cc_nodes)} nodes, "
|
||
f"{ug_s.number_of_edges()} edges", flush=True)
|
||
|
||
# 2. Compute Fiedler on the subgraph
|
||
L_s = nx.laplacian_matrix(ug_s, nodelist=cc_nodes).astype(float)
|
||
v0_s = np.array([ug_s.degree(n) for n in cc_nodes], dtype=float)
|
||
v0_s = v0_s / np.linalg.norm(v0_s)
|
||
|
||
vals, vecs = eigsh(L_s, k=2, which="SM", maxiter=10000, tol=1e-4, v0=v0_s)
|
||
order = np.argsort(vals)
|
||
fiedler_val = vals[order[1]]
|
||
fiedler_vec_real = vecs[:, order[1]]
|
||
sample_f = {cc_nodes[i]: float(fiedler_vec_real[i]) for i in range(len(cc_nodes))}
|
||
print(f" Fiedler eigenvalue: {fiedler_val:.6f} "
|
||
f"zero eig: {vals[order[0]]:.6e}", flush=True)
|
||
|
||
# 3. Extend to all nodes via PageRank-kNN
|
||
pr = nx.pagerank(g, alpha=0.85, max_iter=300)
|
||
pr_vals = {n: pr[n] for n in nodes}
|
||
sample_keys = set(sample_f.keys())
|
||
|
||
fiedler_bins: dict[int, str] = {}
|
||
fiedler_values: dict[int, float] = {}
|
||
k = 5
|
||
for n in nodes:
|
||
if n in sample_f:
|
||
fv = sample_f[n]
|
||
else:
|
||
nearest = sorted(sample_keys,
|
||
key=lambda sn: abs(pr_vals.get(n, 0.0) - pr_vals.get(sn, 0.0)))[:k]
|
||
signs = [1 if sample_f[sn] < 0 else 0 for sn in nearest]
|
||
majority = sum(signs) / len(signs) if nearest else 0.5
|
||
fv = -1.0 if majority > 0.5 else 1.0
|
||
fiedler_values[n] = fv
|
||
fiedler_bins[n] = classify_fiedler_bin(fv)
|
||
|
||
n_extrapolated = len(nodes) - len(sample_f)
|
||
print(f" Fiedler extension: {len(sample_f)} computed, "
|
||
f"{n_extrapolated} PageRank-kNN extrapolated", flush=True)
|
||
|
||
n_neg = sum(1 for v in sample_f.values() if v < 0)
|
||
n_pos = len(sample_f) - n_neg
|
||
collapse_report = {
|
||
"n_total": len(sample_f),
|
||
"n_neg": n_neg,
|
||
"n_pos": n_pos,
|
||
"collapse_ratio": max(n_neg, n_pos) / len(sample_f) if len(sample_f) else 0.0,
|
||
"var_sign": float(np.var([1 if v < 0 else 0 for v in sample_f.values()])),
|
||
"fiedler_eigenvalue": round(fiedler_val, 6),
|
||
"component_nodes": len(sample_f),
|
||
"total_nodes": len(nodes),
|
||
}
|
||
|
||
return fiedler_bins, fiedler_values, collapse_report
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §6 CROSS-TABULATION
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def cross_tabulate(
|
||
pist_hash_to_nodes: dict[str, list[int]],
|
||
node_spectral: dict[int, dict],
|
||
) -> dict:
|
||
tab = {}
|
||
for phash, nodes in pist_hash_to_nodes.items():
|
||
combined_bins = Counter()
|
||
pr_bins = Counter()
|
||
hits_bins = Counter()
|
||
for n in nodes:
|
||
s = node_spectral.get(n, {})
|
||
combined_bins[s.get("spectral_summary", "unknown")] += 1
|
||
pr_bins[s.get("pr_bin", "unknown")] += 1
|
||
hits_bins[s.get("hits_bin", "unknown")] += 1
|
||
total = len(nodes)
|
||
|
||
def _entropy(cnt: Counter) -> float:
|
||
if total == 0:
|
||
return 0.0
|
||
return -sum((c / total) * math.log2(c / total) for c in cnt.values() if c > 0)
|
||
|
||
tab[phash] = {
|
||
"count": total,
|
||
"pagerank_mode": pr_bins.most_common(1)[0][0] if pr_bins else None,
|
||
"hits_mode": hits_bins.most_common(1)[0][0] if hits_bins else None,
|
||
"spectral_entropy": round(_entropy(combined_bins), 4),
|
||
}
|
||
return tab
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
# §7 MAIN
|
||
# ═══════════════════════════════════════════════════════════════════════
|
||
|
||
def main():
|
||
path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/snap_data/cit-HepPh.txt.gz"
|
||
|
||
print(f"Loading {path}...", flush=True)
|
||
g = load_snap_edgelist(path)
|
||
print(f"Graph: {g.number_of_nodes()} nodes, {g.number_of_edges()} edges", flush=True)
|
||
|
||
nodes = sorted(g.nodes())
|
||
|
||
# ── 1. Serialize each paper as citation context text ───────────────
|
||
node_texts = {n: node_context_text(g, n) for n in nodes}
|
||
all_texts = list(node_texts.values())
|
||
|
||
# ── 2. Build PIST global vocabulary ────────────────────────────────
|
||
vocab = build_global_vocabulary(all_texts)
|
||
gvh = hashlib.sha256("|".join(vocab.keys()).encode("utf-8")).hexdigest()
|
||
print(f"Vocabulary: {len(vocab)} tokens hash={gvh}", flush=True)
|
||
|
||
# ── 3. Compute PIST 8×8 matrix per paper ───────────────────────────
|
||
node_pist = {}
|
||
for n in nodes:
|
||
tokens = tokenize(node_texts[n])
|
||
M = build_adjacency_matrix(tokens, vocab)
|
||
node_pist[n] = {"tokens": tokens, "matrix_8x8": M, "matrix_hash": matrix_hash(M)}
|
||
|
||
pist_hash_groups = defaultdict(list)
|
||
for n, info in node_pist.items():
|
||
pist_hash_groups[info["matrix_hash"]].append(n)
|
||
|
||
n_unique = len(pist_hash_groups)
|
||
print(f"Unique PIST matrix hashes: {n_unique} from {len(nodes)} papers", flush=True)
|
||
hash_sizes = [len(v) for v in pist_hash_groups.values()]
|
||
print(f" Hash group sizes: min={min(hash_sizes)} max={max(hash_sizes)} "
|
||
f"median={sorted(hash_sizes)[len(hash_sizes)//2]}", flush=True)
|
||
|
||
# Show a few sample matrices for verification
|
||
for n in nodes[:3]:
|
||
info = node_pist[n]
|
||
flat = [str(c) for row in info["matrix_8x8"] for c in row]
|
||
print(f" node={n} tokens={info['tokens'][:5]}... "
|
||
f"matrix_sum={sum(int(c) for c in flat)} "
|
||
f"nonzero={sum(1 for c in flat if int(c) > 0)}", flush=True)
|
||
|
||
# ── 4. Run spectral survey ─────────────────────────────────────────
|
||
print("PageRank...", flush=True)
|
||
pr = nx.pagerank(g, alpha=0.85, max_iter=300)
|
||
|
||
print("HITS...", flush=True)
|
||
try:
|
||
hubs, auths = nx.hits(g, max_iter=300, normalized=True)
|
||
except nx.PowerIterationFailedConvergence:
|
||
hubs = auths = {n: 1.0 / len(g) for n in g}
|
||
|
||
# Fiedler: diagnostic only (excluded from spectral scoring).
|
||
# Under k-hop projection, the Fiedler sign collapses to the empirical
|
||
# majority class (94.5% single-sign in our regime). See
|
||
# docs/fiedler_non_identifiability.md for the operator-theoretic
|
||
# statement of this non-identifiability condition.
|
||
print("Fiedler (diagnostic)...", flush=True)
|
||
fiedler_bins, fiedler_values, collapse_report = compute_fiedler_full(g)
|
||
collapse_ratio = (
|
||
max(collapse_report["n_neg"], collapse_report["n_pos"])
|
||
/ collapse_report["n_total"]
|
||
) if collapse_report["n_total"] else 0.0
|
||
print(f" Fiedler collapse ratio: {collapse_ratio:.4f} "
|
||
f"(Var(sign)) = {collapse_report['var_sign']:.4f})", flush=True)
|
||
|
||
max_pr = max(pr.values()) or 1.0
|
||
max_auth = max(auths.values()) or 1.0
|
||
max_hub = max(hubs.values()) or 1.0
|
||
|
||
node_spectral = {}
|
||
for n in nodes:
|
||
pr_b = classify_pagerank_bin(pr[n], max_pr)
|
||
hits_b = classify_hits_bin(auths.get(n, 0), max_auth, hubs.get(n, 0), max_hub)
|
||
node_spectral[n] = {
|
||
"pr": round(pr[n], 6), "pr_bin": pr_b,
|
||
"auth": round(auths.get(n, 0), 6),
|
||
"hub": round(hubs.get(n, 0), 6), "hits_bin": hits_b,
|
||
"fiedler": round(fiedler_values.get(n, 0.0), 6),
|
||
"fiedler_bin": fiedler_bins.get(n, "community_unknown"),
|
||
"spectral_summary": spectral_bin_summary(pr_b, hits_b),
|
||
}
|
||
|
||
spectral_bin_counts = Counter(v["spectral_summary"] for v in node_spectral.values())
|
||
print(f"Spectral combos (PR|HITS): {len(spectral_bin_counts)}", flush=True)
|
||
for label, count in spectral_bin_counts.most_common(10):
|
||
print(f" {label}: {count}", flush=True)
|
||
|
||
# ── 5. Cross-tabulate ──────────────────────────────────────────────
|
||
tab = cross_tabulate(pist_hash_groups, node_spectral)
|
||
pure_groups = sum(1 for v in tab.values() if v["spectral_entropy"] == 0.0)
|
||
mixed_groups = len(tab) - pure_groups
|
||
print(f"\nPIST hash groups pure spectral: {pure_groups}/{len(tab)} "
|
||
f"mixed: {mixed_groups}/{len(tab)}", flush=True)
|
||
|
||
# Mean entropy across all groups
|
||
mean_entropy = np.mean([v["spectral_entropy"] for v in tab.values()])
|
||
print(f"Mean spectral entropy across PIST groups: {mean_entropy:.4f}", flush=True)
|
||
|
||
# Diagnose mixing within multi-paper PIST groups.
|
||
pr_pure = 0
|
||
hits_pure = 0
|
||
for phash, nodes_p in pist_hash_groups.items():
|
||
if len(nodes_p) < 2:
|
||
continue
|
||
pr_bins = set(node_spectral[n]["pr_bin"] for n in nodes_p)
|
||
hits_bins = set(node_spectral[n]["hits_bin"] for n in nodes_p)
|
||
if len(pr_bins) == 1: pr_pure += 1
|
||
if len(hits_bins) == 1: hits_pure += 1
|
||
|
||
multi_paper_groups = sum(1 for v in pist_hash_groups.values() if len(v) > 1)
|
||
print(f"\nMulti-paper PIST groups: {multi_paper_groups}", flush=True)
|
||
print(f" PR-bin pure: {pr_pure}/{multi_paper_groups} "
|
||
f"({100*pr_pure/multi_paper_groups:.1f}%)", flush=True)
|
||
print(f" HITS-bin pure: {hits_pure}/{multi_paper_groups} "
|
||
f"({100*hits_pure/multi_paper_groups:.1f}%)", flush=True)
|
||
print(f" Fiedler: excluded from scoring "
|
||
f"(collapse_ratio={collapse_ratio:.4f}, "
|
||
f"Var(sign)={collapse_report['var_sign']:.4f})", flush=True)
|
||
|
||
# ── 6. Output ──────────────────────────────────────────────────────
|
||
predictions = []
|
||
for n in nodes:
|
||
info = node_pist.get(n, {})
|
||
spec = node_spectral.get(n, {})
|
||
predictions.append({
|
||
"node_id": n,
|
||
"matrix_hash": info.get("matrix_hash", ""),
|
||
"pagerank": spec.get("pr", 0), "pr_bin": spec.get("pr_bin", "unknown"),
|
||
"auth": spec.get("auth", 0), "hub": spec.get("hub", 0),
|
||
"hits_bin": spec.get("hits_bin", "unknown"),
|
||
"fiedler": spec.get("fiedler", 0), "fiedler_bin": spec.get("fiedler_bin", "unknown"),
|
||
"spectral_summary": spec.get("spectral_summary", "unknown"),
|
||
})
|
||
|
||
artifact = {
|
||
"schema": "snap_pist_spectral_crossval_v2",
|
||
"claim_boundary": "cross-validation-only;pr+hits-scoring;fiedler-diagnostic-only",
|
||
"source": os.path.basename(path),
|
||
"graph": {"nodes": g.number_of_nodes(), "edges": g.number_of_edges()},
|
||
"pist": {
|
||
"global_vocab_hash": gvh,
|
||
"unique_matrix_hashes": n_unique,
|
||
"matrix_schema": "token_strand_adjacency_8x8_v1",
|
||
},
|
||
"spectral": {
|
||
"distinct_combinations": len(spectral_bin_counts),
|
||
"fiedler_collapse": collapse_report,
|
||
},
|
||
"cross_tabulation": {
|
||
"pist_groups_with_pure_spectral_bins": pure_groups,
|
||
"pist_groups_with_mixed_spectral_bins": mixed_groups,
|
||
"total_pist_groups": len(tab),
|
||
"purity_ratio": round(pure_groups / len(tab), 4) if tab else 0.0,
|
||
"mean_spectral_entropy": round(mean_entropy, 4),
|
||
"scoring_dimensions": "pr_bin|hits_bin (fiedler excluded)",
|
||
},
|
||
}
|
||
|
||
out_path = f"/tmp/snap_pist_spectral_crossval_{os.path.basename(path)}.json"
|
||
with open(out_path, "w") as f:
|
||
json.dump(artifact, f, indent=2, sort_keys=True)
|
||
print(f"\nWrote {out_path}", flush=True)
|
||
|
||
# ── 7. Top PIST groups by size ─────────────────────────────────────
|
||
print("\nTop PIST groups by size:", flush=True)
|
||
for phash, pnodes in sorted(pist_hash_groups.items(), key=lambda x: -len(x[1]))[:25]:
|
||
info = tab[phash]
|
||
spec = node_spectral.get(pnodes[0], {})
|
||
print(f" PIST={phash[:16]} count={len(pnodes):>4} "
|
||
f"PR={spec.get('pr_bin','?'):>10} HITS={spec.get('hits_bin','?'):>10} "
|
||
f"H={info['spectral_entropy']:.3f}", flush=True)
|
||
|
||
return 0
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|