mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
1134 lines
41 KiB
Python
1134 lines
41 KiB
Python
# ==============================================================================
|
|
# COPYRIGHT NO ONE EVERYWHERE LLC (WYOMING HOLDING COMPANY)
|
|
# PROJECT: SOVEREIGN STACK
|
|
# This artifact is entirely proprietary and cryptographically proven.
|
|
# Open-Source usage requires explicit permission from Brandon Scott Schneider.
|
|
# ==============================================================================
|
|
|
|
from __future__ import annotations
|
|
|
|
# [WARDEN BOUNDARY ENFORCEMENT INJECTED]
|
|
import sys
|
|
import os
|
|
try:
|
|
from io_harness_compat import spawn_isolated_process, fetch_network_resource
|
|
except ImportError:
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
|
from io_harness_compat import spawn_isolated_process, fetch_network_resource
|
|
|
|
#!/usr/bin/env python3
|
|
"""High-coverage literature harvesting pipeline with compact archival.
|
|
|
|
This script is a practical stand-in for "fetch everything": it queries multiple
|
|
open sources, deduplicates, ranks relevance, and writes an append-only
|
|
"DeepCompression" archive plus a nibble-style compact index.
|
|
|
|
Sources:
|
|
- Crossref
|
|
- OpenAlex
|
|
- arXiv
|
|
|
|
No third-party dependencies are required.
|
|
"""
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import statistics
|
|
import time
|
|
from collections import Counter
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
import zlib
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timezone
|
|
from typing import Dict, Iterable, List, Optional, Tuple
|
|
|
|
try:
|
|
from scripts.logic_signal_substrate_translation import (
|
|
assert_surface_write_safe,
|
|
logic_signal_substrate_from_archive_domain,
|
|
logic_signal_substrate_from_surface,
|
|
surface_from_logic_signal_substrate,
|
|
)
|
|
except ImportError:
|
|
try:
|
|
from logic_signal_substrate_translation import (
|
|
assert_surface_write_safe,
|
|
logic_signal_substrate_from_archive_domain,
|
|
logic_signal_substrate_from_surface,
|
|
surface_from_logic_signal_substrate,
|
|
)
|
|
except ImportError:
|
|
# Module not available — substrate surface writes disabled.
|
|
# find_low_coverage_sources and other pure functions remain usable.
|
|
assert_surface_write_safe = lambda *a, **kw: None
|
|
logic_signal_substrate_from_archive_domain = lambda *a, **kw: None
|
|
logic_signal_substrate_from_surface = lambda *a, **kw: None
|
|
surface_from_logic_signal_substrate = lambda *a, **kw: {}
|
|
|
|
|
|
USER_AGENT = "graph_os-literature-harvest/1.0 (+local-script)"
|
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
OMNITOKEN_DIR = os.path.join(PROJECT_ROOT, "out", "omnitoken_bridge")
|
|
OMNITOKEN_SURFACE_PATH = os.path.join(OMNITOKEN_DIR, "egress_surface.json")
|
|
|
|
|
|
DEFAULT_QUERIES = [
|
|
"human interaction safety ultra-fast computation",
|
|
"real-time AI systems human factors safety",
|
|
"algorithmic decision support cognitive overload",
|
|
"human-in-the-loop control systems safety",
|
|
"AI alignment human-computer interaction",
|
|
"autonomy override governance AI systems",
|
|
"model latency human trust calibration",
|
|
"psychological effects of conversational AI",
|
|
"adaptive systems intervention threshold",
|
|
"safety-critical machine learning human oversight",
|
|
]
|
|
|
|
|
|
SAFETY_KEYWORDS = [
|
|
"human",
|
|
"interaction",
|
|
"safety",
|
|
"oversight",
|
|
"alignment",
|
|
"trust",
|
|
"cognitive",
|
|
"entrainment",
|
|
"autonomy",
|
|
"handover",
|
|
"intervention",
|
|
"governance",
|
|
"risk",
|
|
"control",
|
|
"error",
|
|
"failsafe",
|
|
"fail-safe",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class Paper:
|
|
source: str
|
|
source_id: str
|
|
title: str
|
|
abstract: str
|
|
authors: List[str]
|
|
year: Optional[int]
|
|
venue: str
|
|
doi: str
|
|
url: str
|
|
query: str
|
|
relevance_score: int = 0
|
|
relevance_bucket_4bit: int = 0
|
|
nibble_fingerprint_hex: str = ""
|
|
|
|
|
|
def now_utc() -> str:
|
|
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
|
|
|
|
def normalize_ws(text: str) -> str:
|
|
return re.sub(r"\s+", " ", (text or "")).strip()
|
|
|
|
|
|
def normalize_title(text: str) -> str:
|
|
text = normalize_ws(text).lower()
|
|
text = re.sub(r"[^a-z0-9 ]+", "", text)
|
|
return text
|
|
|
|
|
|
def safe_int(x: object) -> Optional[int]:
|
|
try:
|
|
return int(x)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def fetch_json(url: str, timeout: float = 25.0, retries: int = 3) -> dict:
|
|
last_exc: Optional[Exception] = None
|
|
for attempt in range(1, retries + 1):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode("utf-8", errors="replace"))
|
|
except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc: # pragma: no cover
|
|
last_exc = exc
|
|
if attempt < retries:
|
|
time.sleep(1.2 * attempt)
|
|
raise RuntimeError(f"Failed to fetch JSON from {url}: {last_exc}")
|
|
|
|
|
|
def fetch_text(url: str, timeout: float = 25.0, retries: int = 3) -> str:
|
|
last_exc: Optional[Exception] = None
|
|
for attempt in range(1, retries + 1):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return resp.read().decode("utf-8", errors="replace")
|
|
except (OSError, urllib.error.URLError) as exc: # pragma: no cover
|
|
last_exc = exc
|
|
if attempt < retries:
|
|
time.sleep(1.2 * attempt)
|
|
raise RuntimeError(f"Failed to fetch text from {url}: {last_exc}")
|
|
|
|
|
|
def score_relevance(title: str, abstract: str) -> int:
|
|
text = f"{title} {abstract}".lower()
|
|
score = 0
|
|
for kw in SAFETY_KEYWORDS:
|
|
if kw in text:
|
|
score += 1
|
|
return score
|
|
|
|
|
|
def bucket_4bit(score: int, max_score: int = 16) -> int:
|
|
score = max(0, score)
|
|
if max_score <= 0:
|
|
return 0
|
|
b = round((min(score, max_score) / max_score) * 15)
|
|
return int(max(0, min(15, b)))
|
|
|
|
|
|
def nibble_fingerprint(title: str, abstract: str) -> str:
|
|
digest = hashlib.sha256((title + "\n" + abstract).encode("utf-8", errors="ignore")).hexdigest()
|
|
# 16 nibbles = compact 64-bit-style fingerprint (hex chars are already nibbles)
|
|
return digest[:16]
|
|
|
|
|
|
def dedupe_key(p: Paper) -> str:
|
|
if p.doi:
|
|
return f"doi:{p.doi.lower()}"
|
|
return f"title:{normalize_title(p.title)}"
|
|
|
|
|
|
def from_crossref(query: str, rows: int) -> Iterable[Paper]:
|
|
q = urllib.parse.quote(query)
|
|
url = (
|
|
"https://api.crossref.org/works"
|
|
f"?query={q}&rows={rows}&select=DOI,title,author,container-title,URL,published-print,published-online,issued"
|
|
)
|
|
data = fetch_json(url)
|
|
items = data.get("message", {}).get("items", [])
|
|
for it in items:
|
|
title = normalize_ws(" ".join(it.get("title", []) if isinstance(it.get("title", []), list) else [str(it.get("title", ""))]))
|
|
venue = normalize_ws(" ".join(it.get("container-title", []) if isinstance(it.get("container-title", []), list) else [str(it.get("container-title", ""))]))
|
|
doi = normalize_ws(it.get("DOI", ""))
|
|
url_out = normalize_ws(it.get("URL", ""))
|
|
|
|
authors = []
|
|
for a in it.get("author", []) or []:
|
|
given = normalize_ws(a.get("given", ""))
|
|
family = normalize_ws(a.get("family", ""))
|
|
full = normalize_ws(f"{given} {family}")
|
|
if full:
|
|
authors.append(full)
|
|
|
|
year = None
|
|
for fld in ("published-print", "published-online", "issued"):
|
|
parts = (((it.get(fld) or {}).get("date-parts") or [[None]])[0] or [None])
|
|
y = safe_int(parts[0])
|
|
if y:
|
|
year = y
|
|
break
|
|
|
|
if title:
|
|
yield Paper(
|
|
source="crossref",
|
|
source_id=doi or url_out or hashlib.md5(title.encode()).hexdigest(),
|
|
title=title,
|
|
abstract="",
|
|
authors=authors,
|
|
year=year,
|
|
venue=venue,
|
|
doi=doi,
|
|
url=url_out,
|
|
query=query,
|
|
)
|
|
|
|
|
|
def from_openalex(query: str, rows: int) -> Iterable[Paper]:
|
|
q = urllib.parse.quote(query)
|
|
url = f"https://api.openalex.org/works?search={q}&per-page={rows}"
|
|
data = fetch_json(url)
|
|
items = data.get("results", [])
|
|
for it in items:
|
|
title = normalize_ws(it.get("display_name", ""))
|
|
abstract_idx = it.get("abstract_inverted_index") or {}
|
|
if abstract_idx:
|
|
max_pos = 0
|
|
for positions in abstract_idx.values():
|
|
for pos in positions:
|
|
if pos > max_pos:
|
|
max_pos = pos
|
|
tokens = [""] * (max_pos + 1)
|
|
for token, positions in abstract_idx.items():
|
|
for pos in positions:
|
|
if 0 <= pos < len(tokens):
|
|
tokens[pos] = token
|
|
abstract = normalize_ws(" ".join(tokens))
|
|
else:
|
|
abstract = ""
|
|
|
|
venue = normalize_ws(((it.get("primary_location") or {}).get("source") or {}).get("display_name", ""))
|
|
year = safe_int(it.get("publication_year"))
|
|
doi = normalize_ws((it.get("doi") or "").replace("https://doi.org/", ""))
|
|
url_out = normalize_ws(it.get("id", ""))
|
|
authors = [
|
|
normalize_ws(((a.get("author") or {}).get("display_name", "")))
|
|
for a in (it.get("authorships") or [])
|
|
]
|
|
authors = [a for a in authors if a]
|
|
|
|
if title:
|
|
yield Paper(
|
|
source="openalex",
|
|
source_id=url_out or doi or hashlib.md5(title.encode()).hexdigest(),
|
|
title=title,
|
|
abstract=abstract,
|
|
authors=authors,
|
|
year=year,
|
|
venue=venue,
|
|
doi=doi,
|
|
url=url_out,
|
|
query=query,
|
|
)
|
|
|
|
|
|
def from_arxiv(query: str, rows: int) -> Iterable[Paper]:
|
|
q = urllib.parse.quote(query)
|
|
url = f"https://export.arxiv.org/api/query?search_query=all:{q}&start=0&max_results={rows}"
|
|
xml_text = fetch_text(url)
|
|
|
|
ns = {
|
|
"atom": "http://www.w3.org/2005/Atom",
|
|
"arxiv": "http://arxiv.org/schemas/atom",
|
|
}
|
|
root = ET.fromstring(xml_text)
|
|
for entry in root.findall("atom:entry", ns):
|
|
title = normalize_ws((entry.findtext("atom:title", default="", namespaces=ns) or ""))
|
|
abstract = normalize_ws((entry.findtext("atom:summary", default="", namespaces=ns) or ""))
|
|
url_out = normalize_ws((entry.findtext("atom:id", default="", namespaces=ns) or ""))
|
|
year = safe_int((entry.findtext("atom:published", default="", namespaces=ns) or "")[:4])
|
|
authors = [
|
|
normalize_ws((a.findtext("atom:name", default="", namespaces=ns) or ""))
|
|
for a in entry.findall("atom:author", ns)
|
|
]
|
|
authors = [a for a in authors if a]
|
|
doi = ""
|
|
for cat in entry.findall("arxiv:doi", ns):
|
|
if cat is not None and (cat.text or "").strip():
|
|
doi = normalize_ws(cat.text)
|
|
break
|
|
|
|
if title:
|
|
yield Paper(
|
|
source="arxiv",
|
|
source_id=url_out or hashlib.md5(title.encode()).hexdigest(),
|
|
title=title,
|
|
abstract=abstract,
|
|
authors=authors,
|
|
year=year,
|
|
venue="arXiv",
|
|
doi=doi,
|
|
url=url_out,
|
|
query=query,
|
|
)
|
|
|
|
|
|
def enrich_scores(papers: Iterable[Paper]) -> List[Paper]:
|
|
out: List[Paper] = []
|
|
for p in papers:
|
|
p.relevance_score = score_relevance(p.title, p.abstract)
|
|
p.relevance_bucket_4bit = bucket_4bit(p.relevance_score)
|
|
p.nibble_fingerprint_hex = nibble_fingerprint(p.title, p.abstract)
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
def dedupe(papers: Iterable[Paper]) -> List[Paper]:
|
|
best: Dict[str, Paper] = {}
|
|
for p in papers:
|
|
k = dedupe_key(p)
|
|
old = best.get(k)
|
|
if old is None or p.relevance_score > old.relevance_score:
|
|
best[k] = p
|
|
return list(best.values())
|
|
|
|
|
|
def find_low_coverage_sources(source_totals: Dict[str, int], min_total_per_source: int) -> Dict[str, int]:
|
|
return {name: total for name, total in source_totals.items() if total < min_total_per_source}
|
|
|
|
|
|
def write_jsonl(path: str, papers: Iterable[Paper]) -> None:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
for p in papers:
|
|
f.write(json.dumps(asdict(p), ensure_ascii=False) + "\n")
|
|
|
|
|
|
def write_csv(path: str, papers: Iterable[Paper]) -> None:
|
|
rows = [asdict(p) for p in papers]
|
|
if not rows:
|
|
return
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", newline="", encoding="utf-8") as f:
|
|
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
|
|
|
|
def write_blackhole_vault(vault_dir: str, papers: List[Paper], run_id: str) -> str:
|
|
return write_blackhole_vault_with_mode(
|
|
vault_dir=vault_dir,
|
|
papers=papers,
|
|
run_id=run_id,
|
|
archive_mode="legacy",
|
|
benchmark_against_legacy=False,
|
|
)
|
|
|
|
|
|
def canonical_archive_row(p: Paper) -> Dict[str, object]:
|
|
"""Return a stable, compact record used for partitioned archive encoding."""
|
|
return {
|
|
"k": dedupe_key(p),
|
|
"b": int(p.relevance_bucket_4bit),
|
|
"f": p.nibble_fingerprint_hex,
|
|
"src": normalize_ws(p.source),
|
|
"t": normalize_ws(p.title),
|
|
"a": normalize_ws(p.abstract),
|
|
"y": p.year if p.year is not None else 0,
|
|
"v": normalize_ws(p.venue),
|
|
"d": normalize_ws(p.doi),
|
|
"u": normalize_ws(p.url),
|
|
"q": normalize_ws(p.query),
|
|
"s": int(p.relevance_score),
|
|
"au": [normalize_ws(author) for author in (p.authors or [])],
|
|
}
|
|
|
|
|
|
def build_legacy_payload_bytes(papers: List[Paper]) -> bytes:
|
|
"""Build legacy JSONL payload bytes for compression."""
|
|
lines = [json.dumps(asdict(p), ensure_ascii=False) for p in papers]
|
|
return ("\n".join(lines) + "\n").encode("utf-8")
|
|
|
|
|
|
def build_partitioned_payload_bytes(papers: List[Paper]) -> bytes:
|
|
"""Build a bucketed, field-partitioned payload that improves locality for zlib."""
|
|
canonical_rows = [canonical_archive_row(p) for p in papers]
|
|
canonical_rows.sort(key=lambda row: (int(row["b"]), str(row["f"]), str(row["k"])))
|
|
|
|
buckets: Dict[int, List[Dict[str, object]]] = {}
|
|
for row in canonical_rows:
|
|
bucket = int(row["b"])
|
|
buckets.setdefault(bucket, []).append(row)
|
|
|
|
bucket_payloads: List[Dict[str, object]] = []
|
|
for bucket in sorted(buckets.keys()):
|
|
rows = buckets[bucket]
|
|
bucket_payloads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"k": [row["k"] for row in rows],
|
|
"f": [row["f"] for row in rows],
|
|
"src": [row["src"] for row in rows],
|
|
"t": [row["t"] for row in rows],
|
|
"a": [row["a"] for row in rows],
|
|
"y": [row["y"] for row in rows],
|
|
"v": [row["v"] for row in rows],
|
|
"d": [row["d"] for row in rows],
|
|
"u": [row["u"] for row in rows],
|
|
"q": [row["q"] for row in rows],
|
|
"s": [row["s"] for row in rows],
|
|
"au": [row["au"] for row in rows],
|
|
}
|
|
)
|
|
|
|
payload = {
|
|
"schema": "deepcompression/partitioned/v1",
|
|
"paper_count": len(papers),
|
|
"bucket_count": len(bucket_payloads),
|
|
"buckets": bucket_payloads,
|
|
}
|
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def build_context_spray_payload_bytes(papers: List[Paper]) -> bytes:
|
|
"""Build context-sensitive payload with local context cells and per-cell delta emissions.
|
|
|
|
Foam strategy:
|
|
- Adaptive merge for tiny cells to avoid over-fragmented context groups
|
|
- Deterministic rotation in each cell to alter adjacency and expose repeated patterns
|
|
- Shared lexicon carry-over so repeated long strings are stored once
|
|
"""
|
|
rows = [canonical_archive_row(p) for p in papers]
|
|
rows.sort(
|
|
key=lambda row: (
|
|
str(row["q"]),
|
|
str(row["src"]),
|
|
int(row["b"]),
|
|
str(row["v"]),
|
|
int(row["y"]),
|
|
str(row["f"]),
|
|
)
|
|
)
|
|
|
|
base_cells: Dict[Tuple[str, str, int, str], List[Dict[str, object]]] = {}
|
|
for row in rows:
|
|
key = (
|
|
str(row["q"]),
|
|
str(row["src"]),
|
|
int(row["b"]),
|
|
str(row["v"]),
|
|
)
|
|
base_cells.setdefault(key, []).append(row)
|
|
|
|
# Merge tiny cells into broader context bins to avoid sparse overhead.
|
|
min_cell_size = 3
|
|
cells: Dict[Tuple[str, str, int, str], List[Dict[str, object]]] = {}
|
|
for key, cell_rows in base_cells.items():
|
|
if len(cell_rows) >= min_cell_size:
|
|
cells[key] = cell_rows
|
|
continue
|
|
|
|
query, _source, bucket, _venue = key
|
|
merged_key = (query, "*", bucket, "*")
|
|
cells.setdefault(merged_key, []).extend(cell_rows)
|
|
|
|
lexicon: List[str] = []
|
|
lexicon_index: Dict[str, int] = {}
|
|
|
|
def lex(value: str) -> int:
|
|
value = normalize_ws(value)
|
|
existing = lexicon_index.get(value)
|
|
if existing is not None:
|
|
return existing
|
|
idx = len(lexicon)
|
|
lexicon.append(value)
|
|
lexicon_index[value] = idx
|
|
return idx
|
|
|
|
def lex_authors(authors: List[object]) -> List[int]:
|
|
return [lex(str(a)) for a in authors]
|
|
|
|
emitted_cells: List[Dict[str, object]] = []
|
|
|
|
def top_ngrams(texts: List[str], min_n: int = 2, max_n: int = 3, top_k: int = 16) -> List[str]:
|
|
counts: Counter[str] = Counter()
|
|
for text in texts:
|
|
words = [w for w in re.split(r"\W+", normalize_ws(text).lower()) if len(w) >= 3]
|
|
for n in range(min_n, max_n + 1):
|
|
if len(words) < n:
|
|
continue
|
|
for i in range(0, len(words) - n + 1):
|
|
phrase = " ".join(words[i : i + n])
|
|
if len(phrase) >= 8:
|
|
counts[phrase] += 1
|
|
ranked = [phrase for phrase, c in counts.most_common(top_k * 3) if c >= 3]
|
|
# Keep longest phrases first so replacement is stable and deterministic.
|
|
ranked.sort(key=lambda x: (-len(x), x))
|
|
return ranked[:top_k]
|
|
|
|
def apply_ngrams(text: str, ngrams: List[str]) -> str:
|
|
out = normalize_ws(text)
|
|
if not out or not ngrams:
|
|
return out
|
|
lowered = out.lower()
|
|
for i, phrase in enumerate(ngrams):
|
|
marker = f"~g{i}~"
|
|
# Case-insensitive, deterministic whole-phrase replacement.
|
|
pattern = re.compile(re.escape(phrase), flags=re.IGNORECASE)
|
|
lowered = pattern.sub(marker, lowered)
|
|
return lowered
|
|
for key in sorted(cells.keys()):
|
|
query, source, bucket, venue = key
|
|
cell_rows = cells[key]
|
|
if not cell_rows:
|
|
continue
|
|
|
|
# Deterministic rotation for context-sensitive adjacency shifts.
|
|
if len(cell_rows) > 1:
|
|
seed = hashlib.sha256(f"{query}|{source}|{bucket}|{venue}".encode("utf-8")).hexdigest()
|
|
offset = int(seed[:4], 16) % len(cell_rows)
|
|
if offset:
|
|
cell_rows = cell_rows[offset:] + cell_rows[:offset]
|
|
|
|
cell_ngrams = top_ngrams(
|
|
[str(r["t"]) for r in cell_rows] + [str(r["a"]) for r in cell_rows],
|
|
min_n=2,
|
|
max_n=3,
|
|
top_k=16,
|
|
)
|
|
|
|
first = cell_rows[0]
|
|
first_full = {
|
|
"k": first["k"],
|
|
"f": first["f"],
|
|
"src": lex(str(first["src"])),
|
|
"t": lex(apply_ngrams(str(first["t"]), cell_ngrams)),
|
|
"a": lex(apply_ngrams(str(first["a"]), cell_ngrams)),
|
|
"y": first["y"],
|
|
"v": lex(str(first["v"])),
|
|
"d": lex(str(first["d"])),
|
|
"u": lex(str(first["u"])),
|
|
"q": lex(str(first["q"])),
|
|
"s": first["s"],
|
|
"au": lex_authors(list(first["au"])),
|
|
}
|
|
|
|
deltas: List[Dict[str, object]] = []
|
|
prev = first
|
|
for row in cell_rows[1:]:
|
|
delta: Dict[str, object] = {
|
|
"k": row["k"],
|
|
"f": row["f"],
|
|
"s": row["s"],
|
|
}
|
|
for field in ("t", "a", "y", "d", "u", "au"):
|
|
if row[field] != prev[field]:
|
|
if field == "au":
|
|
delta[field] = lex_authors(list(row[field]))
|
|
elif field in ("t", "a", "d", "u"):
|
|
if field in ("t", "a"):
|
|
delta[field] = lex(apply_ngrams(str(row[field]), cell_ngrams))
|
|
else:
|
|
delta[field] = lex(str(row[field]))
|
|
else:
|
|
delta[field] = row[field]
|
|
deltas.append(delta)
|
|
prev = row
|
|
|
|
emitted_cells.append(
|
|
{
|
|
"ctx": {
|
|
"q": lex(query),
|
|
"src": lex(source),
|
|
"b": bucket,
|
|
"v": lex(venue),
|
|
},
|
|
"local_ngrams": cell_ngrams,
|
|
"seed": first_full,
|
|
"spray": deltas,
|
|
}
|
|
)
|
|
|
|
payload = {
|
|
"schema": "deepcompression/context-spray/v2",
|
|
"paper_count": len(papers),
|
|
"cell_count": len(emitted_cells),
|
|
"lexicon": lexicon,
|
|
"cells": emitted_cells,
|
|
}
|
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def build_hybrid_payload_bytes(papers: List[Paper]) -> bytes:
|
|
"""Build a hybrid payload:
|
|
- high-entropy fields in partitioned blocks
|
|
- low-entropy repeated fields in context-spray cells
|
|
"""
|
|
rows = [canonical_archive_row(p) for p in papers]
|
|
rows.sort(key=lambda row: (int(row["b"]), str(row["f"]), str(row["k"])))
|
|
|
|
# High-entropy partition: titles/abstracts/urls/dois/authors tend to vary heavily.
|
|
high_partition_buckets: Dict[int, List[Dict[str, object]]] = {}
|
|
for row in rows:
|
|
bucket = int(row["b"])
|
|
high_partition_buckets.setdefault(bucket, []).append(
|
|
{
|
|
"k": row["k"],
|
|
"f": row["f"],
|
|
"t": row["t"],
|
|
"a": row["a"],
|
|
"d": row["d"],
|
|
"u": row["u"],
|
|
"au": row["au"],
|
|
}
|
|
)
|
|
|
|
high_blocks: List[Dict[str, object]] = []
|
|
for bucket in sorted(high_partition_buckets.keys()):
|
|
block_rows = high_partition_buckets[bucket]
|
|
high_blocks.append(
|
|
{
|
|
"bucket": bucket,
|
|
"k": [r["k"] for r in block_rows],
|
|
"f": [r["f"] for r in block_rows],
|
|
"t": [r["t"] for r in block_rows],
|
|
"a": [r["a"] for r in block_rows],
|
|
"d": [r["d"] for r in block_rows],
|
|
"u": [r["u"] for r in block_rows],
|
|
"au": [r["au"] for r in block_rows],
|
|
}
|
|
)
|
|
|
|
# Low-entropy context spray: source/query/venue/year/score are more repeatable.
|
|
low_rows = [
|
|
{
|
|
"k": row["k"],
|
|
"f": row["f"],
|
|
"src": row["src"],
|
|
"q": row["q"],
|
|
"v": row["v"],
|
|
"b": row["b"],
|
|
"y": row["y"],
|
|
"s": row["s"],
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
low_rows.sort(key=lambda row: (str(row["q"]), str(row["src"]), int(row["b"]), str(row["v"]), int(row["y"]), str(row["f"])))
|
|
low_cells: Dict[Tuple[str, str, int, str], List[Dict[str, object]]] = {}
|
|
for row in low_rows:
|
|
key = (str(row["q"]), str(row["src"]), int(row["b"]), str(row["v"]))
|
|
low_cells.setdefault(key, []).append(row)
|
|
|
|
low_emitted: List[Dict[str, object]] = []
|
|
for key in sorted(low_cells.keys()):
|
|
query, source, bucket, venue = key
|
|
cell_rows = low_cells[key]
|
|
first = cell_rows[0]
|
|
deltas: List[Dict[str, object]] = []
|
|
prev = first
|
|
for row in cell_rows[1:]:
|
|
delta: Dict[str, object] = {"k": row["k"], "f": row["f"]}
|
|
for field in ("y", "s"):
|
|
if row[field] != prev[field]:
|
|
delta[field] = row[field]
|
|
deltas.append(delta)
|
|
prev = row
|
|
|
|
low_emitted.append(
|
|
{
|
|
"ctx": {"q": query, "src": source, "b": bucket, "v": venue},
|
|
"seed": first,
|
|
"spray": deltas,
|
|
}
|
|
)
|
|
|
|
payload = {
|
|
"schema": "deepcompression/hybrid/v1",
|
|
"paper_count": len(papers),
|
|
"high_entropy_partitioned": {
|
|
"bucket_count": len(high_blocks),
|
|
"blocks": high_blocks,
|
|
},
|
|
"low_entropy_context_spray": {
|
|
"cell_count": len(low_emitted),
|
|
"cells": low_emitted,
|
|
},
|
|
}
|
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def compress_payload(raw_bytes: bytes) -> Dict[str, object]:
|
|
compressed = zlib.compress(raw_bytes, level=9)
|
|
return {
|
|
"raw_bytes": len(raw_bytes),
|
|
"compressed_bytes": len(compressed),
|
|
"compression_ratio": round(len(compressed) / max(1, len(raw_bytes)), 6),
|
|
"blob": compressed,
|
|
}
|
|
|
|
|
|
def compare_archive_modes(papers: List[Paper]) -> Dict[str, object]:
|
|
"""Compute direct legacy versus partitioned compression metrics on identical inputs."""
|
|
legacy_raw = build_legacy_payload_bytes(papers)
|
|
partitioned_raw = build_partitioned_payload_bytes(papers)
|
|
context_spray_raw = build_context_spray_payload_bytes(papers)
|
|
hybrid_raw = build_hybrid_payload_bytes(papers)
|
|
|
|
legacy_stats = compress_payload(legacy_raw)
|
|
partitioned_stats = compress_payload(partitioned_raw)
|
|
context_spray_stats = compress_payload(context_spray_raw)
|
|
hybrid_stats = compress_payload(hybrid_raw)
|
|
|
|
legacy_ratio = float(legacy_stats["compression_ratio"])
|
|
partitioned_ratio = float(partitioned_stats["compression_ratio"])
|
|
context_spray_ratio = float(context_spray_stats["compression_ratio"])
|
|
hybrid_ratio = float(hybrid_stats["compression_ratio"])
|
|
|
|
candidates = {
|
|
"legacy": int(legacy_stats["compressed_bytes"]),
|
|
"partitioned": int(partitioned_stats["compressed_bytes"]),
|
|
"context_spray": int(context_spray_stats["compressed_bytes"]),
|
|
"hybrid": int(hybrid_stats["compressed_bytes"]),
|
|
}
|
|
auto_winner_mode = min(candidates.keys(), key=lambda m: candidates[m])
|
|
|
|
return {
|
|
"legacy": {
|
|
"raw_bytes": legacy_stats["raw_bytes"],
|
|
"compressed_bytes": legacy_stats["compressed_bytes"],
|
|
"compression_ratio": legacy_ratio,
|
|
},
|
|
"partitioned": {
|
|
"raw_bytes": partitioned_stats["raw_bytes"],
|
|
"compressed_bytes": partitioned_stats["compressed_bytes"],
|
|
"compression_ratio": partitioned_ratio,
|
|
},
|
|
"context_spray": {
|
|
"raw_bytes": context_spray_stats["raw_bytes"],
|
|
"compressed_bytes": context_spray_stats["compressed_bytes"],
|
|
"compression_ratio": context_spray_ratio,
|
|
},
|
|
"hybrid": {
|
|
"raw_bytes": hybrid_stats["raw_bytes"],
|
|
"compressed_bytes": hybrid_stats["compressed_bytes"],
|
|
"compression_ratio": hybrid_ratio,
|
|
},
|
|
"auto_winner": {
|
|
"mode": auto_winner_mode,
|
|
"compressed_bytes": candidates[auto_winner_mode],
|
|
},
|
|
"delta": {
|
|
"partitioned_compressed_bytes_saved": int(legacy_stats["compressed_bytes"]) - int(partitioned_stats["compressed_bytes"]),
|
|
"context_spray_compressed_bytes_saved": int(legacy_stats["compressed_bytes"]) - int(context_spray_stats["compressed_bytes"]),
|
|
"hybrid_compressed_bytes_saved": int(legacy_stats["compressed_bytes"]) - int(hybrid_stats["compressed_bytes"]),
|
|
"compression_ratio_improvement": round(legacy_ratio - partitioned_ratio, 6),
|
|
"improved": partitioned_ratio < legacy_ratio,
|
|
"context_spray_ratio_improvement": round(legacy_ratio - context_spray_ratio, 6),
|
|
"context_spray_improved": context_spray_ratio < legacy_ratio,
|
|
"hybrid_ratio_improvement": round(legacy_ratio - hybrid_ratio, 6),
|
|
"hybrid_improved": hybrid_ratio < legacy_ratio,
|
|
},
|
|
}
|
|
|
|
|
|
def read_archive_domain_from_omnitoken_surface() -> Dict[str, object]:
|
|
if not os.path.exists(OMNITOKEN_SURFACE_PATH):
|
|
return {
|
|
"status": "unavailable",
|
|
"reason": "omnitoken_surface_not_found",
|
|
"surface_path": OMNITOKEN_SURFACE_PATH,
|
|
}
|
|
|
|
try:
|
|
with open(OMNITOKEN_SURFACE_PATH, "r", encoding="utf-8") as f:
|
|
surface = json.load(f)
|
|
except (OSError, json.JSONDecodeError):
|
|
return {
|
|
"status": "unreadable",
|
|
"reason": "invalid_surface_json",
|
|
"surface_path": OMNITOKEN_SURFACE_PATH,
|
|
}
|
|
|
|
if not isinstance(surface, dict):
|
|
return {
|
|
"status": "unreadable",
|
|
"reason": "surface_not_object",
|
|
"surface_path": OMNITOKEN_SURFACE_PATH,
|
|
}
|
|
|
|
surface_bus = surface.get("surface_bus") if isinstance(surface.get("surface_bus"), dict) else {}
|
|
domains = surface_bus.get("domains") if isinstance(surface_bus.get("domains"), dict) else {}
|
|
archive_domain = domains.get("archive_compression") if isinstance(domains.get("archive_compression"), dict) else {}
|
|
|
|
if archive_domain:
|
|
_ = logic_signal_substrate_from_surface(archive_domain)
|
|
return {
|
|
"status": "ok",
|
|
"source": "surface_bus.domains.archive_compression",
|
|
"domain": archive_domain,
|
|
"translation_runtime": {
|
|
"mode": "pure_logic_signal_substrate_internal",
|
|
"logic_signal_substrate_exposed": False,
|
|
},
|
|
}
|
|
|
|
legacy_domain = surface.get("archive_surface") if isinstance(surface.get("archive_surface"), dict) else {}
|
|
if legacy_domain:
|
|
_ = logic_signal_substrate_from_surface(legacy_domain)
|
|
return {
|
|
"status": "ok",
|
|
"source": "archive_surface",
|
|
"domain": legacy_domain,
|
|
"translation_runtime": {
|
|
"mode": "pure_logic_signal_substrate_internal",
|
|
"logic_signal_substrate_exposed": False,
|
|
},
|
|
}
|
|
|
|
return {
|
|
"status": "unavailable",
|
|
"reason": "archive_domain_missing",
|
|
"surface_path": OMNITOKEN_SURFACE_PATH,
|
|
}
|
|
|
|
|
|
def publish_archive_domain_to_omnitoken_surface(archive_domain: Dict[str, object], manifest_path: str) -> None:
|
|
os.makedirs(OMNITOKEN_DIR, exist_ok=True)
|
|
|
|
logic_signal_substrate_state = logic_signal_substrate_from_archive_domain(archive_domain)
|
|
translated_surface_domain = surface_from_logic_signal_substrate(logic_signal_substrate_state)
|
|
|
|
surface: Dict[str, object] = {}
|
|
if os.path.exists(OMNITOKEN_SURFACE_PATH):
|
|
try:
|
|
with open(OMNITOKEN_SURFACE_PATH, "r", encoding="utf-8") as f:
|
|
loaded = json.load(f)
|
|
if isinstance(loaded, dict):
|
|
surface = loaded
|
|
except (OSError, json.JSONDecodeError):
|
|
surface = {}
|
|
|
|
surface_bus = dict(surface.get("surface_bus") or {})
|
|
domains = dict(surface_bus.get("domains") or {})
|
|
domains["archive_compression"] = translated_surface_domain
|
|
surface_bus["schema"] = str(surface_bus.get("schema") or "omnitoken-surface-bus/v1")
|
|
surface_bus["agnostic"] = True
|
|
surface_bus["domains"] = domains
|
|
surface["surface_bus"] = surface_bus
|
|
surface["archive_surface"] = translated_surface_domain
|
|
surface["updated_utc"] = datetime.now(timezone.utc).isoformat()
|
|
surface["archive_manifest_path"] = manifest_path
|
|
|
|
assert_surface_write_safe(surface, scope="omnitoken_surface")
|
|
with open(OMNITOKEN_SURFACE_PATH, "w", encoding="utf-8") as f:
|
|
json.dump(surface, f, indent=2)
|
|
|
|
profile_path_obj = surface.get("profile")
|
|
if isinstance(profile_path_obj, str) and os.path.exists(profile_path_obj):
|
|
try:
|
|
with open(profile_path_obj, "r", encoding="utf-8") as f:
|
|
profile = json.load(f)
|
|
if isinstance(profile, dict):
|
|
p_surface_bus = dict(profile.get("surface_bus") or {})
|
|
p_domains = dict(p_surface_bus.get("domains") or {})
|
|
p_domains["archive_compression"] = translated_surface_domain
|
|
p_surface_bus["schema"] = str(p_surface_bus.get("schema") or "omnitoken-surface-bus/v1")
|
|
p_surface_bus["agnostic"] = True
|
|
p_surface_bus["domains"] = p_domains
|
|
profile["surface_bus"] = p_surface_bus
|
|
profile["archive_surface"] = translated_surface_domain
|
|
profile["updated_utc"] = datetime.now(timezone.utc).isoformat()
|
|
assert_surface_write_safe(profile, scope="omnitoken_profile")
|
|
with open(profile_path_obj, "w", encoding="utf-8") as f:
|
|
json.dump(profile, f, indent=2)
|
|
except (OSError, json.JSONDecodeError):
|
|
pass
|
|
|
|
|
|
def write_blackhole_vault_with_mode(
|
|
vault_dir: str,
|
|
papers: List[Paper],
|
|
run_id: str,
|
|
archive_mode: str,
|
|
benchmark_against_legacy: bool,
|
|
) -> str:
|
|
os.makedirs(vault_dir, exist_ok=True)
|
|
raw_jsonl = os.path.join(vault_dir, f"papers_{run_id}.jsonl")
|
|
write_jsonl(raw_jsonl, papers)
|
|
|
|
benchmark = compare_archive_modes(papers)
|
|
inbound_surface = read_archive_domain_from_omnitoken_surface()
|
|
|
|
if archive_mode == "auto":
|
|
archive_mode = str(benchmark["auto_winner"]["mode"])
|
|
|
|
if archive_mode == "partitioned":
|
|
raw_bytes = build_partitioned_payload_bytes(papers)
|
|
payload_format = "partitioned_bucket_streams"
|
|
elif archive_mode == "context-spray":
|
|
raw_bytes = build_context_spray_payload_bytes(papers)
|
|
payload_format = "context_sensitive_spray_cells"
|
|
elif archive_mode == "hybrid":
|
|
raw_bytes = build_hybrid_payload_bytes(papers)
|
|
payload_format = "hybrid_partitioned_context"
|
|
else:
|
|
raw_bytes = build_legacy_payload_bytes(papers)
|
|
payload_format = "legacy_jsonl"
|
|
|
|
compressed = zlib.compress(raw_bytes, level=9)
|
|
blob_path = os.path.join(vault_dir, f"blackhole_{run_id}.zlib")
|
|
with open(blob_path, "wb") as f:
|
|
f.write(compressed)
|
|
|
|
manifest = {
|
|
"run_id": run_id,
|
|
"created_utc": datetime.now(timezone.utc).isoformat(),
|
|
"paper_count": len(papers),
|
|
"raw_jsonl": raw_jsonl,
|
|
"compressed_blob": blob_path,
|
|
"archive_mode": archive_mode,
|
|
"payload_format": payload_format,
|
|
"raw_bytes": len(raw_bytes),
|
|
"compressed_bytes": len(compressed),
|
|
"compression_ratio": round(len(compressed) / max(1, len(raw_bytes)), 4),
|
|
"nibble_bucket_mean": round(
|
|
statistics.mean((p.relevance_bucket_4bit for p in papers)) if papers else 0.0,
|
|
4,
|
|
),
|
|
"nibble_index": [
|
|
{
|
|
"key": dedupe_key(p),
|
|
"bucket4": p.relevance_bucket_4bit,
|
|
"fingerprint16": p.nibble_fingerprint_hex,
|
|
}
|
|
for p in papers
|
|
],
|
|
"omnitoken_surface_input": inbound_surface,
|
|
}
|
|
|
|
if benchmark_against_legacy:
|
|
manifest["archive_mode_benchmark"] = benchmark
|
|
|
|
manifest_path = os.path.join(vault_dir, f"manifest_{run_id}.json")
|
|
with open(manifest_path, "w", encoding="utf-8") as f:
|
|
json.dump(manifest, f, indent=2)
|
|
|
|
archive_domain = {
|
|
"domain": "archive_compression",
|
|
"selection_policy": "auto_smallest_compressed_bytes",
|
|
"selected_mode": archive_mode,
|
|
"payload_format": payload_format,
|
|
"raw_bytes": int(len(raw_bytes)),
|
|
"compressed_bytes": int(len(compressed)),
|
|
"manifest_path": manifest_path,
|
|
"benchmark": benchmark,
|
|
"updated_utc": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
publish_archive_domain_to_omnitoken_surface(archive_domain, manifest_path)
|
|
|
|
return manifest_path
|
|
|
|
|
|
def gather_all(queries: List[str], rows_per_source: int, dry_run: bool) -> Tuple[List[Paper], Dict[str, int], Dict[str, int]]:
|
|
papers: List[Paper] = []
|
|
source_totals = {"crossref": 0, "openalex": 0, "arxiv": 0}
|
|
source_failures = {"crossref": 0, "openalex": 0, "arxiv": 0}
|
|
|
|
if dry_run:
|
|
for q in queries[:3]:
|
|
papers.append(
|
|
Paper(
|
|
source="dry-run",
|
|
source_id=hashlib.md5(q.encode()).hexdigest(),
|
|
title=f"Synthetic result for: {q}",
|
|
abstract="Human safety oversight and intervention threshold under low latency.",
|
|
authors=["Dry Runner"],
|
|
year=2026,
|
|
venue="Simulation",
|
|
doi="",
|
|
url="",
|
|
query=q,
|
|
)
|
|
)
|
|
return papers, source_totals, source_failures
|
|
|
|
for q in queries:
|
|
try:
|
|
got = list(from_crossref(q, rows_per_source))
|
|
papers.extend(got)
|
|
source_totals["crossref"] += len(got)
|
|
except RuntimeError as exc:
|
|
source_failures["crossref"] += 1
|
|
print(f"[warn] crossref failed for query={q!r}: {exc}")
|
|
|
|
try:
|
|
got = list(from_openalex(q, rows_per_source))
|
|
papers.extend(got)
|
|
source_totals["openalex"] += len(got)
|
|
except RuntimeError as exc:
|
|
source_failures["openalex"] += 1
|
|
print(f"[warn] openalex failed for query={q!r}: {exc}")
|
|
|
|
try:
|
|
got = list(from_arxiv(q, rows_per_source))
|
|
papers.extend(got)
|
|
source_totals["arxiv"] += len(got)
|
|
except (RuntimeError, ET.ParseError) as exc:
|
|
source_failures["arxiv"] += 1
|
|
print(f"[warn] arxiv failed for query={q!r}: {exc}")
|
|
|
|
return papers, source_totals, source_failures
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(description="High-coverage literature harvesting with DeepCompression vault output.")
|
|
p.add_argument("--query", action="append", default=[], help="Query string (repeatable).")
|
|
p.add_argument("--rows-per-source", type=int, default=35, help="Rows per source per query.")
|
|
p.add_argument("--out-dir", default=os.path.join(PROJECT_ROOT, "literature_blackhole"), help="Output directory.")
|
|
p.add_argument("--min-total-per-source", type=int, default=1, help="Fail run if any source returns fewer total records than this.")
|
|
p.add_argument(
|
|
"--archive-mode",
|
|
choices=["legacy", "partitioned", "context-spray", "hybrid", "auto"],
|
|
default="auto",
|
|
help="Archive payload encoding mode.",
|
|
)
|
|
p.add_argument(
|
|
"--benchmark-against-legacy",
|
|
action="store_true",
|
|
help="Include direct legacy-versus-selected-mode compression comparison in manifest.",
|
|
)
|
|
p.add_argument("--dry-run", action="store_true", help="Use synthetic records and skip network calls.")
|
|
return p.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
queries = args.query if args.query else DEFAULT_QUERIES
|
|
|
|
run_id = now_utc()
|
|
all_papers, source_totals, source_failures = gather_all(queries, args.rows_per_source, args.dry_run)
|
|
|
|
if not args.dry_run:
|
|
low_sources = find_low_coverage_sources(source_totals, args.min_total_per_source)
|
|
if low_sources:
|
|
print(json.dumps({
|
|
"error": "insufficient_source_coverage",
|
|
"min_total_per_source": int(args.min_total_per_source),
|
|
"source_totals": source_totals,
|
|
"source_failures": source_failures,
|
|
"low_sources": low_sources,
|
|
}, indent=2))
|
|
return 3
|
|
|
|
scored = enrich_scores(all_papers)
|
|
deduped = dedupe(scored)
|
|
|
|
deduped.sort(key=lambda p: (p.relevance_score, p.year or 0), reverse=True)
|
|
|
|
out_dir = args.out_dir
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
dedup_jsonl = os.path.join(out_dir, f"deduped_{run_id}.jsonl")
|
|
dedup_csv = os.path.join(out_dir, f"deduped_{run_id}.csv")
|
|
write_jsonl(dedup_jsonl, deduped)
|
|
write_csv(dedup_csv, deduped)
|
|
|
|
manifest = write_blackhole_vault_with_mode(
|
|
vault_dir=os.path.join(out_dir, "vault"),
|
|
papers=deduped,
|
|
run_id=run_id,
|
|
archive_mode=args.archive_mode,
|
|
benchmark_against_legacy=bool(args.benchmark_against_legacy),
|
|
)
|
|
|
|
summary = {
|
|
"run_id": run_id,
|
|
"dry_run": bool(args.dry_run),
|
|
"queries": len(queries),
|
|
"raw_records": len(scored),
|
|
"deduped_records": len(deduped),
|
|
"source_totals": source_totals,
|
|
"source_failures": source_failures,
|
|
"top_titles": [p.title for p in deduped[:5]],
|
|
"outputs": {
|
|
"dedup_jsonl": dedup_jsonl,
|
|
"dedup_csv": dedup_csv,
|
|
"manifest": manifest,
|
|
},
|
|
}
|
|
print(json.dumps(summary, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|