mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
256 lines
10 KiB
Python
256 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Break out math-bearing structure from the local arXiv Kaggle metadata zip.
|
|
|
|
This streams the JSONL member inside /home/allaun/Documents/ingest/kaggledataset.zip
|
|
without extracting it. The archive contains paper metadata/abstracts, not the
|
|
full LaTeX source corpus, so equation signals here are abstract/title markers
|
|
and category/domain routes rather than full equation recovery.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import zipfile
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
DEFAULT_ARCHIVE = Path("/home/allaun/Documents/ingest/kaggledataset.zip")
|
|
OUT_DIR = REPO / "shared-data" / "data" / "math_research_databases"
|
|
SUMMARY = OUT_DIR / "arxiv_math_breakout_summary.json"
|
|
CATEGORY_CSV = OUT_DIR / "arxiv_math_breakout_categories.csv"
|
|
MARKER_CSV = OUT_DIR / "arxiv_math_breakout_symbol_markers.csv"
|
|
SAMPLES_JSONL = OUT_DIR / "arxiv_math_breakout_samples.jsonl"
|
|
RECEIPT = OUT_DIR / "arxiv_math_breakout_receipt.json"
|
|
|
|
|
|
MATH_PREFIXES = (
|
|
"math",
|
|
"stat",
|
|
"cs.",
|
|
"nlin",
|
|
"q-bio",
|
|
"q-fin",
|
|
"quant-ph",
|
|
"hep-th",
|
|
"math-ph",
|
|
)
|
|
|
|
LATEX_COMMAND_RE = re.compile(r"\\[A-Za-z]+")
|
|
INLINE_MATH_RE = re.compile(r"\$[^$]{1,160}\$")
|
|
SYMBOL_RE = re.compile(r"(?:[A-Za-z]\^\{?[-+0-9A-Za-z]+}?|[A-Za-z]_\{?[-+0-9A-Za-z]+}?|[=<>≤≥∈∑∫∞⊗⊕±≈≃≅])")
|
|
|
|
|
|
def stable_json(obj: Any) -> str:
|
|
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
|
|
|
|
def sha256_text(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def categories_of(record: dict[str, Any]) -> list[str]:
|
|
raw = record.get("categories")
|
|
if not isinstance(raw, str):
|
|
return []
|
|
return [part for part in raw.split() if part]
|
|
|
|
|
|
def is_math_bearing(categories: list[str]) -> bool:
|
|
return any(cat.startswith(MATH_PREFIXES) for cat in categories)
|
|
|
|
|
|
def text_field(record: dict[str, Any], key: str) -> str:
|
|
value = record.get(key)
|
|
return value if isinstance(value, str) else ""
|
|
|
|
|
|
def marker_counts(text: str) -> Counter[str]:
|
|
counts: Counter[str] = Counter()
|
|
commands = LATEX_COMMAND_RE.findall(text)
|
|
inline = INLINE_MATH_RE.findall(text)
|
|
symbols = SYMBOL_RE.findall(text)
|
|
counts["latex_command"] = len(commands)
|
|
counts["inline_math_span"] = len(inline)
|
|
counts["symbolic_token"] = len(symbols)
|
|
for command in commands[:200]:
|
|
counts[f"cmd:{command}"] += 1
|
|
return counts
|
|
|
|
|
|
def csv_escape(value: Any) -> str:
|
|
text = str(value).replace('"', '""')
|
|
return f'"{text}"'
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--archive", default=str(DEFAULT_ARCHIVE))
|
|
parser.add_argument("--max-records", type=int, default=200_000)
|
|
parser.add_argument("--sample-records", type=int, default=100)
|
|
parser.add_argument("--math-only", action="store_true", default=True)
|
|
args = parser.parse_args()
|
|
|
|
archive = Path(args.archive).expanduser().resolve()
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
archive_hash = sha256_file(archive)
|
|
|
|
total_seen = 0
|
|
selected_seen = 0
|
|
category_counts: Counter[str] = Counter()
|
|
primary_category_counts: Counter[str] = Counter()
|
|
marker_totals: Counter[str] = Counter()
|
|
status_counts: Counter[str] = Counter()
|
|
year_counts: Counter[str] = Counter()
|
|
version_counts: Counter[int] = Counter()
|
|
category_pair_counts: Counter[tuple[str, str]] = Counter()
|
|
samples: list[dict[str, Any]] = []
|
|
category_marker_totals: dict[str, Counter[str]] = defaultdict(Counter)
|
|
|
|
with zipfile.ZipFile(archive) as zf:
|
|
infos = zf.infolist()
|
|
json_members = [info for info in infos if info.filename.endswith(".json")]
|
|
selected_member = json_members[0] if json_members else infos[0]
|
|
with zf.open(selected_member) as handle:
|
|
for raw_line in handle:
|
|
if args.max_records and total_seen >= args.max_records:
|
|
break
|
|
total_seen += 1
|
|
record = json.loads(raw_line)
|
|
cats = categories_of(record)
|
|
if args.math_only and not is_math_bearing(cats):
|
|
continue
|
|
selected_seen += 1
|
|
primary = cats[0] if cats else "uncategorized"
|
|
category_counts.update(cats)
|
|
primary_category_counts[primary] += 1
|
|
for left, right in zip(cats, cats[1:]):
|
|
category_pair_counts[(left, right)] += 1
|
|
|
|
versions = record.get("versions")
|
|
if isinstance(versions, list):
|
|
version_counts[len(versions)] += 1
|
|
if versions and isinstance(versions[0], dict):
|
|
created = str(versions[0].get("created", ""))
|
|
if len(created) >= 4:
|
|
year_counts[created[-4:]] += 1
|
|
|
|
has_doi = bool(record.get("doi"))
|
|
has_journal = bool(record.get("journal-ref"))
|
|
status_counts["has_doi" if has_doi else "missing_doi"] += 1
|
|
status_counts["has_journal_ref" if has_journal else "missing_journal_ref"] += 1
|
|
|
|
combined = "\n".join([text_field(record, "title"), text_field(record, "abstract")])
|
|
markers = marker_counts(combined)
|
|
marker_totals.update(markers)
|
|
category_marker_totals[primary].update(markers)
|
|
|
|
if len(samples) < args.sample_records:
|
|
sample = {
|
|
"id": record.get("id"),
|
|
"title": record.get("title"),
|
|
"primary_category": primary,
|
|
"categories": cats,
|
|
"version_count": len(versions) if isinstance(versions, list) else 0,
|
|
"has_doi": has_doi,
|
|
"has_journal_ref": has_journal,
|
|
"abstract_sha256": sha256_text(text_field(record, "abstract")),
|
|
"marker_counts": {
|
|
key: markers[key]
|
|
for key in ("latex_command", "inline_math_span", "symbolic_token")
|
|
},
|
|
}
|
|
sample["packet_hash"] = sha256_text(stable_json(sample))
|
|
samples.append(sample)
|
|
|
|
category_lines = ["category,count,primary_count,latex_command,inline_math_span,symbolic_token"]
|
|
for category, count in category_counts.most_common():
|
|
marker = category_marker_totals.get(category, Counter())
|
|
category_lines.append(
|
|
",".join(
|
|
[
|
|
csv_escape(category),
|
|
str(count),
|
|
str(primary_category_counts.get(category, 0)),
|
|
str(marker.get("latex_command", 0)),
|
|
str(marker.get("inline_math_span", 0)),
|
|
str(marker.get("symbolic_token", 0)),
|
|
]
|
|
)
|
|
)
|
|
CATEGORY_CSV.write_text("\n".join(category_lines) + "\n", encoding="utf-8")
|
|
|
|
marker_lines = ["marker,count"]
|
|
for marker, count in marker_totals.most_common(500):
|
|
marker_lines.append(f"{csv_escape(marker)},{count}")
|
|
MARKER_CSV.write_text("\n".join(marker_lines) + "\n", encoding="utf-8")
|
|
|
|
SAMPLES_JSONL.write_text("\n".join(stable_json(s) for s in samples) + "\n", encoding="utf-8")
|
|
|
|
summary = {
|
|
"schema": "arxiv_math_breakout_summary_v1",
|
|
"archive_path": str(archive),
|
|
"archive_sha256": archive_hash,
|
|
"max_records": args.max_records,
|
|
"total_records_scanned": total_seen,
|
|
"math_bearing_records": selected_seen,
|
|
"math_bearing_fraction": selected_seen / total_seen if total_seen else 0,
|
|
"category_count": len(category_counts),
|
|
"top_categories": category_counts.most_common(40),
|
|
"top_primary_categories": primary_category_counts.most_common(40),
|
|
"top_category_pairs": [[list(pair), count] for pair, count in category_pair_counts.most_common(40)],
|
|
"marker_totals": dict(marker_totals),
|
|
"top_markers": marker_totals.most_common(80),
|
|
"doi_status": dict(status_counts),
|
|
"top_years": year_counts.most_common(40),
|
|
"version_count_distribution": dict(sorted(version_counts.items())),
|
|
"sample_records": str(SAMPLES_JSONL.relative_to(REPO)),
|
|
"category_csv": str(CATEGORY_CSV.relative_to(REPO)),
|
|
"marker_csv": str(MARKER_CSV.relative_to(REPO)),
|
|
"decision": "HOLD",
|
|
"claim_boundary": (
|
|
"Breakout uses arXiv metadata titles/abstracts/categories only. "
|
|
"It estimates math-bearing route density and symbolic abstract markers, "
|
|
"not full equation extraction from LaTeX sources."
|
|
),
|
|
}
|
|
summary["packet_hash"] = sha256_text(stable_json(summary))
|
|
SUMMARY.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
receipt = {
|
|
"schema": "arxiv_math_breakout_receipt_v1",
|
|
"summary": str(SUMMARY.relative_to(REPO)),
|
|
"archive_path": summary["archive_path"],
|
|
"archive_sha256": summary["archive_sha256"],
|
|
"total_records_scanned": total_seen,
|
|
"math_bearing_records": selected_seen,
|
|
"math_bearing_fraction": summary["math_bearing_fraction"],
|
|
"category_count": summary["category_count"],
|
|
"latex_command_count": marker_totals.get("latex_command", 0),
|
|
"inline_math_span_count": marker_totals.get("inline_math_span", 0),
|
|
"symbolic_token_count": marker_totals.get("symbolic_token", 0),
|
|
"sample_record_count": len(samples),
|
|
"packet_hash": summary["packet_hash"],
|
|
"decision": summary["decision"],
|
|
"claim_boundary": summary["claim_boundary"],
|
|
}
|
|
receipt["receipt_hash"] = sha256_text(stable_json(receipt))
|
|
RECEIPT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps(receipt, indent=2, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|