mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
299 lines
12 KiB
Python
299 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Ingest a local arXiv Kaggle/Croissant metadata descriptor.
|
|
|
|
This does not download the Kaggle archive or arXiv bulk data. It records the
|
|
descriptor as a receipted source surface for later lawful ingestion.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import zipfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
DEFAULT_INPUT = Path("/home/allaun/Documents/ingest/arxiv-metadata.json")
|
|
DEFAULT_ARCHIVE = Path("/home/allaun/Documents/ingest/kaggledataset.zip")
|
|
OUT_DIR = REPO / "shared-data" / "data" / "math_research_databases"
|
|
PACKET = OUT_DIR / "arxiv_metadata_descriptor_packet.json"
|
|
RECEIPT = OUT_DIR / "arxiv_metadata_descriptor_receipt.json"
|
|
ARCHIVE_PACKET = OUT_DIR / "arxiv_kaggle_archive_packet.json"
|
|
ARCHIVE_SAMPLE_JSONL = OUT_DIR / "arxiv_kaggle_archive_sample.jsonl"
|
|
ARCHIVE_SAMPLE_CSV = OUT_DIR / "arxiv_kaggle_archive_sample.csv"
|
|
ARCHIVE_RECEIPT = OUT_DIR / "arxiv_kaggle_archive_receipt.json"
|
|
|
|
|
|
def stable_json(obj: Any) -> str:
|
|
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_text(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def as_list(value: Any) -> list[Any]:
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def text(value: Any) -> str | None:
|
|
return value if isinstance(value, str) else None
|
|
|
|
|
|
def dist_summary(obj: dict[str, Any]) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
for item in as_list(obj.get("distribution")):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
rows.append(
|
|
{
|
|
"id": item.get("@id"),
|
|
"type": item.get("@type"),
|
|
"name": item.get("name"),
|
|
"encoding_format": item.get("encodingFormat"),
|
|
"content_url": item.get("contentUrl"),
|
|
"content_size": item.get("contentSize"),
|
|
"md5": item.get("md5"),
|
|
"includes": item.get("includes"),
|
|
"contained_in": item.get("containedIn", {}).get("@id")
|
|
if isinstance(item.get("containedIn"), dict)
|
|
else None,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--input", default=str(DEFAULT_INPUT))
|
|
parser.add_argument("--archive", default=str(DEFAULT_ARCHIVE))
|
|
parser.add_argument("--sample-records", type=int, default=25)
|
|
args = parser.parse_args()
|
|
|
|
source = Path(args.input).expanduser().resolve()
|
|
raw = source.read_bytes()
|
|
obj = json.loads(raw)
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
license_obj = obj.get("license") if isinstance(obj.get("license"), dict) else {}
|
|
creator_obj = obj.get("creator") if isinstance(obj.get("creator"), dict) else {}
|
|
publisher_obj = obj.get("publisher") if isinstance(obj.get("publisher"), dict) else {}
|
|
catalog_obj = obj.get("includedInDataCatalog") if isinstance(obj.get("includedInDataCatalog"), dict) else {}
|
|
|
|
packet = {
|
|
"schema": "arxiv_metadata_descriptor_packet_v1",
|
|
"rrc_shape_hint": "ArxivMetadataDescriptorSurface",
|
|
"source_path": str(source),
|
|
"source_size_bytes": len(raw),
|
|
"source_sha256": sha256_bytes(raw),
|
|
"dataset_name": obj.get("name"),
|
|
"alternate_name": obj.get("alternateName"),
|
|
"dataset_type": obj.get("@type"),
|
|
"version": obj.get("version"),
|
|
"date_published": obj.get("datePublished"),
|
|
"date_modified": obj.get("dateModified"),
|
|
"url": obj.get("url"),
|
|
"identifier": obj.get("identifier"),
|
|
"cite_as": obj.get("citeAs"),
|
|
"conforms_to": obj.get("conformsTo"),
|
|
"is_accessible_for_free": obj.get("isAccessibleForFree"),
|
|
"license_name": license_obj.get("name"),
|
|
"license_url": license_obj.get("url"),
|
|
"creator": creator_obj.get("name"),
|
|
"publisher": publisher_obj.get("name"),
|
|
"data_catalog": catalog_obj.get("name"),
|
|
"distribution": dist_summary(obj),
|
|
"keywords": as_list(obj.get("keywords")),
|
|
"expected_article_fields": [
|
|
"id",
|
|
"submitter",
|
|
"authors",
|
|
"title",
|
|
"comments",
|
|
"journal-ref",
|
|
"doi",
|
|
"abstract",
|
|
"categories",
|
|
"versions",
|
|
],
|
|
"bulk_surfaces_declared_in_descriptor": [
|
|
"kaggle_metadata_archive",
|
|
"google_cloud_storage_pdf_bucket",
|
|
"google_cloud_storage_source_bucket",
|
|
"arxiv_abs_url_template",
|
|
"arxiv_pdf_url_template",
|
|
],
|
|
"density_markers": [
|
|
"croissant_dataset_descriptor",
|
|
"kaggle_metadata_snapshot",
|
|
"arxiv_category_graph",
|
|
"paper_version_history",
|
|
"abstract_text_surface",
|
|
"latex_source_archive_pointer",
|
|
"pdf_bulk_archive_pointer",
|
|
"knowledge_graph_construction_surface",
|
|
],
|
|
"access_boundary": (
|
|
"Descriptor-only ingest. Metadata license is recorded from the descriptor; "
|
|
"individual papers and full-text/source downloads still follow arXiv/Kaggle/GCS terms."
|
|
),
|
|
"decision": "HOLD",
|
|
"claim_boundary": (
|
|
"This receipt proves local descriptor capture and hashing only. It is not a "
|
|
"receipt for the 1.617 GB metadata archive, the GCS bucket contents, PDFs, "
|
|
"source tarballs, or theorem validity."
|
|
),
|
|
}
|
|
packet["packet_hash"] = sha256_text(stable_json(packet))
|
|
PACKET.write_text(json.dumps(packet, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
receipt = {
|
|
"schema": "arxiv_metadata_descriptor_receipt_v1",
|
|
"packet": str(PACKET.relative_to(REPO)),
|
|
"source_path": packet["source_path"],
|
|
"source_size_bytes": packet["source_size_bytes"],
|
|
"source_sha256": packet["source_sha256"],
|
|
"packet_hash": packet["packet_hash"],
|
|
"dataset_name": packet["dataset_name"],
|
|
"version": packet["version"],
|
|
"distribution_count": len(packet["distribution"]),
|
|
"density_marker_count": len(packet["density_markers"]),
|
|
"decision": packet["decision"],
|
|
"claim_boundary": packet["claim_boundary"],
|
|
}
|
|
receipt["receipt_hash"] = sha256_text(stable_json(receipt))
|
|
RECEIPT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
archive = Path(args.archive).expanduser().resolve()
|
|
archive_receipt: dict[str, Any] | None = None
|
|
if archive.exists():
|
|
archive_hash = sha256_file(archive)
|
|
sample_records: list[dict[str, Any]] = []
|
|
with zipfile.ZipFile(archive) as zf:
|
|
infos = zf.infolist()
|
|
members = [
|
|
{
|
|
"filename": info.filename,
|
|
"compress_size": info.compress_size,
|
|
"file_size": info.file_size,
|
|
"date_time": list(info.date_time),
|
|
"crc": info.CRC,
|
|
}
|
|
for info in infos
|
|
]
|
|
json_members = [info for info in infos if info.filename.endswith(".json")]
|
|
selected = json_members[0] if json_members else infos[0]
|
|
with zf.open(selected) as handle:
|
|
for _ in range(args.sample_records):
|
|
line = handle.readline()
|
|
if not line:
|
|
break
|
|
rec = json.loads(line)
|
|
sample = {
|
|
"id": rec.get("id"),
|
|
"title": rec.get("title"),
|
|
"authors": rec.get("authors"),
|
|
"categories": rec.get("categories"),
|
|
"versions": rec.get("versions"),
|
|
"doi": rec.get("doi"),
|
|
"journal_ref": rec.get("journal-ref"),
|
|
"abstract_sha256": sha256_text(text(rec.get("abstract")) or ""),
|
|
"abstract_bytes": len((text(rec.get("abstract")) or "").encode("utf-8")),
|
|
}
|
|
sample["packet_hash"] = sha256_text(stable_json(sample))
|
|
sample_records.append(sample)
|
|
|
|
ARCHIVE_SAMPLE_JSONL.write_text(
|
|
"\n".join(stable_json(record) for record in sample_records) + "\n", encoding="utf-8"
|
|
)
|
|
csv_lines = [
|
|
"id,title,authors,categories,version_count,doi,journal_ref,abstract_sha256,packet_hash"
|
|
]
|
|
for record in sample_records:
|
|
csv_lines.append(
|
|
",".join(
|
|
[
|
|
json.dumps(record.get("id") or ""),
|
|
json.dumps(record.get("title") or ""),
|
|
json.dumps(record.get("authors") or ""),
|
|
json.dumps(record.get("categories") or ""),
|
|
json.dumps(len(record.get("versions") or [])),
|
|
json.dumps(record.get("doi") or ""),
|
|
json.dumps(record.get("journal_ref") or ""),
|
|
json.dumps(record["abstract_sha256"]),
|
|
json.dumps(record["packet_hash"]),
|
|
]
|
|
)
|
|
)
|
|
ARCHIVE_SAMPLE_CSV.write_text("\n".join(csv_lines) + "\n", encoding="utf-8")
|
|
|
|
archive_packet = {
|
|
"schema": "arxiv_kaggle_archive_packet_v1",
|
|
"rrc_shape_hint": "ArxivKaggleMetadataSnapshot",
|
|
"archive_path": str(archive),
|
|
"archive_size_bytes": archive.stat().st_size,
|
|
"archive_sha256": archive_hash,
|
|
"zip_member_count": len(members),
|
|
"zip_members": members,
|
|
"sample_record_count": len(sample_records),
|
|
"sample_records_jsonl": str(ARCHIVE_SAMPLE_JSONL.relative_to(REPO)),
|
|
"sample_records_csv": str(ARCHIVE_SAMPLE_CSV.relative_to(REPO)),
|
|
"sample_policy": (
|
|
"Stream first records from the JSON member inside the zip without extracting "
|
|
"the 5GB snapshot. Store hashes and routing fields only."
|
|
),
|
|
"density_markers": [
|
|
"kaggle_metadata_archive_verified",
|
|
"arxiv_jsonl_snapshot",
|
|
"category_graph_sample",
|
|
"version_history_sample",
|
|
"abstract_hash_sample",
|
|
],
|
|
"decision": "HOLD",
|
|
"claim_boundary": (
|
|
"Archive hash and bounded sample verified. The full JSONL corpus is not "
|
|
"materialized into the repo, and per-paper full text/PDF/source replay is not claimed."
|
|
),
|
|
}
|
|
archive_packet["packet_hash"] = sha256_text(stable_json(archive_packet))
|
|
ARCHIVE_PACKET.write_text(
|
|
json.dumps(archive_packet, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
)
|
|
archive_receipt = {
|
|
"schema": "arxiv_kaggle_archive_receipt_v1",
|
|
"packet": str(ARCHIVE_PACKET.relative_to(REPO)),
|
|
"archive_path": archive_packet["archive_path"],
|
|
"archive_size_bytes": archive_packet["archive_size_bytes"],
|
|
"archive_sha256": archive_packet["archive_sha256"],
|
|
"zip_member_count": archive_packet["zip_member_count"],
|
|
"sample_record_count": archive_packet["sample_record_count"],
|
|
"packet_hash": archive_packet["packet_hash"],
|
|
"decision": archive_packet["decision"],
|
|
"claim_boundary": archive_packet["claim_boundary"],
|
|
}
|
|
archive_receipt["receipt_hash"] = sha256_text(stable_json(archive_receipt))
|
|
ARCHIVE_RECEIPT.write_text(
|
|
json.dumps(archive_receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
out = {"descriptor_receipt": receipt, "archive_receipt": archive_receipt}
|
|
print(json.dumps(out, indent=2, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|