mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
346 lines
14 KiB
Python
346 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Multi-API rotation fetcher.
|
|
Rotates through: Crossref, OpenAlex, Semantic Scholar, INSPIRE-HEP, Europe PMC, CORE.
|
|
Each API has its own rate limit — we cycle so none gets exhausted.
|
|
Slow, steady, accumulates over time. Writes incrementally to /dev/shm.
|
|
"""
|
|
|
|
import sqlite3
|
|
import urllib.request
|
|
import urllib.parse
|
|
import json
|
|
import time
|
|
import os
|
|
import sys
|
|
import random
|
|
from datetime import datetime
|
|
|
|
SRC = "/home/allaun/physics_equations.db"
|
|
TMP = "/dev/shm/physics_equations.db"
|
|
|
|
if os.path.exists(TMP):
|
|
os.remove(TMP)
|
|
os.system(f"cp {SRC} {TMP}")
|
|
|
|
conn = sqlite3.connect(TMP)
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA synchronous=OFF")
|
|
conn.execute("PRAGMA cache_size=-4000000")
|
|
cur = conn.cursor()
|
|
|
|
# Load domain → equation mapping
|
|
cur.execute("SELECT id, name FROM domains WHERE id IN (SELECT DISTINCT domain_id FROM equations WHERE domain_id IS NOT NULL) ORDER BY id")
|
|
domains = {r[0]: r[1] for r in cur.fetchall()}
|
|
|
|
cur.execute("SELECT domain_id, id FROM equations WHERE domain_id IS NOT NULL")
|
|
domain_eqs = {}
|
|
for d, e in cur.fetchall():
|
|
domain_eqs.setdefault(d, []).append(e)
|
|
|
|
# Find how many verifications each domain already has
|
|
cur.execute("""
|
|
SELECT e.domain_id, COUNT(v.id)
|
|
FROM equations e
|
|
LEFT JOIN verifications v ON v.equation_id = e.id
|
|
WHERE e.domain_id IS NOT NULL
|
|
GROUP BY e.domain_id
|
|
""")
|
|
domain_ver_count = {r[0]: r[1] for r in cur.fetchall()}
|
|
|
|
# Domain search queries (short, effective)
|
|
SEARCHES = {
|
|
1: ("Newton's laws motion", "classical mechanics force acceleration"),
|
|
2: ("universal gravitation inverse square", "Newton law gravitation"),
|
|
3: ("Maxwell equations electromagnetism", "electromagnetic wave propagation"),
|
|
4: ("thermodynamics entropy second law", "Carnot efficiency heat engine"),
|
|
5: ("Schrodinger equation quantum", "Heisenberg uncertainty principle"),
|
|
6: ("general relativity test", "gravitational wave detection"),
|
|
7: ("Standard Model electroweak", "Higgs boson discovery"),
|
|
8: ("Hubble constant cosmology", "cosmic microwave background"),
|
|
9: ("Navier-Stokes turbulence", "Reynolds number flow transition"),
|
|
10: ("Snell law refraction optics", "Fresnel diffraction interference"),
|
|
11: ("speed of sound measurement", "acoustic wave resonance"),
|
|
12: ("BCS superconductivity theory", "Josephson junction effect"),
|
|
13: ("neutrino oscillation flavor", "radioactive decay nuclear"),
|
|
14: ("Chandrasekhar white dwarf limit", "stellar evolution mass luminosity"),
|
|
15: ("Debye plasma screening length", "Alfven wave magnetohydrodynamics"),
|
|
16: ("Noether theorem symmetry", "Fourier transform analysis"),
|
|
17: ("Jarzynski equality fluctuation", "statistical mechanics partition"),
|
|
18: ("Hooke law elasticity stress", "Euler Bernoulli beam bending"),
|
|
19: ("Landauer principle information", "Shannon channel capacity"),
|
|
20: ("Planck constant kilogram redefinition", "atomic clock frequency standard"),
|
|
21: ("Hall-Petch grain boundary strengthening", "Paris law fatigue crack"),
|
|
22: ("X-ray crystallography Bragg law", "Debye-Waller factor thermal"),
|
|
23: ("Shockley diode semiconductor junction", "MOSFET transistor characteristics"),
|
|
24: ("Flory-Huggins polymer solution", "viscoelasticity glass transition"),
|
|
25: ("Langmuir adsorption monolayer", "BET surface area measurement"),
|
|
28: ("Gutenberg-Richter earthquake magnitude", "seismic wave velocity"),
|
|
29: ("geostrophic wind atmospheric", "Rossby wave planetary"),
|
|
30: ("ocean wave dispersion Ekman", "thermohaline circulation deep"),
|
|
31: ("Darcy law groundwater flow", "Richards equation soil moisture"),
|
|
32: ("Hodgkin-Huxley neuron action potential", "Michaelis-Menten enzyme kinetics"),
|
|
33: ("Marcus electron transfer theory", "Arrhenius activation energy reaction"),
|
|
34: ("optical frequency comb femtosecond", "nonlinear optics soliton"),
|
|
35: ("Zeeman effect atomic spectra", "Lamb shift hydrogen fine structure"),
|
|
36: ("non-Newtonian power-law fluid", "Bingham yield stress rheology"),
|
|
37: ("Archard wear coefficient tribology", "Reynolds lubrication equation"),
|
|
38: ("Janssen effect granular silo", "angle of repose granular flow"),
|
|
39: ("Coulomb blockade single electron", "graphene Dirac cone dispersion"),
|
|
40: ("Bell inequality quantum entanglement", "quantum error correction surface code"),
|
|
41: ("Lorenz attractor deterministic chaos", "Feigenbaum period doubling universality"),
|
|
42: ("MRI Bloch equation T1 T2", "proton therapy Bragg peak"),
|
|
43: ("Bethe-Bloch stopping power", "radiation dosimetry cavity theory"),
|
|
44: ("solar cell Shockley-Queisser efficiency", "Betz wind turbine limit"),
|
|
45: ("Parker spiral solar wind magnetic", "magnetosphere radiation belt"),
|
|
46: ("Chapman-Jouguet detonation wave", "Rankine-Hugoniot shock compression"),
|
|
47: ("negative index metamaterial refraction", "transformation optics invisibility"),
|
|
48: ("sonar equation underwater propagation", "ocean acoustic tomography"),
|
|
49: ("PID controller feedback stability", "Nyquist criterion control theory"),
|
|
}
|
|
|
|
# ================================================================
|
|
# API FETCHERS
|
|
# ================================================================
|
|
def fetch_crossref(query, limit=5):
|
|
"""Crossref API — polite pool with email in User-Agent."""
|
|
papers = []
|
|
try:
|
|
url = "https://api.crossref.org/works"
|
|
params = {
|
|
"query": query,
|
|
"rows": limit,
|
|
"sort": "relevance",
|
|
"filter": "type:journal-article",
|
|
}
|
|
full = url + "?" + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(full, headers={
|
|
"User-Agent": "PhysicsDB/1.0 (mailto:research@example.com)",
|
|
"Accept": "application/json",
|
|
})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
for item in data.get("message", {}).get("items", []):
|
|
title_list = item.get("title", [])
|
|
title = title_list[0] if title_list else ""
|
|
year = item.get("created", {}).get("date-parts", [[0]])[0][0]
|
|
doi = item.get("DOI", "")
|
|
journal = item.get("container-title", [""])[0] if item.get("container-title") else ""
|
|
if title:
|
|
papers.append({"title": title[:250], "year": year, "doi": doi, "journal": journal, "source": "Crossref"})
|
|
except Exception:
|
|
pass
|
|
return papers
|
|
|
|
def fetch_openalex(query, limit=5):
|
|
"""OpenAlex API — open, no key needed, ~10 req/sec polite."""
|
|
papers = []
|
|
try:
|
|
url = "https://api.openalex.org/works"
|
|
params = {"search": query, "per_page": limit, "sort": "cited_by_count:desc"}
|
|
full = url + "?" + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(full, headers={
|
|
"User-Agent": "mailto:research@example.com",
|
|
"Accept": "application/json",
|
|
})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
for item in data.get("results", []):
|
|
title = item.get("title", "")
|
|
year = item.get("publication_year") or 0
|
|
doi = item.get("doi", "") or ""
|
|
journal = ""
|
|
if item.get("primary_location") and item["primary_location"].get("source"):
|
|
journal = item["primary_location"]["source"].get("display_name", "")
|
|
if title:
|
|
papers.append({"title": title[:250], "year": year, "doi": doi, "journal": journal, "source": "OpenAlex"})
|
|
except Exception:
|
|
pass
|
|
return papers
|
|
|
|
def fetch_inspirehep(query, limit=5):
|
|
"""INSPIRE-HEP API — HEP papers, great for QFT/nuclear/astro domains."""
|
|
papers = []
|
|
try:
|
|
url = "https://inspirehep.net/api/literature"
|
|
params = {"q": query, "size": limit, "sort": "mostrecent"}
|
|
full = url + "?" + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(full, headers={
|
|
"User-Agent": "PhysicsDB/1.0",
|
|
"Accept": "application/json",
|
|
})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
for item in data.get("hits", {}).get("hits", []):
|
|
meta = item.get("metadata", {})
|
|
title_el = meta.get("titles", [{}])
|
|
title = title_el[0].get("title", "") if title_el else ""
|
|
year = meta.get("publication_info", [{}])
|
|
yr = year[0].get("year", 0) if year else 0
|
|
dois = meta.get("dois", [{}])
|
|
doi = dois[0].get("value", "") if dois else ""
|
|
if title:
|
|
papers.append({"title": title[:250], "year": yr, "doi": doi, "journal": "INSPIRE-HEP", "source": "INSPIRE-HEP"})
|
|
except Exception:
|
|
pass
|
|
return papers
|
|
|
|
def fetch_europepmc(query, limit=5):
|
|
"""Europe PMC API — biomedical/life sciences papers (biophys, medphys, etc)."""
|
|
papers = []
|
|
try:
|
|
url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
|
|
params = {"query": query, "resultType": "core", "pageSize": limit, "format": "json"}
|
|
full = url + "?" + urllib.parse.urlencode(params)
|
|
req = urllib.request.Request(full, headers={"User-Agent": "PhysicsDB/1.0"})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
for item in data.get("resultList", {}).get("result", []):
|
|
title = item.get("title", "")
|
|
year = int(item.get("firstPublicationDate", "0")[:4]) if item.get("firstPublicationDate") else 0
|
|
doi = item.get("doi", "")
|
|
journal = item.get("journalTitle", "")
|
|
if title:
|
|
papers.append({"title": title[:250], "year": year, "doi": doi, "journal": journal, "source": "EuropePMC"})
|
|
except Exception:
|
|
pass
|
|
return papers
|
|
|
|
def fetch_core(query, limit=5):
|
|
"""CORE.ac.uk API — open access repository aggregator."""
|
|
papers = []
|
|
try:
|
|
url = "https://api.core.ac.uk/v3/search/works"
|
|
body = json.dumps({"q": query, "limit": limit}).encode()
|
|
req = urllib.request.Request(url, data=body, headers={
|
|
"User-Agent": "PhysicsDB/1.0",
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json",
|
|
})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
for item in data.get("results", []):
|
|
title = item.get("title", "")
|
|
year = item.get("yearPublished") or 0
|
|
doi = item.get("doi", "")
|
|
journal = item.get("publisher", "")
|
|
if title:
|
|
papers.append({"title": title[:250], "year": year, "doi": doi, "journal": journal, "source": "CORE"})
|
|
except Exception:
|
|
pass
|
|
return papers
|
|
|
|
# ================================================================
|
|
# API rotation — 5 APIs, 1 query each cycle, 3s between queries
|
|
# ================================================================
|
|
APIS = [
|
|
("Crossref", fetch_crossref, 1.5),
|
|
("OpenAlex", fetch_openalex, 2.0),
|
|
("CrossRef2", fetch_crossref, 1.5),
|
|
("INSPIRE", fetch_inspirehep, 1.5),
|
|
("EuropePMC", fetch_europepmc, 1.5),
|
|
("OpenAlex2", fetch_openalex, 2.0),
|
|
("Crossref3", fetch_crossref, 1.5),
|
|
("INSPIRE2", fetch_inspirehep, 1.5),
|
|
]
|
|
|
|
# Get domains needing more papers (prioritize those with fewest)
|
|
gap_domains = sorted(
|
|
[(did, domain_ver_count.get(did, 0)) for did in SEARCHES.keys()],
|
|
key=lambda x: x[1]
|
|
)
|
|
|
|
print(f"Multi-API fetch — {len(APIS)} API slots, {len(gap_domains)} domains")
|
|
print(f" Least-covered domains: {', '.join(domains[d] for d,_ in gap_domains[:5])}")
|
|
print(f" APIs: {', '.join(a[0] for a in APIS)}")
|
|
print()
|
|
|
|
total = 0
|
|
insert_rows = []
|
|
start = time.time()
|
|
api_idx = 0
|
|
|
|
while gap_domains and (time.time() - start) < 7200: # 2 hour max
|
|
dom_id, _ = gap_domains.pop(0)
|
|
|
|
if dom_id not in SEARCHES:
|
|
continue
|
|
|
|
name = domains.get(dom_id, f"dom_{dom_id}")
|
|
eqs = domain_eqs.get(dom_id, [None])
|
|
|
|
queries = SEARCHES[dom_id]
|
|
# Pick one query — alternate between the two
|
|
q = queries[0] if total % 2 == 0 else queries[1]
|
|
|
|
api_name, fetcher, delay = APIS[api_idx % len(APIS)]
|
|
api_idx += 1
|
|
|
|
papers = fetcher(q, limit=5)
|
|
|
|
for i, p in enumerate(papers):
|
|
eq_id = eqs[i % len(eqs)] if eqs else None
|
|
insert_rows.append((
|
|
eq_id, p["title"],
|
|
f"{p['source']}: {p['journal']}" if p.get('journal') else p['source'],
|
|
p["year"], p.get("doi", p['source']), p['source'],
|
|
))
|
|
total += 1
|
|
|
|
t = time.time() - start
|
|
marker = "✓" if papers else "○"
|
|
print(f" {marker} [{api_name:10s}] {name:28s} → {len(papers):2d} papers | {total:4d} total | {t:.0f}s", flush=True)
|
|
|
|
# Re-add domain if it got zero papers (retry later with other query)
|
|
if not papers:
|
|
gap_domains.append((dom_id, 0))
|
|
|
|
# Flush every 10 batches
|
|
if len(insert_rows) >= 50:
|
|
cur.executemany(
|
|
"""INSERT INTO verifications (equation_id, test_name, experiment, year, precision_level, status)
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
insert_rows,
|
|
)
|
|
conn.commit()
|
|
print(f" ⟳ Flushed {len(insert_rows)} records ({total} total) [{time.time()-start:.0f}s]", flush=True)
|
|
insert_rows = []
|
|
|
|
time.sleep(delay)
|
|
|
|
# Final flush
|
|
if insert_rows:
|
|
cur.executemany(
|
|
"INSERT INTO verifications (equation_id, test_name, experiment, year, precision_level, status) VALUES (?,?,?,?,?,?)",
|
|
insert_rows,
|
|
)
|
|
conn.commit()
|
|
|
|
# ================================================================
|
|
# STATS
|
|
# ================================================================
|
|
cur.execute("SELECT COUNT(*) FROM verifications")
|
|
total_ver = cur.fetchone()[0]
|
|
cur.execute("SELECT status, COUNT(*) FROM verifications GROUP BY status ORDER BY COUNT(*) DESC")
|
|
print(f"\n═══ VERIFICATIONS: {total_ver} ═══")
|
|
for row in cur.fetchall():
|
|
print(f" {row[0]:30s}: {row[1]:6d}")
|
|
|
|
cur.execute("""
|
|
SELECT d.name, COUNT(v.id) as n
|
|
FROM verifications v
|
|
JOIN equations e ON v.equation_id = e.id
|
|
JOIN domains d ON e.domain_id = d.id
|
|
GROUP BY d.id ORDER BY n DESC
|
|
""")
|
|
print(f"\n═══ DOMAINS BY COVERAGE ═══")
|
|
for row in cur.fetchall():
|
|
pct = row[1] / max(total_ver, 1) * 100
|
|
bar = "█" * int(pct / 2)
|
|
print(f" {row[0]:28s} {row[1]:4d} {bar}")
|
|
|
|
conn.close()
|
|
os.system(f"cp {TMP} {SRC}")
|
|
elapsed = time.time() - start
|
|
print(f"\n✓ Complete — {total_ver} verifications in {elapsed:.0f}s ({elapsed/60:.1f} min)")
|
|
print(f" Database: {SRC}")
|