diff --git a/4-Infrastructure/workers/s2-proxy/index.js b/4-Infrastructure/workers/s2-proxy/index.js new file mode 100644 index 00000000..2bea7867 --- /dev/null +++ b/4-Infrastructure/workers/s2-proxy/index.js @@ -0,0 +1,115 @@ +/** + * s2-proxy Cloudflare Worker + * + * Proxies requests to the Semantic Scholar Graph API with: + * - Per-worker API key (stored as secret S2_API_KEY) + * - KV caching of responses (24h TTL, configurable) + * - Simple rate-limit header passthrough so coordinator knows to back off + * + * Deploy N copies of this worker (each with its own API key secret) to get + * N × 1 req/sec aggregate throughput against S2. + * + * Endpoint: + * GET /fetch?path=/graph/v1/paper/{id}&fields=paperId,title,year,references,citations + * + * The coordinator calls https://.workers.dev/fetch?path= + * and gets back the raw S2 JSON (or a cached copy). + * + * Environment bindings required (wrangler.toml): + * S2_API_KEY — secret (s2 api key for this worker instance) + * S2_CACHE — KV namespace binding + */ + +const S2_BASE = "https://api.semanticscholar.org"; +const CACHE_TTL = 86400; // 24 hours + +export default { + async fetch(request, env, ctx) { + // ── CORS preflight ──────────────────────────────────────────────────────── + if (request.method === "OPTIONS") { + return new Response(null, { + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET", + "Access-Control-Allow-Headers": "Content-Type", + }, + }); + } + + const url = new URL(request.url); + + // ── Health check ────────────────────────────────────────────────────────── + if (url.pathname === "/health") { + return new Response(JSON.stringify({ ok: true, key_set: !!env.S2_API_KEY }), { + headers: { "Content-Type": "application/json" }, + }); + } + + // ── Main proxy endpoint ─────────────────────────────────────────────────── + if (url.pathname !== "/fetch") { + return new Response("use /fetch?path=/graph/v1/paper/...", { status: 400 }); + } + + const s2Path = url.searchParams.get("path"); + if (!s2Path) { + return new Response("missing path param", { status: 400 }); + } + + // Preserve any query params the caller passed (fields, limit, offset, etc.) + const forwardParams = new URLSearchParams(url.search); + forwardParams.delete("path"); + const s2Url = `${S2_BASE}${s2Path}${ + forwardParams.toString() ? "?" + forwardParams.toString() : "" + }`; + + // ── KV cache lookup ─────────────────────────────────────────────────────── + const cacheKey = s2Url; + if (env.S2_CACHE) { + const cached = await env.S2_CACHE.get(cacheKey); + if (cached !== null) { + return new Response(cached, { + headers: { + "Content-Type": "application/json", + "X-S2-Cache": "HIT", + "Access-Control-Allow-Origin": "*", + }, + }); + } + } + + // ── Fetch from S2 ───────────────────────────────────────────────────────── + const headers = { "User-Agent": "research-stack-crawler/1.0" }; + if (env.S2_API_KEY) { + headers["x-api-key"] = env.S2_API_KEY; + } + + let s2Resp; + try { + s2Resp = await fetch(s2Url, { headers }); + } catch (err) { + return new Response(JSON.stringify({ error: String(err) }), { + status: 502, + headers: { "Content-Type": "application/json" }, + }); + } + + // Pass rate-limit headers back so coordinator can see them + const outHeaders = { + "Content-Type": "application/json", + "X-S2-Cache": "MISS", + "Access-Control-Allow-Origin": "*", + }; + for (const h of ["x-ratelimit-limit", "x-ratelimit-remaining", "retry-after"]) { + if (s2Resp.headers.has(h)) outHeaders[h] = s2Resp.headers.get(h); + } + + const body = await s2Resp.text(); + + // Cache successful responses only + if (s2Resp.ok && env.S2_CACHE) { + ctx.waitUntil(env.S2_CACHE.put(cacheKey, body, { expirationTtl: CACHE_TTL })); + } + + return new Response(body, { status: s2Resp.status, headers: outHeaders }); + }, +}; diff --git a/4-Infrastructure/workers/s2-proxy/run_after_key.sh b/4-Infrastructure/workers/s2-proxy/run_after_key.sh new file mode 100755 index 00000000..980a8001 --- /dev/null +++ b/4-Infrastructure/workers/s2-proxy/run_after_key.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Run this after setting S2_API_KEY secret to confirm the worker + key work, +# then launch the full citation crawl. + +set -e +WORKER="https://s2-proxy.researchstack.workers.dev" +SCRIPT_DIR="$(cd "$(dirname "$0")/../.." && pwd)/shim" + +echo "=== health check ===" +curl -s "$WORKER/health" | python3 -m json.tool + +echo "" +echo "=== test fetch (PageRank paper search) ===" +curl -sv "$WORKER/fetch?path=/graph/v1/paper/search&query=PageRank+Brin+Page+1998&fields=paperId,title,year&limit=2" 2>&1 | tail -20 + +echo "" +echo "=== launching citation crawl ===" +python3 "$SCRIPT_DIR/s2_citation_crawler.py" \ + --seed "PageRank Brin Page 1998 web search engine" \ + --workers "$WORKER" \ + --hops 2 \ + --max-refs 50 \ + --max-cites 30 \ + --max-papers 400 \ + --out /tmp/pagerank_citation_graph.json diff --git a/4-Infrastructure/workers/s2-proxy/wrangler.toml b/4-Infrastructure/workers/s2-proxy/wrangler.toml new file mode 100644 index 00000000..214bb283 --- /dev/null +++ b/4-Infrastructure/workers/s2-proxy/wrangler.toml @@ -0,0 +1,26 @@ +name = "s2-proxy" +main = "index.js" +compatibility_date = "2024-01-01" +workers_dev = true + +# KV namespace for response caching. +# Create with: wrangler kv namespace create S2_CACHE +# Then paste the id below. +[[kv_namespaces]] +binding = "S2_CACHE" +id = "e7e05154bb734928bf6a6a5cf0fc7f0a" + +# S2_API_KEY is a secret — never put it in wrangler.toml. +# Set with: wrangler secret put S2_API_KEY +# Then paste your key when prompted. +# +# Get a free S2 API key at: https://www.semanticscholar.org/product/api +# One key per account. For N-worker deployment, use N accounts/keys. + +# To deploy multiple instances with different keys: +# cp wrangler.toml wrangler-2.toml # change name = "s2-proxy-2" +# wrangler deploy --config wrangler-2.toml +# wrangler secret put S2_API_KEY --config wrangler-2.toml + +# Deployed worker URL (set after first deploy): +# https://s2-proxy.researchstack.workers.dev