feat(infra): Semantic Scholar Cloudflare worker proxy

Add s2-proxy worker (index.js, run_after_key.sh, wrangler.toml) for
Semantic Scholar API proxying.

4-Infrastructure/workers/s2-proxy/index.js

@@ -0,0 +1,115 @@
+/**
+ * s2-proxy Cloudflare Worker
+ *
+ * Proxies requests to the Semantic Scholar Graph API with:
+ *   - Per-worker API key (stored as secret S2_API_KEY)
+ *   - KV caching of responses (24h TTL, configurable)
+ *   - Simple rate-limit header passthrough so coordinator knows to back off
+ *
+ * Deploy N copies of this worker (each with its own API key secret) to get
+ * N × 1 req/sec aggregate throughput against S2.
+ *
+ * Endpoint:
+ *   GET /fetch?path=/graph/v1/paper/{id}&fields=paperId,title,year,references,citations
+ *
+ * The coordinator calls  https://<worker>.workers.dev/fetch?path=<s2_path>
+ * and gets back the raw S2 JSON (or a cached copy).
+ *
+ * Environment bindings required (wrangler.toml):
+ *   S2_API_KEY  — secret (s2 api key for this worker instance)
+ *   S2_CACHE    — KV namespace binding
+ */
+
+const S2_BASE = "https://api.semanticscholar.org";
+const CACHE_TTL = 86400; // 24 hours
+
+export default {
+  async fetch(request, env, ctx) {
+    // ── CORS preflight ────────────────────────────────────────────────────────
+    if (request.method === "OPTIONS") {
+      return new Response(null, {
+        headers: {
+          "Access-Control-Allow-Origin": "*",
+          "Access-Control-Allow-Methods": "GET",
+          "Access-Control-Allow-Headers": "Content-Type",
+        },
+      });
+    }
+
+    const url = new URL(request.url);
+
+    // ── Health check ──────────────────────────────────────────────────────────
+    if (url.pathname === "/health") {
+      return new Response(JSON.stringify({ ok: true, key_set: !!env.S2_API_KEY }), {
+        headers: { "Content-Type": "application/json" },
+      });
+    }
+
+    // ── Main proxy endpoint ───────────────────────────────────────────────────
+    if (url.pathname !== "/fetch") {
+      return new Response("use /fetch?path=/graph/v1/paper/...", { status: 400 });
+    }
+
+    const s2Path = url.searchParams.get("path");
+    if (!s2Path) {
+      return new Response("missing path param", { status: 400 });
+    }
+
+    // Preserve any query params the caller passed (fields, limit, offset, etc.)
+    const forwardParams = new URLSearchParams(url.search);
+    forwardParams.delete("path");
+    const s2Url = `${S2_BASE}${s2Path}${
+      forwardParams.toString() ? "?" + forwardParams.toString() : ""
+    }`;
+
+    // ── KV cache lookup ───────────────────────────────────────────────────────
+    const cacheKey = s2Url;
+    if (env.S2_CACHE) {
+      const cached = await env.S2_CACHE.get(cacheKey);
+      if (cached !== null) {
+        return new Response(cached, {
+          headers: {
+            "Content-Type": "application/json",
+            "X-S2-Cache": "HIT",
+            "Access-Control-Allow-Origin": "*",
+          },
+        });
+      }
+    }
+
+    // ── Fetch from S2 ─────────────────────────────────────────────────────────
+    const headers = { "User-Agent": "research-stack-crawler/1.0" };
+    if (env.S2_API_KEY) {
+      headers["x-api-key"] = env.S2_API_KEY;
+    }
+
+    let s2Resp;
+    try {
+      s2Resp = await fetch(s2Url, { headers });
+    } catch (err) {
+      return new Response(JSON.stringify({ error: String(err) }), {
+        status: 502,
+        headers: { "Content-Type": "application/json" },
+      });
+    }
+
+    // Pass rate-limit headers back so coordinator can see them
+    const outHeaders = {
+      "Content-Type": "application/json",
+      "X-S2-Cache": "MISS",
+      "Access-Control-Allow-Origin": "*",
+    };
+    for (const h of ["x-ratelimit-limit", "x-ratelimit-remaining", "retry-after"]) {
+      if (s2Resp.headers.has(h)) outHeaders[h] = s2Resp.headers.get(h);
+    }
+
+    const body = await s2Resp.text();
+
+    // Cache successful responses only
+    if (s2Resp.ok && env.S2_CACHE) {
+      ctx.waitUntil(env.S2_CACHE.put(cacheKey, body, { expirationTtl: CACHE_TTL }));
+    }
+
+    return new Response(body, { status: s2Resp.status, headers: outHeaders });
+  },
+};

4-Infrastructure/workers/s2-proxy/run_after_key.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Run this after setting S2_API_KEY secret to confirm the worker + key work,
+# then launch the full citation crawl.
+
+set -e
+WORKER="https://s2-proxy.researchstack.workers.dev"
+SCRIPT_DIR="$(cd "$(dirname "$0")/../.." && pwd)/shim"
+
+echo "=== health check ==="
+curl -s "$WORKER/health" | python3 -m json.tool
+
+echo ""
+echo "=== test fetch (PageRank paper search) ==="
+curl -sv "$WORKER/fetch?path=/graph/v1/paper/search&query=PageRank+Brin+Page+1998&fields=paperId,title,year&limit=2" 2>&1 | tail -20
+
+echo ""
+echo "=== launching citation crawl ==="
+python3 "$SCRIPT_DIR/s2_citation_crawler.py" \
+  --seed "PageRank Brin Page 1998 web search engine" \
+  --workers "$WORKER" \
+  --hops 2 \
+  --max-refs 50 \
+  --max-cites 30 \
+  --max-papers 400 \
+  --out /tmp/pagerank_citation_graph.json

4-Infrastructure/workers/s2-proxy/wrangler.toml

@@ -0,0 +1,26 @@
+name = "s2-proxy"
+main = "index.js"
+compatibility_date = "2024-01-01"
+workers_dev = true
+
+# KV namespace for response caching.
+# Create with: wrangler kv namespace create S2_CACHE
+# Then paste the id below.
+[[kv_namespaces]]
+binding = "S2_CACHE"
+id = "e7e05154bb734928bf6a6a5cf0fc7f0a"
+
+# S2_API_KEY is a secret — never put it in wrangler.toml.
+# Set with: wrangler secret put S2_API_KEY
+# Then paste your key when prompted.
+#
+# Get a free S2 API key at: https://www.semanticscholar.org/product/api
+# One key per account. For N-worker deployment, use N accounts/keys.
+
+# To deploy multiple instances with different keys:
+#   cp wrangler.toml wrangler-2.toml  # change name = "s2-proxy-2"
+#   wrangler deploy --config wrangler-2.toml
+#   wrangler secret put S2_API_KEY --config wrangler-2.toml
+
+# Deployed worker URL (set after first deploy):
+# https://s2-proxy.researchstack.workers.dev