mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
allaun
55d3c281c4
Add s2-proxy worker (index.js, run_after_key.sh, wrangler.toml) for Semantic Scholar API proxying.
25 lines
808 B
Bash
Executable file
25 lines
808 B
Bash
Executable file
#!/usr/bin/env bash
|
|
# Run this after setting S2_API_KEY secret to confirm the worker + key work,
|
|
# then launch the full citation crawl.
|
|
|
|
set -e
|
|
WORKER="https://s2-proxy.researchstack.workers.dev"
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")/../.." && pwd)/shim"
|
|
|
|
echo "=== health check ==="
|
|
curl -s "$WORKER/health" | python3 -m json.tool
|
|
|
|
echo ""
|
|
echo "=== test fetch (PageRank paper search) ==="
|
|
curl -sv "$WORKER/fetch?path=/graph/v1/paper/search&query=PageRank+Brin+Page+1998&fields=paperId,title,year&limit=2" 2>&1 | tail -20
|
|
|
|
echo ""
|
|
echo "=== launching citation crawl ==="
|
|
python3 "$SCRIPT_DIR/s2_citation_crawler.py" \
|
|
--seed "PageRank Brin Page 1998 web search engine" \
|
|
--workers "$WORKER" \
|
|
--hops 2 \
|
|
--max-refs 50 \
|
|
--max-cites 30 \
|
|
--max-papers 400 \
|
|
--out /tmp/pagerank_citation_graph.json
|