allaun/Research-Stack
1
0
Fork 0
mirror of https://github.com/allaunthefox/Research-Stack.git synced 2026-08-18 17:30:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(infra): escape Gremlin properties and fix ENE shell escaping

Browse source 
- distribute_extraction.py: add gremlin_escape() for special chars,
  add --batch-size async mode with submitAsync()
- load_cornfield.py: pipe SQL via stdin to avoid shell metachar issues
- populate_ene_tables.py: same stdin pipe fix for neon SSH calls
This commit is contained in:
allaun 2026-06-22 04:18:24 -05:00
parent 7dfa40d754
commit b258418782
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
scripts/load_cornfield.py
View file

@ -35,11 +35,11 @@ CONTAINER = "arxiv-pg"
# ── Postgres helpers ────────────────────────────────────────────────────────── # ── Postgres helpers ──────────────────────────────────────────────────────────
def neon(sql: str, timeout: int = 60) -> str: def neon(sql: str, timeout: int = 60) -> str:
"""Run SQL on neon-64gb arxiv DB, return stdout.""" """Run SQL on neon-64gb arxiv DB, return stdout. Uses stdin pipe to avoid shell escaping."""
r = subprocess.run( r = subprocess.run(
["ssh", NEON_HOST, ["ssh", NEON_HOST,
f"podman exec {CONTAINER} psql -U postgres -d arxiv -t -A -c \"{sql}\""], f"podman exec -i {CONTAINER} psql -U postgres -d arxiv -t -A"],
capture_output=True, text=True, timeout=timeout, input=sql, capture_output=True, text=True, timeout=timeout,
) )
if r.returncode != 0: if r.returncode != 0:
print(f" NEON ERR: {r.stderr[:200]}") print(f" NEON ERR: {r.stderr[:200]}")