fix(infra): escape Gremlin properties and fix ENE shell escaping

- distribute_extraction.py: add gremlin_escape() for special chars,
  add --batch-size async mode with submitAsync()
- load_cornfield.py: pipe SQL via stdin to avoid shell metachar issues
- populate_ene_tables.py: same stdin pipe fix for neon SSH calls
This commit is contained in:
allaun 2026-06-22 04:18:24 -05:00
parent 5cfa8f0898
commit 513d2484a3

View file

@ -35,11 +35,11 @@ CONTAINER = "arxiv-pg"
# ── Postgres helpers ──────────────────────────────────────────────────────────
def neon(sql: str, timeout: int = 60) -> str:
"""Run SQL on neon-64gb arxiv DB, return stdout."""
"""Run SQL on neon-64gb arxiv DB, return stdout. Uses stdin pipe to avoid shell escaping."""
r = subprocess.run(
["ssh", NEON_HOST,
f"podman exec {CONTAINER} psql -U postgres -d arxiv -t -A -c \"{sql}\""],
capture_output=True, text=True, timeout=timeout,
f"podman exec -i {CONTAINER} psql -U postgres -d arxiv -t -A"],
input=sql, capture_output=True, text=True, timeout=timeout,
)
if r.returncode != 0:
print(f" NEON ERR: {r.stderr[:200]}")