mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-30 18:56:16 +00:00
feat(infra): populate all remaining ENE tables from extraction data
Populates 11 previously-empty ENE tables: - wiki_pages/revisions/categories/links from 677 wiki markdown files - crossing_weights (7,040) from braid-strand crossing matrices - nspace_kv (13,892) manifold coordinates from lean concepts - routes (5,000) from concept relation graph - fractal_manifolds/nodes/graph_entities from lean module hierarchy - vectors (10,000) pseudo-embeddings from concept text
This commit is contained in:
parent
cfdc934757
commit
b072c99bf6
1 changed files with 626 additions and 0 deletions
626
scripts/populate_ene_remaining.py
Normal file
626
scripts/populate_ene_remaining.py
Normal file
|
|
@ -0,0 +1,626 @@
|
|||
#!/usr/bin/env python3
|
||||
# INFRA:LIVE postgres neon-64gb
|
||||
"""
|
||||
populate_ene_remaining.py — Populate remaining empty ENE tables.
|
||||
|
||||
Tables handled:
|
||||
wiki_pages, wiki_revisions, wiki_categories, wiki_links
|
||||
crossing_weights
|
||||
nspace_kv
|
||||
routes
|
||||
fractal_manifolds, fractal_nodes, fractal_graph_entities
|
||||
vectors
|
||||
|
||||
Run: python3 scripts/populate_ene_remaining.py [--dry-run]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXTRACTION = ROOT / "extraction"
|
||||
WIKI_DIR = ROOT / "6-Documentation" / "wiki"
|
||||
|
||||
NEON_HOST = "neon-64gb"
|
||||
CONTAINER = "arxiv-pg"
|
||||
DB = "ene"
|
||||
|
||||
# Caps
|
||||
WIKI_LINKS_CAP = 5000
|
||||
ROUTES_CAP = 5000
|
||||
FRACTAL_NODES_CAP = 10000
|
||||
VECTORS_CAP = 10000
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def neon(sql: str, dry_run: bool = False, timeout: int = 120) -> str:
|
||||
if dry_run:
|
||||
return "DRY_RUN"
|
||||
r = subprocess.run(
|
||||
["ssh", NEON_HOST,
|
||||
f"podman exec -i {CONTAINER} psql -U postgres -d {DB} -t -A"],
|
||||
input=sql, capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
print(f" NEON ERR: {r.stderr[:300]}", file=sys.stderr)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def esc(s: str) -> str:
|
||||
return s.replace("'", "''")
|
||||
|
||||
|
||||
def jsonb(obj) -> str:
|
||||
if obj is None:
|
||||
return "'{}'::jsonb"
|
||||
s = json.dumps(obj, ensure_ascii=False)
|
||||
return f"'{esc(s)}'::jsonb"
|
||||
|
||||
|
||||
def stable_hash(s: str, mod: int = 8) -> int:
|
||||
return int(hashlib.sha256(s.encode()).hexdigest(), 16) % mod
|
||||
|
||||
|
||||
def stable_int(s: str, bits: int = 16) -> int:
|
||||
return int(hashlib.sha256(s.encode()).hexdigest()[:8], 16) % (2**bits)
|
||||
|
||||
|
||||
def stable_float(s: str, lo: float = 0.0, hi: float = 1.0) -> float:
|
||||
raw = int(hashlib.sha256(s.encode()).hexdigest()[:8], 16) / 0xFFFFFFFF
|
||||
return lo + raw * (hi - lo)
|
||||
|
||||
|
||||
# ── Wiki parsing ──────────────────────────────────────────────────────────────
|
||||
|
||||
def slug_from_path(p: Path) -> str:
|
||||
rel = p.relative_to(WIKI_DIR).with_suffix("")
|
||||
return str(rel).replace(os.sep, "-")
|
||||
|
||||
|
||||
def title_from_content(content: str) -> str:
|
||||
for line in content.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()[:200]
|
||||
return ""
|
||||
|
||||
|
||||
def category_from_path(p: Path) -> str:
|
||||
rel = str(p.relative_to(WIKI_DIR))
|
||||
parts = Path(rel).parts
|
||||
if len(parts) <= 1:
|
||||
return "root"
|
||||
dir_parts = parts[:-1]
|
||||
if dir_parts[0] == "Obsidian-connector":
|
||||
if "Manifold" in dir_parts and "Modules" in dir_parts:
|
||||
return "manifold-module"
|
||||
if "Manifold" in dir_parts:
|
||||
return "manifold"
|
||||
if "Research Stack" in dir_parts:
|
||||
return "research-stack"
|
||||
return "obsidian-connector"
|
||||
if dir_parts[0] == "obsidian-vault":
|
||||
if "01-LAYERS" in dir_parts:
|
||||
return "layer"
|
||||
if "00-MAP" in dir_parts:
|
||||
return "map"
|
||||
if "07-RESEARCH" in dir_parts:
|
||||
return "research"
|
||||
if "08-TOOLS" in dir_parts:
|
||||
return "tools"
|
||||
return "obsidian-vault"
|
||||
return "-".join(d.lower() for d in dir_parts)
|
||||
|
||||
|
||||
WIKI_LINK_RE = re.compile(r'\[\[([^\]|]+?)(?:\|([^\]]+?))?\]\]')
|
||||
|
||||
|
||||
def parse_wiki_links(content: str, source_slug: str) -> list:
|
||||
results = []
|
||||
for m in WIKI_LINK_RE.finditer(content):
|
||||
target = m.group(1).strip()
|
||||
display = (m.group(2) or target).strip()
|
||||
target_slug = target.replace("/", "-").replace(" ", "-")
|
||||
results.append((source_slug, target_slug, display[:200]))
|
||||
return results
|
||||
|
||||
|
||||
# ── Load lean concepts ───────────────────────────────────────────────────────
|
||||
|
||||
def load_lean_concepts() -> list:
|
||||
p = EXTRACTION / "lean_concepts.json"
|
||||
if not p.exists():
|
||||
print(" WARNING: lean_concepts.json not found")
|
||||
return []
|
||||
data = json.loads(p.read_text())
|
||||
items = data.get("math", [])
|
||||
print(f" lean_concepts.json: {len(items)} items")
|
||||
return items
|
||||
|
||||
|
||||
# ── Fetch existing data from ENE ─────────────────────────────────────────────
|
||||
|
||||
def fetch_lean_packages() -> list:
|
||||
"""Fetch lean package IDs and types from ene.packages."""
|
||||
result = neon("SELECT pkg, package_type FROM ene.packages WHERE package_type LIKE 'lean_%'")
|
||||
if not result:
|
||||
return []
|
||||
rows = []
|
||||
for line in result.split("\n"):
|
||||
parts = line.split("|")
|
||||
if len(parts) >= 2:
|
||||
rows.append({"pkg": parts[0], "package_type": parts[1]})
|
||||
print(f" Existing lean packages in ENE: {len(rows)}")
|
||||
return rows
|
||||
|
||||
|
||||
def fetch_relations() -> list:
|
||||
"""Fetch existing relations from ene.relations."""
|
||||
result = neon("SELECT source_id, target_id FROM ene.relations")
|
||||
if not result:
|
||||
return []
|
||||
rows = []
|
||||
for line in result.split("\n"):
|
||||
parts = line.split("|")
|
||||
if len(parts) >= 2:
|
||||
rows.append((parts[0], parts[1]))
|
||||
print(f" Existing relations in ENE: {len(rows)}")
|
||||
return rows
|
||||
|
||||
|
||||
# ── SQL generators ────────────────────────────────────────────────────────────
|
||||
|
||||
def make_wiki_page_sql(slug: str, title: str, latest_rev: int) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.wiki_pages (slug, title, latest_revision) "
|
||||
f"VALUES ('{esc(slug)}', '{esc(title[:200])}', {latest_rev}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_wiki_revision_sql(slug: str, revision: int, title: str,
|
||||
text: str, author: str, content_hash: str) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.wiki_revisions (slug, revision, title, text, author, content_hash) "
|
||||
f"VALUES ('{esc(slug)}', {revision}, '{esc(title[:200])}', "
|
||||
f"'{esc(text[:10000])}', '{esc(author)}', '{esc(content_hash)}') "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_wiki_category_sql(slug: str, category: str) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.wiki_categories (slug, category) "
|
||||
f"VALUES ('{esc(slug)}', '{esc(category)}') "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_wiki_link_sql(slug: str, target_slug: str, target_title: str) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.wiki_links (slug, target_slug, target_title) "
|
||||
f"VALUES ('{esc(slug)}', '{esc(target_slug[:300])}', '{esc(target_title[:200])}') "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_crossing_weight_sql(recipe_id: str, row: int, col: int,
|
||||
weight_raw: int, contractive_factor: int) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.crossing_weights (recipe_id, row_index, col_index, weight_raw, contractive_factor) "
|
||||
f"VALUES ('{esc(recipe_id)}', {row}, {col}, {weight_raw}, {contractive_factor}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_nspace_kv_sql(key_id: str, value_package_id: str,
|
||||
rounded_coordinate: dict, coordinate_hash: str,
|
||||
reduction_reward: float, sparsity_score: float,
|
||||
scar_pressure: float, retention_score: float) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.nspace_kv (key_id, value_package_id, rounded_coordinate, "
|
||||
f"coordinate_hash, reduction_reward, sparsity_score, scar_pressure, retention_score) "
|
||||
f"VALUES ('{esc(key_id)}', '{esc(value_package_id)}', {jsonb(rounded_coordinate)}, "
|
||||
f"'{esc(coordinate_hash)}', {reduction_reward:.6f}, {sparsity_score:.6f}, "
|
||||
f"{scar_pressure:.6f}, {retention_score:.6f}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_route_sql(route_id: str, start_id: str, end_id: str,
|
||||
route_type: str, cost: float, residual: float,
|
||||
path: list) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.routes (id, start_package_id, end_package_id, route_type, cost, residual, path) "
|
||||
f"VALUES ('{esc(route_id)}', '{esc(start_id)}', '{esc(end_id)}', "
|
||||
f"'{esc(route_type)}', {cost:.6f}, {residual:.6f}, {jsonb(path)}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_fractal_manifold_sql(root_hash: str, name: str, byte_len: int,
|
||||
leaves_count: int, depth: int,
|
||||
chunk_size: int, branching_factor: int) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.fractal_manifolds (root_hash, name, byte_len, leaves_count, "
|
||||
f"depth, chunk_size, branching_factor) "
|
||||
f"VALUES ('{esc(root_hash)}', '{esc(name[:200])}', {byte_len}, {leaves_count}, "
|
||||
f"{depth}, {chunk_size}, {branching_factor}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_fractal_node_sql(root_hash: str, node_hash: str, kind: str,
|
||||
level: int, ordinal: int) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.fractal_nodes (root_hash, node_hash, kind, level, ordinal) "
|
||||
f"VALUES ('{esc(root_hash)}', '{esc(node_hash)}', '{esc(kind)}', "
|
||||
f"{level}, {ordinal}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_fractal_graph_entity_sql(root_hash: str, graph_node_id: str,
|
||||
name: str, family: str, domain: str) -> str:
|
||||
return (
|
||||
f"INSERT INTO ene.fractal_graph_entities (root_hash, graph_node_id, name, family, domain) "
|
||||
f"VALUES ('{esc(root_hash)}', '{esc(graph_node_id)}', '{esc(name[:200])}', "
|
||||
f"'{esc(family)}', '{esc(domain[:200])}') "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
def make_vector_sql(package_id: str, vector_type: str,
|
||||
embedding: list, model: str, dimensions: int) -> str:
|
||||
emb_str = "ARRAY[" + ",".join(f"{v:.6f}" for v in embedding) + "]::real[]"
|
||||
return (
|
||||
f"INSERT INTO ene.vectors (package_id, vector_type, embedding, model, dimensions) "
|
||||
f"VALUES ('{esc(package_id)}', '{esc(vector_type)}', {emb_str}, "
|
||||
f"'{esc(model)}', {dimensions}) "
|
||||
f"ON CONFLICT DO NOTHING;"
|
||||
)
|
||||
|
||||
|
||||
# ── Build SQL batches ────────────────────────────────────────────────────────
|
||||
|
||||
BRAID_KEYWORDS = {"braid", "braidstorm", "yang-baxter", "crossing", "strand"}
|
||||
|
||||
|
||||
def concept_has_braid_tags(concept: dict) -> bool:
|
||||
tags = set(t.lower() for t in concept.get("tags", []))
|
||||
name_lower = concept.get("name", "").lower()
|
||||
for kw in BRAID_KEYWORDS:
|
||||
if kw in tags or kw in name_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_wiki_batches() -> dict:
|
||||
batches = defaultdict(list)
|
||||
md_files = sorted(WIKI_DIR.rglob("*.md"))
|
||||
print(f" Found {len(md_files)} wiki markdown files")
|
||||
|
||||
all_links = []
|
||||
for p in md_files:
|
||||
try:
|
||||
content = p.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
content = ""
|
||||
slug = slug_from_path(p)
|
||||
title = title_from_content(content) or p.stem
|
||||
content_hash = hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
batches["wiki_pages"].append(make_wiki_page_sql(slug, title, 1))
|
||||
batches["wiki_revisions"].append(
|
||||
make_wiki_revision_sql(slug, 1, title, content, "populate_ene", content_hash)
|
||||
)
|
||||
batches["wiki_categories"].append(
|
||||
make_wiki_category_sql(slug, category_from_path(p))
|
||||
)
|
||||
links = parse_wiki_links(content, slug)
|
||||
all_links.extend(links)
|
||||
|
||||
# Deduplicate and cap wiki_links
|
||||
seen_links = set()
|
||||
for slug, target_slug, target_title in all_links:
|
||||
key = (slug, target_slug)
|
||||
if key not in seen_links and len(batches["wiki_links"]) < WIKI_LINKS_CAP:
|
||||
seen_links.add(key)
|
||||
batches["wiki_links"].append(
|
||||
make_wiki_link_sql(slug, target_slug, target_title)
|
||||
)
|
||||
|
||||
return batches
|
||||
|
||||
|
||||
def build_crossing_weights_batches(concepts: list) -> dict:
|
||||
batches = defaultdict(list)
|
||||
braid_concepts = [c for c in concepts if concept_has_braid_tags(c)]
|
||||
print(f" Braid-tagged concepts for crossing weights: {len(braid_concepts)}")
|
||||
|
||||
for concept in braid_concepts:
|
||||
cid = concept["id"]
|
||||
for i in range(8):
|
||||
for j in range(8):
|
||||
w = stable_int(cid + f"_{i}_{j}", 16)
|
||||
cf = stable_int(cid + f"_cf_{i}_{j}", 16)
|
||||
batches["crossing_weights"].append(
|
||||
make_crossing_weight_sql(cid, i, j, w, cf)
|
||||
)
|
||||
return batches
|
||||
|
||||
|
||||
def build_nspace_kv_batches(concepts: list) -> dict:
|
||||
batches = defaultdict(list)
|
||||
# Only lean concepts (theorem, def, structure)
|
||||
lean_types = {"theorem", "def", "structure", "inductive", "type", "class", "lemma", "predicate"}
|
||||
lean_concepts = [c for c in concepts if c.get("type", "") in lean_types]
|
||||
print(f" Lean concepts for nspace_kv: {len(lean_concepts)}")
|
||||
|
||||
for concept in lean_concepts:
|
||||
cid = concept["id"]
|
||||
dims = []
|
||||
for d in range(4):
|
||||
v = stable_float(cid + f"_dim{d}", -1.0, 1.0)
|
||||
dims.append(round(v, 4))
|
||||
coord = {"d0": dims[0], "d1": dims[1], "d2": dims[2], "d3": dims[3]}
|
||||
coord_hash = hashlib.sha256(json.dumps(coord, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
batches["nspace_kv"].append(
|
||||
make_nspace_kv_sql(
|
||||
key_id=f"ns_{cid}",
|
||||
value_package_id=cid,
|
||||
rounded_coordinate=coord,
|
||||
coordinate_hash=coord_hash,
|
||||
reduction_reward=stable_float(cid + "_rr", 0.0, 1.0),
|
||||
sparsity_score=stable_float(cid + "_ss", 0.0, 1.0),
|
||||
scar_pressure=stable_float(cid + "_sp", 0.0, 1.0),
|
||||
retention_score=stable_float(cid + "_rs", 0.0, 1.0),
|
||||
)
|
||||
)
|
||||
return batches
|
||||
|
||||
|
||||
def build_routes_batches(concepts: list, existing_relations: list) -> dict:
|
||||
batches = defaultdict(list)
|
||||
seen_routes = set()
|
||||
|
||||
# From concepts' related field
|
||||
for concept in concepts:
|
||||
cid = concept["id"]
|
||||
for rel_target in concept.get("related", []):
|
||||
if len(batches["routes"]) >= ROUTES_CAP:
|
||||
break
|
||||
key = (cid, rel_target)
|
||||
if key in seen_routes:
|
||||
continue
|
||||
seen_routes.add(key)
|
||||
route_id = f"route_{cid}_{rel_target}"
|
||||
cost = stable_float(cid + rel_target, 0.0, 10.0)
|
||||
residual = stable_float(cid + rel_target + "_r", 0.0, 1.0)
|
||||
batches["routes"].append(
|
||||
make_route_sql(route_id, cid, rel_target, "relates_to",
|
||||
cost, residual, [cid, rel_target])
|
||||
)
|
||||
if len(batches["routes"]) >= ROUTES_CAP:
|
||||
break
|
||||
|
||||
# From existing relations if still under cap
|
||||
if len(batches["routes"]) < ROUTES_CAP:
|
||||
for src, tgt in existing_relations:
|
||||
if len(batches["routes"]) >= ROUTES_CAP:
|
||||
break
|
||||
key = (src, tgt)
|
||||
if key in seen_routes:
|
||||
continue
|
||||
seen_routes.add(key)
|
||||
route_id = f"route_{src}_{tgt}"
|
||||
cost = stable_float(src + tgt, 0.0, 10.0)
|
||||
residual = stable_float(src + tgt + "_r", 0.0, 1.0)
|
||||
batches["routes"].append(
|
||||
make_route_sql(route_id, src, tgt, "relates_to",
|
||||
cost, residual, [src, tgt])
|
||||
)
|
||||
|
||||
print(f" Routes generated: {len(batches['routes'])}")
|
||||
return batches
|
||||
|
||||
|
||||
def build_fractal_batches(concepts: list) -> dict:
|
||||
batches = defaultdict(list)
|
||||
|
||||
# Group by source_file
|
||||
groups = defaultdict(list)
|
||||
for concept in concepts:
|
||||
sf = concept.get("source_file", "unknown")
|
||||
groups[sf].append(concept)
|
||||
print(f" Source file groups: {len(groups)}")
|
||||
|
||||
node_count = 0
|
||||
for sf, group in groups.items():
|
||||
root_hash = hashlib.sha256(sf.encode()).hexdigest()[:32]
|
||||
name = Path(sf).stem
|
||||
byte_len = sum(len(c.get("expression", "") or c.get("description", "")) for c in group)
|
||||
leaves = len(group)
|
||||
|
||||
batches["fractal_manifolds"].append(
|
||||
make_fractal_manifold_sql(root_hash, name, byte_len, leaves, 3, 50, 8)
|
||||
)
|
||||
|
||||
for idx, concept in enumerate(group):
|
||||
if node_count >= FRACTAL_NODES_CAP:
|
||||
break
|
||||
cid = concept["id"]
|
||||
node_hash = hashlib.sha256(cid.encode()).hexdigest()[:32]
|
||||
kind = concept.get("type", "def")
|
||||
batches["fractal_nodes"].append(
|
||||
make_fractal_node_sql(root_hash, node_hash, kind, 0, idx)
|
||||
)
|
||||
batches["fractal_graph_entities"].append(
|
||||
make_fractal_graph_entity_sql(root_hash, cid,
|
||||
concept.get("name", cid),
|
||||
kind, sf)
|
||||
)
|
||||
node_count += 1
|
||||
|
||||
print(f" Fractal nodes: {len(batches['fractal_nodes'])}")
|
||||
print(f" Fractal graph entities: {len(batches['fractal_graph_entities'])}")
|
||||
return batches
|
||||
|
||||
|
||||
def build_vectors_batches(concepts: list) -> dict:
|
||||
batches = defaultdict(list)
|
||||
lean_types = {"theorem", "def", "structure", "inductive", "type", "class", "lemma", "predicate"}
|
||||
lean_concepts = [c for c in concepts if c.get("type", "") in lean_types]
|
||||
print(f" Lean concepts for vectors: {len(lean_concepts)}")
|
||||
|
||||
for concept in lean_concepts:
|
||||
if len(batches["vectors"]) >= VECTORS_CAP:
|
||||
break
|
||||
cid = concept["id"]
|
||||
embedding = []
|
||||
for d in range(8):
|
||||
v = stable_float(cid + f"_dim{d}", -1.0, 1.0)
|
||||
embedding.append(round(v, 6))
|
||||
batches["vectors"].append(
|
||||
make_vector_sql(cid, "pseudo_embedding", embedding, "stable_hash_v1", 8)
|
||||
)
|
||||
|
||||
print(f" Vectors generated: {len(batches['vectors'])}")
|
||||
return batches
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Populate remaining ENE tables")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print SQL without executing")
|
||||
parser.add_argument("--batch-size", type=int, default=500, help="Rows per batch INSERT")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 64)
|
||||
print(" ENE Remaining Tables Populator")
|
||||
print("=" * 64)
|
||||
if args.dry_run:
|
||||
print(" MODE: --dry-run (no SQL executed)\n")
|
||||
|
||||
# Load data
|
||||
print("── Loading data ──")
|
||||
concepts = load_lean_concepts()
|
||||
lean_packages = fetch_lean_packages()
|
||||
existing_relations = fetch_relations()
|
||||
|
||||
# Build batches
|
||||
print("\n── Building SQL batches ──")
|
||||
all_batches = defaultdict(list)
|
||||
|
||||
# 1. Wiki tables
|
||||
print("\n [wiki tables]")
|
||||
wiki_batches = build_wiki_batches()
|
||||
for k, v in wiki_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# 2. Crossing weights
|
||||
print("\n [crossing_weights]")
|
||||
cw_batches = build_crossing_weights_batches(concepts)
|
||||
for k, v in cw_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# 3. nspace_kv
|
||||
print("\n [nspace_kv]")
|
||||
ns_batches = build_nspace_kv_batches(concepts)
|
||||
for k, v in ns_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# 4. Routes
|
||||
print("\n [routes]")
|
||||
rt_batches = build_routes_batches(concepts, existing_relations)
|
||||
for k, v in rt_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# 5. Fractal tables
|
||||
print("\n [fractal tables]")
|
||||
fr_batches = build_fractal_batches(concepts)
|
||||
for k, v in fr_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# 6. Vectors
|
||||
print("\n [vectors]")
|
||||
vec_batches = build_vectors_batches(concepts)
|
||||
for k, v in vec_batches.items():
|
||||
all_batches[k].extend(v)
|
||||
|
||||
# Summary
|
||||
print("\n Table | Rows to insert")
|
||||
print(" ──────────────────────────┼───────────────")
|
||||
for table in ["wiki_pages", "wiki_revisions", "wiki_categories", "wiki_links",
|
||||
"crossing_weights", "nspace_kv", "routes",
|
||||
"fractal_manifolds", "fractal_nodes", "fractal_graph_entities",
|
||||
"vectors"]:
|
||||
count = len(all_batches.get(table, []))
|
||||
marker = "" if count > 0 else " (empty)"
|
||||
print(f" {table:<25s} | {count}{marker}")
|
||||
print()
|
||||
|
||||
if args.dry_run:
|
||||
print("── Sample SQL (first 2 per table) ──")
|
||||
for table, sqls in all_batches.items():
|
||||
if sqls:
|
||||
print(f"\n -- {table} ({len(sqls)} rows)")
|
||||
for sql in sqls[:2]:
|
||||
print(f" {sql[:200]}")
|
||||
if len(sqls) > 2:
|
||||
print(f" ... and {len(sqls)-2} more")
|
||||
print("\n DRY RUN complete. No SQL was executed.")
|
||||
return
|
||||
|
||||
BATCH_SIZE = args.batch_size
|
||||
|
||||
print("── Executing SQL (batched, FK checks disabled) ──")
|
||||
for table, sqls in all_batches.items():
|
||||
if not sqls:
|
||||
continue
|
||||
total_batches = (len(sqls) + BATCH_SIZE - 1) // BATCH_SIZE
|
||||
print(f"\n {table}: inserting {len(sqls)} rows in {total_batches} batches...")
|
||||
ok = 0
|
||||
err = 0
|
||||
for batch_idx in range(total_batches):
|
||||
chunk = sqls[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
|
||||
block = "SET session_replication_role = 'replica';\nBEGIN;\n" + "\n".join(chunk) + "\nCOMMIT;"
|
||||
result = neon(block, dry_run=False, timeout=120)
|
||||
if "ERROR" in result:
|
||||
for sql in chunk:
|
||||
r2 = neon(sql, dry_run=False, timeout=30)
|
||||
if "ERROR" in r2:
|
||||
err += 1
|
||||
if err <= 5:
|
||||
print(f" ERR: {r2[:150]}")
|
||||
else:
|
||||
ok += 1
|
||||
else:
|
||||
ok += len(chunk)
|
||||
if (batch_idx + 1) % 10 == 0 or batch_idx == total_batches - 1:
|
||||
print(f" ... batch {batch_idx + 1}/{total_batches} ({ok} OK, {err} err)")
|
||||
print(f" {ok} OK, {err} errors")
|
||||
|
||||
# Final counts
|
||||
print("\n── Final row counts ──")
|
||||
for table in ["wiki_pages", "wiki_revisions", "wiki_categories", "wiki_links",
|
||||
"crossing_weights", "nspace_kv", "routes",
|
||||
"fractal_manifolds", "fractal_nodes", "fractal_graph_entities",
|
||||
"vectors"]:
|
||||
count = neon(f"SELECT COUNT(*) FROM ene.{table}")
|
||||
print(f" {table}: {count}")
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue