mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
cb11e4ac27
Lean sorry audit (lake build passes, 3539 jobs): - FixedPointBridge: 10 sorrys quarantined with TODO(lean-port) — all blocked on Float→Q bridge lemmas (Q0_16/Q16_16 round-trip error bounds) - HyperbolicStateSurface: 3 sorrys quarantined — need Q16_16.sqrt error-bound and Q16_16.add_pos_of_pos lemmas - CostEffectiveVerification: 1 sorry quarantined; also fixed pre-existing struct/structure typo, Array.Repr, Real.abs syntax, and Bool/Prop mismatch - MMRFAMMUnification: 1 sorry quarantined — Array.foldl induction lemma missing - WaveformTeleport: constantWaveformAtFixedPoint_base native_decide was numerically false; replaced with sorry + TODO(lean-port) - RcloneIntegration: startTask_pending_non_increasing PROVED — only sorry fully closed, using List.partition_eq_filter_filter + List.filter_sublist - DiffusionSNRBias, GPUVerificationMetaprobe, QFactor, SSMS: already properly quarantined; verified build passes Infrastructure additions: - ene_wiki_body_reingest.py: 5-source priority resolver for ene.wiki_revisions text="" gap (TiddlyWiki → filesystem → Notion → package description → stub) - zfs-pool-setup.sh: stackcache pool (500G sparse vdev) with hot/warm/cold thermal-zone dataset hierarchy; requires reboot to 7.0.9-1-cachyos kernel Docs: - ROADMAP.md: mark Lean→Verilog/FPGA targets as LONG-TERM in Phase 6 - UNIFIED_SIGNAL_ARCHITECTURE.md: add FPGA-column deferral notice Generated with [Devin](https://cli.devin.ai/docs) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
365 lines
12 KiB
Python
365 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = [
|
|
# "boto3",
|
|
# "psycopg2-binary",
|
|
# ]
|
|
# ///
|
|
"""
|
|
ENE Wiki Body Re-ingestion Shim.
|
|
|
|
All 278 ene.wiki_revisions rows currently have text = "" or "[object Promise]"
|
|
(a JavaScript async bug from the original ingestion pipeline).
|
|
|
|
This shim finds real content for each wiki page slug from the following
|
|
sources, in priority order:
|
|
|
|
1. knowledge.tiddlywiki_pages — exact title match (body field)
|
|
2. Local filesystem — for "X → Y" title pattern, read the
|
|
actual file or directory listing from
|
|
/home/allaun/Research Stack/X/Y
|
|
3. knowledge.documents — Notion content match by title (if content != '')
|
|
4. ene.packages description — description field (fallback)
|
|
5. Synthesized stub — title + slug as minimal placeholder
|
|
|
|
Updates ene.wiki_revisions.text in-place (only rows where text is empty or
|
|
"[object Promise]") and writes a receipt to ingestion.receipts.
|
|
|
|
Run with:
|
|
cd "/home/allaun/Research Stack"
|
|
uv run 4-Infrastructure/shim/ene_wiki_body_reingest.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import boto3
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
log = logging.getLogger("ene_wiki_body_reingest")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# RDS connection (IAM auth)
|
|
# ---------------------------------------------------------------------------
|
|
RDS_HOST = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
|
RDS_PORT = int(os.environ.get("RDS_PORT", "5432"))
|
|
RDS_USER = os.environ.get("RDS_USER", "postgres")
|
|
RDS_DBNAME = os.environ.get("RDS_DBNAME", "postgres")
|
|
AWS_REGION = os.environ.get("AWS_REGION", "us-east-1")
|
|
|
|
RESEARCH_STACK = Path(os.environ.get("RESEARCH_STACK", "/home/allaun/Research Stack"))
|
|
|
|
# Maximum bytes to read from a single local file (avoid ingesting huge blobs)
|
|
MAX_FILE_BYTES = 64 * 1024 # 64 KB
|
|
|
|
|
|
def connect() -> psycopg2.extensions.connection:
|
|
token = boto3.client("rds", region_name=AWS_REGION).generate_db_auth_token(
|
|
DBHostname=RDS_HOST, Port=RDS_PORT, DBUsername=RDS_USER, Region=AWS_REGION
|
|
)
|
|
return psycopg2.connect(
|
|
host=RDS_HOST, port=RDS_PORT, user=RDS_USER,
|
|
password=token, dbname=RDS_DBNAME, sslmode="require", connect_timeout=10
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source 1: TiddlyWiki pages (knowledge.tiddlywiki_pages)
|
|
# ---------------------------------------------------------------------------
|
|
def load_tiddlywiki_index(conn) -> dict[str, str]:
|
|
"""Return {lower(title): body} for all tiddlywiki pages with non-empty body."""
|
|
idx: dict[str, str] = {}
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT title, body FROM knowledge.tiddlywiki_pages WHERE body IS NOT NULL AND body != ''"
|
|
)
|
|
for title, body in cur.fetchall():
|
|
idx[title.lower().strip()] = body
|
|
log.info("TiddlyWiki index loaded: %d pages", len(idx))
|
|
return idx
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source 2: Local filesystem
|
|
# ---------------------------------------------------------------------------
|
|
def title_to_fs_path(title: str) -> Optional[Path]:
|
|
"""'A → B' wiki title → /home/allaun/Research Stack/A/B."""
|
|
if " → " not in title:
|
|
return None
|
|
parent, child = title.split(" → ", 1)
|
|
return RESEARCH_STACK / parent.strip() / child.strip()
|
|
|
|
|
|
def read_fs_content(path: Path) -> Optional[str]:
|
|
"""
|
|
Read content from a filesystem path:
|
|
- If it's a file, return its text (truncated to MAX_FILE_BYTES).
|
|
- If it's a directory, return a compact directory listing with file sizes.
|
|
- If it doesn't exist, return None.
|
|
"""
|
|
if not path.exists():
|
|
return None
|
|
|
|
if path.is_file():
|
|
try:
|
|
raw = path.read_bytes()[:MAX_FILE_BYTES]
|
|
text = raw.decode("utf-8", errors="replace")
|
|
suffix = f"\n\n[truncated at {MAX_FILE_BYTES // 1024} KB]" if len(raw) == MAX_FILE_BYTES else ""
|
|
return text + suffix
|
|
except Exception as exc:
|
|
log.debug("Could not read file %s: %s", path, exc)
|
|
return None
|
|
|
|
if path.is_dir():
|
|
lines = [f"# {path.name}/\n"]
|
|
try:
|
|
entries = sorted(path.iterdir())
|
|
except PermissionError:
|
|
return None
|
|
files = [e for e in entries if e.is_file()]
|
|
dirs = [e for e in entries if e.is_dir()]
|
|
if dirs:
|
|
lines.append("## Subdirectories")
|
|
for d in dirs[:50]:
|
|
lines.append(f"- `{d.name}/`")
|
|
if files:
|
|
lines.append("\n## Files")
|
|
for f in files[:100]:
|
|
try:
|
|
sz = f.stat().st_size
|
|
lines.append(f"- `{f.name}` ({sz:,} bytes)")
|
|
except OSError:
|
|
lines.append(f"- `{f.name}`")
|
|
return "\n".join(lines)
|
|
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source 3: Notion documents (knowledge.documents)
|
|
# ---------------------------------------------------------------------------
|
|
def load_notion_index(conn) -> dict[str, str]:
|
|
"""Return {lower(title): content} for Notion docs with non-empty content."""
|
|
idx: dict[str, str] = {}
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT title, content FROM knowledge.documents "
|
|
"WHERE source='notion' AND content IS NOT NULL AND content != ''"
|
|
)
|
|
for title, content in cur.fetchall():
|
|
key = title.lower().strip()
|
|
# Keep the longest content if there are duplicates
|
|
if key not in idx or len(content) > len(idx[key]):
|
|
idx[key] = content
|
|
log.info("Notion document index loaded: %d entries", len(idx))
|
|
return idx
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source 4: ENE packages description
|
|
# ---------------------------------------------------------------------------
|
|
def load_packages_index(conn) -> dict[str, str]:
|
|
"""Return {slug: description} from ene.packages for wiki/* packages."""
|
|
idx: dict[str, str] = {}
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT pkg, description FROM ene.packages "
|
|
"WHERE pkg LIKE 'ene/wiki/%' AND description IS NOT NULL AND description != ''"
|
|
)
|
|
for pkg, description in cur.fetchall():
|
|
slug = pkg.removeprefix("ene/wiki/")
|
|
idx[slug] = description
|
|
log.info("ENE packages index loaded: %d wiki entries", len(idx))
|
|
return idx
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Content resolution
|
|
# ---------------------------------------------------------------------------
|
|
def resolve_content(
|
|
slug: str,
|
|
title: str,
|
|
tiddly_idx: dict[str, str],
|
|
notion_idx: dict[str, str],
|
|
pkg_idx: dict[str, str],
|
|
) -> tuple[str, str]:
|
|
"""
|
|
Return (content, source_tag) for the given wiki revision.
|
|
source_tag is one of: tiddlywiki, filesystem, notion, package, generated
|
|
"""
|
|
title_lower = title.lower().strip()
|
|
|
|
# --- Source 1: TiddlyWiki exact match ---
|
|
if title_lower in tiddly_idx:
|
|
return tiddly_idx[title_lower], "tiddlywiki"
|
|
|
|
# --- Source 2: Filesystem (for "X → Y" titles) ---
|
|
fs_path = title_to_fs_path(title)
|
|
if fs_path is not None:
|
|
fs_content = read_fs_content(fs_path)
|
|
if fs_content:
|
|
return fs_content, "filesystem"
|
|
|
|
# --- Source 3: Notion documents ---
|
|
if title_lower in notion_idx:
|
|
return notion_idx[title_lower], "notion"
|
|
|
|
# --- Source 4: ENE packages description ---
|
|
if slug in pkg_idx:
|
|
desc = pkg_idx[slug]
|
|
return f"# {title}\n\n{desc}", "package"
|
|
|
|
# --- Source 5: Generated stub ---
|
|
stub = (
|
|
f"# {title}\n\n"
|
|
f"*Stub page — no source content found during re-ingestion.*\n\n"
|
|
f"**Slug:** `{slug}`\n"
|
|
)
|
|
return stub, "generated"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Receipt helper
|
|
# ---------------------------------------------------------------------------
|
|
def record_receipt(conn, status: str, metadata: dict, error: str | None = None) -> None:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO ingestion.receipts
|
|
(receipt_id, shim_name, status, metadata, error_detail, ran_at)
|
|
VALUES (%s, %s, %s, %s, %s, now())
|
|
""",
|
|
(
|
|
str(uuid.uuid4()),
|
|
"ene_wiki_body_reingest",
|
|
status,
|
|
json.dumps(metadata),
|
|
error,
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
def main() -> None:
|
|
log.info("Connecting to RDS…")
|
|
conn = connect()
|
|
conn.autocommit = False
|
|
|
|
# Build source indexes (read-only queries, no transaction needed)
|
|
log.info("Building source indexes…")
|
|
tiddly_idx = load_tiddlywiki_index(conn)
|
|
notion_idx = load_notion_index(conn)
|
|
pkg_idx = load_packages_index(conn)
|
|
|
|
# Fetch all wiki revisions (we update ALL of them to fix the [object Promise] bug too)
|
|
log.info("Fetching all wiki revisions…")
|
|
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
|
cur.execute(
|
|
"SELECT slug, revision, title, text FROM ene.wiki_revisions ORDER BY slug, revision"
|
|
)
|
|
revisions = cur.fetchall()
|
|
|
|
log.info("Total revisions to process: %d", len(revisions))
|
|
|
|
# Stats counters
|
|
stats: dict[str, int] = {
|
|
"total": len(revisions),
|
|
"updated": 0,
|
|
"skipped_already_good": 0,
|
|
"tiddlywiki": 0,
|
|
"filesystem": 0,
|
|
"notion": 0,
|
|
"package": 0,
|
|
"generated": 0,
|
|
}
|
|
|
|
BATCH = 50
|
|
batch_count = 0
|
|
|
|
for rev in revisions:
|
|
slug = rev["slug"]
|
|
revision = rev["revision"]
|
|
title = rev["title"]
|
|
current = rev["text"] or ""
|
|
|
|
# Decide whether this row needs updating:
|
|
# Bad if it's empty string OR the literal "[object Promise]" (including repeated newlines of it)
|
|
is_bad = (
|
|
current.strip() == ""
|
|
or current.strip() == "[object Promise]"
|
|
or (current.replace("[object Promise]", "").replace("\n", "").strip() == "")
|
|
)
|
|
|
|
if not is_bad:
|
|
stats["skipped_already_good"] += 1
|
|
continue
|
|
|
|
# Resolve content from sources
|
|
content, source_tag = resolve_content(slug, title, tiddly_idx, notion_idx, pkg_idx)
|
|
|
|
stats[source_tag] += 1
|
|
stats["updated"] += 1
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE ene.wiki_revisions SET text = %s WHERE slug = %s AND revision = %s",
|
|
(content, slug, revision),
|
|
)
|
|
|
|
batch_count += 1
|
|
if batch_count % BATCH == 0:
|
|
conn.commit()
|
|
log.info(
|
|
" Committed %d/%d revisions updated so far…",
|
|
stats["updated"],
|
|
stats["total"],
|
|
)
|
|
|
|
conn.commit()
|
|
log.info("Final commit done.")
|
|
|
|
log.info(
|
|
"Summary: total=%d updated=%d skipped=%d | "
|
|
"tiddlywiki=%d filesystem=%d notion=%d package=%d generated=%d",
|
|
stats["total"],
|
|
stats["updated"],
|
|
stats["skipped_already_good"],
|
|
stats["tiddlywiki"],
|
|
stats["filesystem"],
|
|
stats["notion"],
|
|
stats["package"],
|
|
stats["generated"],
|
|
)
|
|
|
|
record_receipt(conn, "success", {
|
|
"pages_updated": stats["updated"],
|
|
"skipped_already_good": stats["skipped_already_good"],
|
|
"sources": {
|
|
"tiddlywiki": stats["tiddlywiki"],
|
|
"filesystem": stats["filesystem"],
|
|
"notion": stats["notion"],
|
|
"package": stats["package"],
|
|
"generated": stats["generated"],
|
|
},
|
|
"total_revisions": stats["total"],
|
|
})
|
|
|
|
conn.close()
|
|
log.info("Done.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|