feat: add RDS probe tool, credential server, and Notion/Linear ingestion pipeline

New infrastructure components:
- rds_probe: Rust database inspection tool with IAM auth
- credential_server.py: REST credential provider server
- credential_provider.py: credential resolution chain
- ene_rds_fractal_fold.py / ene_rds_wiki_layer.py: RDS-backed ENE layers
- import_dumps_to_rds.py / export_linear_from_rds.py: ingestion pipeline
- recover_credential_server.sh: deployment script (sanitized)

Sanitize hardcoded secrets across codebase:
- Strip API keys from recover_credential_server.sh → env var lookups
- Replace hardcoded Wolfram appid (HYJE3R3R63) → env var in 5 scripts
- Strip fallback key values from config/index.js
- Add .claude/ and optimized_basis_v3.bin to .gitignore

Ingested 2,685 records into RDS: 2,421 Linear issues + 264 wiki pages
This commit is contained in:
Brandon Schneider 2026-05-18 00:31:22 -05:00
parent 6c1d739a04
commit ba1e4cf191
19 changed files with 4858 additions and 5 deletions

4
.gitignore vendored
View file

@ -1,4 +1,8 @@
.env
.claude/
optimized_basis_v3.bin
4-Infrastructure/deploy/
**/node_modules/
**/__pycache__/
*.db

View file

@ -0,0 +1,305 @@
from __future__ import annotations
import json
import os
import socket
import urllib.parse
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CREDENTIAL_PROVIDER_VERSION = "0.4"
# Remote credential server (microVM) — all nodes query this as primary
_cred_server_url = os.environ.get("RS_CREDENTIAL_SERVER", "http://100.101.247.127:8444")
# Config file path — local fallback if remote is unreachable
_cred_config_path = os.environ.get(
"RS_CREDENTIAL_CONFIG",
"/etc/rs-surface/credentials.json",
)
RDS_HOST = os.environ.get(
"RDS_HOST",
"database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com",
)
RDS_PORT = int(os.environ.get("RDS_PORT", "5432"))
RDS_USER = os.environ.get("RDS_USER", "postgres")
RDS_IAM_TOKEN = os.environ.get("RDS_IAM_TOKEN", "")
@dataclass
class Credential:
provider: str
key_name: str
value: str
metadata: dict[str, Any] = field(default_factory=dict)
PROVIDER_ENV_MAP: dict[str, dict[str, str]] = {
"deepseek": {
"env_var": "DEEPSEEK_API_KEY",
"description": "DeepSeek LLM API",
},
"quandela": {
"env_var": "QUANDELA_API_KEY",
"description": "Quandela quantum cloud",
},
"wolfram_alpha": {
"env_var": "WOLFRAM_ALPHA_APPID",
"description": "Wolfram Alpha API",
},
"notion": {
"env_var": "NOTION_API_KEY",
"description": "Notion API",
},
"linear": {
"env_var": "LINEAR_API_KEY",
"description": "Linear API",
},
"gemini": {
"env_var": "GEMINI_API_KEY",
"description": "Google Gemini API",
},
"ollama": {
"env_var": "OLLAMA_API_KEY",
"description": "Ollama local API",
},
"brave_search": {
"env_var": "BRAVE_API_KEY",
"description": "Brave Search API",
},
"neural_endeavor": {
"env_var": "ENE_ENCRYPTION_KEY",
"description": "ENE encryption master key",
},
"bedrock": {
"env_var": "AWS_BEARER_TOKEN_BEDROCK",
"description": "Amazon Bedrock API",
},
"venice": {
"env_var": "VENICE_API_KEY",
"description": "Venice AI API",
},
}
def _rds_connect():
import psycopg2
import os
# Primary check: RDS_IAM_TOKEN env var
password = RDS_IAM_TOKEN
if not password:
# Generate dynamic IAM authentication token using boto3
try:
import boto3
region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
client = boto3.client('rds', region_name=region)
password = client.generate_db_auth_token(
DBHostname=RDS_HOST,
Port=RDS_PORT,
DBUsername=RDS_USER,
Region=region
)
except Exception as e:
# Fallback to standard environment password or log warning
pass
if not password:
password = os.environ.get("RDS_PASSWORD", "")
return psycopg2.connect(
host=RDS_HOST,
port=RDS_PORT,
user=RDS_USER,
password=password,
dbname="postgres",
connect_timeout=10,
sslmode="require",
)
def _load_from_rds() -> list[Credential]:
try:
conn = _rds_connect()
cur = conn.cursor()
cur.execute("""
SELECT pkg, provider, encode(encrypted_payload, 'escape'), classification
FROM credential_store.credentials
WHERE is_active = TRUE
ORDER BY provider
""")
creds = []
for pkg, provider, payload_raw, classification in cur.fetchall():
creds.append(Credential(
provider=provider,
key_name=pkg,
value=payload_raw,
metadata={
"source": "rds",
"pkg": pkg,
"classification": classification,
},
))
conn.close()
return creds
except Exception as e:
return []
def _load_from_remote() -> list[Credential]:
"""Fetch credentials from the remote credential server (microVM)."""
import urllib.request
import urllib.error
# Skip self-query: don't try to connect to ourselves
self_hosts = {"localhost", "127.0.0.1", "0.0.0.0", "", socket.gethostname()}
netloc = urllib.parse.urlparse(_cred_server_url).hostname or ""
if netloc in self_hosts:
return []
try:
req = urllib.request.Request(_cred_server_url + "/credentials")
with urllib.request.urlopen(req, timeout=5) as resp:
manifest = json.loads(resp.read().decode("utf-8"))
except (urllib.error.URLError, json.JSONDecodeError, OSError):
return []
providers = manifest.get("providers", [])
creds: list[Credential] = []
for p in providers:
name = p.get("name")
if not name:
continue
try:
req = urllib.request.Request(f"{_cred_server_url}/credentials/{name}")
with urllib.request.urlopen(req, timeout=5) as resp:
data = json.loads(resp.read().decode("utf-8"))
except (urllib.error.URLError, json.JSONDecodeError, OSError):
continue
if data.get("ok"):
creds.append(Credential(
provider=name,
key_name=data.get("key_name", f"remote/{name}"),
value=data["key"],
metadata={"source": "remote", "server": _cred_server_url},
))
return creds
def _load_from_config() -> list[Credential]:
"""Read credentials from a JSON config file.
Format:
{
"deepseek": "sk-...",
"bedrock": "ABSK...",
"ollama": "..."
}
"""
path = Path(_cred_config_path)
if not path.exists():
return []
try:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError):
return []
if not isinstance(data, dict):
return []
creds: list[Credential] = []
for provider_name, value in data.items():
if not isinstance(value, str) or not value.strip():
continue
cfg = PROVIDER_ENV_MAP.get(provider_name, {})
creds.append(Credential(
provider=provider_name,
key_name=f"config/{provider_name}",
value=value.strip(),
metadata={
"description": cfg.get("description", provider_name),
"source": "config_file",
"file": str(path),
},
))
return creds
def _load_from_env() -> list[Credential]:
creds: list[Credential] = []
for provider, cfg in PROVIDER_ENV_MAP.items():
value = os.getenv(cfg["env_var"])
if not value:
continue
creds.append(Credential(
provider=provider,
key_name=cfg["env_var"],
value=value,
metadata={"description": cfg["description"], "source": "env"},
))
return creds
def load_credentials() -> list[Credential]:
creds = _load_from_rds()
if creds:
return creds
creds = _load_from_remote()
if creds:
return creds
creds = _load_from_config()
if creds:
return creds
return _load_from_env()
def _detect_backend() -> str:
if _load_from_rds():
return "rds"
if _load_from_remote():
return "remote"
if _load_from_config():
return "config_file"
return "env"
def credential_status() -> dict[str, Any]:
creds = load_credentials()
return {
"ok": True,
"node": os.environ.get("RS_SURFACE_NODE_ID", "unknown"),
"provider_version": CREDENTIAL_PROVIDER_VERSION,
"backend": _detect_backend(),
"available_providers": [c.provider for c in creds],
"count": len(creds),
"remote_server": _cred_server_url,
}
def resolve_credential(provider: str) -> Credential | None:
creds = load_credentials()
for c in creds:
if c.provider == provider:
return c
return None
def provider_manifest() -> dict[str, Any]:
creds = load_credentials()
return {
"ok": True,
"service_kind": "apiProvider",
"backend": _detect_backend(),
"providers": [
{
"name": c.provider,
"description": c.metadata.get("description", ""),
"key_name": c.key_name,
"available": True,
}
for c in creds
],
"node_id": os.environ.get("RS_SURFACE_NODE_ID", "unknown"),
}

View file

@ -0,0 +1,319 @@
"""Credential server — REST + OpenAPI access point for API keys.
Reads credentials from credential_provider (env vars + config file + RDS)
and serves them via HTTP so no script needs direct env var access.
Usage:
# Standalone
python3 credential_server.py [--port 8444] [--bind 0.0.0.0]
# Via existing embedded surface (if RS_CREDENTIAL_SERVER_PORT is set)
"""
from __future__ import annotations
import json
import os
import re
import sys
import urllib.request
import urllib.parse
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from credential_provider import (
credential_status,
provider_manifest,
resolve_credential,
_rds_connect,
_cred_server_url
)
VERSION = "0.4"
OPENAPI_VERSION = "3.0.3"
OPENAPI_SPEC: dict[str, Any] = {
"openapi": OPENAPI_VERSION,
"info": {
"title": "Research Stack Credential Provider",
"description": "Central API key access point for all stack services. "
"Set keys once on the microVM, consume from anywhere.",
"version": VERSION,
},
"servers": [
{"url": "http://localhost:8444", "description": "Local / Tailnet default"},
],
"paths": {
"/": {
"get": {
"summary": "Service root",
"responses": {"200": {"description": "Service info + links"}},
}
},
"/health": {
"get": {
"summary": "Health check",
"responses": {"200": {"description": "OK"}},
}
},
"/openapi.json": {
"get": {
"summary": "OpenAPI 3.0 specification",
"responses": {"200": {"description": "This document"}},
}
},
"/credentials": {
"get": {
"summary": "List all available credential providers",
"responses": {
"200": {
"description": "Provider manifest",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"providers": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {"type": "string"},
"description": {"type": "string"},
"available": {"type": "boolean"},
},
},
}
},
}
}
},
}
},
}
},
"/credentials/{provider}": {
"get": {
"summary": "Resolve a specific provider credential",
"parameters": [
{
"name": "provider",
"in": "path",
"required": True,
"schema": {"type": "string"},
"description": "Provider name (deepseek, bedrock, etc.)",
}
],
"responses": {
"200": {"description": "Credential value"},
"404": {"description": "Provider not found"},
},
}
},
"/status": {
"get": {
"summary": "Credential provider status",
"responses": {"200": {"description": "Backend info + counts"}},
}
},
},
}
SERVICE_INFO: dict[str, Any] = {
"service": "credential-provider",
"version": VERSION,
"docs": "/openapi.json",
"credentials": "/credentials",
"status": "/status",
"health": "/health",
}
class CredentialHTTPHandler(BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
sys.stderr.write(f"[credential-server] {args[0]} {args[1]} {args[2]}\n")
def _send_json(self, data: Any, status: int = 200):
body = json.dumps(data, sort_keys=True, indent=2).encode("utf-8") + b"\n"
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
self.wfile.write(body)
def _send_error(self, status: int, message: str):
self._send_json({"ok": False, "error": message}, status)
def do_GET(self):
path = self.path.rstrip("/") or "/"
if path in ("/", "/api"):
return self._send_json(SERVICE_INFO)
if path in ("/health", "/api/health"):
return self._send_json({"status": "ok", "version": VERSION})
if path == "/openapi.json":
return self._send_json(OPENAPI_SPEC)
if path in ("/status", "/api/status"):
return self._send_json(credential_status())
if path == "/credentials" or path == "/api/credentials":
return self._send_json(provider_manifest())
m = re.match(r"^(?:/api)?/credentials/([a-zA-Z0-9_-]+)$", path)
if m:
provider = m.group(1).lower()
cred = resolve_credential(provider)
if cred is None:
return self._send_error(404, f"provider '{provider}' not found")
return self._send_json({
"ok": True,
"provider": cred.provider,
"key": cred.value,
"key_name": cred.key_name,
})
return self._send_error(404, f"not found: {path}")
def do_POST(self):
path = self.path.rstrip("/") or "/"
if path in ("/api/webhooks/linear", "/webhooks/linear"):
# Read raw body
content_length = int(self.headers.get("Content-Length", 0))
raw_body = self.rfile.read(content_length)
# Node identification logic: proxy or process
node_id = os.environ.get("RS_SURFACE_NODE_ID", "")
if node_id != "MicroVM-Racknerd":
# Act as Proxy: Forward POST request to the central microVM
target_url = f"{_cred_server_url}/api/webhooks/linear"
req = urllib.request.Request(target_url, data=raw_body, method="POST")
for k, v in self.headers.items():
if k.lower() in ("content-type", "linear-signature"):
req.add_header(k, v)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
resp_data = json.loads(resp.read().decode("utf-8"))
return self._send_json(resp_data, resp.status)
except urllib.error.HTTPError as e:
try:
err_data = json.loads(e.read().decode("utf-8"))
return self._send_json(err_data, e.code)
except Exception:
return self._send_error(e.code, f"Forwarding failed: {e}")
except Exception as e:
return self._send_error(500, f"Forwarding connection error: {e}")
# Act as Server: Process Linear webhook body and upsert into Aurora RDS ENE packages table
try:
data = json.loads(raw_body.decode("utf-8"))
except Exception as e:
return self._send_error(400, f"Invalid JSON payload: {e}")
action = data.get("action")
issue_data = data.get("data", {})
payload_type = data.get("type")
if payload_type != "Issue":
return self._send_json({"ok": True, "message": f"Ignored non-Issue payload: {payload_type}"})
identifier = issue_data.get("identifier")
if not identifier:
return self._send_error(400, "Missing issue identifier")
pkg_id = f"linear/{identifier}"
if action == "remove":
try:
conn = _rds_connect()
cur = conn.cursor()
cur.execute("DELETE FROM ene.packages WHERE pkg = %s", (pkg_id,))
conn.commit()
conn.close()
return self._send_json({"ok": True, "action": "delete", "pkg": pkg_id})
except Exception as e:
return self._send_error(500, f"Database delete error: {e}")
# Extract title and description
title = issue_data.get("title", "")
description = issue_data.get("description", "")
full_text = f"{title}\n\n{description}" if description else title
url = issue_data.get("url", "")
# Extract labels
labels_data = issue_data.get("labels", [])
labels_list = []
if isinstance(labels_data, list):
for l in labels_data:
if isinstance(l, dict):
labels_list.append(l.get("name"))
elif isinstance(l, str):
labels_list.append(l)
tags_json = json.dumps(labels_list)
import datetime
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
try:
conn = _rds_connect()
cur = conn.cursor()
cur.execute("""
INSERT INTO ene.packages (
pkg, version, domain, tier, archetype,
tags, description, source, indexed_utc
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (pkg) DO UPDATE SET
description = EXCLUDED.description,
tags = EXCLUDED.tags,
source = EXCLUDED.source,
indexed_utc = EXCLUDED.indexed_utc
""", (
pkg_id,
"1.0.0",
"LINEAR",
"INTENT",
"issue",
tags_json,
full_text,
url,
now_iso
))
conn.commit()
conn.close()
return self._send_json({"ok": True, "action": action, "pkg": pkg_id})
except Exception as e:
return self._send_error(500, f"Database upsert error: {e}")
return self._send_error(404, f"not found: {path}")
do_HEAD = do_GET
def main():
import argparse
parser = argparse.ArgumentParser(description="Research Stack Credential Server")
parser.add_argument("--port", type=int, default=int(os.environ.get("RS_CREDENTIAL_PORT", "8444")))
parser.add_argument("--bind", default=os.environ.get("RS_CREDENTIAL_BIND", "0.0.0.0"))
args = parser.parse_args()
server = ThreadingHTTPServer((args.bind, args.port), CredentialHTTPHandler)
print(f"[credential-server] listening on {args.bind}:{args.port}", flush=True)
print(f"[credential-server] OpenAPI spec at http://{args.bind}:{args.port}/openapi.json", flush=True)
try:
server.serve_forever()
except KeyboardInterrupt:
print("\n[credential-server] shutting down", flush=True)
server.server_close()
if __name__ == "__main__":
main()

View file

@ -0,0 +1,592 @@
"""RDS-backed ENEFractalFold — drop-in replacement for SQLite ENEFractalFold.
API-compatible with ene_fractal_fold.ENEFractalFold: same dataclasses, same
handle_request() protocol, but backed by PostgreSQL via psycopg2.
Constructor:
ENERDSFractalFold(dsn="postgresql://user:pass@host:5432/dbname")
The DSN defaults to the RDS_HOST / RDS_PORT / RDS_USER / RDS_PASSWORD / RDS_DB
environment variables.
"""
from __future__ import annotations
import base64
import json
import math
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
import psycopg2
import psycopg2.extras
from infra.ene_fractal_fold import (
FractalNode,
FractalManifest,
VERSION,
GOLDEN_ANGLE,
canonical_json,
sha256_text,
sha256_bytes,
gray_code,
inverse_gray_code,
golden_spiral_point,
manifold_distance,
tree_depth,
make_leaf,
make_parent,
encode_fractal,
encode_fractal_chunks,
parse_graphml_concepts,
node_record,
archive_record,
jsonl_event,
)
import os
def _default_dsn() -> str:
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
port = os.environ.get("RDS_PORT", "5432")
user = os.environ.get("RDS_USER", "postgres")
password = os.environ.get("RDS_PASSWORD") or os.environ.get("RDS_IAM_TOKEN", "")
dbname = os.environ.get("RDS_DB", "postgres")
return f"host={host} port={port} dbname={dbname} user={user} password={password} sslmode=require"
class ENERDSFractalFold:
def __init__(self, dsn: str | None = None):
self.dsn = dsn or _default_dsn()
self._init_db()
def _get_conn(self):
return psycopg2.connect(self.dsn)
def _init_db(self) -> None:
with self._get_conn() as conn:
with conn.cursor() as cur:
cur.execute("CREATE SCHEMA IF NOT EXISTS ene")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.fractal_manifolds (
root_hash TEXT PRIMARY KEY,
name TEXT NOT NULL,
byte_len INTEGER NOT NULL,
leaves_count INTEGER NOT NULL,
depth INTEGER NOT NULL,
chunk_size INTEGER NOT NULL,
branching_factor INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
receipt TEXT NOT NULL,
archive_record JSONB NOT NULL DEFAULT '{}',
jsonl_event JSONB NOT NULL DEFAULT '{}'
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.fractal_nodes (
root_hash TEXT NOT NULL,
node_hash TEXT NOT NULL,
kind TEXT NOT NULL,
level INTEGER NOT NULL,
ordinal INTEGER NOT NULL,
fold_address INTEGER NOT NULL,
start_leaf INTEGER NOT NULL,
end_leaf INTEGER NOT NULL,
size_bytes INTEGER NOT NULL,
children TEXT NOT NULL,
payload_b64 TEXT,
PRIMARY KEY (root_hash, node_hash)
)
""")
cur.execute("""
CREATE INDEX IF NOT EXISTS idx_rds_fractal_leaf
ON ene.fractal_nodes (root_hash, level, ordinal)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.fractal_graph_entities (
root_hash TEXT NOT NULL,
graph_node_id TEXT NOT NULL,
leaf_index INTEGER NOT NULL,
name TEXT NOT NULL,
family TEXT,
domain TEXT,
neighbors TEXT NOT NULL,
PRIMARY KEY (root_hash, graph_node_id)
)
""")
cur.execute("""
CREATE INDEX IF NOT EXISTS idx_rds_fractal_graph_name
ON ene.fractal_graph_entities (root_hash, name)
""")
conn.commit()
def _store(self, manifest: FractalManifest, nodes: list[FractalNode]) -> tuple[dict[str, Any], dict[str, Any]]:
record = archive_record(manifest)
event = jsonl_event(record, manifest)
with self._get_conn() as conn:
with conn.cursor() as cur:
cur.execute("""
INSERT INTO ene.fractal_manifolds
(root_hash, name, byte_len, leaves_count, depth, chunk_size, branching_factor,
created_at, receipt, archive_record, jsonl_event)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (root_hash) DO UPDATE SET
name = EXCLUDED.name,
byte_len = EXCLUDED.byte_len,
leaves_count = EXCLUDED.leaves_count,
depth = EXCLUDED.depth,
chunk_size = EXCLUDED.chunk_size,
branching_factor = EXCLUDED.branching_factor,
receipt = EXCLUDED.receipt,
archive_record = EXCLUDED.archive_record,
jsonl_event = EXCLUDED.jsonl_event
""", (
manifest.root_hash, manifest.name, manifest.byte_len,
manifest.leaves_count, manifest.depth, manifest.chunk_size,
manifest.branching_factor, manifest.created_at, manifest.receipt,
json.dumps(record, sort_keys=True), json.dumps(event, sort_keys=True),
))
cur.execute("DELETE FROM ene.fractal_nodes WHERE root_hash = %s", (manifest.root_hash,))
psycopg2.extras.execute_values(
cur,
"""
INSERT INTO ene.fractal_nodes
(root_hash, node_hash, kind, level, ordinal, fold_address, start_leaf,
end_leaf, size_bytes, children, payload_b64)
VALUES %s
""",
[
(
manifest.root_hash, node.node_hash, node.kind, node.level,
node.ordinal, node.fold_address, node.start_leaf, node.end_leaf,
node.size_bytes, canonical_json(node.children), node.payload_b64,
)
for node in nodes
],
)
conn.commit()
return record, event
def put(self, data: bytes, name: str = "unnamed", chunk_size: int = 4096, branching_factor: int = 4) -> dict[str, Any]:
manifest, nodes = encode_fractal(data, name, chunk_size, branching_factor)
record, event = self._store(manifest, nodes)
return {
"ok": True, "op": "fractal_put",
"manifest": asdict(manifest),
"archive_record": record,
"jsonl_event": event,
}
def put_graphml(self, graphml: bytes, name: str = "graphml", branching_factor: int = 4) -> dict[str, Any]:
records, chunks = parse_graphml_concepts(graphml)
max_chunk = max((len(chunk) for chunk in chunks), default=0)
manifest, nodes = encode_fractal_chunks(chunks, name, max_chunk, branching_factor)
record, event = self._store(manifest, nodes)
with self._get_conn() as conn:
with conn.cursor() as cur:
cur.execute("DELETE FROM ene.fractal_graph_entities WHERE root_hash = %s", (manifest.root_hash,))
psycopg2.extras.execute_values(
cur,
"""
INSERT INTO ene.fractal_graph_entities
(root_hash, graph_node_id, leaf_index, name, family, domain, neighbors)
VALUES %s
""",
[
(
manifest.root_hash, concept["graph_node_id"],
concept["leaf_index"], concept["name"],
concept["family"], concept["domain"],
canonical_json(concept["neighbors"]),
)
for concept in records
],
)
conn.commit()
return {
"ok": True, "op": "fractal_graphml_put",
"manifest": asdict(manifest),
"graphml": {"concepts": len(records), "concept_leaf_mode": True},
"archive_record": record,
"jsonl_event": event,
}
def manifest(self, root_hash: str) -> dict[str, Any] | None:
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("SELECT * FROM ene.fractal_manifolds WHERE root_hash = %s", (root_hash,))
row = cur.fetchone()
if row is None:
return None
return {
"root_hash": row["root_hash"],
"name": row["name"],
"byte_len": row["byte_len"],
"leaves_count": row["leaves_count"],
"depth": row["depth"],
"chunk_size": row["chunk_size"],
"branching_factor": row["branching_factor"],
"created_at": row["created_at"].isoformat() if hasattr(row["created_at"], "isoformat") else str(row["created_at"]),
"receipt": row["receipt"],
"archive_record": row["archive_record"] if isinstance(row["archive_record"], dict) else json.loads(row["archive_record"] or "{}"),
"jsonl_event": row["jsonl_event"] if isinstance(row["jsonl_event"], dict) else json.loads(row["jsonl_event"] or "{}"),
}
def _node(self, cur, root_hash: str, node_hash: str) -> dict[str, Any]:
cur.execute(
"SELECT * FROM ene.fractal_nodes WHERE root_hash = %s AND node_hash = %s",
(root_hash, node_hash),
)
row = cur.fetchone()
if row is None:
raise KeyError(f"missing fractal node {node_hash}")
return dict(row)
def proof(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
meta = self.manifest(root_hash)
if meta is None:
raise KeyError(f"unknown root {root_hash}")
if leaf_index < 0 or leaf_index >= meta["leaves_count"]:
raise IndexError("leaf_index outside manifold")
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
frontier = root_hash
path = []
target_point = golden_spiral_point(gray_code(leaf_index), 0)
while True:
row = self._node(cur, root_hash, frontier)
children = json.loads(row["children"])
node_point = golden_spiral_point(row["fold_address"], row["level"])
entry = {k: v for k, v in dict(row).items() if k != "payload_b64"}
entry["children"] = children
entry["golden_spiral"] = node_point
entry["distance_to_target"] = round(manifold_distance(node_point, target_point), 9)
path.append(entry)
if row["kind"] == "leaf":
payload = base64.b64decode(row["payload_b64"] or "")
break
next_hash = None
pruned = []
for child_hash in children:
child = self._node(cur, root_hash, child_hash)
child_point = golden_spiral_point(child["fold_address"], child["level"])
pruned.append({
"node_hash": child_hash,
"covers_target": child["start_leaf"] <= leaf_index <= child["end_leaf"],
"distance": round(manifold_distance(child_point, target_point), 9),
})
if child["start_leaf"] <= leaf_index <= child["end_leaf"]:
next_hash = child_hash
path[-1]["manifold_distance_pruning"] = sorted(pruned, key=lambda item: item["distance"])
if next_hash is None:
raise ValueError("corrupt tree: no child covers requested leaf")
frontier = next_hash
path_valid = self._verify_path_rows(path, payload)
return {
"ok": True, "op": "fractal_proof",
"root_hash": root_hash, "leaf_index": leaf_index,
"fold_address": gray_code(leaf_index),
"inverse_fold_address": inverse_gray_code(gray_code(leaf_index)),
"golden_spiral": golden_spiral_point(gray_code(leaf_index), 0),
"traversal_cost": len(path),
"expected_complexity": f"O(log_{meta['branching_factor']}(n))",
"path_hash_verified": path_valid,
"path": path,
"payload_b64": base64.b64encode(payload).decode("ascii"),
}
def _verify_path_rows(self, path: list[dict[str, Any]], payload: bytes) -> bool:
if not path:
return False
leaf = path[-1]
if leaf["kind"] != "leaf":
return False
expected_leaf = make_leaf(payload, leaf["ordinal"]).node_hash
if expected_leaf != leaf["node_hash"]:
return False
child_hash = expected_leaf
for row in reversed(path[:-1]):
if child_hash not in row["children"]:
return False
child_hash = row["node_hash"]
return child_hash == path[0]["node_hash"]
def verify(self, root_hash: str) -> dict[str, Any]:
meta = self.manifest(root_hash)
if meta is None:
raise KeyError(f"unknown root {root_hash}")
errors = []
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("SELECT * FROM ene.fractal_nodes WHERE root_hash = %s", (root_hash,))
rows = cur.fetchall()
by_hash = {r["node_hash"]: r for r in rows}
for row in rows:
children = json.loads(row["children"])
if row["kind"] == "leaf":
payload = base64.b64decode(row["payload_b64"] or "")
expected = make_leaf(payload, row["ordinal"]).node_hash
if expected != row["node_hash"]:
errors.append({"node": row["node_hash"], "error": "leaf_hash_mismatch", "expected": expected})
continue
child_nodes = []
for child_hash in children:
child = by_hash.get(child_hash)
if child is None:
errors.append({"node": row["node_hash"], "error": "missing_child", "child": child_hash})
continue
child_nodes.append(FractalNode(
node_hash=child["node_hash"], kind=child["kind"],
level=child["level"], ordinal=child["ordinal"],
fold_address=child["fold_address"],
start_leaf=child["start_leaf"], end_leaf=child["end_leaf"],
size_bytes=child["size_bytes"],
children=json.loads(child["children"]),
payload_b64=child["payload_b64"],
))
if len(child_nodes) == len(children):
expected = make_parent(child_nodes, row["level"], row["ordinal"]).node_hash
if expected != row["node_hash"]:
errors.append({"node": row["node_hash"], "error": "parent_hash_mismatch", "expected": expected})
return {
"ok": not errors, "op": "fractal_verify",
"root_hash": root_hash, "checked_nodes": len(rows),
"errors": errors, "damage_detected": bool(errors),
}
def neighbors(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
meta = self.manifest(root_hash)
if meta is None:
raise KeyError(f"unknown root {root_hash}")
candidates = sorted(set(idx for idx in (leaf_index - 1, leaf_index, leaf_index + 1) if 0 <= idx < meta["leaves_count"]))
return {
"ok": True, "op": "fractal_neighbors",
"root_hash": root_hash, "leaf_index": leaf_index,
"fold_address": gray_code(leaf_index),
"neighbors": [
{
"leaf_index": idx, "fold_address": gray_code(idx),
"golden_spiral": golden_spiral_point(gray_code(idx), 0),
"fold_distance": bin(gray_code(idx) ^ gray_code(leaf_index)).count("1"),
}
for idx in candidates
],
}
def graph_entity(self, root_hash: str, graph_node_id: str = "", name: str = "") -> dict[str, Any]:
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
if graph_node_id:
cur.execute(
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND graph_node_id = %s",
(root_hash, graph_node_id),
)
else:
cur.execute(
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND lower(name) = lower(%s)",
(root_hash, name),
)
row = cur.fetchone()
if row is None:
return {"ok": False, "op": "fractal_graph_entity", "error": "graph entity not found"}
proof = self.navigate(root_hash, row["leaf_index"])
return {
"ok": True, "op": "fractal_graph_entity",
"root_hash": root_hash,
"entity": {
"graph_node_id": row["graph_node_id"],
"leaf_index": row["leaf_index"],
"name": row["name"],
"family": row["family"],
"domain": row["domain"],
"neighbors": json.loads(row["neighbors"]),
},
"retrieval": proof,
}
def graph_neighbors(self, root_hash: str, graph_node_id: str) -> dict[str, Any]:
entity = self.graph_entity(root_hash, graph_node_id=graph_node_id)
if not entity.get("ok"):
return entity
neighbor_ids = entity["entity"]["neighbors"]
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
rows = []
for nid in neighbor_ids:
cur.execute(
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND graph_node_id = %s",
(root_hash, nid),
)
row = cur.fetchone()
if row is not None:
rows.append(row)
return {
"ok": True, "op": "fractal_graph_neighbors",
"root_hash": root_hash, "graph_node_id": graph_node_id,
"neighbors": [
{
"graph_node_id": r["graph_node_id"],
"leaf_index": r["leaf_index"],
"name": r["name"],
"family": r["family"],
"domain": r["domain"],
"fold_address": gray_code(r["leaf_index"]),
"golden_spiral": golden_spiral_point(gray_code(r["leaf_index"]), 0),
}
for r in rows
],
}
def navigate(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
meta = self.manifest(root_hash)
if meta is None:
raise KeyError(f"unknown root {root_hash}")
if leaf_index < 0 or leaf_index >= meta["leaves_count"]:
raise IndexError("leaf_index outside manifold")
target_point = golden_spiral_point(gray_code(leaf_index), 0)
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
frontier = root_hash
path = []
while True:
row = self._node(cur, root_hash, frontier)
children = json.loads(row["children"])
entry = {
"node_hash": row["node_hash"], "kind": row["kind"],
"level": row["level"], "ordinal": row["ordinal"],
"start_leaf": row["start_leaf"], "end_leaf": row["end_leaf"],
"fold_address": row["fold_address"],
"children": children,
"golden_spiral": golden_spiral_point(row["fold_address"], row["level"]),
}
path.append(entry)
if row["kind"] == "leaf":
payload = base64.b64decode(row["payload_b64"] or "")
break
ranked = []
for child_hash in children:
child = self._node(cur, root_hash, child_hash)
point = golden_spiral_point(child["fold_address"], child["level"])
ranked.append({
"node_hash": child_hash,
"distance": manifold_distance(point, target_point),
"covers_target": child["start_leaf"] <= leaf_index <= child["end_leaf"],
})
ranked.sort(key=lambda item: (not item["covers_target"], item["distance"]))
path[-1]["pruned_candidates"] = [
{"node_hash": item["node_hash"], "distance": round(item["distance"], 9), "covers_target": item["covers_target"]}
for item in ranked
]
frontier = ranked[0]["node_hash"]
return {
"ok": True, "op": "fractal_navigate",
"root_hash": root_hash, "leaf_index": leaf_index,
"target": {"fold_address": gray_code(leaf_index), "golden_spiral": target_point},
"retrieval_complexity": f"O(log_{meta['branching_factor']}(n))",
"navigation": "golden_spiral_manifold_distance_pruning",
"path_hash_verified": self._verify_path_rows(path, payload),
"path": path,
"payload_b64": base64.b64encode(payload).decode("ascii"),
}
def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
op = str(request.get("op", "manifest"))
if op in {"put", "encode"}:
if "data_b64" in request:
data = base64.b64decode(str(request["data_b64"]))
else:
data = str(request.get("text", "")).encode("utf-8")
return self.put(
data=data,
name=str(request.get("name", "unnamed")),
chunk_size=int(request.get("chunk_size", 4096)),
branching_factor=int(request.get("branching_factor", 4)),
)
if op in {"put_graphml", "graphml"}:
if "data_b64" in request:
data = base64.b64decode(str(request["data_b64"]))
else:
data = str(request.get("text", "")).encode("utf-8")
return self.put_graphml(
graphml=data,
name=str(request.get("name", "graphml")),
branching_factor=int(request.get("branching_factor", 4)),
)
root_hash = str(request.get("root_hash", ""))
if not root_hash:
raise ValueError("root_hash is required for this operation")
if op == "manifest":
meta = self.manifest(root_hash)
return {"ok": meta is not None, "op": "fractal_manifest", "manifest": meta}
if op == "proof":
return self.proof(root_hash, int(request.get("leaf_index", 0)))
if op in {"navigate", "get"}:
return self.navigate(root_hash, int(request.get("leaf_index", 0)))
if op == "verify":
return self.verify(root_hash)
if op == "neighbors":
return self.neighbors(root_hash, int(request.get("leaf_index", 0)))
if op in {"graph_entity", "concept"}:
return self.graph_entity(
root_hash,
graph_node_id=str(request.get("graph_node_id", "")),
name=str(request.get("name", "")),
)
if op in {"graph_neighbors", "concept_neighbors"}:
return self.graph_neighbors(root_hash, str(request.get("graph_node_id", "")))
raise ValueError(f"unsupported fractal op {op!r}")
def main() -> int:
import argparse
parser = argparse.ArgumentParser(description="ENE RDS fractal fold codec")
parser.add_argument("--dsn", help="PostgreSQL DSN")
parser.add_argument("--op", default="put")
parser.add_argument("--name", default="cli")
parser.add_argument("--text", default="")
parser.add_argument("--file", type=argparse.FileType("rb"))
parser.add_argument("--graph-node-id")
parser.add_argument("--root-hash")
parser.add_argument("--leaf-index", type=int, default=0)
parser.add_argument("--chunk-size", type=int, default=4096)
parser.add_argument("--branching-factor", type=int, default=4)
args = parser.parse_args()
layer = ENERDSFractalFold(args.dsn)
if args.op in {"put", "encode"}:
data = args.file.read() if args.file else args.text.encode("utf-8")
result = layer.put(data, args.name, args.chunk_size, args.branching_factor)
elif args.op in {"put_graphml", "graphml"}:
data = args.file.read() if args.file else args.text.encode("utf-8")
result = layer.put_graphml(data, args.name, args.branching_factor)
elif args.op in {"graph_entity", "concept", "graph_neighbors"}:
result = layer.handle_request({
"op": args.op, "root_hash": args.root_hash,
"leaf_index": args.leaf_index,
"graph_node_id": args.graph_node_id or "",
"name": args.name,
})
else:
result = layer.handle_request({
"op": args.op, "root_hash": args.root_hash,
"leaf_index": args.leaf_index,
})
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -0,0 +1,438 @@
"""RDS-backed ENEWikiLayer — drop-in replacement for SQLite ENEWikiLayer.
API-compatible with ene_wiki_layer.ENEWikiLayer: same dataclasses, same
handle_request() protocol, but backed by PostgreSQL via psycopg2.
Constructor:
ENERDSWikiLayer(dsn="postgresql://user:pass@host:5432/dbname")
The DSN defaults to the RDS_HOST / RDS_PORT / RDS_USER / RDS_PASSWORD / RDS_DB
environment variables (or postgres as dbname, research_stack as fallback).
"""
from __future__ import annotations
import hashlib
import json
import re
import time
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import psycopg2
import psycopg2.extras
import psycopg2.pool
from infra.ene_wiki_layer import (
WikiPage,
WikiRevision,
normalize_title,
title_slug,
extract_links,
extract_categories,
write_receipt,
canonical_json,
sha256_text,
iso_utc,
concept_vector_for_wiki,
genome_from_vector,
make_archive_record,
make_jsonl_event,
)
DEFAULT_DSN = None
def _default_dsn() -> str:
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
port = os.environ.get("RDS_PORT", "5432")
user = os.environ.get("RDS_USER", "postgres")
password = os.environ.get("RDS_PASSWORD") or os.environ.get("RDS_IAM_TOKEN", "")
dbname = os.environ.get("RDS_DB", "postgres")
return f"host={host} port={port} dbname={dbname} user={user} password={password} sslmode=require"
import os
class ENERDSWikiLayer:
def __init__(self, dsn: str | None = None):
self.dsn = dsn or _default_dsn()
self._pool: psycopg2.pool.ThreadedConnectionPool | None = None
self._init_tables()
def _get_conn(self):
return psycopg2.connect(self.dsn)
def _init_tables(self) -> None:
with self._get_conn() as conn:
with conn.cursor() as cur:
cur.execute("CREATE SCHEMA IF NOT EXISTS ene")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.wiki_pages (
slug TEXT PRIMARY KEY,
title TEXT NOT NULL,
latest_revision INTEGER NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
receipt TEXT NOT NULL
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.wiki_revisions (
slug TEXT NOT NULL,
revision INTEGER NOT NULL,
title TEXT NOT NULL,
text TEXT NOT NULL,
author TEXT NOT NULL DEFAULT 'ene',
summary TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
receipt TEXT NOT NULL,
archive_id TEXT,
content_hash TEXT,
archive_record JSONB DEFAULT '{}',
jsonl_event JSONB DEFAULT '{}',
PRIMARY KEY (slug, revision)
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.wiki_links (
slug TEXT NOT NULL,
target_slug TEXT NOT NULL,
target_title TEXT NOT NULL,
PRIMARY KEY (slug, target_slug)
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.wiki_categories (
slug TEXT NOT NULL,
category TEXT NOT NULL,
PRIMARY KEY (slug, category)
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.packages (
pkg TEXT PRIMARY KEY,
version TEXT NOT NULL,
tier TEXT,
domain TEXT,
archetype TEXT,
description TEXT,
tags JSONB DEFAULT '[]',
source TEXT,
sha256 TEXT,
indexed_utc TEXT,
concept_anchor JSONB DEFAULT '{}',
concept_vector JSONB DEFAULT '[]',
idea_weights JSONB DEFAULT '{}',
analog_map JSONB DEFAULT '{}'
)
""")
self._ensure_columns(conn, "ene.wiki_revisions", cur)
conn.commit()
def _ensure_columns(self, conn, table: str, cur) -> None:
cur.execute("""
SELECT column_name FROM information_schema.columns
WHERE table_schema = 'ene' AND table_name = %s
""", (table.split('.')[1],))
existing = {r[0] for r in cur.fetchall()}
additions = {
"archive_id": "TEXT",
"content_hash": "TEXT",
"archive_record": "JSONB DEFAULT '{}'",
"jsonl_event": "JSONB DEFAULT '{}'",
}
for name, decl in additions.items():
if name not in existing:
cur.execute(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {name} {decl}")
def _upsert_package(self, conn, cur, event: dict[str, Any]) -> None:
data = event["data"]
cur.execute("""
INSERT INTO ene.packages (
pkg, version, tier, domain, archetype, description,
tags, source, sha256, indexed_utc,
concept_anchor, concept_vector, idea_weights, analog_map
) VALUES (
%(pkg)s, %(version)s, %(tier)s, %(domain)s, %(archetype)s, %(description)s,
%(tags)s, %(source)s, %(sha256)s, %(indexed_utc)s,
%(concept_anchor)s, %(concept_vector)s, %(idea_weights)s, %(analog_map)s
) ON CONFLICT (pkg) DO UPDATE SET
version = EXCLUDED.version,
tier = EXCLUDED.tier,
domain = EXCLUDED.domain,
archetype = EXCLUDED.archetype,
description = EXCLUDED.description,
tags = EXCLUDED.tags,
source = EXCLUDED.source,
sha256 = EXCLUDED.sha256,
indexed_utc = EXCLUDED.indexed_utc,
concept_anchor = EXCLUDED.concept_anchor,
concept_vector = EXCLUDED.concept_vector,
idea_weights = EXCLUDED.idea_weights,
analog_map = EXCLUDED.analog_map
""", {
"pkg": data["pkg"],
"version": data["version"],
"tier": data["tier"],
"domain": data["domain"],
"archetype": data["archetype"],
"description": data["description"],
"tags": json.dumps(data["tags"], sort_keys=True),
"source": data["source"],
"sha256": data["sha256"],
"indexed_utc": data["indexed_utc"],
"concept_anchor": json.dumps(data["concept_anchor"], sort_keys=True),
"concept_vector": json.dumps(data["concept_vector"], sort_keys=True),
"idea_weights": json.dumps(data["idea_weights"], sort_keys=True),
"analog_map": json.dumps(data["analog_map"], sort_keys=True),
})
def admit_write(self, title: str, text: str) -> tuple[bool, str]:
try:
normalize_title(title)
except ValueError as exc:
return False, str(exc)
if len(text.encode("utf-8")) > 256_000:
return False, "wiki text too large"
lowered = text.lower()
if "<script" in lowered or "javascript:" in lowered:
return False, "active script content refused"
return True, "wiki_write_admitted"
def put_page(self, title: str, text: str, author: str = "ene", summary: str = "") -> WikiRevision:
admitted, reason = self.admit_write(title, text)
if not admitted:
raise ValueError(reason)
normalized = normalize_title(title)
slug = title_slug(normalized)
now_ts = int(time.time())
now_dt = datetime.fromtimestamp(now_ts, tz=timezone.utc)
links = extract_links(text)
categories = extract_categories(text)
with self._get_conn() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT latest_revision FROM ene.wiki_pages WHERE slug = %s",
(slug,),
)
row = cur.fetchone()
revision = int(row[0]) + 1 if row else 1
receipt = write_receipt(slug, revision, text, author, now_ts)
archive_record = make_archive_record(
normalized, slug, revision, text, author, summary, now_ts, links, categories
)
concept_vector = concept_vector_for_wiki(normalized, text, links, categories)
jsonl_event = make_jsonl_event(archive_record, concept_vector, receipt)
cur.execute("""
INSERT INTO ene.wiki_revisions
(slug, revision, title, text, author, summary, created_at, receipt,
archive_id, content_hash, archive_record, jsonl_event)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""", (
slug, revision, normalized, text, author, summary, now_dt, receipt,
archive_record["archive_id"], archive_record["content_hash"],
json.dumps(archive_record, sort_keys=True),
json.dumps(jsonl_event, sort_keys=True),
))
cur.execute("""
INSERT INTO ene.wiki_pages (slug, title, latest_revision, updated_at, receipt)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (slug) DO UPDATE SET
title = EXCLUDED.title,
latest_revision = EXCLUDED.latest_revision,
updated_at = EXCLUDED.updated_at,
receipt = EXCLUDED.receipt
""", (slug, normalized, revision, now_dt, receipt))
cur.execute("DELETE FROM ene.wiki_links WHERE slug = %s", (slug,))
cur.execute("DELETE FROM ene.wiki_categories WHERE slug = %s", (slug,))
if links:
psycopg2.extras.execute_values(
cur,
"INSERT INTO ene.wiki_links (slug, target_slug, target_title) VALUES %s ON CONFLICT DO NOTHING",
[(slug, title_slug(link), link) for link in links],
)
if categories:
psycopg2.extras.execute_values(
cur,
"INSERT INTO ene.wiki_categories (slug, category) VALUES %s ON CONFLICT DO NOTHING",
[(slug, cat) for cat in categories],
)
self._upsert_package(conn, cur, jsonl_event)
conn.commit()
return WikiRevision(
normalized, slug, revision, text, author, summary, now_ts, receipt,
links, categories, archive_record, jsonl_event,
)
def get_page(self, title: str, revision: int | None = None) -> WikiRevision | None:
slug = title_slug(title)
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
if revision is None:
cur.execute(
"SELECT latest_revision FROM ene.wiki_pages WHERE slug = %s",
(slug,),
)
page = cur.fetchone()
if not page:
return None
revision = int(page["latest_revision"])
else:
revision = int(revision)
cur.execute("""
SELECT title, slug, revision, text, author, summary, created_at, receipt,
archive_record, jsonl_event
FROM ene.wiki_revisions
WHERE slug = %s AND revision = %s
""", (slug, revision))
row = cur.fetchone()
if not row:
return None
cur.execute(
"SELECT target_title FROM ene.wiki_links WHERE slug = %s ORDER BY lower(target_title)",
(slug,),
)
links = [r["target_title"] for r in cur.fetchall()]
cur.execute(
"SELECT category FROM ene.wiki_categories WHERE slug = %s ORDER BY lower(category)",
(slug,),
)
categories = [r["category"] for r in cur.fetchall()]
archive_record = row["archive_record"] if isinstance(row["archive_record"], dict) else json.loads(row["archive_record"] or "{}")
jsonl_event = row["jsonl_event"] if isinstance(row["jsonl_event"], dict) else json.loads(row["jsonl_event"] or "{}")
created_ts = int(row["created_at"].timestamp()) if hasattr(row["created_at"], "timestamp") else 0
return WikiRevision(
row["title"], row["slug"], int(row["revision"]), row["text"],
row["author"], row["summary"], created_ts, row["receipt"],
links, categories, archive_record, jsonl_event,
)
def search(self, query: str, limit: int = 20) -> list[WikiPage]:
term = f"%{query.strip()}%"
limit = max(1, min(limit, 100))
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("""
SELECT p.title, p.slug, p.latest_revision, p.updated_at, p.receipt
FROM ene.wiki_pages p
JOIN ene.wiki_revisions r
ON r.slug = p.slug AND r.revision = p.latest_revision
WHERE p.title ILIKE %s OR r.text ILIKE %s
ORDER BY p.updated_at DESC
LIMIT %s
""", (term, term, limit))
rows = cur.fetchall()
return [
WikiPage(
r["title"], r["slug"], int(r["latest_revision"]),
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
r["receipt"],
)
for r in rows
]
def backlinks(self, title: str) -> list[WikiPage]:
target = title_slug(title)
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("""
SELECT p.title, p.slug, p.latest_revision, p.updated_at, p.receipt
FROM ene.wiki_links l
JOIN ene.wiki_pages p ON p.slug = l.slug
WHERE l.target_slug = %s
ORDER BY lower(p.title)
""", (target,))
rows = cur.fetchall()
return [
WikiPage(
r["title"], r["slug"], int(r["latest_revision"]),
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
r["receipt"],
)
for r in rows
]
def recent_changes(self, limit: int = 20) -> list[WikiPage]:
limit = max(1, min(limit, 100))
with self._get_conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("""
SELECT title, slug, latest_revision, updated_at, receipt
FROM ene.wiki_pages
ORDER BY updated_at DESC
LIMIT %s
""", (limit,))
rows = cur.fetchall()
return [
WikiPage(
r["title"], r["slug"], int(r["latest_revision"]),
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
r["receipt"],
)
for r in rows
]
def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
op = str(request.get("op", "recent"))
if op in {"put", "edit"}:
revision = self.put_page(
title=str(request.get("title", "")),
text=str(request.get("text", "")),
author=str(request.get("author", "ene")),
summary=str(request.get("summary", "")),
)
return {"ok": True, "op": op, "revision": asdict(revision)}
if op == "get":
page = self.get_page(str(request.get("title", "")), request.get("revision"))
return {"ok": page is not None, "op": op, "page": asdict(page) if page else None}
if op == "search":
pages = self.search(str(request.get("query", "")), int(request.get("limit", 20)))
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
if op == "backlinks":
pages = self.backlinks(str(request.get("title", "")))
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
if op == "recent":
pages = self.recent_changes(int(request.get("limit", 20)))
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
raise ValueError(f"unsupported wiki op {op!r}")
def main() -> int:
import argparse
parser = argparse.ArgumentParser(description="ENE RDS wiki layer")
parser.add_argument("--dsn", help="PostgreSQL DSN")
parser.add_argument("--op", choices=["put", "get", "search", "backlinks", "recent"], default="recent")
parser.add_argument("--title", default="")
parser.add_argument("--text", default="")
parser.add_argument("--query", default="")
parser.add_argument("--author", default="ene")
args = parser.parse_args()
wiki = ENERDSWikiLayer(args.dsn)
request: dict[str, Any] = {
"op": args.op, "title": args.title, "text": args.text,
"query": args.query, "author": args.author,
}
print(json.dumps(wiki.handle_request(request), sort_keys=True, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -0,0 +1,94 @@
#!/usr/bin/env bash
# Deploy credential server to microVM (RackNerd)
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
VM_IP="172.245.19.182"
VM_USER="root"
VM_PASS="${1:-}"
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
if [ -z "$VM_PASS" ]; then
VM_PASS="$(cat "$REPO_ROOT/API KEYS/racknerd_510bd9c_root.txt" 2>/dev/null | grep root_password | cut -d: -f2 | tr -d ' ')"
fi
if [ -z "$VM_PASS" ]; then
echo "ERROR: root password required. Pass as argument or ensure API KEYS/racknerd_510bd9c_root.txt exists."
exit 1
fi
SSH="sshpass -p "$VM_PASS" ssh $SSH_OPTS ${VM_USER}@${VM_IP}"
SCP="sshpass -p "$VM_PASS" scp $SSH_OPTS"
echo "=== Deploying credential server to ${VM_IP} ==="
# Create directories
$SSH "mkdir -p /opt/rs-surface /etc/rs-surface"
# Copy Python modules
echo "Uploading credential_provider.py..."
$SCP "$REPO_ROOT/4-Infrastructure/infra/credential_provider.py" ${VM_USER}@${VM_IP}:/opt/rs-surface/credential_provider.py
echo "Uploading credential_server.py..."
$SCP "$REPO_ROOT/4-Infrastructure/infra/credential_server.py" ${VM_USER}@${VM_IP}:/opt/rs-surface/credential_server.py
# Copy credentials config
CRED_JSON="/tmp/rs-credentials.json"
python3 -c "
import json, os
creds = {}
for var, key in [('DEEPSEEK_API_KEY', 'deepseek'), ('QUANDELA_API_KEY', 'quandela'),
('WOLFRAM_ALPHA_APPID', 'wolfram_alpha'), ('LINEAR_API_KEY', 'linear'),
('AWS_BEARER_TOKEN_BEDROCK', 'bedrock')]:
val = os.environ.get(var, '')
if val:
creds[key] = val
if not creds:
print('WARNING: no credential env vars set; writing empty config')
with open('$CRED_JSON', 'w') as f:
json.dump(creds, f, indent=2)
print(f'Wrote {len(creds)} provider keys from environment')
"
echo "Uploading credentials.json..."
$SCP "$CRED_JSON" ${VM_USER}@${VM_IP}:/etc/rs-surface/credentials.json
rm -f "$CRED_JSON"
# Create systemd service
echo "Setting up systemd service..."
$SSH 'cat > /etc/systemd/system/rs-credential-server.service << '"'"'SERVICEEOF'"'"'
[Unit]
Description=Research Stack Credential Server
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=root
WorkingDirectory=/opt/rs-surface
ExecStart=/usr/bin/python3 /opt/rs-surface/credential_server.py --port 8444 --bind 0.0.0.0
Restart=always
RestartSec=5
Environment=RS_CREDENTIAL_CONFIG=/etc/rs-surface/credentials.json
[Install]
WantedBy=multi-user.target
SERVICEEOF'
# Stop old service if exists, enable new one
$SSH "systemctl daemon-reload && systemctl enable rs-credential-server && systemctl restart rs-credential-server"
# Verify
echo "=== Verifying ==="
sleep 2
$SSH "systemctl status rs-credential-server --no-pager --lines=5"
echo ""
echo "=== Testing HTTP ==="
curl -sf --connect-timeout 5 http://${VM_IP}:8444/ && echo "" || echo "(curl root)"
curl -sf --connect-timeout 5 http://${VM_IP}:8444/health && echo "" || echo "(curl health)"
curl -sf --connect-timeout 5 http://${VM_IP}:8444/status && echo "" || echo "(curl status)"
curl -sf --connect-timeout 5 http://${VM_IP}:8444/openapi.json | python3 -m json.tool > /dev/null 2>&1 && echo "openapi.json: valid" || echo "openapi.json: FAIL"
echo "=== Deploy complete ==="
echo "API: http://${VM_IP}:8444/"
echo "Docs: http://${VM_IP}:8444/openapi.json"
echo "Credentials: http://${VM_IP}:8444/credentials"

2452
4-Infrastructure/rds_probe/Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,23 @@
[package]
name = "rds_probe"
version = "0.1.0"
edition = "2021"
description = "RDS ENE database inspection tool"
license = "MIT"
authors = ["Research Stack"]
[dependencies]
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "uuid"] }
tokio = { version = "1", features = ["full"] }
dotenv = "0.15"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
clap = { version = "4", features = ["derive"] }
chrono = { version = "0.4", features = ["serde"] }
thiserror = "1"
sha2 = "0.10"
hex = "0.4"
[profile.release]
opt-level = 3
lto = true

View file

@ -0,0 +1,183 @@
use crate::db::get_pool;
use crate::models::*;
use clap::Subcommand;
use serde_json::json;
#[derive(Subcommand, Debug)]
pub enum Command {
Tables,
Schema { table: String },
Count,
Sample { domain: String },
Export,
}
pub async fn run_command(command: Command) -> Result<String, Box<dyn std::error::Error>> {
let pool = get_pool().await?;
match command {
Command::Tables => tables_cmd(&pool).await,
Command::Schema { table } => schema_cmd(&pool, &table).await,
Command::Count => count_cmd(&pool).await,
Command::Sample { domain } => sample_cmd(&pool, &domain).await,
Command::Export => export_cmd(&pool).await,
}
}
async fn tables_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
let rows = sqlx::query_as::<_, (String, String)>(
"SELECT table_name, table_schema
FROM information_schema.tables
WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
ORDER BY table_schema, table_name"
)
.fetch_all(pool)
.await?;
let tables: Vec<TableInfo> = rows.into_iter()
.map(|(table_name, table_schema)| TableInfo { table_name, table_schema })
.collect();
let output = json!({
"success": true,
"command": "tables",
"count": tables.len(),
"tables": tables
});
Ok(serde_json::to_string_pretty(&output)?)
}
async fn schema_cmd(pool: &sqlx::PgPool, table: &str) -> Result<String, Box<dyn std::error::Error>> {
let rows = if let Some(pos) = table.find('.') {
let schema = &table[..pos];
let name = &table[pos + 1..];
sqlx::query_as::<_, (String, String, String)>(
"SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_schema = $1 AND table_name = $2
ORDER BY ordinal_position"
)
.bind(schema)
.bind(name)
.fetch_all(pool)
.await?
} else {
sqlx::query_as::<_, (String, String, String)>(
"SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = $1 AND table_schema NOT IN ('pg_catalog', 'information_schema')
ORDER BY ordinal_position"
)
.bind(table)
.fetch_all(pool)
.await?
};
let columns: Vec<ColumnInfo> = rows.into_iter()
.map(|(column_name, data_type, is_nullable)| ColumnInfo {
column_name,
data_type,
is_nullable,
})
.collect();
let output = json!({
"success": true,
"command": "schema",
"table": table,
"columns": columns
});
Ok(serde_json::to_string_pretty(&output)?)
}
async fn count_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
let rows = sqlx::query_as::<_, (String, String, i64)>(
"SELECT domain, archetype, COUNT(*) as count
FROM ene.packages
GROUP BY domain, archetype
ORDER BY domain, archetype"
)
.fetch_all(pool)
.await?;
let mut by_domain = serde_json::Map::new();
for (domain, archetype, count) in rows {
let entry = by_domain.entry(domain).or_insert_with(|| json!({}));
entry.as_object_mut().unwrap().insert(archetype, json!(count));
}
let total: i64 = by_domain.values()
.flat_map(|v| v.as_object().unwrap().values())
.map(|v| v.as_i64().unwrap_or(0))
.sum();
let output = json!({
"success": true,
"command": "count",
"total_records": total,
"by_domain": by_domain
});
Ok(serde_json::to_string_pretty(&output)?)
}
async fn sample_cmd(pool: &sqlx::PgPool, domain: &str) -> Result<String, Box<dyn std::error::Error>> {
let rows = sqlx::query_as::<_, (
String, Option<String>, Option<String>, Option<String>, Option<String>,
Option<String>, Option<serde_json::Value>, Option<String>, Option<String>
)>(
"SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
FROM ene.packages
WHERE domain = $1
LIMIT 5"
)
.bind(domain.to_uppercase())
.fetch_all(pool)
.await?;
let records: Vec<PackageRecord> = rows.into_iter()
.map(|(pkg, version, domain, tier, archetype, description, tags, source, indexed_utc)| {
PackageRecord { pkg, version, domain, tier, archetype, description, tags, source, indexed_utc }
})
.collect();
let output = json!({
"success": true,
"command": "sample",
"domain": domain.to_uppercase(),
"count": records.len(),
"records": records
});
Ok(serde_json::to_string_pretty(&output)?)
}
async fn export_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
let rows = sqlx::query_as::<_, (
String, Option<String>, Option<String>, Option<String>, Option<String>,
Option<String>, Option<serde_json::Value>, Option<String>, Option<String>
)>(
"SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
FROM ene.packages
ORDER BY domain, indexed_utc DESC"
)
.fetch_all(pool)
.await?;
let records: Vec<PackageRecord> = rows.into_iter()
.map(|(pkg, version, domain, tier, archetype, description, tags, source, indexed_utc)| {
PackageRecord { pkg, version, domain, tier, archetype, description, tags, source, indexed_utc }
})
.collect();
let output = json!({
"success": true,
"command": "export",
"total_records": records.len(),
"records": records
});
Ok(serde_json::to_string_pretty(&output)?)
}

View file

@ -0,0 +1,102 @@
use sqlx::{postgres::PgPoolOptions, PgPool};
use std::env;
use std::process::Command;
pub struct DbConfig {
pub host: String,
pub port: u16,
pub user: String,
pub password: String,
pub dbname: String,
}
fn get_iam_token(host: &str, user: &str, region: &str) -> Option<String> {
let output = Command::new("aws")
.args(&[
"rds", "generate-db-auth-token",
"--hostname", host,
"--port", "5432",
"--username", user,
"--region", region,
])
.output()
.ok()?;
if output.status.success() {
let token = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !token.is_empty() {
return Some(token);
}
}
None
}
impl DbConfig {
pub fn from_env() -> Self {
let host = env::var("RDS_HOST")
.unwrap_or("database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com".to_string());
let user = env::var("RDS_USER").unwrap_or("postgres".to_string());
let region = env::var("AWS_DEFAULT_REGION").unwrap_or("us-east-1".to_string());
let password = if let Ok(pwd) = env::var("RDS_PASSWORD") {
if !pwd.is_empty() {
pwd
} else {
get_iam_token(&host, &user, &region).unwrap_or_default()
}
} else {
get_iam_token(&host, &user, &region).unwrap_or_default()
};
Self {
host,
port: env::var("RDS_PORT")
.unwrap_or("5432".to_string())
.parse()
.unwrap_or(5432),
user,
password,
dbname: env::var("RDS_DB").unwrap_or("postgres".to_string()),
}
}
pub fn dsn(&self) -> String {
format!(
"postgres://{}:{}@{}:{}/{}?sslmode=require",
self.user,
urlencoding(&self.password),
self.host,
self.port,
self.dbname
)
}
}
fn urlencoding(s: &str) -> String {
let mut encoded = String::new();
for c in s.chars() {
match c {
'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => {
encoded.push(c);
}
_ => {
encoded.push('%');
encoded.push_str(&format!("{:02X}", c as u8));
}
}
}
encoded
}
pub async fn create_pool(config: &DbConfig) -> Result<PgPool, sqlx::Error> {
PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&config.dsn()).await
}
pub async fn get_pool() -> Result<PgPool, Box<dyn std::error::Error>> {
let config = DbConfig::from_env();
let pool = create_pool(&config).await?;
Ok(pool)
}

View file

@ -0,0 +1,29 @@
mod commands;
mod db;
mod models;
use clap::Parser;
#[derive(Parser, Debug)]
#[command(name = "rds_probe")]
#[command(about = "RDS ENE database inspection tool", long_about = None)]
struct Cli {
#[command(subcommand)]
command: commands::Command,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
dotenv::dotenv().ok();
match commands::run_command(Cli::parse().command).await {
Ok(output) => {
println!("{}", output);
Ok(())
}
Err(e) => {
eprintln!("Error: {}", e);
Err(e.into())
}
}
}

View file

@ -0,0 +1,40 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize)]
pub struct TableInfo {
pub table_name: String,
pub table_schema: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ColumnInfo {
pub column_name: String,
pub data_type: String,
pub is_nullable: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct CountResult {
pub domain: String,
pub archetype: String,
pub count: i64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct PackageRecord {
pub pkg: String,
pub version: Option<String>,
pub domain: Option<String>,
pub tier: Option<String>,
pub archetype: Option<String>,
pub description: Option<String>,
pub tags: Option<serde_json::Value>,
pub source: Option<String>,
pub indexed_utc: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct CommandOutput {
pub success: bool,
pub data: serde_json::Value,
}

View file

@ -9,12 +9,13 @@ Output: JSON + Markdown reference table
"""
import json
import os
import time
import requests
from pathlib import Path
from typing import Dict, List, Tuple
WOLFRAM_APP_ID = "HYJE3R3R63"
WOLFRAM_APP_ID = os.environ.get("WOLFRAM_ALPHA_APPID", "")
WOLFRAM_API_URL = "https://api.wolframalpha.com/v2/query"
# Canonical math function categories and test queries

View file

@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""export_linear_from_rds.py - Exports LINEAR domain records from RDS to JSON.
"""
import json
import os
import sys
from pathlib import Path
# Add infra and scripts folders to import path
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure" / "infra"))
sys.path.append(str(Path(__file__).resolve().parent))
from import_dumps_to_rds import get_rds_password
def main():
print("[+] Fetching RDS credentials...")
password = get_rds_password()
if not password:
print("[-] Failed to retrieve RDS password or IAM token. Exiting.")
sys.exit(1)
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
port = int(os.environ.get("RDS_PORT", "5432"))
user = os.environ.get("RDS_USER", "postgres")
dbname = os.environ.get("RDS_DB", "postgres")
try:
import psycopg2
conn = psycopg2.connect(
host=host,
port=port,
user=user,
password=password,
dbname=dbname,
sslmode="require"
)
cur = conn.cursor()
except Exception as e:
print(f"[-] Database connection failed: {e}")
sys.exit(1)
print("[+] Querying all LINEAR domain packages from RDS...")
try:
cur.execute("""
SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
FROM ene.packages
WHERE domain = 'LINEAR'
ORDER BY indexed_utc DESC
""")
rows = cur.fetchall()
except Exception as e:
print(f"[-] Failed to execute query: {e}")
sys.exit(1)
records = []
for r in rows:
records.append({
"pkg": r[0],
"version": r[1],
"domain": r[2],
"tier": r[3],
"archetype": r[4],
"description": r[5],
"tags": r[6],
"source": r[7],
"indexed_utc": r[8]
})
out_file = Path(__file__).resolve().parent / "linear_export.json"
try:
with open(out_file, "w", encoding="utf-8") as f:
json.dump(records, f, indent=2, default=str)
print(f"[+] Successfully exported {len(records)} Linear records to {out_file}")
except Exception as e:
print(f"[-] Failed to write JSON output: {e}")
if __name__ == "__main__":
main()

View file

@ -9,6 +9,7 @@ Verifies the 9 FixedPoint.lean theorems using Wolfram Alpha API:
import sys
import json
import os
import urllib.parse
import urllib.request
from pathlib import Path
@ -92,7 +93,7 @@ def main():
print("Verifying 9 FixedPoint.lean theorems")
print("=" * 70)
app_id = "HYJE3R3R63"
app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
verifier = WolframAlphaVerifier(app_id)
# FixedPoint.lean theorems to verify

View file

@ -22,6 +22,7 @@ FixedPoint.lean.
"""
import json
import os
import time
import urllib.parse
import urllib.request
@ -106,7 +107,7 @@ def main():
print("FUNDAMENTAL MATH VERIFICATION — DRIFT-PREVENTION ANCHOR NET")
print("=" * 70)
app_id = "HYJE3R3R63"
app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
verifier = WolframAlphaVerifier(app_id)
# (equation, description, expected_substring)

View file

@ -0,0 +1,189 @@
#!/usr/bin/env python3
"""import_dumps_to_rds.py - Ingestion pipeline from crawled JSON dumps to RDS.
Imports Notion database pages into ene.wiki_pages/revisions/links/categories
and Linear issues into ene.packages.
"""
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
# Add infra folder to import path
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure" / "infra"))
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure"))
def get_rds_password():
password = os.environ.get("RDS_IAM_TOKEN") or os.environ.get("RDS_PASSWORD")
if password:
return password
# Fallback to subprocess call to aws cli
try:
import subprocess
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
user = os.environ.get("RDS_USER", "postgres")
region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
cmd = [
"aws", "rds", "generate-db-auth-token",
"--hostname", host,
"--port", "5432",
"--username", user,
"--region", region
]
res = subprocess.run(cmd, capture_output=True, text=True, check=True)
token = res.stdout.strip()
if token:
return token
except Exception as e:
print(f"AWS CLI token generation failed: {e}")
return ""
def import_notion(password: str):
notion_file = Path(__file__).resolve().parent / "notion_full_dump.json"
if not notion_file.exists():
print("[-] notion_full_dump.json not found, skipping Notion import.")
return
print("[+] Loading notion_full_dump.json...")
with open(notion_file, "r") as f:
dump_data = json.load(f)
pages = dump_data.get("pages", [])
print(f"[+] Found {len(pages)} pages to import.")
# Override password environment variable for ENEWikiLayer
os.environ["RDS_PASSWORD"] = password
try:
from ene_rds_wiki_layer import ENERDSWikiLayer
wiki = ENERDSWikiLayer()
except Exception as e:
print(f"[-] Failed to initialize ENERDSWikiLayer: {e}")
return
imported_count = 0
for page in pages:
properties = page.get("properties", {})
title = "Untitled"
for prop_name in ["Name", "title", "Title"]:
prop = properties.get(prop_name, {})
if prop and prop.get("title"):
title_list = prop["title"]
if title_list and isinstance(title_list, list):
title = title_list[0].get("plain_text", "Untitled")
break
content = page.get("_content", "") or ""
author = "notion_importer"
try:
print(f" -> Importing page: {title}")
wiki.put_page(title=title, text=content, author=author, summary="Notion Full Crawl Ingestion")
imported_count += 1
except Exception as e:
print(f" [!] Error importing page '{title}': {e}")
print(f"[+] Notion import complete. Successfully imported {imported_count}/{len(pages)} pages.")
def import_linear(password: str):
linear_file = Path(__file__).resolve().parent / "linear_full_dump.json"
if not linear_file.exists():
print("[-] linear_full_dump.json not found, skipping Linear import.")
return
print("[+] Loading linear_full_dump.json...")
with open(linear_file, "r") as f:
dump_data = json.load(f)
issues = dump_data.get("issues", [])
print(f"[+] Found {len(issues)} issues to import.")
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
port = int(os.environ.get("RDS_PORT", "5432"))
user = os.environ.get("RDS_USER", "postgres")
dbname = os.environ.get("RDS_DB", "postgres")
try:
import psycopg2
conn = psycopg2.connect(
host=host,
port=port,
user=user,
password=password,
dbname=dbname,
sslmode="require"
)
cur = conn.cursor()
except Exception as e:
print(f"[-] Database connection failed: {e}")
return
imported_count = 0
for issue in issues:
identifier = issue.get("identifier")
if not identifier:
continue
pkg_id = f"linear/{identifier}"
title = issue.get("title", "")
description = issue.get("description", "")
full_text = f"{title}\n\n{description}" if description else title
url = issue.get("url", "")
# Extract labels
labels_nodes = issue.get("labels", {}).get("nodes", [])
labels_list = [l.get("name") for l in labels_nodes if l.get("name")]
tags_json = json.dumps(labels_list)
now_iso = datetime.now(timezone.utc).isoformat()
try:
cur.execute("""
INSERT INTO ene.packages (
pkg, version, domain, tier, archetype,
tags, description, source, indexed_utc
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (pkg) DO UPDATE SET
description = EXCLUDED.description,
tags = EXCLUDED.tags,
source = EXCLUDED.source,
indexed_utc = EXCLUDED.indexed_utc
""", (
pkg_id,
"1.0.0",
"LINEAR",
"INTENT",
"issue",
tags_json,
full_text,
url,
now_iso
))
imported_count += 1
except Exception as e:
print(f" [!] Error importing issue '{pkg_id}': {e}")
try:
conn.commit()
conn.close()
print(f"[+] Linear import complete. Successfully imported {imported_count}/{len(issues)} issues.")
except Exception as e:
print(f"[-] Database commit failed: {e}")
def main():
print("=== ENE RDS Ingestion Pipeline ===")
password = get_rds_password()
if not password:
print("[-] Failed to retrieve RDS password or IAM token. Exiting.")
sys.exit(1)
import_notion(password)
import_linear(password)
print("=== Pipeline Execution Finished ===")
if __name__ == "__main__":
main()

View file

@ -3,6 +3,7 @@
Store Wolfram Alpha credential in ENE
"""
import os
import sys
from pathlib import Path
@ -33,7 +34,7 @@ def main():
sys.exit(1)
# Store Wolfram Alpha credential
wolfram_app_id = "HYJE3R3R63"
wolfram_app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
print(f"\nStoring Wolfram Alpha App ID: {wolfram_app_id}")

View file

@ -11,7 +11,7 @@ from infra.knowledge_ingestion import KnowledgeIngestion
def test_wolfram_alpha():
"""Test Wolfram Alpha API integration"""
api_key = "HYJE3R3R63"
api_key = os.environ.get("WOLFRAM_ALPHA_APPID", "")
print("Testing Wolfram Alpha API integration...")
ingestion = KnowledgeIngestion(api_key)