mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
feat: add RDS probe tool, credential server, and Notion/Linear ingestion pipeline
New infrastructure components: - rds_probe: Rust database inspection tool with IAM auth - credential_server.py: REST credential provider server - credential_provider.py: credential resolution chain - ene_rds_fractal_fold.py / ene_rds_wiki_layer.py: RDS-backed ENE layers - import_dumps_to_rds.py / export_linear_from_rds.py: ingestion pipeline - recover_credential_server.sh: deployment script (sanitized) Sanitize hardcoded secrets across codebase: - Strip API keys from recover_credential_server.sh → env var lookups - Replace hardcoded Wolfram appid (HYJE3R3R63) → env var in 5 scripts - Strip fallback key values from config/index.js - Add .claude/ and optimized_basis_v3.bin to .gitignore Ingested 2,685 records into RDS: 2,421 Linear issues + 264 wiki pages
This commit is contained in:
parent
10d4c65ee7
commit
c5b6a31a8b
19 changed files with 4858 additions and 5 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -1,4 +1,8 @@
|
|||
.env
|
||||
.claude/
|
||||
optimized_basis_v3.bin
|
||||
4-Infrastructure/deploy/
|
||||
|
||||
**/node_modules/
|
||||
**/__pycache__/
|
||||
*.db
|
||||
|
|
|
|||
305
4-Infrastructure/infra/credential_provider.py
Normal file
305
4-Infrastructure/infra/credential_provider.py
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
CREDENTIAL_PROVIDER_VERSION = "0.4"
|
||||
|
||||
# Remote credential server (microVM) — all nodes query this as primary
|
||||
_cred_server_url = os.environ.get("RS_CREDENTIAL_SERVER", "http://100.101.247.127:8444")
|
||||
|
||||
# Config file path — local fallback if remote is unreachable
|
||||
_cred_config_path = os.environ.get(
|
||||
"RS_CREDENTIAL_CONFIG",
|
||||
"/etc/rs-surface/credentials.json",
|
||||
)
|
||||
|
||||
RDS_HOST = os.environ.get(
|
||||
"RDS_HOST",
|
||||
"database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com",
|
||||
)
|
||||
RDS_PORT = int(os.environ.get("RDS_PORT", "5432"))
|
||||
RDS_USER = os.environ.get("RDS_USER", "postgres")
|
||||
RDS_IAM_TOKEN = os.environ.get("RDS_IAM_TOKEN", "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Credential:
|
||||
provider: str
|
||||
key_name: str
|
||||
value: str
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
PROVIDER_ENV_MAP: dict[str, dict[str, str]] = {
|
||||
"deepseek": {
|
||||
"env_var": "DEEPSEEK_API_KEY",
|
||||
"description": "DeepSeek LLM API",
|
||||
},
|
||||
"quandela": {
|
||||
"env_var": "QUANDELA_API_KEY",
|
||||
"description": "Quandela quantum cloud",
|
||||
},
|
||||
"wolfram_alpha": {
|
||||
"env_var": "WOLFRAM_ALPHA_APPID",
|
||||
"description": "Wolfram Alpha API",
|
||||
},
|
||||
"notion": {
|
||||
"env_var": "NOTION_API_KEY",
|
||||
"description": "Notion API",
|
||||
},
|
||||
"linear": {
|
||||
"env_var": "LINEAR_API_KEY",
|
||||
"description": "Linear API",
|
||||
},
|
||||
"gemini": {
|
||||
"env_var": "GEMINI_API_KEY",
|
||||
"description": "Google Gemini API",
|
||||
},
|
||||
"ollama": {
|
||||
"env_var": "OLLAMA_API_KEY",
|
||||
"description": "Ollama local API",
|
||||
},
|
||||
"brave_search": {
|
||||
"env_var": "BRAVE_API_KEY",
|
||||
"description": "Brave Search API",
|
||||
},
|
||||
"neural_endeavor": {
|
||||
"env_var": "ENE_ENCRYPTION_KEY",
|
||||
"description": "ENE encryption master key",
|
||||
},
|
||||
"bedrock": {
|
||||
"env_var": "AWS_BEARER_TOKEN_BEDROCK",
|
||||
"description": "Amazon Bedrock API",
|
||||
},
|
||||
"venice": {
|
||||
"env_var": "VENICE_API_KEY",
|
||||
"description": "Venice AI API",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _rds_connect():
|
||||
import psycopg2
|
||||
import os
|
||||
|
||||
# Primary check: RDS_IAM_TOKEN env var
|
||||
password = RDS_IAM_TOKEN
|
||||
|
||||
if not password:
|
||||
# Generate dynamic IAM authentication token using boto3
|
||||
try:
|
||||
import boto3
|
||||
region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
|
||||
client = boto3.client('rds', region_name=region)
|
||||
password = client.generate_db_auth_token(
|
||||
DBHostname=RDS_HOST,
|
||||
Port=RDS_PORT,
|
||||
DBUsername=RDS_USER,
|
||||
Region=region
|
||||
)
|
||||
except Exception as e:
|
||||
# Fallback to standard environment password or log warning
|
||||
pass
|
||||
|
||||
if not password:
|
||||
password = os.environ.get("RDS_PASSWORD", "")
|
||||
|
||||
return psycopg2.connect(
|
||||
host=RDS_HOST,
|
||||
port=RDS_PORT,
|
||||
user=RDS_USER,
|
||||
password=password,
|
||||
dbname="postgres",
|
||||
connect_timeout=10,
|
||||
sslmode="require",
|
||||
)
|
||||
|
||||
|
||||
def _load_from_rds() -> list[Credential]:
|
||||
try:
|
||||
conn = _rds_connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT pkg, provider, encode(encrypted_payload, 'escape'), classification
|
||||
FROM credential_store.credentials
|
||||
WHERE is_active = TRUE
|
||||
ORDER BY provider
|
||||
""")
|
||||
creds = []
|
||||
for pkg, provider, payload_raw, classification in cur.fetchall():
|
||||
creds.append(Credential(
|
||||
provider=provider,
|
||||
key_name=pkg,
|
||||
value=payload_raw,
|
||||
metadata={
|
||||
"source": "rds",
|
||||
"pkg": pkg,
|
||||
"classification": classification,
|
||||
},
|
||||
))
|
||||
conn.close()
|
||||
return creds
|
||||
except Exception as e:
|
||||
return []
|
||||
|
||||
|
||||
def _load_from_remote() -> list[Credential]:
|
||||
"""Fetch credentials from the remote credential server (microVM)."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
# Skip self-query: don't try to connect to ourselves
|
||||
self_hosts = {"localhost", "127.0.0.1", "0.0.0.0", "", socket.gethostname()}
|
||||
netloc = urllib.parse.urlparse(_cred_server_url).hostname or ""
|
||||
if netloc in self_hosts:
|
||||
return []
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(_cred_server_url + "/credentials")
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
manifest = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
||||
return []
|
||||
|
||||
providers = manifest.get("providers", [])
|
||||
creds: list[Credential] = []
|
||||
for p in providers:
|
||||
name = p.get("name")
|
||||
if not name:
|
||||
continue
|
||||
try:
|
||||
req = urllib.request.Request(f"{_cred_server_url}/credentials/{name}")
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
||||
continue
|
||||
if data.get("ok"):
|
||||
creds.append(Credential(
|
||||
provider=name,
|
||||
key_name=data.get("key_name", f"remote/{name}"),
|
||||
value=data["key"],
|
||||
metadata={"source": "remote", "server": _cred_server_url},
|
||||
))
|
||||
return creds
|
||||
|
||||
|
||||
def _load_from_config() -> list[Credential]:
|
||||
"""Read credentials from a JSON config file.
|
||||
|
||||
Format:
|
||||
{
|
||||
"deepseek": "sk-...",
|
||||
"bedrock": "ABSK...",
|
||||
"ollama": "..."
|
||||
}
|
||||
"""
|
||||
path = Path(_cred_config_path)
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
creds: list[Credential] = []
|
||||
for provider_name, value in data.items():
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
continue
|
||||
cfg = PROVIDER_ENV_MAP.get(provider_name, {})
|
||||
creds.append(Credential(
|
||||
provider=provider_name,
|
||||
key_name=f"config/{provider_name}",
|
||||
value=value.strip(),
|
||||
metadata={
|
||||
"description": cfg.get("description", provider_name),
|
||||
"source": "config_file",
|
||||
"file": str(path),
|
||||
},
|
||||
))
|
||||
return creds
|
||||
|
||||
|
||||
def _load_from_env() -> list[Credential]:
|
||||
creds: list[Credential] = []
|
||||
for provider, cfg in PROVIDER_ENV_MAP.items():
|
||||
value = os.getenv(cfg["env_var"])
|
||||
if not value:
|
||||
continue
|
||||
creds.append(Credential(
|
||||
provider=provider,
|
||||
key_name=cfg["env_var"],
|
||||
value=value,
|
||||
metadata={"description": cfg["description"], "source": "env"},
|
||||
))
|
||||
return creds
|
||||
|
||||
|
||||
def load_credentials() -> list[Credential]:
|
||||
creds = _load_from_rds()
|
||||
if creds:
|
||||
return creds
|
||||
creds = _load_from_remote()
|
||||
if creds:
|
||||
return creds
|
||||
creds = _load_from_config()
|
||||
if creds:
|
||||
return creds
|
||||
return _load_from_env()
|
||||
|
||||
|
||||
def _detect_backend() -> str:
|
||||
if _load_from_rds():
|
||||
return "rds"
|
||||
if _load_from_remote():
|
||||
return "remote"
|
||||
if _load_from_config():
|
||||
return "config_file"
|
||||
return "env"
|
||||
|
||||
|
||||
def credential_status() -> dict[str, Any]:
|
||||
creds = load_credentials()
|
||||
return {
|
||||
"ok": True,
|
||||
"node": os.environ.get("RS_SURFACE_NODE_ID", "unknown"),
|
||||
"provider_version": CREDENTIAL_PROVIDER_VERSION,
|
||||
"backend": _detect_backend(),
|
||||
"available_providers": [c.provider for c in creds],
|
||||
"count": len(creds),
|
||||
"remote_server": _cred_server_url,
|
||||
}
|
||||
|
||||
|
||||
def resolve_credential(provider: str) -> Credential | None:
|
||||
creds = load_credentials()
|
||||
for c in creds:
|
||||
if c.provider == provider:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def provider_manifest() -> dict[str, Any]:
|
||||
creds = load_credentials()
|
||||
return {
|
||||
"ok": True,
|
||||
"service_kind": "apiProvider",
|
||||
"backend": _detect_backend(),
|
||||
"providers": [
|
||||
{
|
||||
"name": c.provider,
|
||||
"description": c.metadata.get("description", ""),
|
||||
"key_name": c.key_name,
|
||||
"available": True,
|
||||
}
|
||||
for c in creds
|
||||
],
|
||||
"node_id": os.environ.get("RS_SURFACE_NODE_ID", "unknown"),
|
||||
}
|
||||
319
4-Infrastructure/infra/credential_server.py
Normal file
319
4-Infrastructure/infra/credential_server.py
Normal file
|
|
@ -0,0 +1,319 @@
|
|||
"""Credential server — REST + OpenAPI access point for API keys.
|
||||
|
||||
Reads credentials from credential_provider (env vars + config file + RDS)
|
||||
and serves them via HTTP so no script needs direct env var access.
|
||||
|
||||
Usage:
|
||||
# Standalone
|
||||
python3 credential_server.py [--port 8444] [--bind 0.0.0.0]
|
||||
|
||||
# Via existing embedded surface (if RS_CREDENTIAL_SERVER_PORT is set)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from credential_provider import (
|
||||
credential_status,
|
||||
provider_manifest,
|
||||
resolve_credential,
|
||||
_rds_connect,
|
||||
_cred_server_url
|
||||
)
|
||||
|
||||
VERSION = "0.4"
|
||||
OPENAPI_VERSION = "3.0.3"
|
||||
|
||||
OPENAPI_SPEC: dict[str, Any] = {
|
||||
"openapi": OPENAPI_VERSION,
|
||||
"info": {
|
||||
"title": "Research Stack Credential Provider",
|
||||
"description": "Central API key access point for all stack services. "
|
||||
"Set keys once on the microVM, consume from anywhere.",
|
||||
"version": VERSION,
|
||||
},
|
||||
"servers": [
|
||||
{"url": "http://localhost:8444", "description": "Local / Tailnet default"},
|
||||
],
|
||||
"paths": {
|
||||
"/": {
|
||||
"get": {
|
||||
"summary": "Service root",
|
||||
"responses": {"200": {"description": "Service info + links"}},
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"summary": "Health check",
|
||||
"responses": {"200": {"description": "OK"}},
|
||||
}
|
||||
},
|
||||
"/openapi.json": {
|
||||
"get": {
|
||||
"summary": "OpenAPI 3.0 specification",
|
||||
"responses": {"200": {"description": "This document"}},
|
||||
}
|
||||
},
|
||||
"/credentials": {
|
||||
"get": {
|
||||
"summary": "List all available credential providers",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Provider manifest",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"providers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"available": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
"/credentials/{provider}": {
|
||||
"get": {
|
||||
"summary": "Resolve a specific provider credential",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "provider",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "Provider name (deepseek, bedrock, etc.)",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {"description": "Credential value"},
|
||||
"404": {"description": "Provider not found"},
|
||||
},
|
||||
}
|
||||
},
|
||||
"/status": {
|
||||
"get": {
|
||||
"summary": "Credential provider status",
|
||||
"responses": {"200": {"description": "Backend info + counts"}},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
SERVICE_INFO: dict[str, Any] = {
|
||||
"service": "credential-provider",
|
||||
"version": VERSION,
|
||||
"docs": "/openapi.json",
|
||||
"credentials": "/credentials",
|
||||
"status": "/status",
|
||||
"health": "/health",
|
||||
}
|
||||
|
||||
|
||||
class CredentialHTTPHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
sys.stderr.write(f"[credential-server] {args[0]} {args[1]} {args[2]}\n")
|
||||
|
||||
def _send_json(self, data: Any, status: int = 200):
|
||||
body = json.dumps(data, sort_keys=True, indent=2).encode("utf-8") + b"\n"
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_error(self, status: int, message: str):
|
||||
self._send_json({"ok": False, "error": message}, status)
|
||||
|
||||
def do_GET(self):
|
||||
path = self.path.rstrip("/") or "/"
|
||||
|
||||
if path in ("/", "/api"):
|
||||
return self._send_json(SERVICE_INFO)
|
||||
|
||||
if path in ("/health", "/api/health"):
|
||||
return self._send_json({"status": "ok", "version": VERSION})
|
||||
|
||||
if path == "/openapi.json":
|
||||
return self._send_json(OPENAPI_SPEC)
|
||||
|
||||
if path in ("/status", "/api/status"):
|
||||
return self._send_json(credential_status())
|
||||
|
||||
if path == "/credentials" or path == "/api/credentials":
|
||||
return self._send_json(provider_manifest())
|
||||
|
||||
m = re.match(r"^(?:/api)?/credentials/([a-zA-Z0-9_-]+)$", path)
|
||||
if m:
|
||||
provider = m.group(1).lower()
|
||||
cred = resolve_credential(provider)
|
||||
if cred is None:
|
||||
return self._send_error(404, f"provider '{provider}' not found")
|
||||
return self._send_json({
|
||||
"ok": True,
|
||||
"provider": cred.provider,
|
||||
"key": cred.value,
|
||||
"key_name": cred.key_name,
|
||||
})
|
||||
|
||||
return self._send_error(404, f"not found: {path}")
|
||||
|
||||
def do_POST(self):
|
||||
path = self.path.rstrip("/") or "/"
|
||||
|
||||
if path in ("/api/webhooks/linear", "/webhooks/linear"):
|
||||
# Read raw body
|
||||
content_length = int(self.headers.get("Content-Length", 0))
|
||||
raw_body = self.rfile.read(content_length)
|
||||
|
||||
# Node identification logic: proxy or process
|
||||
node_id = os.environ.get("RS_SURFACE_NODE_ID", "")
|
||||
if node_id != "MicroVM-Racknerd":
|
||||
# Act as Proxy: Forward POST request to the central microVM
|
||||
target_url = f"{_cred_server_url}/api/webhooks/linear"
|
||||
req = urllib.request.Request(target_url, data=raw_body, method="POST")
|
||||
for k, v in self.headers.items():
|
||||
if k.lower() in ("content-type", "linear-signature"):
|
||||
req.add_header(k, v)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
resp_data = json.loads(resp.read().decode("utf-8"))
|
||||
return self._send_json(resp_data, resp.status)
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
err_data = json.loads(e.read().decode("utf-8"))
|
||||
return self._send_json(err_data, e.code)
|
||||
except Exception:
|
||||
return self._send_error(e.code, f"Forwarding failed: {e}")
|
||||
except Exception as e:
|
||||
return self._send_error(500, f"Forwarding connection error: {e}")
|
||||
|
||||
# Act as Server: Process Linear webhook body and upsert into Aurora RDS ENE packages table
|
||||
try:
|
||||
data = json.loads(raw_body.decode("utf-8"))
|
||||
except Exception as e:
|
||||
return self._send_error(400, f"Invalid JSON payload: {e}")
|
||||
|
||||
action = data.get("action")
|
||||
issue_data = data.get("data", {})
|
||||
payload_type = data.get("type")
|
||||
|
||||
if payload_type != "Issue":
|
||||
return self._send_json({"ok": True, "message": f"Ignored non-Issue payload: {payload_type}"})
|
||||
|
||||
identifier = issue_data.get("identifier")
|
||||
if not identifier:
|
||||
return self._send_error(400, "Missing issue identifier")
|
||||
|
||||
pkg_id = f"linear/{identifier}"
|
||||
|
||||
if action == "remove":
|
||||
try:
|
||||
conn = _rds_connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("DELETE FROM ene.packages WHERE pkg = %s", (pkg_id,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return self._send_json({"ok": True, "action": "delete", "pkg": pkg_id})
|
||||
except Exception as e:
|
||||
return self._send_error(500, f"Database delete error: {e}")
|
||||
|
||||
# Extract title and description
|
||||
title = issue_data.get("title", "")
|
||||
description = issue_data.get("description", "")
|
||||
full_text = f"{title}\n\n{description}" if description else title
|
||||
url = issue_data.get("url", "")
|
||||
|
||||
# Extract labels
|
||||
labels_data = issue_data.get("labels", [])
|
||||
labels_list = []
|
||||
if isinstance(labels_data, list):
|
||||
for l in labels_data:
|
||||
if isinstance(l, dict):
|
||||
labels_list.append(l.get("name"))
|
||||
elif isinstance(l, str):
|
||||
labels_list.append(l)
|
||||
tags_json = json.dumps(labels_list)
|
||||
|
||||
import datetime
|
||||
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
|
||||
try:
|
||||
conn = _rds_connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
INSERT INTO ene.packages (
|
||||
pkg, version, domain, tier, archetype,
|
||||
tags, description, source, indexed_utc
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (pkg) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
tags = EXCLUDED.tags,
|
||||
source = EXCLUDED.source,
|
||||
indexed_utc = EXCLUDED.indexed_utc
|
||||
""", (
|
||||
pkg_id,
|
||||
"1.0.0",
|
||||
"LINEAR",
|
||||
"INTENT",
|
||||
"issue",
|
||||
tags_json,
|
||||
full_text,
|
||||
url,
|
||||
now_iso
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return self._send_json({"ok": True, "action": action, "pkg": pkg_id})
|
||||
except Exception as e:
|
||||
return self._send_error(500, f"Database upsert error: {e}")
|
||||
|
||||
return self._send_error(404, f"not found: {path}")
|
||||
|
||||
do_HEAD = do_GET
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Research Stack Credential Server")
|
||||
parser.add_argument("--port", type=int, default=int(os.environ.get("RS_CREDENTIAL_PORT", "8444")))
|
||||
parser.add_argument("--bind", default=os.environ.get("RS_CREDENTIAL_BIND", "0.0.0.0"))
|
||||
args = parser.parse_args()
|
||||
|
||||
server = ThreadingHTTPServer((args.bind, args.port), CredentialHTTPHandler)
|
||||
print(f"[credential-server] listening on {args.bind}:{args.port}", flush=True)
|
||||
print(f"[credential-server] OpenAPI spec at http://{args.bind}:{args.port}/openapi.json", flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\n[credential-server] shutting down", flush=True)
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
592
4-Infrastructure/infra/ene_rds_fractal_fold.py
Normal file
592
4-Infrastructure/infra/ene_rds_fractal_fold.py
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
"""RDS-backed ENEFractalFold — drop-in replacement for SQLite ENEFractalFold.
|
||||
|
||||
API-compatible with ene_fractal_fold.ENEFractalFold: same dataclasses, same
|
||||
handle_request() protocol, but backed by PostgreSQL via psycopg2.
|
||||
|
||||
Constructor:
|
||||
ENERDSFractalFold(dsn="postgresql://user:pass@host:5432/dbname")
|
||||
|
||||
The DSN defaults to the RDS_HOST / RDS_PORT / RDS_USER / RDS_PASSWORD / RDS_DB
|
||||
environment variables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
|
||||
from infra.ene_fractal_fold import (
|
||||
FractalNode,
|
||||
FractalManifest,
|
||||
VERSION,
|
||||
GOLDEN_ANGLE,
|
||||
canonical_json,
|
||||
sha256_text,
|
||||
sha256_bytes,
|
||||
gray_code,
|
||||
inverse_gray_code,
|
||||
golden_spiral_point,
|
||||
manifold_distance,
|
||||
tree_depth,
|
||||
make_leaf,
|
||||
make_parent,
|
||||
encode_fractal,
|
||||
encode_fractal_chunks,
|
||||
parse_graphml_concepts,
|
||||
node_record,
|
||||
archive_record,
|
||||
jsonl_event,
|
||||
)
|
||||
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def _default_dsn() -> str:
|
||||
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
||||
port = os.environ.get("RDS_PORT", "5432")
|
||||
user = os.environ.get("RDS_USER", "postgres")
|
||||
password = os.environ.get("RDS_PASSWORD") or os.environ.get("RDS_IAM_TOKEN", "")
|
||||
dbname = os.environ.get("RDS_DB", "postgres")
|
||||
return f"host={host} port={port} dbname={dbname} user={user} password={password} sslmode=require"
|
||||
|
||||
|
||||
class ENERDSFractalFold:
|
||||
def __init__(self, dsn: str | None = None):
|
||||
self.dsn = dsn or _default_dsn()
|
||||
self._init_db()
|
||||
|
||||
def _get_conn(self):
|
||||
return psycopg2.connect(self.dsn)
|
||||
|
||||
def _init_db(self) -> None:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("CREATE SCHEMA IF NOT EXISTS ene")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.fractal_manifolds (
|
||||
root_hash TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
byte_len INTEGER NOT NULL,
|
||||
leaves_count INTEGER NOT NULL,
|
||||
depth INTEGER NOT NULL,
|
||||
chunk_size INTEGER NOT NULL,
|
||||
branching_factor INTEGER NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
receipt TEXT NOT NULL,
|
||||
archive_record JSONB NOT NULL DEFAULT '{}',
|
||||
jsonl_event JSONB NOT NULL DEFAULT '{}'
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.fractal_nodes (
|
||||
root_hash TEXT NOT NULL,
|
||||
node_hash TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
level INTEGER NOT NULL,
|
||||
ordinal INTEGER NOT NULL,
|
||||
fold_address INTEGER NOT NULL,
|
||||
start_leaf INTEGER NOT NULL,
|
||||
end_leaf INTEGER NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
children TEXT NOT NULL,
|
||||
payload_b64 TEXT,
|
||||
PRIMARY KEY (root_hash, node_hash)
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_rds_fractal_leaf
|
||||
ON ene.fractal_nodes (root_hash, level, ordinal)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.fractal_graph_entities (
|
||||
root_hash TEXT NOT NULL,
|
||||
graph_node_id TEXT NOT NULL,
|
||||
leaf_index INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
family TEXT,
|
||||
domain TEXT,
|
||||
neighbors TEXT NOT NULL,
|
||||
PRIMARY KEY (root_hash, graph_node_id)
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_rds_fractal_graph_name
|
||||
ON ene.fractal_graph_entities (root_hash, name)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
def _store(self, manifest: FractalManifest, nodes: list[FractalNode]) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
record = archive_record(manifest)
|
||||
event = jsonl_event(record, manifest)
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO ene.fractal_manifolds
|
||||
(root_hash, name, byte_len, leaves_count, depth, chunk_size, branching_factor,
|
||||
created_at, receipt, archive_record, jsonl_event)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (root_hash) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
byte_len = EXCLUDED.byte_len,
|
||||
leaves_count = EXCLUDED.leaves_count,
|
||||
depth = EXCLUDED.depth,
|
||||
chunk_size = EXCLUDED.chunk_size,
|
||||
branching_factor = EXCLUDED.branching_factor,
|
||||
receipt = EXCLUDED.receipt,
|
||||
archive_record = EXCLUDED.archive_record,
|
||||
jsonl_event = EXCLUDED.jsonl_event
|
||||
""", (
|
||||
manifest.root_hash, manifest.name, manifest.byte_len,
|
||||
manifest.leaves_count, manifest.depth, manifest.chunk_size,
|
||||
manifest.branching_factor, manifest.created_at, manifest.receipt,
|
||||
json.dumps(record, sort_keys=True), json.dumps(event, sort_keys=True),
|
||||
))
|
||||
|
||||
cur.execute("DELETE FROM ene.fractal_nodes WHERE root_hash = %s", (manifest.root_hash,))
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"""
|
||||
INSERT INTO ene.fractal_nodes
|
||||
(root_hash, node_hash, kind, level, ordinal, fold_address, start_leaf,
|
||||
end_leaf, size_bytes, children, payload_b64)
|
||||
VALUES %s
|
||||
""",
|
||||
[
|
||||
(
|
||||
manifest.root_hash, node.node_hash, node.kind, node.level,
|
||||
node.ordinal, node.fold_address, node.start_leaf, node.end_leaf,
|
||||
node.size_bytes, canonical_json(node.children), node.payload_b64,
|
||||
)
|
||||
for node in nodes
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
return record, event
|
||||
|
||||
def put(self, data: bytes, name: str = "unnamed", chunk_size: int = 4096, branching_factor: int = 4) -> dict[str, Any]:
|
||||
manifest, nodes = encode_fractal(data, name, chunk_size, branching_factor)
|
||||
record, event = self._store(manifest, nodes)
|
||||
return {
|
||||
"ok": True, "op": "fractal_put",
|
||||
"manifest": asdict(manifest),
|
||||
"archive_record": record,
|
||||
"jsonl_event": event,
|
||||
}
|
||||
|
||||
def put_graphml(self, graphml: bytes, name: str = "graphml", branching_factor: int = 4) -> dict[str, Any]:
|
||||
records, chunks = parse_graphml_concepts(graphml)
|
||||
max_chunk = max((len(chunk) for chunk in chunks), default=0)
|
||||
manifest, nodes = encode_fractal_chunks(chunks, name, max_chunk, branching_factor)
|
||||
record, event = self._store(manifest, nodes)
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM ene.fractal_graph_entities WHERE root_hash = %s", (manifest.root_hash,))
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"""
|
||||
INSERT INTO ene.fractal_graph_entities
|
||||
(root_hash, graph_node_id, leaf_index, name, family, domain, neighbors)
|
||||
VALUES %s
|
||||
""",
|
||||
[
|
||||
(
|
||||
manifest.root_hash, concept["graph_node_id"],
|
||||
concept["leaf_index"], concept["name"],
|
||||
concept["family"], concept["domain"],
|
||||
canonical_json(concept["neighbors"]),
|
||||
)
|
||||
for concept in records
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
return {
|
||||
"ok": True, "op": "fractal_graphml_put",
|
||||
"manifest": asdict(manifest),
|
||||
"graphml": {"concepts": len(records), "concept_leaf_mode": True},
|
||||
"archive_record": record,
|
||||
"jsonl_event": event,
|
||||
}
|
||||
|
||||
def manifest(self, root_hash: str) -> dict[str, Any] | None:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("SELECT * FROM ene.fractal_manifolds WHERE root_hash = %s", (root_hash,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"root_hash": row["root_hash"],
|
||||
"name": row["name"],
|
||||
"byte_len": row["byte_len"],
|
||||
"leaves_count": row["leaves_count"],
|
||||
"depth": row["depth"],
|
||||
"chunk_size": row["chunk_size"],
|
||||
"branching_factor": row["branching_factor"],
|
||||
"created_at": row["created_at"].isoformat() if hasattr(row["created_at"], "isoformat") else str(row["created_at"]),
|
||||
"receipt": row["receipt"],
|
||||
"archive_record": row["archive_record"] if isinstance(row["archive_record"], dict) else json.loads(row["archive_record"] or "{}"),
|
||||
"jsonl_event": row["jsonl_event"] if isinstance(row["jsonl_event"], dict) else json.loads(row["jsonl_event"] or "{}"),
|
||||
}
|
||||
|
||||
def _node(self, cur, root_hash: str, node_hash: str) -> dict[str, Any]:
|
||||
cur.execute(
|
||||
"SELECT * FROM ene.fractal_nodes WHERE root_hash = %s AND node_hash = %s",
|
||||
(root_hash, node_hash),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
raise KeyError(f"missing fractal node {node_hash}")
|
||||
return dict(row)
|
||||
|
||||
def proof(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
|
||||
meta = self.manifest(root_hash)
|
||||
if meta is None:
|
||||
raise KeyError(f"unknown root {root_hash}")
|
||||
if leaf_index < 0 or leaf_index >= meta["leaves_count"]:
|
||||
raise IndexError("leaf_index outside manifold")
|
||||
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
frontier = root_hash
|
||||
path = []
|
||||
target_point = golden_spiral_point(gray_code(leaf_index), 0)
|
||||
while True:
|
||||
row = self._node(cur, root_hash, frontier)
|
||||
children = json.loads(row["children"])
|
||||
node_point = golden_spiral_point(row["fold_address"], row["level"])
|
||||
entry = {k: v for k, v in dict(row).items() if k != "payload_b64"}
|
||||
entry["children"] = children
|
||||
entry["golden_spiral"] = node_point
|
||||
entry["distance_to_target"] = round(manifold_distance(node_point, target_point), 9)
|
||||
path.append(entry)
|
||||
|
||||
if row["kind"] == "leaf":
|
||||
payload = base64.b64decode(row["payload_b64"] or "")
|
||||
break
|
||||
|
||||
next_hash = None
|
||||
pruned = []
|
||||
for child_hash in children:
|
||||
child = self._node(cur, root_hash, child_hash)
|
||||
child_point = golden_spiral_point(child["fold_address"], child["level"])
|
||||
pruned.append({
|
||||
"node_hash": child_hash,
|
||||
"covers_target": child["start_leaf"] <= leaf_index <= child["end_leaf"],
|
||||
"distance": round(manifold_distance(child_point, target_point), 9),
|
||||
})
|
||||
if child["start_leaf"] <= leaf_index <= child["end_leaf"]:
|
||||
next_hash = child_hash
|
||||
path[-1]["manifold_distance_pruning"] = sorted(pruned, key=lambda item: item["distance"])
|
||||
if next_hash is None:
|
||||
raise ValueError("corrupt tree: no child covers requested leaf")
|
||||
frontier = next_hash
|
||||
|
||||
path_valid = self._verify_path_rows(path, payload)
|
||||
return {
|
||||
"ok": True, "op": "fractal_proof",
|
||||
"root_hash": root_hash, "leaf_index": leaf_index,
|
||||
"fold_address": gray_code(leaf_index),
|
||||
"inverse_fold_address": inverse_gray_code(gray_code(leaf_index)),
|
||||
"golden_spiral": golden_spiral_point(gray_code(leaf_index), 0),
|
||||
"traversal_cost": len(path),
|
||||
"expected_complexity": f"O(log_{meta['branching_factor']}(n))",
|
||||
"path_hash_verified": path_valid,
|
||||
"path": path,
|
||||
"payload_b64": base64.b64encode(payload).decode("ascii"),
|
||||
}
|
||||
|
||||
def _verify_path_rows(self, path: list[dict[str, Any]], payload: bytes) -> bool:
|
||||
if not path:
|
||||
return False
|
||||
leaf = path[-1]
|
||||
if leaf["kind"] != "leaf":
|
||||
return False
|
||||
expected_leaf = make_leaf(payload, leaf["ordinal"]).node_hash
|
||||
if expected_leaf != leaf["node_hash"]:
|
||||
return False
|
||||
child_hash = expected_leaf
|
||||
for row in reversed(path[:-1]):
|
||||
if child_hash not in row["children"]:
|
||||
return False
|
||||
child_hash = row["node_hash"]
|
||||
return child_hash == path[0]["node_hash"]
|
||||
|
||||
def verify(self, root_hash: str) -> dict[str, Any]:
|
||||
meta = self.manifest(root_hash)
|
||||
if meta is None:
|
||||
raise KeyError(f"unknown root {root_hash}")
|
||||
errors = []
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("SELECT * FROM ene.fractal_nodes WHERE root_hash = %s", (root_hash,))
|
||||
rows = cur.fetchall()
|
||||
by_hash = {r["node_hash"]: r for r in rows}
|
||||
for row in rows:
|
||||
children = json.loads(row["children"])
|
||||
if row["kind"] == "leaf":
|
||||
payload = base64.b64decode(row["payload_b64"] or "")
|
||||
expected = make_leaf(payload, row["ordinal"]).node_hash
|
||||
if expected != row["node_hash"]:
|
||||
errors.append({"node": row["node_hash"], "error": "leaf_hash_mismatch", "expected": expected})
|
||||
continue
|
||||
child_nodes = []
|
||||
for child_hash in children:
|
||||
child = by_hash.get(child_hash)
|
||||
if child is None:
|
||||
errors.append({"node": row["node_hash"], "error": "missing_child", "child": child_hash})
|
||||
continue
|
||||
child_nodes.append(FractalNode(
|
||||
node_hash=child["node_hash"], kind=child["kind"],
|
||||
level=child["level"], ordinal=child["ordinal"],
|
||||
fold_address=child["fold_address"],
|
||||
start_leaf=child["start_leaf"], end_leaf=child["end_leaf"],
|
||||
size_bytes=child["size_bytes"],
|
||||
children=json.loads(child["children"]),
|
||||
payload_b64=child["payload_b64"],
|
||||
))
|
||||
if len(child_nodes) == len(children):
|
||||
expected = make_parent(child_nodes, row["level"], row["ordinal"]).node_hash
|
||||
if expected != row["node_hash"]:
|
||||
errors.append({"node": row["node_hash"], "error": "parent_hash_mismatch", "expected": expected})
|
||||
return {
|
||||
"ok": not errors, "op": "fractal_verify",
|
||||
"root_hash": root_hash, "checked_nodes": len(rows),
|
||||
"errors": errors, "damage_detected": bool(errors),
|
||||
}
|
||||
|
||||
def neighbors(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
|
||||
meta = self.manifest(root_hash)
|
||||
if meta is None:
|
||||
raise KeyError(f"unknown root {root_hash}")
|
||||
candidates = sorted(set(idx for idx in (leaf_index - 1, leaf_index, leaf_index + 1) if 0 <= idx < meta["leaves_count"]))
|
||||
return {
|
||||
"ok": True, "op": "fractal_neighbors",
|
||||
"root_hash": root_hash, "leaf_index": leaf_index,
|
||||
"fold_address": gray_code(leaf_index),
|
||||
"neighbors": [
|
||||
{
|
||||
"leaf_index": idx, "fold_address": gray_code(idx),
|
||||
"golden_spiral": golden_spiral_point(gray_code(idx), 0),
|
||||
"fold_distance": bin(gray_code(idx) ^ gray_code(leaf_index)).count("1"),
|
||||
}
|
||||
for idx in candidates
|
||||
],
|
||||
}
|
||||
|
||||
def graph_entity(self, root_hash: str, graph_node_id: str = "", name: str = "") -> dict[str, Any]:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
if graph_node_id:
|
||||
cur.execute(
|
||||
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND graph_node_id = %s",
|
||||
(root_hash, graph_node_id),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND lower(name) = lower(%s)",
|
||||
(root_hash, name),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return {"ok": False, "op": "fractal_graph_entity", "error": "graph entity not found"}
|
||||
proof = self.navigate(root_hash, row["leaf_index"])
|
||||
return {
|
||||
"ok": True, "op": "fractal_graph_entity",
|
||||
"root_hash": root_hash,
|
||||
"entity": {
|
||||
"graph_node_id": row["graph_node_id"],
|
||||
"leaf_index": row["leaf_index"],
|
||||
"name": row["name"],
|
||||
"family": row["family"],
|
||||
"domain": row["domain"],
|
||||
"neighbors": json.loads(row["neighbors"]),
|
||||
},
|
||||
"retrieval": proof,
|
||||
}
|
||||
|
||||
def graph_neighbors(self, root_hash: str, graph_node_id: str) -> dict[str, Any]:
|
||||
entity = self.graph_entity(root_hash, graph_node_id=graph_node_id)
|
||||
if not entity.get("ok"):
|
||||
return entity
|
||||
neighbor_ids = entity["entity"]["neighbors"]
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
rows = []
|
||||
for nid in neighbor_ids:
|
||||
cur.execute(
|
||||
"SELECT * FROM ene.fractal_graph_entities WHERE root_hash = %s AND graph_node_id = %s",
|
||||
(root_hash, nid),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
return {
|
||||
"ok": True, "op": "fractal_graph_neighbors",
|
||||
"root_hash": root_hash, "graph_node_id": graph_node_id,
|
||||
"neighbors": [
|
||||
{
|
||||
"graph_node_id": r["graph_node_id"],
|
||||
"leaf_index": r["leaf_index"],
|
||||
"name": r["name"],
|
||||
"family": r["family"],
|
||||
"domain": r["domain"],
|
||||
"fold_address": gray_code(r["leaf_index"]),
|
||||
"golden_spiral": golden_spiral_point(gray_code(r["leaf_index"]), 0),
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
def navigate(self, root_hash: str, leaf_index: int) -> dict[str, Any]:
|
||||
meta = self.manifest(root_hash)
|
||||
if meta is None:
|
||||
raise KeyError(f"unknown root {root_hash}")
|
||||
if leaf_index < 0 or leaf_index >= meta["leaves_count"]:
|
||||
raise IndexError("leaf_index outside manifold")
|
||||
|
||||
target_point = golden_spiral_point(gray_code(leaf_index), 0)
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
frontier = root_hash
|
||||
path = []
|
||||
while True:
|
||||
row = self._node(cur, root_hash, frontier)
|
||||
children = json.loads(row["children"])
|
||||
entry = {
|
||||
"node_hash": row["node_hash"], "kind": row["kind"],
|
||||
"level": row["level"], "ordinal": row["ordinal"],
|
||||
"start_leaf": row["start_leaf"], "end_leaf": row["end_leaf"],
|
||||
"fold_address": row["fold_address"],
|
||||
"children": children,
|
||||
"golden_spiral": golden_spiral_point(row["fold_address"], row["level"]),
|
||||
}
|
||||
path.append(entry)
|
||||
|
||||
if row["kind"] == "leaf":
|
||||
payload = base64.b64decode(row["payload_b64"] or "")
|
||||
break
|
||||
|
||||
ranked = []
|
||||
for child_hash in children:
|
||||
child = self._node(cur, root_hash, child_hash)
|
||||
point = golden_spiral_point(child["fold_address"], child["level"])
|
||||
ranked.append({
|
||||
"node_hash": child_hash,
|
||||
"distance": manifold_distance(point, target_point),
|
||||
"covers_target": child["start_leaf"] <= leaf_index <= child["end_leaf"],
|
||||
})
|
||||
ranked.sort(key=lambda item: (not item["covers_target"], item["distance"]))
|
||||
path[-1]["pruned_candidates"] = [
|
||||
{"node_hash": item["node_hash"], "distance": round(item["distance"], 9), "covers_target": item["covers_target"]}
|
||||
for item in ranked
|
||||
]
|
||||
frontier = ranked[0]["node_hash"]
|
||||
|
||||
return {
|
||||
"ok": True, "op": "fractal_navigate",
|
||||
"root_hash": root_hash, "leaf_index": leaf_index,
|
||||
"target": {"fold_address": gray_code(leaf_index), "golden_spiral": target_point},
|
||||
"retrieval_complexity": f"O(log_{meta['branching_factor']}(n))",
|
||||
"navigation": "golden_spiral_manifold_distance_pruning",
|
||||
"path_hash_verified": self._verify_path_rows(path, payload),
|
||||
"path": path,
|
||||
"payload_b64": base64.b64encode(payload).decode("ascii"),
|
||||
}
|
||||
|
||||
def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||
op = str(request.get("op", "manifest"))
|
||||
if op in {"put", "encode"}:
|
||||
if "data_b64" in request:
|
||||
data = base64.b64decode(str(request["data_b64"]))
|
||||
else:
|
||||
data = str(request.get("text", "")).encode("utf-8")
|
||||
return self.put(
|
||||
data=data,
|
||||
name=str(request.get("name", "unnamed")),
|
||||
chunk_size=int(request.get("chunk_size", 4096)),
|
||||
branching_factor=int(request.get("branching_factor", 4)),
|
||||
)
|
||||
if op in {"put_graphml", "graphml"}:
|
||||
if "data_b64" in request:
|
||||
data = base64.b64decode(str(request["data_b64"]))
|
||||
else:
|
||||
data = str(request.get("text", "")).encode("utf-8")
|
||||
return self.put_graphml(
|
||||
graphml=data,
|
||||
name=str(request.get("name", "graphml")),
|
||||
branching_factor=int(request.get("branching_factor", 4)),
|
||||
)
|
||||
root_hash = str(request.get("root_hash", ""))
|
||||
if not root_hash:
|
||||
raise ValueError("root_hash is required for this operation")
|
||||
if op == "manifest":
|
||||
meta = self.manifest(root_hash)
|
||||
return {"ok": meta is not None, "op": "fractal_manifest", "manifest": meta}
|
||||
if op == "proof":
|
||||
return self.proof(root_hash, int(request.get("leaf_index", 0)))
|
||||
if op in {"navigate", "get"}:
|
||||
return self.navigate(root_hash, int(request.get("leaf_index", 0)))
|
||||
if op == "verify":
|
||||
return self.verify(root_hash)
|
||||
if op == "neighbors":
|
||||
return self.neighbors(root_hash, int(request.get("leaf_index", 0)))
|
||||
if op in {"graph_entity", "concept"}:
|
||||
return self.graph_entity(
|
||||
root_hash,
|
||||
graph_node_id=str(request.get("graph_node_id", "")),
|
||||
name=str(request.get("name", "")),
|
||||
)
|
||||
if op in {"graph_neighbors", "concept_neighbors"}:
|
||||
return self.graph_neighbors(root_hash, str(request.get("graph_node_id", "")))
|
||||
raise ValueError(f"unsupported fractal op {op!r}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="ENE RDS fractal fold codec")
|
||||
parser.add_argument("--dsn", help="PostgreSQL DSN")
|
||||
parser.add_argument("--op", default="put")
|
||||
parser.add_argument("--name", default="cli")
|
||||
parser.add_argument("--text", default="")
|
||||
parser.add_argument("--file", type=argparse.FileType("rb"))
|
||||
parser.add_argument("--graph-node-id")
|
||||
parser.add_argument("--root-hash")
|
||||
parser.add_argument("--leaf-index", type=int, default=0)
|
||||
parser.add_argument("--chunk-size", type=int, default=4096)
|
||||
parser.add_argument("--branching-factor", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
layer = ENERDSFractalFold(args.dsn)
|
||||
if args.op in {"put", "encode"}:
|
||||
data = args.file.read() if args.file else args.text.encode("utf-8")
|
||||
result = layer.put(data, args.name, args.chunk_size, args.branching_factor)
|
||||
elif args.op in {"put_graphml", "graphml"}:
|
||||
data = args.file.read() if args.file else args.text.encode("utf-8")
|
||||
result = layer.put_graphml(data, args.name, args.branching_factor)
|
||||
elif args.op in {"graph_entity", "concept", "graph_neighbors"}:
|
||||
result = layer.handle_request({
|
||||
"op": args.op, "root_hash": args.root_hash,
|
||||
"leaf_index": args.leaf_index,
|
||||
"graph_node_id": args.graph_node_id or "",
|
||||
"name": args.name,
|
||||
})
|
||||
else:
|
||||
result = layer.handle_request({
|
||||
"op": args.op, "root_hash": args.root_hash,
|
||||
"leaf_index": args.leaf_index,
|
||||
})
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
438
4-Infrastructure/infra/ene_rds_wiki_layer.py
Normal file
438
4-Infrastructure/infra/ene_rds_wiki_layer.py
Normal file
|
|
@ -0,0 +1,438 @@
|
|||
"""RDS-backed ENEWikiLayer — drop-in replacement for SQLite ENEWikiLayer.
|
||||
|
||||
API-compatible with ene_wiki_layer.ENEWikiLayer: same dataclasses, same
|
||||
handle_request() protocol, but backed by PostgreSQL via psycopg2.
|
||||
|
||||
Constructor:
|
||||
ENERDSWikiLayer(dsn="postgresql://user:pass@host:5432/dbname")
|
||||
|
||||
The DSN defaults to the RDS_HOST / RDS_PORT / RDS_USER / RDS_PASSWORD / RDS_DB
|
||||
environment variables (or postgres as dbname, research_stack as fallback).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
import psycopg2.pool
|
||||
|
||||
from infra.ene_wiki_layer import (
|
||||
WikiPage,
|
||||
WikiRevision,
|
||||
normalize_title,
|
||||
title_slug,
|
||||
extract_links,
|
||||
extract_categories,
|
||||
write_receipt,
|
||||
canonical_json,
|
||||
sha256_text,
|
||||
iso_utc,
|
||||
concept_vector_for_wiki,
|
||||
genome_from_vector,
|
||||
make_archive_record,
|
||||
make_jsonl_event,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_DSN = None
|
||||
|
||||
|
||||
def _default_dsn() -> str:
|
||||
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
||||
port = os.environ.get("RDS_PORT", "5432")
|
||||
user = os.environ.get("RDS_USER", "postgres")
|
||||
password = os.environ.get("RDS_PASSWORD") or os.environ.get("RDS_IAM_TOKEN", "")
|
||||
dbname = os.environ.get("RDS_DB", "postgres")
|
||||
return f"host={host} port={port} dbname={dbname} user={user} password={password} sslmode=require"
|
||||
|
||||
|
||||
import os
|
||||
|
||||
|
||||
class ENERDSWikiLayer:
|
||||
def __init__(self, dsn: str | None = None):
|
||||
self.dsn = dsn or _default_dsn()
|
||||
self._pool: psycopg2.pool.ThreadedConnectionPool | None = None
|
||||
self._init_tables()
|
||||
|
||||
def _get_conn(self):
|
||||
return psycopg2.connect(self.dsn)
|
||||
|
||||
def _init_tables(self) -> None:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("CREATE SCHEMA IF NOT EXISTS ene")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.wiki_pages (
|
||||
slug TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
latest_revision INTEGER NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
receipt TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.wiki_revisions (
|
||||
slug TEXT NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
author TEXT NOT NULL DEFAULT 'ene',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
receipt TEXT NOT NULL,
|
||||
archive_id TEXT,
|
||||
content_hash TEXT,
|
||||
archive_record JSONB DEFAULT '{}',
|
||||
jsonl_event JSONB DEFAULT '{}',
|
||||
PRIMARY KEY (slug, revision)
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.wiki_links (
|
||||
slug TEXT NOT NULL,
|
||||
target_slug TEXT NOT NULL,
|
||||
target_title TEXT NOT NULL,
|
||||
PRIMARY KEY (slug, target_slug)
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.wiki_categories (
|
||||
slug TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
PRIMARY KEY (slug, category)
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ene.packages (
|
||||
pkg TEXT PRIMARY KEY,
|
||||
version TEXT NOT NULL,
|
||||
tier TEXT,
|
||||
domain TEXT,
|
||||
archetype TEXT,
|
||||
description TEXT,
|
||||
tags JSONB DEFAULT '[]',
|
||||
source TEXT,
|
||||
sha256 TEXT,
|
||||
indexed_utc TEXT,
|
||||
concept_anchor JSONB DEFAULT '{}',
|
||||
concept_vector JSONB DEFAULT '[]',
|
||||
idea_weights JSONB DEFAULT '{}',
|
||||
analog_map JSONB DEFAULT '{}'
|
||||
)
|
||||
""")
|
||||
self._ensure_columns(conn, "ene.wiki_revisions", cur)
|
||||
conn.commit()
|
||||
|
||||
def _ensure_columns(self, conn, table: str, cur) -> None:
|
||||
cur.execute("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'ene' AND table_name = %s
|
||||
""", (table.split('.')[1],))
|
||||
existing = {r[0] for r in cur.fetchall()}
|
||||
additions = {
|
||||
"archive_id": "TEXT",
|
||||
"content_hash": "TEXT",
|
||||
"archive_record": "JSONB DEFAULT '{}'",
|
||||
"jsonl_event": "JSONB DEFAULT '{}'",
|
||||
}
|
||||
for name, decl in additions.items():
|
||||
if name not in existing:
|
||||
cur.execute(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {name} {decl}")
|
||||
|
||||
def _upsert_package(self, conn, cur, event: dict[str, Any]) -> None:
|
||||
data = event["data"]
|
||||
cur.execute("""
|
||||
INSERT INTO ene.packages (
|
||||
pkg, version, tier, domain, archetype, description,
|
||||
tags, source, sha256, indexed_utc,
|
||||
concept_anchor, concept_vector, idea_weights, analog_map
|
||||
) VALUES (
|
||||
%(pkg)s, %(version)s, %(tier)s, %(domain)s, %(archetype)s, %(description)s,
|
||||
%(tags)s, %(source)s, %(sha256)s, %(indexed_utc)s,
|
||||
%(concept_anchor)s, %(concept_vector)s, %(idea_weights)s, %(analog_map)s
|
||||
) ON CONFLICT (pkg) DO UPDATE SET
|
||||
version = EXCLUDED.version,
|
||||
tier = EXCLUDED.tier,
|
||||
domain = EXCLUDED.domain,
|
||||
archetype = EXCLUDED.archetype,
|
||||
description = EXCLUDED.description,
|
||||
tags = EXCLUDED.tags,
|
||||
source = EXCLUDED.source,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
indexed_utc = EXCLUDED.indexed_utc,
|
||||
concept_anchor = EXCLUDED.concept_anchor,
|
||||
concept_vector = EXCLUDED.concept_vector,
|
||||
idea_weights = EXCLUDED.idea_weights,
|
||||
analog_map = EXCLUDED.analog_map
|
||||
""", {
|
||||
"pkg": data["pkg"],
|
||||
"version": data["version"],
|
||||
"tier": data["tier"],
|
||||
"domain": data["domain"],
|
||||
"archetype": data["archetype"],
|
||||
"description": data["description"],
|
||||
"tags": json.dumps(data["tags"], sort_keys=True),
|
||||
"source": data["source"],
|
||||
"sha256": data["sha256"],
|
||||
"indexed_utc": data["indexed_utc"],
|
||||
"concept_anchor": json.dumps(data["concept_anchor"], sort_keys=True),
|
||||
"concept_vector": json.dumps(data["concept_vector"], sort_keys=True),
|
||||
"idea_weights": json.dumps(data["idea_weights"], sort_keys=True),
|
||||
"analog_map": json.dumps(data["analog_map"], sort_keys=True),
|
||||
})
|
||||
|
||||
def admit_write(self, title: str, text: str) -> tuple[bool, str]:
|
||||
try:
|
||||
normalize_title(title)
|
||||
except ValueError as exc:
|
||||
return False, str(exc)
|
||||
if len(text.encode("utf-8")) > 256_000:
|
||||
return False, "wiki text too large"
|
||||
lowered = text.lower()
|
||||
if "<script" in lowered or "javascript:" in lowered:
|
||||
return False, "active script content refused"
|
||||
return True, "wiki_write_admitted"
|
||||
|
||||
def put_page(self, title: str, text: str, author: str = "ene", summary: str = "") -> WikiRevision:
|
||||
admitted, reason = self.admit_write(title, text)
|
||||
if not admitted:
|
||||
raise ValueError(reason)
|
||||
|
||||
normalized = normalize_title(title)
|
||||
slug = title_slug(normalized)
|
||||
now_ts = int(time.time())
|
||||
now_dt = datetime.fromtimestamp(now_ts, tz=timezone.utc)
|
||||
links = extract_links(text)
|
||||
categories = extract_categories(text)
|
||||
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT latest_revision FROM ene.wiki_pages WHERE slug = %s",
|
||||
(slug,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
revision = int(row[0]) + 1 if row else 1
|
||||
receipt = write_receipt(slug, revision, text, author, now_ts)
|
||||
archive_record = make_archive_record(
|
||||
normalized, slug, revision, text, author, summary, now_ts, links, categories
|
||||
)
|
||||
concept_vector = concept_vector_for_wiki(normalized, text, links, categories)
|
||||
jsonl_event = make_jsonl_event(archive_record, concept_vector, receipt)
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO ene.wiki_revisions
|
||||
(slug, revision, title, text, author, summary, created_at, receipt,
|
||||
archive_id, content_hash, archive_record, jsonl_event)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
slug, revision, normalized, text, author, summary, now_dt, receipt,
|
||||
archive_record["archive_id"], archive_record["content_hash"],
|
||||
json.dumps(archive_record, sort_keys=True),
|
||||
json.dumps(jsonl_event, sort_keys=True),
|
||||
))
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO ene.wiki_pages (slug, title, latest_revision, updated_at, receipt)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (slug) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
latest_revision = EXCLUDED.latest_revision,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
receipt = EXCLUDED.receipt
|
||||
""", (slug, normalized, revision, now_dt, receipt))
|
||||
|
||||
cur.execute("DELETE FROM ene.wiki_links WHERE slug = %s", (slug,))
|
||||
cur.execute("DELETE FROM ene.wiki_categories WHERE slug = %s", (slug,))
|
||||
|
||||
if links:
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO ene.wiki_links (slug, target_slug, target_title) VALUES %s ON CONFLICT DO NOTHING",
|
||||
[(slug, title_slug(link), link) for link in links],
|
||||
)
|
||||
if categories:
|
||||
psycopg2.extras.execute_values(
|
||||
cur,
|
||||
"INSERT INTO ene.wiki_categories (slug, category) VALUES %s ON CONFLICT DO NOTHING",
|
||||
[(slug, cat) for cat in categories],
|
||||
)
|
||||
|
||||
self._upsert_package(conn, cur, jsonl_event)
|
||||
conn.commit()
|
||||
|
||||
return WikiRevision(
|
||||
normalized, slug, revision, text, author, summary, now_ts, receipt,
|
||||
links, categories, archive_record, jsonl_event,
|
||||
)
|
||||
|
||||
def get_page(self, title: str, revision: int | None = None) -> WikiRevision | None:
|
||||
slug = title_slug(title)
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
if revision is None:
|
||||
cur.execute(
|
||||
"SELECT latest_revision FROM ene.wiki_pages WHERE slug = %s",
|
||||
(slug,),
|
||||
)
|
||||
page = cur.fetchone()
|
||||
if not page:
|
||||
return None
|
||||
revision = int(page["latest_revision"])
|
||||
else:
|
||||
revision = int(revision)
|
||||
|
||||
cur.execute("""
|
||||
SELECT title, slug, revision, text, author, summary, created_at, receipt,
|
||||
archive_record, jsonl_event
|
||||
FROM ene.wiki_revisions
|
||||
WHERE slug = %s AND revision = %s
|
||||
""", (slug, revision))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
cur.execute(
|
||||
"SELECT target_title FROM ene.wiki_links WHERE slug = %s ORDER BY lower(target_title)",
|
||||
(slug,),
|
||||
)
|
||||
links = [r["target_title"] for r in cur.fetchall()]
|
||||
|
||||
cur.execute(
|
||||
"SELECT category FROM ene.wiki_categories WHERE slug = %s ORDER BY lower(category)",
|
||||
(slug,),
|
||||
)
|
||||
categories = [r["category"] for r in cur.fetchall()]
|
||||
|
||||
archive_record = row["archive_record"] if isinstance(row["archive_record"], dict) else json.loads(row["archive_record"] or "{}")
|
||||
jsonl_event = row["jsonl_event"] if isinstance(row["jsonl_event"], dict) else json.loads(row["jsonl_event"] or "{}")
|
||||
created_ts = int(row["created_at"].timestamp()) if hasattr(row["created_at"], "timestamp") else 0
|
||||
|
||||
return WikiRevision(
|
||||
row["title"], row["slug"], int(row["revision"]), row["text"],
|
||||
row["author"], row["summary"], created_ts, row["receipt"],
|
||||
links, categories, archive_record, jsonl_event,
|
||||
)
|
||||
|
||||
def search(self, query: str, limit: int = 20) -> list[WikiPage]:
|
||||
term = f"%{query.strip()}%"
|
||||
limit = max(1, min(limit, 100))
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT p.title, p.slug, p.latest_revision, p.updated_at, p.receipt
|
||||
FROM ene.wiki_pages p
|
||||
JOIN ene.wiki_revisions r
|
||||
ON r.slug = p.slug AND r.revision = p.latest_revision
|
||||
WHERE p.title ILIKE %s OR r.text ILIKE %s
|
||||
ORDER BY p.updated_at DESC
|
||||
LIMIT %s
|
||||
""", (term, term, limit))
|
||||
rows = cur.fetchall()
|
||||
return [
|
||||
WikiPage(
|
||||
r["title"], r["slug"], int(r["latest_revision"]),
|
||||
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
|
||||
r["receipt"],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def backlinks(self, title: str) -> list[WikiPage]:
|
||||
target = title_slug(title)
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT p.title, p.slug, p.latest_revision, p.updated_at, p.receipt
|
||||
FROM ene.wiki_links l
|
||||
JOIN ene.wiki_pages p ON p.slug = l.slug
|
||||
WHERE l.target_slug = %s
|
||||
ORDER BY lower(p.title)
|
||||
""", (target,))
|
||||
rows = cur.fetchall()
|
||||
return [
|
||||
WikiPage(
|
||||
r["title"], r["slug"], int(r["latest_revision"]),
|
||||
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
|
||||
r["receipt"],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def recent_changes(self, limit: int = 20) -> list[WikiPage]:
|
||||
limit = max(1, min(limit, 100))
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT title, slug, latest_revision, updated_at, receipt
|
||||
FROM ene.wiki_pages
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT %s
|
||||
""", (limit,))
|
||||
rows = cur.fetchall()
|
||||
return [
|
||||
WikiPage(
|
||||
r["title"], r["slug"], int(r["latest_revision"]),
|
||||
int(r["updated_at"].timestamp()) if hasattr(r["updated_at"], "timestamp") else 0,
|
||||
r["receipt"],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def handle_request(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||
op = str(request.get("op", "recent"))
|
||||
if op in {"put", "edit"}:
|
||||
revision = self.put_page(
|
||||
title=str(request.get("title", "")),
|
||||
text=str(request.get("text", "")),
|
||||
author=str(request.get("author", "ene")),
|
||||
summary=str(request.get("summary", "")),
|
||||
)
|
||||
return {"ok": True, "op": op, "revision": asdict(revision)}
|
||||
if op == "get":
|
||||
page = self.get_page(str(request.get("title", "")), request.get("revision"))
|
||||
return {"ok": page is not None, "op": op, "page": asdict(page) if page else None}
|
||||
if op == "search":
|
||||
pages = self.search(str(request.get("query", "")), int(request.get("limit", 20)))
|
||||
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
|
||||
if op == "backlinks":
|
||||
pages = self.backlinks(str(request.get("title", "")))
|
||||
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
|
||||
if op == "recent":
|
||||
pages = self.recent_changes(int(request.get("limit", 20)))
|
||||
return {"ok": True, "op": op, "pages": [asdict(page) for page in pages]}
|
||||
raise ValueError(f"unsupported wiki op {op!r}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="ENE RDS wiki layer")
|
||||
parser.add_argument("--dsn", help="PostgreSQL DSN")
|
||||
parser.add_argument("--op", choices=["put", "get", "search", "backlinks", "recent"], default="recent")
|
||||
parser.add_argument("--title", default="")
|
||||
parser.add_argument("--text", default="")
|
||||
parser.add_argument("--query", default="")
|
||||
parser.add_argument("--author", default="ene")
|
||||
args = parser.parse_args()
|
||||
|
||||
wiki = ENERDSWikiLayer(args.dsn)
|
||||
request: dict[str, Any] = {
|
||||
"op": args.op, "title": args.title, "text": args.text,
|
||||
"query": args.query, "author": args.author,
|
||||
}
|
||||
print(json.dumps(wiki.handle_request(request), sort_keys=True, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
94
4-Infrastructure/infra/recover_credential_server.sh
Normal file
94
4-Infrastructure/infra/recover_credential_server.sh
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env bash
|
||||
# Deploy credential server to microVM (RackNerd)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
VM_IP="172.245.19.182"
|
||||
VM_USER="root"
|
||||
VM_PASS="${1:-}"
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
|
||||
|
||||
if [ -z "$VM_PASS" ]; then
|
||||
VM_PASS="$(cat "$REPO_ROOT/API KEYS/racknerd_510bd9c_root.txt" 2>/dev/null | grep root_password | cut -d: -f2 | tr -d ' ')"
|
||||
fi
|
||||
|
||||
if [ -z "$VM_PASS" ]; then
|
||||
echo "ERROR: root password required. Pass as argument or ensure API KEYS/racknerd_510bd9c_root.txt exists."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SSH="sshpass -p "$VM_PASS" ssh $SSH_OPTS ${VM_USER}@${VM_IP}"
|
||||
SCP="sshpass -p "$VM_PASS" scp $SSH_OPTS"
|
||||
|
||||
echo "=== Deploying credential server to ${VM_IP} ==="
|
||||
|
||||
# Create directories
|
||||
$SSH "mkdir -p /opt/rs-surface /etc/rs-surface"
|
||||
|
||||
# Copy Python modules
|
||||
echo "Uploading credential_provider.py..."
|
||||
$SCP "$REPO_ROOT/4-Infrastructure/infra/credential_provider.py" ${VM_USER}@${VM_IP}:/opt/rs-surface/credential_provider.py
|
||||
|
||||
echo "Uploading credential_server.py..."
|
||||
$SCP "$REPO_ROOT/4-Infrastructure/infra/credential_server.py" ${VM_USER}@${VM_IP}:/opt/rs-surface/credential_server.py
|
||||
|
||||
# Copy credentials config
|
||||
CRED_JSON="/tmp/rs-credentials.json"
|
||||
python3 -c "
|
||||
import json, os
|
||||
creds = {}
|
||||
for var, key in [('DEEPSEEK_API_KEY', 'deepseek'), ('QUANDELA_API_KEY', 'quandela'),
|
||||
('WOLFRAM_ALPHA_APPID', 'wolfram_alpha'), ('LINEAR_API_KEY', 'linear'),
|
||||
('AWS_BEARER_TOKEN_BEDROCK', 'bedrock')]:
|
||||
val = os.environ.get(var, '')
|
||||
if val:
|
||||
creds[key] = val
|
||||
if not creds:
|
||||
print('WARNING: no credential env vars set; writing empty config')
|
||||
with open('$CRED_JSON', 'w') as f:
|
||||
json.dump(creds, f, indent=2)
|
||||
print(f'Wrote {len(creds)} provider keys from environment')
|
||||
"
|
||||
echo "Uploading credentials.json..."
|
||||
$SCP "$CRED_JSON" ${VM_USER}@${VM_IP}:/etc/rs-surface/credentials.json
|
||||
rm -f "$CRED_JSON"
|
||||
|
||||
# Create systemd service
|
||||
echo "Setting up systemd service..."
|
||||
$SSH 'cat > /etc/systemd/system/rs-credential-server.service << '"'"'SERVICEEOF'"'"'
|
||||
[Unit]
|
||||
Description=Research Stack Credential Server
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/rs-surface
|
||||
ExecStart=/usr/bin/python3 /opt/rs-surface/credential_server.py --port 8444 --bind 0.0.0.0
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
Environment=RS_CREDENTIAL_CONFIG=/etc/rs-surface/credentials.json
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
SERVICEEOF'
|
||||
|
||||
# Stop old service if exists, enable new one
|
||||
$SSH "systemctl daemon-reload && systemctl enable rs-credential-server && systemctl restart rs-credential-server"
|
||||
|
||||
# Verify
|
||||
echo "=== Verifying ==="
|
||||
sleep 2
|
||||
$SSH "systemctl status rs-credential-server --no-pager --lines=5"
|
||||
echo ""
|
||||
echo "=== Testing HTTP ==="
|
||||
curl -sf --connect-timeout 5 http://${VM_IP}:8444/ && echo "" || echo "(curl root)"
|
||||
curl -sf --connect-timeout 5 http://${VM_IP}:8444/health && echo "" || echo "(curl health)"
|
||||
curl -sf --connect-timeout 5 http://${VM_IP}:8444/status && echo "" || echo "(curl status)"
|
||||
curl -sf --connect-timeout 5 http://${VM_IP}:8444/openapi.json | python3 -m json.tool > /dev/null 2>&1 && echo "openapi.json: valid" || echo "openapi.json: FAIL"
|
||||
|
||||
echo "=== Deploy complete ==="
|
||||
echo "API: http://${VM_IP}:8444/"
|
||||
echo "Docs: http://${VM_IP}:8444/openapi.json"
|
||||
echo "Credentials: http://${VM_IP}:8444/credentials"
|
||||
2452
4-Infrastructure/rds_probe/Cargo.lock
generated
Normal file
2452
4-Infrastructure/rds_probe/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
23
4-Infrastructure/rds_probe/Cargo.toml
Normal file
23
4-Infrastructure/rds_probe/Cargo.toml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
[package]
|
||||
name = "rds_probe"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "RDS ENE database inspection tool"
|
||||
license = "MIT"
|
||||
authors = ["Research Stack"]
|
||||
|
||||
[dependencies]
|
||||
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "uuid"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
dotenv = "0.15"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
thiserror = "1"
|
||||
sha2 = "0.10"
|
||||
hex = "0.4"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = true
|
||||
183
4-Infrastructure/rds_probe/src/commands.rs
Normal file
183
4-Infrastructure/rds_probe/src/commands.rs
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
use crate::db::get_pool;
|
||||
use crate::models::*;
|
||||
use clap::Subcommand;
|
||||
use serde_json::json;
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum Command {
|
||||
Tables,
|
||||
Schema { table: String },
|
||||
Count,
|
||||
Sample { domain: String },
|
||||
Export,
|
||||
}
|
||||
|
||||
pub async fn run_command(command: Command) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let pool = get_pool().await?;
|
||||
|
||||
match command {
|
||||
Command::Tables => tables_cmd(&pool).await,
|
||||
Command::Schema { table } => schema_cmd(&pool, &table).await,
|
||||
Command::Count => count_cmd(&pool).await,
|
||||
Command::Sample { domain } => sample_cmd(&pool, &domain).await,
|
||||
Command::Export => export_cmd(&pool).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn tables_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let rows = sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT table_name, table_schema
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
|
||||
ORDER BY table_schema, table_name"
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let tables: Vec<TableInfo> = rows.into_iter()
|
||||
.map(|(table_name, table_schema)| TableInfo { table_name, table_schema })
|
||||
.collect();
|
||||
|
||||
let output = json!({
|
||||
"success": true,
|
||||
"command": "tables",
|
||||
"count": tables.len(),
|
||||
"tables": tables
|
||||
});
|
||||
|
||||
Ok(serde_json::to_string_pretty(&output)?)
|
||||
}
|
||||
|
||||
async fn schema_cmd(pool: &sqlx::PgPool, table: &str) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let rows = if let Some(pos) = table.find('.') {
|
||||
let schema = &table[..pos];
|
||||
let name = &table[pos + 1..];
|
||||
sqlx::query_as::<_, (String, String, String)>(
|
||||
"SELECT column_name, data_type, is_nullable
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = $1 AND table_name = $2
|
||||
ORDER BY ordinal_position"
|
||||
)
|
||||
.bind(schema)
|
||||
.bind(name)
|
||||
.fetch_all(pool)
|
||||
.await?
|
||||
} else {
|
||||
sqlx::query_as::<_, (String, String, String)>(
|
||||
"SELECT column_name, data_type, is_nullable
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = $1 AND table_schema NOT IN ('pg_catalog', 'information_schema')
|
||||
ORDER BY ordinal_position"
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_all(pool)
|
||||
.await?
|
||||
};
|
||||
|
||||
let columns: Vec<ColumnInfo> = rows.into_iter()
|
||||
.map(|(column_name, data_type, is_nullable)| ColumnInfo {
|
||||
column_name,
|
||||
data_type,
|
||||
is_nullable,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let output = json!({
|
||||
"success": true,
|
||||
"command": "schema",
|
||||
"table": table,
|
||||
"columns": columns
|
||||
});
|
||||
|
||||
Ok(serde_json::to_string_pretty(&output)?)
|
||||
}
|
||||
|
||||
async fn count_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let rows = sqlx::query_as::<_, (String, String, i64)>(
|
||||
"SELECT domain, archetype, COUNT(*) as count
|
||||
FROM ene.packages
|
||||
GROUP BY domain, archetype
|
||||
ORDER BY domain, archetype"
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let mut by_domain = serde_json::Map::new();
|
||||
for (domain, archetype, count) in rows {
|
||||
let entry = by_domain.entry(domain).or_insert_with(|| json!({}));
|
||||
entry.as_object_mut().unwrap().insert(archetype, json!(count));
|
||||
}
|
||||
|
||||
let total: i64 = by_domain.values()
|
||||
.flat_map(|v| v.as_object().unwrap().values())
|
||||
.map(|v| v.as_i64().unwrap_or(0))
|
||||
.sum();
|
||||
|
||||
let output = json!({
|
||||
"success": true,
|
||||
"command": "count",
|
||||
"total_records": total,
|
||||
"by_domain": by_domain
|
||||
});
|
||||
|
||||
Ok(serde_json::to_string_pretty(&output)?)
|
||||
}
|
||||
|
||||
async fn sample_cmd(pool: &sqlx::PgPool, domain: &str) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let rows = sqlx::query_as::<_, (
|
||||
String, Option<String>, Option<String>, Option<String>, Option<String>,
|
||||
Option<String>, Option<serde_json::Value>, Option<String>, Option<String>
|
||||
)>(
|
||||
"SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
|
||||
FROM ene.packages
|
||||
WHERE domain = $1
|
||||
LIMIT 5"
|
||||
)
|
||||
.bind(domain.to_uppercase())
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let records: Vec<PackageRecord> = rows.into_iter()
|
||||
.map(|(pkg, version, domain, tier, archetype, description, tags, source, indexed_utc)| {
|
||||
PackageRecord { pkg, version, domain, tier, archetype, description, tags, source, indexed_utc }
|
||||
})
|
||||
.collect();
|
||||
|
||||
let output = json!({
|
||||
"success": true,
|
||||
"command": "sample",
|
||||
"domain": domain.to_uppercase(),
|
||||
"count": records.len(),
|
||||
"records": records
|
||||
});
|
||||
|
||||
Ok(serde_json::to_string_pretty(&output)?)
|
||||
}
|
||||
|
||||
async fn export_cmd(pool: &sqlx::PgPool) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let rows = sqlx::query_as::<_, (
|
||||
String, Option<String>, Option<String>, Option<String>, Option<String>,
|
||||
Option<String>, Option<serde_json::Value>, Option<String>, Option<String>
|
||||
)>(
|
||||
"SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
|
||||
FROM ene.packages
|
||||
ORDER BY domain, indexed_utc DESC"
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let records: Vec<PackageRecord> = rows.into_iter()
|
||||
.map(|(pkg, version, domain, tier, archetype, description, tags, source, indexed_utc)| {
|
||||
PackageRecord { pkg, version, domain, tier, archetype, description, tags, source, indexed_utc }
|
||||
})
|
||||
.collect();
|
||||
|
||||
let output = json!({
|
||||
"success": true,
|
||||
"command": "export",
|
||||
"total_records": records.len(),
|
||||
"records": records
|
||||
});
|
||||
|
||||
Ok(serde_json::to_string_pretty(&output)?)
|
||||
}
|
||||
102
4-Infrastructure/rds_probe/src/db.rs
Normal file
102
4-Infrastructure/rds_probe/src/db.rs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
use sqlx::{postgres::PgPoolOptions, PgPool};
|
||||
use std::env;
|
||||
use std::process::Command;
|
||||
|
||||
pub struct DbConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub user: String,
|
||||
pub password: String,
|
||||
pub dbname: String,
|
||||
}
|
||||
|
||||
fn get_iam_token(host: &str, user: &str, region: &str) -> Option<String> {
|
||||
let output = Command::new("aws")
|
||||
.args(&[
|
||||
"rds", "generate-db-auth-token",
|
||||
"--hostname", host,
|
||||
"--port", "5432",
|
||||
"--username", user,
|
||||
"--region", region,
|
||||
])
|
||||
.output()
|
||||
.ok()?;
|
||||
|
||||
if output.status.success() {
|
||||
let token = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
if !token.is_empty() {
|
||||
return Some(token);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
impl DbConfig {
|
||||
pub fn from_env() -> Self {
|
||||
let host = env::var("RDS_HOST")
|
||||
.unwrap_or("database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com".to_string());
|
||||
let user = env::var("RDS_USER").unwrap_or("postgres".to_string());
|
||||
let region = env::var("AWS_DEFAULT_REGION").unwrap_or("us-east-1".to_string());
|
||||
|
||||
let password = if let Ok(pwd) = env::var("RDS_PASSWORD") {
|
||||
if !pwd.is_empty() {
|
||||
pwd
|
||||
} else {
|
||||
get_iam_token(&host, &user, ®ion).unwrap_or_default()
|
||||
}
|
||||
} else {
|
||||
get_iam_token(&host, &user, ®ion).unwrap_or_default()
|
||||
};
|
||||
|
||||
Self {
|
||||
host,
|
||||
port: env::var("RDS_PORT")
|
||||
.unwrap_or("5432".to_string())
|
||||
.parse()
|
||||
.unwrap_or(5432),
|
||||
user,
|
||||
password,
|
||||
dbname: env::var("RDS_DB").unwrap_or("postgres".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dsn(&self) -> String {
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}?sslmode=require",
|
||||
self.user,
|
||||
urlencoding(&self.password),
|
||||
self.host,
|
||||
self.port,
|
||||
self.dbname
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn urlencoding(s: &str) -> String {
|
||||
let mut encoded = String::new();
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => {
|
||||
encoded.push(c);
|
||||
}
|
||||
_ => {
|
||||
encoded.push('%');
|
||||
encoded.push_str(&format!("{:02X}", c as u8));
|
||||
}
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
pub async fn create_pool(config: &DbConfig) -> Result<PgPool, sqlx::Error> {
|
||||
PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.acquire_timeout(std::time::Duration::from_secs(30))
|
||||
.connect(&config.dsn()).await
|
||||
}
|
||||
|
||||
pub async fn get_pool() -> Result<PgPool, Box<dyn std::error::Error>> {
|
||||
let config = DbConfig::from_env();
|
||||
let pool = create_pool(&config).await?;
|
||||
Ok(pool)
|
||||
}
|
||||
29
4-Infrastructure/rds_probe/src/main.rs
Normal file
29
4-Infrastructure/rds_probe/src/main.rs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
mod commands;
|
||||
mod db;
|
||||
mod models;
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "rds_probe")]
|
||||
#[command(about = "RDS ENE database inspection tool", long_about = None)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: commands::Command,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dotenv::dotenv().ok();
|
||||
|
||||
match commands::run_command(Cli::parse().command).await {
|
||||
Ok(output) => {
|
||||
println!("{}", output);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error: {}", e);
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
40
4-Infrastructure/rds_probe/src/models.rs
Normal file
40
4-Infrastructure/rds_probe/src/models.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct TableInfo {
|
||||
pub table_name: String,
|
||||
pub table_schema: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ColumnInfo {
|
||||
pub column_name: String,
|
||||
pub data_type: String,
|
||||
pub is_nullable: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct CountResult {
|
||||
pub domain: String,
|
||||
pub archetype: String,
|
||||
pub count: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct PackageRecord {
|
||||
pub pkg: String,
|
||||
pub version: Option<String>,
|
||||
pub domain: Option<String>,
|
||||
pub tier: Option<String>,
|
||||
pub archetype: Option<String>,
|
||||
pub description: Option<String>,
|
||||
pub tags: Option<serde_json::Value>,
|
||||
pub source: Option<String>,
|
||||
pub indexed_utc: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct CommandOutput {
|
||||
pub success: bool,
|
||||
pub data: serde_json::Value,
|
||||
}
|
||||
|
|
@ -9,12 +9,13 @@ Output: JSON + Markdown reference table
|
|||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
WOLFRAM_APP_ID = "HYJE3R3R63"
|
||||
WOLFRAM_APP_ID = os.environ.get("WOLFRAM_ALPHA_APPID", "")
|
||||
WOLFRAM_API_URL = "https://api.wolframalpha.com/v2/query"
|
||||
|
||||
# Canonical math function categories and test queries
|
||||
|
|
|
|||
79
5-Applications/scripts/export_linear_from_rds.py
Executable file
79
5-Applications/scripts/export_linear_from_rds.py
Executable file
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env python3
|
||||
"""export_linear_from_rds.py - Exports LINEAR domain records from RDS to JSON.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add infra and scripts folders to import path
|
||||
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure" / "infra"))
|
||||
sys.path.append(str(Path(__file__).resolve().parent))
|
||||
|
||||
from import_dumps_to_rds import get_rds_password
|
||||
|
||||
def main():
|
||||
print("[+] Fetching RDS credentials...")
|
||||
password = get_rds_password()
|
||||
if not password:
|
||||
print("[-] Failed to retrieve RDS password or IAM token. Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
||||
port = int(os.environ.get("RDS_PORT", "5432"))
|
||||
user = os.environ.get("RDS_USER", "postgres")
|
||||
dbname = os.environ.get("RDS_DB", "postgres")
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(
|
||||
host=host,
|
||||
port=port,
|
||||
user=user,
|
||||
password=password,
|
||||
dbname=dbname,
|
||||
sslmode="require"
|
||||
)
|
||||
cur = conn.cursor()
|
||||
except Exception as e:
|
||||
print(f"[-] Database connection failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("[+] Querying all LINEAR domain packages from RDS...")
|
||||
try:
|
||||
cur.execute("""
|
||||
SELECT pkg, version, domain, tier, archetype, description, tags, source, indexed_utc
|
||||
FROM ene.packages
|
||||
WHERE domain = 'LINEAR'
|
||||
ORDER BY indexed_utc DESC
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
except Exception as e:
|
||||
print(f"[-] Failed to execute query: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
records = []
|
||||
for r in rows:
|
||||
records.append({
|
||||
"pkg": r[0],
|
||||
"version": r[1],
|
||||
"domain": r[2],
|
||||
"tier": r[3],
|
||||
"archetype": r[4],
|
||||
"description": r[5],
|
||||
"tags": r[6],
|
||||
"source": r[7],
|
||||
"indexed_utc": r[8]
|
||||
})
|
||||
|
||||
out_file = Path(__file__).resolve().parent / "linear_export.json"
|
||||
try:
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, indent=2, default=str)
|
||||
print(f"[+] Successfully exported {len(records)} Linear records to {out_file}")
|
||||
except Exception as e:
|
||||
print(f"[-] Failed to write JSON output: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -9,6 +9,7 @@ Verifies the 9 FixedPoint.lean theorems using Wolfram Alpha API:
|
|||
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
|
@ -92,7 +93,7 @@ def main():
|
|||
print("Verifying 9 FixedPoint.lean theorems")
|
||||
print("=" * 70)
|
||||
|
||||
app_id = "HYJE3R3R63"
|
||||
app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
|
||||
verifier = WolframAlphaVerifier(app_id)
|
||||
|
||||
# FixedPoint.lean theorems to verify
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ FixedPoint.lean.
|
|||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
|
@ -106,7 +107,7 @@ def main():
|
|||
print("FUNDAMENTAL MATH VERIFICATION — DRIFT-PREVENTION ANCHOR NET")
|
||||
print("=" * 70)
|
||||
|
||||
app_id = "HYJE3R3R63"
|
||||
app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
|
||||
verifier = WolframAlphaVerifier(app_id)
|
||||
|
||||
# (equation, description, expected_substring)
|
||||
|
|
|
|||
189
5-Applications/scripts/import_dumps_to_rds.py
Executable file
189
5-Applications/scripts/import_dumps_to_rds.py
Executable file
|
|
@ -0,0 +1,189 @@
|
|||
#!/usr/bin/env python3
|
||||
"""import_dumps_to_rds.py - Ingestion pipeline from crawled JSON dumps to RDS.
|
||||
|
||||
Imports Notion database pages into ene.wiki_pages/revisions/links/categories
|
||||
and Linear issues into ene.packages.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Add infra folder to import path
|
||||
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure" / "infra"))
|
||||
sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "4-Infrastructure"))
|
||||
|
||||
def get_rds_password():
|
||||
password = os.environ.get("RDS_IAM_TOKEN") or os.environ.get("RDS_PASSWORD")
|
||||
if password:
|
||||
return password
|
||||
|
||||
# Fallback to subprocess call to aws cli
|
||||
try:
|
||||
import subprocess
|
||||
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
||||
user = os.environ.get("RDS_USER", "postgres")
|
||||
region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
|
||||
cmd = [
|
||||
"aws", "rds", "generate-db-auth-token",
|
||||
"--hostname", host,
|
||||
"--port", "5432",
|
||||
"--username", user,
|
||||
"--region", region
|
||||
]
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
||||
token = res.stdout.strip()
|
||||
if token:
|
||||
return token
|
||||
except Exception as e:
|
||||
print(f"AWS CLI token generation failed: {e}")
|
||||
|
||||
return ""
|
||||
|
||||
def import_notion(password: str):
|
||||
notion_file = Path(__file__).resolve().parent / "notion_full_dump.json"
|
||||
if not notion_file.exists():
|
||||
print("[-] notion_full_dump.json not found, skipping Notion import.")
|
||||
return
|
||||
|
||||
print("[+] Loading notion_full_dump.json...")
|
||||
with open(notion_file, "r") as f:
|
||||
dump_data = json.load(f)
|
||||
|
||||
pages = dump_data.get("pages", [])
|
||||
print(f"[+] Found {len(pages)} pages to import.")
|
||||
|
||||
# Override password environment variable for ENEWikiLayer
|
||||
os.environ["RDS_PASSWORD"] = password
|
||||
|
||||
try:
|
||||
from ene_rds_wiki_layer import ENERDSWikiLayer
|
||||
wiki = ENERDSWikiLayer()
|
||||
except Exception as e:
|
||||
print(f"[-] Failed to initialize ENERDSWikiLayer: {e}")
|
||||
return
|
||||
|
||||
imported_count = 0
|
||||
for page in pages:
|
||||
properties = page.get("properties", {})
|
||||
title = "Untitled"
|
||||
for prop_name in ["Name", "title", "Title"]:
|
||||
prop = properties.get(prop_name, {})
|
||||
if prop and prop.get("title"):
|
||||
title_list = prop["title"]
|
||||
if title_list and isinstance(title_list, list):
|
||||
title = title_list[0].get("plain_text", "Untitled")
|
||||
break
|
||||
|
||||
content = page.get("_content", "") or ""
|
||||
author = "notion_importer"
|
||||
|
||||
try:
|
||||
print(f" -> Importing page: {title}")
|
||||
wiki.put_page(title=title, text=content, author=author, summary="Notion Full Crawl Ingestion")
|
||||
imported_count += 1
|
||||
except Exception as e:
|
||||
print(f" [!] Error importing page '{title}': {e}")
|
||||
|
||||
print(f"[+] Notion import complete. Successfully imported {imported_count}/{len(pages)} pages.")
|
||||
|
||||
def import_linear(password: str):
|
||||
linear_file = Path(__file__).resolve().parent / "linear_full_dump.json"
|
||||
if not linear_file.exists():
|
||||
print("[-] linear_full_dump.json not found, skipping Linear import.")
|
||||
return
|
||||
|
||||
print("[+] Loading linear_full_dump.json...")
|
||||
with open(linear_file, "r") as f:
|
||||
dump_data = json.load(f)
|
||||
|
||||
issues = dump_data.get("issues", [])
|
||||
print(f"[+] Found {len(issues)} issues to import.")
|
||||
|
||||
host = os.environ.get("RDS_HOST", "database-1-instance-1.cghu8yqogqwo.us-east-1.rds.amazonaws.com")
|
||||
port = int(os.environ.get("RDS_PORT", "5432"))
|
||||
user = os.environ.get("RDS_USER", "postgres")
|
||||
dbname = os.environ.get("RDS_DB", "postgres")
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(
|
||||
host=host,
|
||||
port=port,
|
||||
user=user,
|
||||
password=password,
|
||||
dbname=dbname,
|
||||
sslmode="require"
|
||||
)
|
||||
cur = conn.cursor()
|
||||
except Exception as e:
|
||||
print(f"[-] Database connection failed: {e}")
|
||||
return
|
||||
|
||||
imported_count = 0
|
||||
for issue in issues:
|
||||
identifier = issue.get("identifier")
|
||||
if not identifier:
|
||||
continue
|
||||
|
||||
pkg_id = f"linear/{identifier}"
|
||||
title = issue.get("title", "")
|
||||
description = issue.get("description", "")
|
||||
full_text = f"{title}\n\n{description}" if description else title
|
||||
url = issue.get("url", "")
|
||||
|
||||
# Extract labels
|
||||
labels_nodes = issue.get("labels", {}).get("nodes", [])
|
||||
labels_list = [l.get("name") for l in labels_nodes if l.get("name")]
|
||||
tags_json = json.dumps(labels_list)
|
||||
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT INTO ene.packages (
|
||||
pkg, version, domain, tier, archetype,
|
||||
tags, description, source, indexed_utc
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (pkg) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
tags = EXCLUDED.tags,
|
||||
source = EXCLUDED.source,
|
||||
indexed_utc = EXCLUDED.indexed_utc
|
||||
""", (
|
||||
pkg_id,
|
||||
"1.0.0",
|
||||
"LINEAR",
|
||||
"INTENT",
|
||||
"issue",
|
||||
tags_json,
|
||||
full_text,
|
||||
url,
|
||||
now_iso
|
||||
))
|
||||
imported_count += 1
|
||||
except Exception as e:
|
||||
print(f" [!] Error importing issue '{pkg_id}': {e}")
|
||||
|
||||
try:
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"[+] Linear import complete. Successfully imported {imported_count}/{len(issues)} issues.")
|
||||
except Exception as e:
|
||||
print(f"[-] Database commit failed: {e}")
|
||||
|
||||
def main():
|
||||
print("=== ENE RDS Ingestion Pipeline ===")
|
||||
password = get_rds_password()
|
||||
if not password:
|
||||
print("[-] Failed to retrieve RDS password or IAM token. Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
import_notion(password)
|
||||
import_linear(password)
|
||||
print("=== Pipeline Execution Finished ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
Store Wolfram Alpha credential in ENE
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ def main():
|
|||
sys.exit(1)
|
||||
|
||||
# Store Wolfram Alpha credential
|
||||
wolfram_app_id = "HYJE3R3R63"
|
||||
wolfram_app_id = os.environ.get("WOLFRAM_ALPHA_APPID", "")
|
||||
|
||||
print(f"\nStoring Wolfram Alpha App ID: {wolfram_app_id}")
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from infra.knowledge_ingestion import KnowledgeIngestion
|
|||
|
||||
def test_wolfram_alpha():
|
||||
"""Test Wolfram Alpha API integration"""
|
||||
api_key = "HYJE3R3R63"
|
||||
api_key = os.environ.get("WOLFRAM_ALPHA_APPID", "")
|
||||
|
||||
print("Testing Wolfram Alpha API integration...")
|
||||
ingestion = KnowledgeIngestion(api_key)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue