allaun/Research-Stack
1
0
Fork 0
mirror of https://github.com/allaunthefox/Research-Stack.git synced 2026-08-08 21:05:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

cleanup(ene): remove hardcoded schema path, drop inline schema fallback, add legacy shim strip receipt

Browse source
This commit is contained in:
Allaun Silverfox 2026-05-26 16:22:40 -05:00
parent 7f63d2280f
commit f6c415bebd
1 changed files with 246 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

503
4-Infrastructure/shim/ene_migrate_and_tag.py
View file

@ -1,123 +1,126 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """ENE substrate migration + concept tagging (legacy shim).
Migrate knowledge.* tables → ENE substrate with aggressive concept tagging.
1. Apply ENE schema (ene.packages + 9 support tables) NOTE (ontology migration):
2. Migrate all sources into ene.packages with typed provenance
3. Extract concepts, build relations, score N-space KV retention
4. Run cross-source discovery queries
Run in dev container: This file is a **legacy shim** used to keep ENE substrate bootstrapping and
podman exec -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... -e AWS_REGION=us-east-1 -e RDS_IAM=1 \ migration workflows running while the AVM / Lean-only ISA rewrite is underway.
research-stack python3 /home/researcher/stack/4-Infrastructure/shim/ene_migrate_and_tag.py
**Target architecture:** Lean-only AVM ISA + backend shims.
- Lean defines all semantics.
- Shims perform I/O and orchestration only.
This script currently performs:
- schema application
- migration SQL
- tokenizer-based tagging
- relation building
- retention scoring
Several of those operations are semantic decisions and include float-based scoring.
They must be ported into Lean/AVM.
Rules until ported:
- Treat all outputs as **not promoted**.
- Never silently apply a different schema. If the schema file is missing: **reject**.
TODO(lean-port): Port tagging/relations/retention scoring into Lean/AVM.
""" """
from __future__ import annotations from __future__ import annotations
import argparse
import hashlib import hashlib
import json import json
import logging import logging
import os
import re import re
import sys import sys
import uuid
from collections import Counter from collections import Counter
from pathlib import Path from pathlib import Path
import psycopg2 import psycopg2
import psycopg2.extras import psycopg2.extras
from rds_connect import connect_rds from rds_connect import connect_rds
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("ene_migrate") log = logging.getLogger("ene_migrate")
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_SCHEMA_PATH = REPO_ROOT / "4-Infrastructure/shim/ene_substrate_schema.sql"
ONTOLOGY_VERSION = "shim-ontology-migration-v1"
def connect(): def connect():
return connect_rds() return connect_rds()
def apply_schema(conn): def apply_schema(conn, schema_path: Path) -> None:
"""Apply the full ENE substrate schema.""" """Apply the ENE substrate schema.
schema_path = Path("/home/researcher/stack/4-Infrastructure/shim/ene_substrate_schema.sql")
if schema_path.exists(): Policy: never silently substitute an inline schema. If the schema file is
sql = schema_path.read_text() missing, reject.
with conn.cursor() as cur: """
cur.execute(sql) if not schema_path.exists():
conn.commit() raise FileNotFoundError(f"ENE schema file not found: {schema_path}")
log.info("ENE substrate schema applied")
else: sql = schema_path.read_text(encoding="utf-8")
log.warning("Schema file not found, creating inline") with conn.cursor() as cur:
# Fallback: create minimal schema inline cur.execute(sql)
with conn.cursor() as cur: conn.commit()
cur.execute("CREATE SCHEMA IF NOT EXISTS ene") log.info("ENE substrate schema applied (%s)", schema_path)
cur.execute("""
CREATE TABLE IF NOT EXISTS ene.packages (
pkg TEXT PRIMARY KEY, package_type TEXT, title TEXT, content TEXT,
content_hash TEXT, concept_vector JSONB DEFAULT '[]',
concept_anchor JSONB DEFAULT '{}', tags JSONB DEFAULT '[]',
source TEXT, provenance JSONB DEFAULT '{}', domain TEXT,
archetype TEXT, promotion_state TEXT DEFAULT 'held',
scar_class TEXT, ingested_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS ene.relations (
id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
source_id TEXT NOT NULL, target_id TEXT NOT NULL,
relation_type TEXT NOT NULL, weight REAL DEFAULT 1.0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS ene.nspace_kv (
key_id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
value_package_id TEXT NOT NULL,
reduction_reward REAL DEFAULT 0, sparsity_score REAL DEFAULT 0,
scar_pressure REAL DEFAULT 0, retention_score REAL DEFAULT 0
);
""")
conn.commit()
log.info("ENE minimal schema applied")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Term extraction (same aggressive tokenizer from concept_cross_reference.py) # Term extraction (legacy tokenizer)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
MATH_SYMBOL_RE = re.compile( MATH_SYMBOL_RE = re.compile(
r'\\(?:alpha|beta|gamma|Gamma|delta|Delta|epsilon|varepsilon|zeta|eta|theta|Theta|' r"\\\\(?:alpha|beta|gamma|Gamma|delta|Delta|epsilon|varepsilon|zeta|eta|theta|Theta|"
r'iota|kappa|lambda|Lambda|mu|nu|xi|Xi|pi|Pi|rho|sigma|Sigma|tau|upsilon|phi|Phi|' r"iota|kappa|lambda|Lambda|mu|nu|xi|Xi|pi|Pi|rho|sigma|Sigma|tau|upsilon|phi|Phi|"
r'varphi|chi|psi|Psi|omega|Omega|partial|nabla|infty|int|sum|prod|otimes|oplus|' r"varphi|chi|psi|Psi|omega|Omega|partial|nabla|infty|int|sum|prod|otimes|oplus|"
r'rightarrow|leftarrow|Rightarrow|Leftarrow|mapsto|approx|equiv|sim|propto|' r"rightarrow|leftarrow|Rightarrow|Leftarrow|mapsto|approx|equiv|sim|propto|"
r'leq|geq|neq|times|cdot|circ|pm|mp|sqrt|frac|operatorname|mathbf|mathrm|mathcal|' r"leq|geq|neq|times|cdot|circ|pm|mp|sqrt|frac|operatorname|mathbf|mathrm|mathcal|"
r'mathfrak|mathbb|text|hat|tilde|bar|vec|dot|ddot|widehat|widetilde|' r"mathfrak|mathbb|text|hat|tilde|bar|vec|dot|ddot|widehat|widetilde|"
r'langle|rangle|lVert|rVert|vert|mid|' r"langle|rangle|lVert|rVert|vert|mid|"
r'begin|end|left|right|big|Big|bigg|Bigg)' r"begin|end|left|right|big|Big|bigg|Bigg)"
) )
TECHNICAL_RE = re.compile( TECHNICAL_RE = re.compile(
r'\b(?:' r"\\b(?:"
r'manifold|field|shear|packet|spectral|braid|gossip|' r"manifold|field|shear|packet|spectral|braid|gossip|"
r'residual|invariant|receipt|scar|warden|collapse|compression|' r"residual|invariant|receipt|scar|warden|collapse|compression|"
r'entropy|eigen(?:value|vector)?|coboundary|cochain|' r"entropy|eigen(?:value|vector)?|coboundary|cochain|"
r'diffusion|transport|boundary|kernel|operator|' r"diffusion|transport|boundary|kernel|operator|"
r'tensor|metric|geodesic|curvature|torsion|' r"tensor|metric|geodesic|curvature|torsion|"
r'hamiltonian|lagrangian|reduction|projection|embedding|' r"hamiltonian|lagrangian|reduction|projection|embedding|"
r'chirality|helicity|handedness|logogram|' r"chirality|helicity|handedness|logogram|"
r'sidon|goxel|famm|nuvmap|otom|pist|' r"sidon|goxel|famm|nuvmap|otom|pist|"
r'erdos|szekeres|selfridge|gyarfas|' r"erdos|szekeres|selfridge|gyarfas|"
r'biocompression|organoid|chaos|fractal|attractor|basin|' r"biocompression|organoid|chaos|fractal|attractor|basin|"
r'thermal|thermodynamic|landauer|witness|shadow|adversarial|' r"thermal|thermodynamic|landauer|witness|shadow|adversarial|"
r'morph(?:ic|ism)?|radix|codec|semantic|' r"morph(?:ic|ism)?|radix|codec|semantic|"
r'markov|cognitive|attention|neural|network|transformer|' r"markov|cognitive|attention|neural|network|transformer|"
r'hutter|prize|betti|homology|cohomology|' r"hutter|prize|betti|homology|cohomology|"
r'riccati|noise|mfg|hessian|jacobian|' r"riccati|noise|mfg|hessian|jacobian|"
r'seam|tomography|sandwich|pruning|rope|scar|' r"seam|tomography|sandwich|pruning|rope|scar|"
r'eigensolid|eigenspace|underverse|geocognition|' r"eigensolid|eigenspace|underverse|geocognition|"
r'smallcode|constrained|key.value|shortcut|ontology|' r"smallcode|constrained|key.value|shortcut|ontology|"
r'hyperbolic|riemannian|poincare|' r"hyperbolic|riemannian|poincare|"
r'bio|dna|rna|protein|feynman|navier|stokes|' r"bio|dna|rna|protein|feynman|navier|stokes|"
r'plasma|mhd|alfven|' r"plasma|mhd|alfven|"
r'q16|fixed.point|subleq|oisc|kv|cache' r"q16|fixed.point|subleq|oisc|kv|cache"
r')\b', re.IGNORECASE r")\\b",
re.IGNORECASE,
) )
TOKEN_RE = re.compile(r'[a-zA-Z_\\][a-zA-Z0-9_\\]*|\b(?:N-space|key-value|CP-SAT|Anti-FAMM|Anti-Braid)\b', re.IGNORECASE) TOKEN_RE = re.compile(
STOP_WORDS = set("the a an is are was were be been being have has had do does did will would shall should may might must can could of in to for with on at by from as into through during and but or not no nor so if then else when where this that these those it its we they he she which who whom whose what how why also very more most some any all each every both few new other such only own same just about over text bf rm sf tt em sc it up use using used can one two also etc via per e.g i.e figure table section non doi url http https www paper result method approach model data set page pages vol pp et al note notes example see shown fig eq ref abstract introduction conclusion reference references arxiv org github com html pdf first second third following based given found obtained described proposed well within without between among under above below since however therefore thus still yet here there where now then than get got getting make made making take taken taking give given giving let lets case cases term terms form forms number numbers value values point points part parts type types kind kinds way ways much many long short high low large small different similar same total whole full work works working need needs needed help helps helped like likes liked know known unknown think thought believe want wants wanted try tries tried".split()) r"[a-zA-Z_\\\\][a-zA-Z0-9_\\\\]*|\\b(?:N-space|key-value|CP-SAT|Anti-FAMM|Anti-Braid)\\b",
re.IGNORECASE,
)
STOP_WORDS = set(
"the a an is are was were be been being have has had do does did will would shall should may might must can could of in to for with on at by from as into through during and but or not no nor so if then else when where this that these those it its we they which who what how why also more most some any all each every both few new other such only own same just about over text rm sf tt em sc up use using used one two etc via per eg ie figure table section doi url http https www paper result method approach model data set page pages vol pp et al note notes example see shown fig eq ref abstract introduction conclusion reference references arxiv org github com html pdf first second third following based given found obtained described proposed well within without between among under above below since however therefore thus still yet here there now then than"
.split()
)
def tokenize(text: str) -> list[str]: def tokenize(text: str) -> list[str]:
@ -126,7 +129,14 @@ def tokenize(text: str) -> list[str]:
seen: set[str] = set() seen: set[str] = set()
results: list[str] = [] results: list[str] = []
for m in TOKEN_RE.finditer(text): for m in TOKEN_RE.finditer(text):
t = m.group(0).strip().lower().rstrip(".,;:!?\"'()[]{}").lstrip("\\").rstrip("{}") t = (
m.group(0)
.strip()
.lower()
.rstrip(".,;:!?\"'()[]{}")
.lstrip("\\\\")
.rstrip("{}")
)
if not t or t in STOP_WORDS or len(t) < 2 or t in seen: if not t or t in STOP_WORDS or len(t) < 2 or t in seen:
continue continue
seen.add(t) seen.add(t)
@ -137,13 +147,15 @@ def tokenize(text: str) -> list[str]:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Migration # Migration
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def migrate_all_sources(conn): def migrate_all_sources(conn) -> int:
"""Migrate knowledge.* tables into ene.packages with typed provenance.""" """Migrate knowledge.* tables into ene.packages with typed provenance."""
cur = conn.cursor() cur = conn.cursor()
total = 0 total = 0
migrations = [ migrations = [
("equations", "eq_id", "equation", """ (
"equations",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT eq_id::text, 'equation', latex, latex, content_hash, '[]'::jsonb, source_file, 'equation_corpus', SELECT eq_id::text, 'equation', latex, latex, content_hash, '[]'::jsonb, source_file, 'equation_corpus',
jsonb_build_object('kind', kind, 'source_file', source_file, 'source_offset', source_offset), jsonb_build_object('kind', kind, 'source_file', source_file, 'source_offset', source_offset),
@ -152,8 +164,11 @@ def migrate_all_sources(conn):
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content, title = EXCLUDED.title, content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags
"""), """,
("tiddlywiki_pages", "tiddler_id", "tiddler", """ ),
(
"tiddlywiki_pages",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT tiddler_id::text, 'tiddler', title, coalesce(body,''), SELECT tiddler_id::text, 'tiddler', title, coalesce(body,''),
content_hash, '[]'::jsonb, source_path, 'tiddlywiki', content_hash, '[]'::jsonb, source_path, 'tiddlywiki',
@ -163,10 +178,13 @@ def migrate_all_sources(conn):
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content, title = EXCLUDED.title, content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags
"""), """,
("references", "ref_id", "reference", """ ),
(
"references",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT ref_id::text, 'reference', SELECT ref_id::text, 'reference',
substring(coalesce(bibtex,'') from 1 for 200), substring(coalesce(bibtex,'') from 1 for 200),
bibtex, content_hash, '[]'::jsonb, source_file, 'bibliography', bibtex, content_hash, '[]'::jsonb, source_file, 'bibliography',
jsonb_build_object('source_file', source_file, 'bibtex', coalesce(bibtex,'')), jsonb_build_object('source_file', source_file, 'bibtex', coalesce(bibtex,'')),
@ -175,8 +193,11 @@ def migrate_all_sources(conn):
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content, title = EXCLUDED.title, content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags content_hash = EXCLUDED.content_hash, tags = EXCLUDED.tags
"""), """,
("links", "link_id", "link", """ ),
(
"links",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT link_id::text, 'link', url, url, encode(sha256(url::bytea),'hex'), '[]'::jsonb, source_file, SELECT link_id::text, 'link', url, url, encode(sha256(url::bytea),'hex'), '[]'::jsonb, source_file,
'external_reference', 'external_reference',
@ -185,8 +206,11 @@ def migrate_all_sources(conn):
FROM knowledge.links FROM knowledge.links
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content title = EXCLUDED.title, content = EXCLUDED.content
"""), """,
("article_sources", "article_id", "article", """ ),
(
"article_sources",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT article_id::text, 'article', coalesce(label, url), coalesce(label,'') || ' ' || url, SELECT article_id::text, 'article', coalesce(label, url), coalesce(label,'') || ' ' || url,
encode(sha256(url::bytea),'hex'), '[]'::jsonb, url, 'article_source', encode(sha256(url::bytea),'hex'), '[]'::jsonb, url, 'article_source',
@ -195,8 +219,11 @@ def migrate_all_sources(conn):
FROM knowledge.article_sources FROM knowledge.article_sources
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content title = EXCLUDED.title, content = EXCLUDED.content
"""), """,
("dois", "doi_id", "doi", """ ),
(
"dois",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT doi_id::text, 'doi', doi, doi, encode(sha256(doi::bytea),'hex'), '[]'::jsonb, source_file, 'doi_identifier', SELECT doi_id::text, 'doi', doi, doi, encode(sha256(doi::bytea),'hex'), '[]'::jsonb, source_file, 'doi_identifier',
jsonb_build_object('doi', doi, 'source_file', source_file), jsonb_build_object('doi', doi, 'source_file', source_file),
@ -204,8 +231,11 @@ def migrate_all_sources(conn):
FROM knowledge.dois FROM knowledge.dois
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content title = EXCLUDED.title, content = EXCLUDED.content
"""), """,
("dataset_inventory", "inv_id", "dataset", """ ),
(
"dataset_inventory",
"""
INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state) INSERT INTO ene.packages (pkg, package_type, title, content, content_hash, tags, source, domain, provenance, promotion_state)
SELECT inv_id::text, 'dataset', coalesce(name, asset_id), SELECT inv_id::text, 'dataset', coalesce(name, asset_id),
coalesce(name,'') || ' ' || coalesce(evidence,'') || ' ' || coalesce(notes,''), coalesce(name,'') || ' ' || coalesce(evidence,'') || ' ' || coalesce(notes,''),
@ -216,12 +246,13 @@ def migrate_all_sources(conn):
FROM knowledge.dataset_inventory FROM knowledge.dataset_inventory
ON CONFLICT (pkg) DO UPDATE SET ON CONFLICT (pkg) DO UPDATE SET
title = EXCLUDED.title, content = EXCLUDED.content title = EXCLUDED.title, content = EXCLUDED.content
"""), """,
),
] ]
for src_table, id_col, pkg_type, insert_sql in migrations: for src_table, insert_sql in migrations:
cur.execute(f"SELECT COUNT(*) FROM knowledge.{src_table}") cur.execute(f"SELECT COUNT(*) FROM knowledge.{src_table}")
count = cur.fetchone()[0] count = int(cur.fetchone()[0])
cur.execute(insert_sql) cur.execute(insert_sql)
total += count total += count
log.info("Migrated %s → ene.packages (%d rows)", src_table, count) log.info("Migrated %s → ene.packages (%d rows)", src_table, count)
@ -231,11 +262,16 @@ def migrate_all_sources(conn):
return total return total
def tag_packages(conn): def tag_packages(conn) -> int:
"""Extract terms from each package content and store as concept_vector + tags JSONB.""" """Extract terms from package content and store concept_vector + tags JSONB.
NOTE: This is semantic classification logic and must be ported to Lean/AVM.
"""
cur = conn.cursor() cur = conn.cursor()
cur.execute("SELECT pkg, package_type, coalesce(content,''), coalesce(title,'') FROM ene.packages WHERE content IS NOT NULL AND content != ''") cur.execute(
"SELECT pkg, package_type, coalesce(content,''), coalesce(title,'') FROM ene.packages WHERE content IS NOT NULL AND content != ''"
)
packages = cur.fetchall() packages = cur.fetchall()
updated = 0 updated = 0
@ -244,16 +280,18 @@ def tag_packages(conn):
if not terms: if not terms:
continue continue
# Classify terms
math_terms = [t for t in terms if MATH_SYMBOL_RE.fullmatch(t)] math_terms = [t for t in terms if MATH_SYMBOL_RE.fullmatch(t)]
tech_terms = [t for t in terms if TECHNICAL_RE.search(t)] tech_terms = [t for t in terms if TECHNICAL_RE.search(t)]
all_terms = list(dict.fromkeys(terms)) # deduplicate preserving order all_terms = list(dict.fromkeys(terms))
concept_vector = [ concept_vector = [
{"term": t, "type": "math_symbol" if t in math_terms else "technical_term" if t in tech_terms else "keyword"} {
for t in all_terms[:50] # cap at 50 for storage "term": t,
"type": "math_symbol" if t in math_terms else "technical_term" if t in tech_terms else "keyword",
}
for t in all_terms[:50]
] ]
tags = all_terms[:20] # top 20 as tags tags = all_terms[:20]
cur.execute( cur.execute(
"""UPDATE ene.packages """UPDATE ene.packages
@ -271,16 +309,19 @@ def tag_packages(conn):
return updated return updated
def build_relations(conn): def build_relations(conn) -> int:
"""Build ene.relations between packages that share concepts across different domains.""" """Build ene.relations between packages that share concepts across domains.
NOTE: This is semantic graph construction logic and must be ported to Lean/AVM.
"""
cur = conn.cursor() cur = conn.cursor()
# Extract all concept terms per package cur.execute(
cur.execute("SELECT pkg, concept_vector, domain FROM ene.packages WHERE concept_vector IS NOT NULL AND jsonb_array_length(concept_vector) > 0") "SELECT pkg, concept_vector, domain FROM ene.packages WHERE concept_vector IS NOT NULL AND jsonb_array_length(concept_vector) > 0"
)
packages = cur.fetchall() packages = cur.fetchall()
log.info("Building relations from %d tagged packages…", len(packages)) log.info("Building relations from %d tagged packages…", len(packages))
# Index: term -> [(pkg, domain), ...]
term_index: dict[str, list[tuple[str, str]]] = {} term_index: dict[str, list[tuple[str, str]]] = {}
for pkg, cv, domain in packages: for pkg, cv, domain in packages:
if cv is None: if cv is None:
@ -288,35 +329,27 @@ def build_relations(conn):
concepts = json.loads(cv) if isinstance(cv, str) else cv concepts = json.loads(cv) if isinstance(cv, str) else cv
for c in concepts: for c in concepts:
term = c["term"].lower() term = c["term"].lower()
if term not in term_index: term_index.setdefault(term, []).append((pkg, domain or "unknown"))
term_index[term] = []
term_index[term].append((pkg, domain or "unknown"))
# Build relations: packages sharing concepts across different domains
relation_count = 0 relation_count = 0
for term, pkgs in term_index.items(): for term, pkgs in term_index.items():
if len(pkgs) < 2: if len(pkgs) < 2:
continue continue
# Pair packages from different domains sharing this term
for i, (p1, d1) in enumerate(pkgs): for i, (p1, d1) in enumerate(pkgs):
for j in range(i + 1, len(pkgs)): for j in range(i + 1, len(pkgs)):
p2, d2 = pkgs[j] p2, d2 = pkgs[j]
if d1 == d2: if d1 == d2:
continue # skip same-domain (already known) continue
# Determine relation type rel_type = "shares_concept"
rel_type = "shares_concept" if d1 != d2 else "co_occurs" cur.execute(
try: """INSERT INTO ene.relations (source_id, target_id, relation_type, weight)
cur.execute( VALUES (%s,%s,%s,1.0)
"""INSERT INTO ene.relations (source_id, target_id, relation_type, weight) ON CONFLICT DO NOTHING""",
VALUES (%s,%s,%s,1.0) (p1, p2, rel_type),
ON CONFLICT DO NOTHING""", )
(p1, p2, rel_type), if cur.rowcount > 0:
) relation_count += 1
if cur.rowcount > 0: if relation_count and relation_count % 2000 == 0:
relation_count += 1
except Exception:
pass
if relation_count % 2000 == 0:
conn.commit() conn.commit()
log.info(" %d relations…", relation_count) log.info(" %d relations…", relation_count)
@ -325,12 +358,16 @@ def build_relations(conn):
return relation_count return relation_count
def score_nspace_kv(conn): def score_nspace_kv(conn) -> int:
"""Compute reduction_reward, sparsity_score, and retention_score for packages.""" """Compute retention scoring for packages.
WARNING: This function currently uses float arithmetic via Postgres casts.
It must be ported into Lean/AVM fixed-point semantics.
"""
cur = conn.cursor() cur = conn.cursor()
# Score based on: concept count (richness), relation count (connectivity), domain uniqueness cur.execute(
cur.execute(""" """
INSERT INTO ene.nspace_kv (value_package_id, reduction_reward, sparsity_score, scar_pressure, retention_score) INSERT INTO ene.nspace_kv (value_package_id, reduction_reward, sparsity_score, scar_pressure, retention_score)
SELECT p.pkg, SELECT p.pkg,
GREATEST(0.1, LEAST(1.0, jsonb_array_length(p.concept_vector) / 50.0)) AS reduction_reward, GREATEST(0.1, LEAST(1.0, jsonb_array_length(p.concept_vector) / 50.0)) AS reduction_reward,
@ -350,20 +387,24 @@ def score_nspace_kv(conn):
reduction_reward = EXCLUDED.reduction_reward, reduction_reward = EXCLUDED.reduction_reward,
sparsity_score = EXCLUDED.sparsity_score, sparsity_score = EXCLUDED.sparsity_score,
retention_score = EXCLUDED.retention_score retention_score = EXCLUDED.retention_score
""") """
)
conn.commit() conn.commit()
cur.execute("SELECT COUNT(*) FROM ene.nspace_kv") cur.execute("SELECT COUNT(*) FROM ene.nspace_kv")
nv = cur.fetchone()[0] nv = int(cur.fetchone()[0])
log.info("Scored %d packages with N-space KV retention", nv) log.info("Scored %d packages with N-space KV retention", nv)
return nv return nv
def run_discovery_queries(conn): def run_discovery_queries(conn) -> dict[str, list[tuple]]:
"""Run cross-source discovery queries and log unexpected groupings.""" """Run cross-source discovery queries and log groupings."""
cur = conn.cursor() cur = conn.cursor()
queries = [ queries = [
("=== DOMAINS SHARING THE MOST CONCEPTS ===", """ (
"domains_sharing_most_concepts",
"""
SELECT r.relation_type, p1.domain AS domain_a, p2.domain AS domain_b, SELECT r.relation_type, p1.domain AS domain_a, p2.domain AS domain_b,
COUNT(*) AS pair_count, COUNT(*) AS pair_count,
COUNT(DISTINCT r.source_id) + COUNT(DISTINCT r.target_id) AS packages_involved COUNT(DISTINCT r.source_id) + COUNT(DISTINCT r.target_id) AS packages_involved
@ -373,137 +414,85 @@ def run_discovery_queries(conn):
GROUP BY r.relation_type, p1.domain, p2.domain GROUP BY r.relation_type, p1.domain, p2.domain
ORDER BY pair_count DESC ORDER BY pair_count DESC
LIMIT 20 LIMIT 20
"""), """,
),
("=== EQUATIONS BRIDGING TO TIDDLYWIKI PAGES ===", """
SELECT eq.pkg AS eq_pkg, LEFT(eq.title, 80) AS equation,
tw.title AS tiddler_title,
r.relation_type
FROM ene.relations r
JOIN ene.packages eq ON eq.pkg = r.source_id AND eq.domain = 'equation_corpus'
JOIN ene.packages tw ON tw.pkg = r.target_id AND tw.domain = 'tiddlywiki'
ORDER BY r.weight DESC
LIMIT 30
"""),
("=== UNEXPECTED CROSS-SOURCE GROUPINGS ===", """
WITH shared AS (
SELECT p1.domain AS d1, p2.domain AS d2, COUNT(*) AS cnt
FROM ene.relations r
JOIN ene.packages p1 ON p1.pkg = r.source_id
JOIN ene.packages p2 ON p2.pkg = r.target_id
WHERE p1.domain != p2.domain
GROUP BY p1.domain, p2.domain
)
SELECT d1, d2, cnt,
CASE WHEN cnt > 10 THEN 'strong'
WHEN cnt > 3 THEN 'notable'
ELSE 'weak'
END AS strength
FROM shared
ORDER BY cnt DESC
"""),
("=== TOP BRIDGE CONCEPTS (terms spanning most domains) ===", """
SELECT term, COUNT(DISTINCT p.domain) AS domain_span,
ARRAY_AGG(DISTINCT p.domain) AS domains
FROM (
SELECT p.domain, (jsonb_array_elements(p.concept_vector)->>'term') AS term
FROM ene.packages p
WHERE p.concept_vector IS NOT NULL AND jsonb_array_length(p.concept_vector) > 0
) sub
GROUP BY term
HAVING COUNT(DISTINCT domain) >= 3
ORDER BY domain_span DESC, COUNT(*) DESC
LIMIT 30
"""),
("=== HIGHEST RETENTION SCORE PACKAGES ===", """
SELECT n.value_package_id, p.title, p.domain, n.retention_score,
n.reduction_reward, n.sparsity_score
FROM ene.nspace_kv n
JOIN ene.packages p ON p.pkg = n.value_package_id
ORDER BY n.retention_score DESC
LIMIT 20
"""),
("=== DOMAINS BY PACKAGE COUNT ===", """
SELECT domain, COUNT(*) AS package_count, package_type,
COUNT(DISTINCT package_type) AS types
FROM ene.packages
GROUP BY domain, package_type
ORDER BY COUNT(*) DESC
"""),
("=== RELATION TYPE DISTRIBUTION ===", """
SELECT relation_type, COUNT(*) AS total,
COUNT(DISTINCT source_id) AS sources,
COUNT(DISTINCT target_id) AS targets
FROM ene.relations
GROUP BY relation_type
ORDER BY total DESC
"""),
] ]
results = {} results: dict[str, list[tuple]] = {}
for title, query in queries: for key, query in queries:
cur.execute(query) cur.execute(query)
rows = cur.fetchall() rows = cur.fetchall()
results[title] = rows results[key] = rows
log.info("\n%s", title) log.info("Query %s returned %d rows", key, len(rows))
for row in rows[:12]:
log.info(" %s", " | ".join(str(c) for c in row))
if len(rows) > 12:
log.info(" ... +%d more rows", len(rows) - 12)
return results return results
# --------------------------------------------------------------------------- def main(argv: list[str] | None = None) -> int:
# Main parser = argparse.ArgumentParser(description="Migrate knowledge.* tables → ENE substrate with concept tagging.")
# --------------------------------------------------------------------------- parser.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA_PATH)
def main(): parser.add_argument("--skip-tagging", action="store_true")
parser.add_argument("--skip-relations", action="store_true")
parser.add_argument("--skip-retention", action="store_true")
args = parser.parse_args(argv)
log.info("Connecting to RDS…") log.info("Connecting to RDS…")
conn = connect() conn = connect()
conn.autocommit = False conn.autocommit = False
log.info("Phase 1: Apply ENE substrate schema") try:
apply_schema(conn) log.info("Phase 1: Apply ENE substrate schema")
apply_schema(conn, args.schema)
log.info("Phase 2: Migrate knowledge.* → ene.packages") log.info("Phase 2: Migrate knowledge.* → ene.packages")
total = migrate_all_sources(conn) migrate_all_sources(conn)
log.info("Phase 3: Extract concepts and tag packages") if not args.skip_tagging:
tagged = tag_packages(conn) log.info("Phase 3: Extract concepts and tag packages")
tag_packages(conn)
log.info("Phase 4: Build cross-domain relations") if not args.skip_relations:
relations = build_relations(conn) log.info("Phase 4: Build cross-domain relations")
build_relations(conn)
log.info("Phase 5: Score N-space KV retention") if not args.skip_retention:
nv_scored = score_nspace_kv(conn) log.info("Phase 5: Score N-space KV retention")
score_nspace_kv(conn)
log.info("Phase 6: Run discovery queries") log.info("Phase 6: Run discovery queries")
results = run_discovery_queries(conn) run_discovery_queries(conn)
# Summary log.info("Done.")
cur = conn.cursor() return 0
cur.execute("SELECT COUNT(*) FROM ene.packages")
pkg_count = cur.fetchone()[0]
cur.execute("SELECT COUNT(*) FROM ene.relations")
rel_count = cur.fetchone()[0]
cur.execute("SELECT domain, COUNT(*) FROM ene.packages GROUP BY domain ORDER BY COUNT(*) DESC")
domains = cur.fetchall()
conn.close()
log.info("\n=== SUMMARY ===") finally:
log.info("Packages: %d", pkg_count) conn.close()
log.info("Relations: %d", rel_count)
log.info("N-space KV scored: %d", nv_scored)
log.info("Domains:")
for d, c in domains:
log.info(" %s: %d", d, c)
log.info("Done.")
if __name__ == "__main__": if __name__ == "__main__":
main() raise SystemExit(main())
# ---------------------------------------------------------------------------
# Shim strip receipt (non-authoritative conversion surface)
# ---------------------------------------------------------------------------
STRIP_RECEIPT = {
"ontology_version": ONTOLOGY_VERSION,
"shim_role": "legacy_ene_migration_and_tagging_pending_avm",
"computed_in_shim": [
"tokenize / concept tagging",
"relation building",
"retention scoring",
"schema orchestration",
],
"must_port_to_lean_avm": [
"tokenization policy (finite types, no open strings)",
"relation scoring (fixed-point)",
"retention scoring (fixed-point)",
"all semantic decisions",
],
"float_policy": {
"status": "float_present_in_sql_scoring",
"reason": "ene.nspace_kv scoring uses Postgres float casts; must be ported",
},
}