mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
528 lines
18 KiB
Python
528 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
# ==============================================================================
|
|
# COPYRIGHT NO ONE EVERYWHERE LLC (WYOMING HOLDING COMPANY)
|
|
# PROJECT: SOVEREIGN STACK
|
|
# This artifact is entirely proprietary and cryptographically proven.
|
|
# Open-Source usage requires explicit permission from Brandon Scott Schneider.
|
|
# ==============================================================================
|
|
# PTOS: LAYER=RULE / DOMAIN=DATA / CONDITION=STABLE / STAGE=ACTIVE / SOURCE=CODE
|
|
"""
|
|
Substrate Index Repair Pass
|
|
===========================
|
|
Fixes several categories of data point mismatches:
|
|
|
|
1. Backfill meta_capsule_hash — 231 capsules without SHA256
|
|
2. Derive missing idea_weights + concept_anchor + foam_score
|
|
for nodes that have descriptions/tags but no semantic weights
|
|
3. Backfill compression_ratio into waveprobe_sig dicts where derivable
|
|
4. Audit tier/domain overload
|
|
5. Reclassify legacy DATA or NULL PTOS domains when a deterministic
|
|
operational-domain inference is confident enough
|
|
|
|
Does NOT promote RESEARCH → canonical tiers (that requires human review)
|
|
but it does repair operational-domain catch-alls so the backlog is smaller
|
|
and more queryable.
|
|
|
|
Usage:
|
|
python3 5-Applications/scripts/repair_index.py [--dry-run]
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(REPO / "scripts"))
|
|
|
|
from substrate_git_index import _open_db, _PHI, _concept_vector_from_weights
|
|
|
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
|
|
|
_STOP = {"the","and","for","that","this","with","from","are","was","were",
|
|
"have","been","will","would","could","should","which","their","there",
|
|
"these","those","what","when","where","how","all","any","can","not",
|
|
"but","also","its","into","over","each","more","other","used","using"}
|
|
|
|
_PTOS_DOMAINS = (
|
|
"COMPUTE",
|
|
"TOKEN",
|
|
"RULE",
|
|
"STORE",
|
|
"POWER",
|
|
"COMMS",
|
|
"MATERIAL",
|
|
"DATA",
|
|
"CLOCK",
|
|
"TEST",
|
|
)
|
|
|
|
_PTOS_DOMAIN_KEYWORDS: dict[str, set[str]] = {
|
|
"COMPUTE": {
|
|
"compute", "compression", "codec", "encode", "decode", "algorithm",
|
|
"runtime", "kernel", "vm", "opcode", "instruction", "graph", "dag",
|
|
"lut", "tsm", "substrate", "metafoam", "hutter", "script", "source",
|
|
"interface", "executor", "smoother", "math", "mathematics",
|
|
"geometry", "topology", "theorem", "proof", "equation", "invariant",
|
|
"torsion", "mobius", "polytope", "patamathematics", "pataphysics",
|
|
},
|
|
"TOKEN": {
|
|
"token", "omnitoken", "wallet", "mint", "burn", "settlement",
|
|
},
|
|
"RULE": {
|
|
"rule", "govern", "legal", "ethic", "policy", "constraint", "audit",
|
|
"patent", "license", "compliance", "safety", "protocol", "rights",
|
|
"eula", "review",
|
|
},
|
|
"STORE": {
|
|
"store", "storage", "archive", "manifest", "capsule", "vault",
|
|
"database", "sqlite", "index", "query", "schema", "table", "row",
|
|
"column", "persist", "metadata", "fingerprint", "build", "target",
|
|
},
|
|
"POWER": {
|
|
"power", "energy", "thermal", "heat", "battery", "voltage", "current",
|
|
"econom", "market", "incentive", "budget", "bom",
|
|
},
|
|
"COMMS": {
|
|
"comms", "communic", "network", "protocol", "radio", "signal",
|
|
"packet", "route", "mesh", "http", "api", "bridge", "bus", "ipc",
|
|
"transport",
|
|
},
|
|
"MATERIAL": {
|
|
"material", "chem", "molecule", "atomic", "atom", "orbital",
|
|
"superconductor", "silicon", "wafer", "pcb", "hdl", "asic", "chip",
|
|
"lithography", "nanowire", "qchem", "hardware", "risc", "kda",
|
|
},
|
|
"DATA": {
|
|
"data", "dataset", "json", "csv", "session", "transcript",
|
|
"document", "text", "note", "record", "corpus", "import", "chat",
|
|
"summary",
|
|
},
|
|
"CLOCK": {
|
|
"clock", "time", "timing", "tick", "latency", "temporal", "phase",
|
|
"schedule",
|
|
},
|
|
"TEST": {
|
|
"test", "tests", "testing", "validation", "verify", "verified",
|
|
"benchmark", "assert", "rigor", "experiment", "ab_test", "run_matrix",
|
|
},
|
|
}
|
|
|
|
_CONCEPT_TO_PTOS = {
|
|
"compression": "COMPUTE",
|
|
"substrate": "COMPUTE",
|
|
"mathematics": "COMPUTE",
|
|
"computation": "COMPUTE",
|
|
"governance": "RULE",
|
|
"containment": "RULE",
|
|
"cryptography": "RULE",
|
|
"economics": "POWER",
|
|
"physics": "MATERIAL",
|
|
"chemistry": "MATERIAL",
|
|
"biology": "MATERIAL",
|
|
"hardware": "MATERIAL",
|
|
"linguistics": "COMMS",
|
|
"geography": "DATA",
|
|
"music": "DATA",
|
|
"neuroscience": "DATA",
|
|
"research": "DATA",
|
|
}
|
|
|
|
_PTOS_HEADER_RE = re.compile(
|
|
r"PTOS:\s*LAYER=\w+\s*/\s*DOMAIN=(?P<domain>\w+)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
def _extract_weights(text: str) -> dict:
|
|
tokens = re.findall(r"\b[a-zA-Z][a-zA-Z_]{4,}\b", text.lower())
|
|
counts = Counter(t for t in tokens if t not in _STOP)
|
|
if not counts:
|
|
return {}
|
|
max_c = counts.most_common(1)[0][1]
|
|
weights = {}
|
|
for word, count in counts.most_common(25):
|
|
score = min(1.0, (count / max_c) * 0.85)
|
|
if score >= 0.10:
|
|
weights[word] = round(score, 3)
|
|
return weights
|
|
|
|
|
|
def _read_head(path: Path, limit: int = 8192) -> str:
|
|
try:
|
|
with open(path, "rb") as f:
|
|
raw = f.read(limit)
|
|
return raw.decode("utf-8", errors="replace")
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _load_json(raw: str | None):
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return json.loads(raw)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _keyword_score(tokens: Counter[str], keywords: set[str]) -> float:
|
|
score = 0.0
|
|
for token, count in tokens.items():
|
|
for keyword in keywords:
|
|
if token.startswith(keyword):
|
|
score += min(count, 6)
|
|
break
|
|
return score
|
|
|
|
|
|
def _infer_domain_from_row(
|
|
pkg: str,
|
|
module: str | None,
|
|
archetype: str | None,
|
|
description: str | None,
|
|
tags_raw: str | None,
|
|
files_raw: str | None,
|
|
session_id: str | None,
|
|
concept_anchor_raw: str | None,
|
|
) -> tuple[str, float]:
|
|
"""Infer a better PTOS domain for legacy DATA rows."""
|
|
tags = _load_json(tags_raw) or []
|
|
files = _load_json(files_raw) or []
|
|
anchor = _load_json(concept_anchor_raw) or {}
|
|
|
|
candidate_refs: list[str] = []
|
|
if session_id:
|
|
candidate_refs.append(session_id)
|
|
candidate_refs.extend(str(f) for f in files[:8])
|
|
|
|
resolved_paths: list[Path] = []
|
|
seen: set[str] = set()
|
|
for raw in candidate_refs:
|
|
if not raw:
|
|
continue
|
|
p = Path(raw)
|
|
probes = [p]
|
|
if not p.is_absolute():
|
|
probes.append(REPO / p)
|
|
for probe in probes:
|
|
try:
|
|
rp = probe.resolve()
|
|
except Exception:
|
|
rp = probe
|
|
key = str(rp)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
if rp.exists():
|
|
resolved_paths.append(rp)
|
|
break
|
|
|
|
header_domain = None
|
|
content_snippets: list[str] = []
|
|
for path in resolved_paths[:3]:
|
|
head = _read_head(path)
|
|
if head:
|
|
content_snippets.append(head)
|
|
if header_domain is None:
|
|
m = _PTOS_HEADER_RE.search(head)
|
|
if m:
|
|
domain = m.group("domain").upper()
|
|
if domain in _PTOS_DOMAINS:
|
|
header_domain = domain
|
|
if header_domain is not None:
|
|
return header_domain, 100.0
|
|
|
|
path_hint_parts: list[str] = []
|
|
for path in resolved_paths:
|
|
if path.suffix == ".json" and path.name.endswith(".metadata.json"):
|
|
continue
|
|
try:
|
|
rel = path.relative_to(REPO)
|
|
path_hint_parts.append(str(rel))
|
|
except Exception:
|
|
path_hint_parts.append(str(path))
|
|
path_hint = " ".join(path_hint_parts).lower()
|
|
|
|
if any(part in path_hint for part in ("/target/", ".fingerprint/", "build-script", "run-build-script", ".rustc_info")):
|
|
return "STORE", 12.0
|
|
if any(part in path_hint for part in ("/src/", ".rs", ".py", ".c", ".sh")) and not any(
|
|
part in path_hint for part in ("/tests/", "validation", "benchmark", "ab_test")
|
|
):
|
|
# Source files default to COMPUTE unless strong test/rule cues override.
|
|
early_compute = True
|
|
else:
|
|
early_compute = False
|
|
|
|
if any(part in path_hint for part in ("validation", "verification", "/tests/", "benchmark", "ab_test", "run_matrix")):
|
|
return "TEST", 11.0
|
|
if any(part in path_hint for part in ("audit", "review", "policy", "legal", "ethic", "patent", "license", "safety_protocol", "safety")):
|
|
return "RULE", 10.0
|
|
if any(part in path_hint for part in ("bom", "pcb", "hdl", "asic", "chip", "wafer", "material", "qchem")):
|
|
return "MATERIAL", 10.0
|
|
if any(part in path_hint for part in ("manifest", "vault", "storage", "sqlite", "index", "archive_summary")):
|
|
return "STORE", 10.0
|
|
if early_compute:
|
|
return "COMPUTE", 9.5
|
|
|
|
text = " ".join(
|
|
filter(
|
|
None,
|
|
[
|
|
pkg,
|
|
module or "",
|
|
archetype or "",
|
|
description or "",
|
|
" ".join(map(str, tags)),
|
|
str(anchor.get("domain", "")),
|
|
path_hint,
|
|
*content_snippets,
|
|
],
|
|
)
|
|
).lower()
|
|
tokens = Counter(re.findall(r"\b[a-zA-Z][a-zA-Z_]{2,}\b", text))
|
|
weights = _extract_weights(text)
|
|
|
|
scores = {domain: 0.0 for domain in _PTOS_DOMAINS}
|
|
for domain, keywords in _PTOS_DOMAIN_KEYWORDS.items():
|
|
scores[domain] += _keyword_score(tokens, keywords)
|
|
for word, weight in weights.items():
|
|
if any(word.startswith(keyword) for keyword in keywords):
|
|
scores[domain] += 2.0 * float(weight)
|
|
|
|
if early_compute:
|
|
scores["COMPUTE"] += 4.0
|
|
|
|
anchor_domain = str(anchor.get("domain", "")).lower()
|
|
mapped = _CONCEPT_TO_PTOS.get(anchor_domain)
|
|
if mapped:
|
|
scores[mapped] += 3.0
|
|
|
|
best = max(scores, key=lambda d: scores[d])
|
|
if scores[best] <= 0.0:
|
|
return "DATA", 0.0
|
|
return best, scores[best]
|
|
|
|
def _detect_anchor(pkg: str, domain: str, description: str, tags_raw) -> dict:
|
|
text = f"{pkg} {description or ''}".lower()
|
|
tags = json.loads(tags_raw) if tags_raw else []
|
|
tag_str = " ".join(tags).lower()
|
|
|
|
if any(k in text+tag_str for k in ("hash","merkle","sha256","dag","crypto","attest")):
|
|
dom = "compression"
|
|
elif any(k in text+tag_str for k in ("hardware","kda","circuit","pcb","hdl","risc")):
|
|
dom = "hardware"
|
|
elif any(k in text+tag_str for k in ("hutter","enwik","compress","codec","lzma","entropy")):
|
|
dom = "compression"
|
|
elif any(k in text+tag_str for k in ("storage","store","archive","dedup","lut")):
|
|
dom = "substrate"
|
|
elif domain and domain != "DATA":
|
|
dom = domain.lower()
|
|
else:
|
|
dom = "research"
|
|
|
|
return {"domain": dom, "concept": re.sub(r"[^a-z0-9]+","_",pkg)[:60], "resolution": "FORMING"}
|
|
|
|
|
|
# ── repair passes ─────────────────────────────────────────────────────────────
|
|
|
|
def pass_capsule_hash(conn, dry_run: bool) -> int:
|
|
"""Compute SHA256 of meta_capsule string for all rows missing the hash."""
|
|
rows = conn.execute(
|
|
"SELECT rowid, pkg, meta_capsule FROM packages "
|
|
"WHERE meta_capsule IS NOT NULL AND meta_capsule != '' "
|
|
"AND (meta_capsule_hash IS NULL OR meta_capsule_hash = '')"
|
|
).fetchall()
|
|
|
|
updated = 0
|
|
for rowid, pkg, capsule in rows:
|
|
h = hashlib.sha256(capsule.encode()).hexdigest()
|
|
if not dry_run:
|
|
conn.execute(
|
|
"UPDATE packages SET meta_capsule_hash=? WHERE rowid=?", (h, rowid)
|
|
)
|
|
updated += 1
|
|
|
|
if not dry_run:
|
|
conn.commit()
|
|
print(f" capsule_hash: backfilled {updated} rows")
|
|
return updated
|
|
|
|
|
|
def pass_compression_ratio(conn, dry_run: bool) -> int:
|
|
"""Derive compression_ratio = 8 / bpb_zlib and write back into meta_capsule JSON."""
|
|
rows = conn.execute(
|
|
"SELECT rowid, pkg, meta_capsule FROM packages "
|
|
"WHERE meta_capsule LIKE '%waveprobe_sig%'"
|
|
).fetchall()
|
|
|
|
updated = 0
|
|
for rowid, pkg, capsule in rows:
|
|
try:
|
|
d = json.loads(capsule)
|
|
except Exception:
|
|
continue
|
|
ws = d.get("waveprobe_sig", {})
|
|
if ws.get("compression_ratio") is not None:
|
|
continue
|
|
bpb = ws.get("bpb_zlib")
|
|
if not bpb or bpb <= 0:
|
|
continue
|
|
ws["compression_ratio"] = round(8.0 / bpb, 4)
|
|
new_capsule = json.dumps(d)
|
|
new_hash = hashlib.sha256(new_capsule.encode()).hexdigest()
|
|
if not dry_run:
|
|
conn.execute(
|
|
"UPDATE packages SET meta_capsule=?, meta_capsule_hash=? WHERE rowid=?",
|
|
(new_capsule, new_hash, rowid)
|
|
)
|
|
updated += 1
|
|
|
|
if not dry_run:
|
|
conn.commit()
|
|
print(f" compression_ratio: derived for {updated} rows")
|
|
return updated
|
|
|
|
|
|
def pass_semantic_weights(conn, dry_run: bool) -> int:
|
|
"""Rebuild idea_weights + concept_anchor + foam_score for nodes missing them."""
|
|
rows = conn.execute(
|
|
"SELECT rowid, pkg, domain, description, tags, files "
|
|
"FROM packages "
|
|
"WHERE idea_weights IS NULL OR idea_weights = '' OR idea_weights = '{}'"
|
|
).fetchall()
|
|
|
|
updated = 0
|
|
for rowid, pkg, domain, description, tags_raw, files_raw in rows:
|
|
# Build text signal from all available fields
|
|
files = json.loads(files_raw) if files_raw else []
|
|
file_stems = " ".join(Path(f).stem.replace("-","_").replace(" ","_")
|
|
for f in files)
|
|
text = f"{pkg} {description or ''} {file_stems}"
|
|
|
|
weights = _extract_weights(text)
|
|
if not weights:
|
|
# Minimal fallback from pkg name tokens
|
|
weights = {t: 0.5 for t in re.findall(r"[a-z]{4,}", pkg.lower())[:5]}
|
|
if not weights:
|
|
print(f" [skip] {pkg}: no extractable weights")
|
|
continue
|
|
|
|
vec = _concept_vector_from_weights(weights)
|
|
anchor = _detect_anchor(pkg, domain, description or "", tags_raw)
|
|
foam = round(sum(weights.values()) / max(len(weights), 1) * _PHI, 4)
|
|
|
|
if not dry_run:
|
|
conn.execute(
|
|
"UPDATE packages SET idea_weights=?, concept_vector=?, "
|
|
"concept_anchor=?, foam_score=? WHERE rowid=?",
|
|
(json.dumps(weights), json.dumps(vec),
|
|
json.dumps(anchor), foam, rowid)
|
|
)
|
|
updated += 1
|
|
print(f" rebuilt: {pkg} foam={foam:.3f} "
|
|
f"axes={sum(1 for x in vec if x > 0.01)}/14")
|
|
|
|
if not dry_run:
|
|
conn.commit()
|
|
print(f" semantic_weights: rebuilt for {updated} rows")
|
|
return updated
|
|
|
|
|
|
def pass_tier_domain_audit(conn) -> None:
|
|
"""Report tier/domain overload — does not modify data."""
|
|
total = conn.execute("SELECT COUNT(*) FROM packages").fetchone()[0]
|
|
research = conn.execute(
|
|
"SELECT COUNT(*) FROM packages WHERE tier='RESEARCH'"
|
|
).fetchone()[0]
|
|
data_dom = conn.execute(
|
|
"SELECT COUNT(*) FROM packages WHERE domain='DATA'"
|
|
).fetchone()[0]
|
|
print(f" RESEARCH tier: {research}/{total} "
|
|
f"({100*research//total}%) — schema says transitional/intake only")
|
|
print(f" DATA domain: {data_dom}/{total} "
|
|
f"({100*data_dom//total}%) — schema expects domain-specific classification")
|
|
print(f" Action needed: human tier-promotion pass "
|
|
f"(RESEARCH → SINGULARITY/PLASMA/CRYSTALLINE/FOAM/GOVERNANCE)")
|
|
|
|
# Show canonical tier distribution for non-RESEARCH rows
|
|
rows = conn.execute(
|
|
"SELECT tier, COUNT(*) FROM packages WHERE tier != 'RESEARCH' "
|
|
"GROUP BY tier ORDER BY 2 DESC"
|
|
).fetchall()
|
|
print(f" Promoted nodes already:")
|
|
for tier, n in rows:
|
|
print(f" {tier}: {n}")
|
|
|
|
|
|
def pass_data_domain_reclassify(conn, dry_run: bool) -> int:
|
|
"""Promote legacy DATA/NULL rows to more specific PTOS domains when confident."""
|
|
rows = conn.execute(
|
|
"SELECT rowid, pkg, module, archetype, description, tags, files, "
|
|
"session_id, concept_anchor "
|
|
"FROM packages WHERE domain='DATA' OR domain IS NULL OR domain=''"
|
|
).fetchall()
|
|
|
|
updated = 0
|
|
for rowid, pkg, module, archetype, description, tags_raw, files_raw, session_id, concept_anchor_raw in rows:
|
|
inferred, score = _infer_domain_from_row(
|
|
pkg, module, archetype, description, tags_raw, files_raw, session_id, concept_anchor_raw
|
|
)
|
|
if inferred == "DATA":
|
|
continue
|
|
if score < 6.0:
|
|
continue
|
|
if not dry_run:
|
|
conn.execute(
|
|
"UPDATE packages SET domain=? WHERE rowid=?",
|
|
(inferred, rowid),
|
|
)
|
|
updated += 1
|
|
|
|
if not dry_run:
|
|
conn.commit()
|
|
print(f" data_domain_reclassify: promoted {updated} rows")
|
|
return updated
|
|
|
|
|
|
# ── main ─────────────────────────────────────────────────────────────────────
|
|
|
|
def run(dry_run: bool = False) -> None:
|
|
conn = _open_db()
|
|
|
|
print(f"Substrate Index Repair {'[DRY RUN]' if dry_run else '[LIVE]'}")
|
|
print(f"DB: {REPO / 'substrate_index.db'}")
|
|
total = conn.execute("SELECT COUNT(*) FROM packages").fetchone()[0]
|
|
print(f"Total rows: {total}\n")
|
|
|
|
print("Pass 1: meta_capsule_hash backfill")
|
|
pass_capsule_hash(conn, dry_run)
|
|
|
|
print("\nPass 2: compression_ratio derivation")
|
|
pass_compression_ratio(conn, dry_run)
|
|
|
|
print("\nPass 3: semantic weights rebuild")
|
|
pass_semantic_weights(conn, dry_run)
|
|
|
|
print("\nPass 4: tier/domain overload audit")
|
|
pass_tier_domain_audit(conn)
|
|
|
|
print("\nPass 5: legacy DATA domain reclassification")
|
|
pass_data_domain_reclassify(conn, dry_run)
|
|
|
|
print("\nPost-pass audit")
|
|
pass_tier_domain_audit(conn)
|
|
|
|
conn.close()
|
|
print("\nDone.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--dry-run", action="store_true")
|
|
args = p.parse_args()
|
|
run(dry_run=args.dry_run)
|