mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
390 lines
17 KiB
Python
390 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""Build a custom-equation awareness manifest for the local LLM.
|
||
|
||
This script inventories equation-bearing artifacts across the Research Stack and
|
||
turns them into compact curriculum records. The goal is awareness and routing,
|
||
not proof: every extracted equation keeps its source path, line/key, hash,
|
||
claim boundary, and primitive/bucket hints when available.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
|
||
DEFAULT_ROOTS = [
|
||
Path("4-Infrastructure/shim"),
|
||
Path("0-Core-Formalism/otom"),
|
||
Path("0-Core-Formalism/lean/Semantics/Semantics"),
|
||
Path("6-Documentation/tiddlywiki-local/wiki/tiddlers"),
|
||
Path("6-Documentation/docs"),
|
||
Path("6-Documentation/papers/OTOM"),
|
||
]
|
||
|
||
NAME_PATTERNS = [
|
||
"*equation*",
|
||
"*Equation*",
|
||
"*SMN*",
|
||
"*Semantic Mass*",
|
||
"*4primitive*",
|
||
"*MasterEquation*",
|
||
"*EquationTranslation*",
|
||
"*FieldEquation*",
|
||
"*GCLField*",
|
||
"*HachimojiEquation*",
|
||
"*WitnessGrammar*",
|
||
"*UnderversePacket*",
|
||
]
|
||
|
||
TEXT_SUFFIXES = {".md", ".tid", ".lean", ".tex", ".txt", ".mmd"}
|
||
JSON_SUFFIXES = {".json"}
|
||
|
||
LINE_RE = re.compile(
|
||
r"(equation|formula|display|display_equation|master equation|semantic mass number|SMN|u_t|argmin|min_|def\s+|structure\s+|inductive\s+|abbrev\s+)",
|
||
re.IGNORECASE,
|
||
)
|
||
SYMBOLIC_LINE_MARKERS = ("ρ(", "G =", "G=", "Γ", "C =", "C=", "AᵀA", "UΛUᵀ")
|
||
|
||
STOP_PATH_PARTS = {"__pycache__"}
|
||
GENERATED_NAME_MARKERS = (
|
||
"_receipt.json",
|
||
"_curriculum.jsonl",
|
||
"_manifest.jsonl",
|
||
"physics_math_llm_sft.jsonl",
|
||
)
|
||
|
||
|
||
def sha256_bytes(data: bytes) -> str:
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def safe_read(path: Path) -> bytes:
|
||
return path.read_bytes()
|
||
|
||
|
||
def iter_candidate_files(roots: list[Path]) -> list[Path]:
|
||
seen: set[Path] = set()
|
||
candidates: list[Path] = []
|
||
for root in roots:
|
||
if not root.exists():
|
||
continue
|
||
for pattern in NAME_PATTERNS:
|
||
for path in root.rglob(pattern):
|
||
if not path.is_file():
|
||
continue
|
||
if any(part in STOP_PATH_PARTS for part in path.parts):
|
||
continue
|
||
if any(marker in path.name for marker in GENERATED_NAME_MARKERS):
|
||
continue
|
||
if path.suffix not in TEXT_SUFFIXES | JSON_SUFFIXES:
|
||
continue
|
||
if path not in seen:
|
||
seen.add(path)
|
||
candidates.append(path)
|
||
return sorted(candidates)
|
||
|
||
|
||
def classify_text(text: str) -> str:
|
||
lower = text.lower()
|
||
stripped = lower.strip()
|
||
if "semantic mass number" in lower or "smn" in lower:
|
||
return "semantic_load"
|
||
if stripped in {r"\begin{equation}", r"\end{equation}", "display_equation", "equation forest:"}:
|
||
return "registry_schema"
|
||
if any(term in lower for term in ["receipt:", "status:", "type ", "display_equation: string", "equation_id", "source_path", "durable source", "kernel id", "registry role"]):
|
||
return "registry_schema"
|
||
if lower.startswith(("# ", "## ", "### ", "!! ")) or lower.endswith(" equations") or lower.endswith(" equation"):
|
||
return "heading_or_doc"
|
||
if any(term in lower for term in ["equation sniffer", "sniffer", "scent", "witness grammar", "equation graphs", "probe candidate equation regions", "residual motifs", "adapter candidates"]):
|
||
return "equation_sniffer"
|
||
if any(term in lower for term in ["equation forest", "kernel registry", "kernelclass", "equationkernel", "kernelassignment", "fuse_existing_equation_kernels"]):
|
||
return "equation_forest_control"
|
||
if any(term in lower for term in ["equation type:", "geocognition", "publicclaim", "external validation gate", "quarantine bin", "confidence cap"]):
|
||
return "equation_atlas"
|
||
if any(term in lower for term in ["betti", "homology", "rank h_", "rank h", "hole", "loop", "cavity", "underverse", "complement-space", "forbidden", "excluded", "residual", "gamma(g)", "torsion", "curvature", "zeta(1/2"]):
|
||
return "topology"
|
||
if any(term in lower for term in ["u_t", "u_x", "u_xx", "navier", "stokes", "burgers", "viscous", "fluid", "laplacian", "partial", "gradient", "torque", "tau_", "theta", "wave equation"]):
|
||
return "pde_dynamics"
|
||
if any(term in lower for term in ["selector", "metamaterial", "polarization", "chirality", "spin", "material", "active", "phase circulation", "eta_thg", "psi_g", "gouy"]):
|
||
return "material_selector"
|
||
if any(term in lower for term in ["diat", "sidon", "shell", "k²", "sqrt", "floor", "a(n)", "b(n)", "genome18", "18-bit"]):
|
||
return "integer_geometry"
|
||
if any(term in lower for term in ["s(t)", "w_i", "h_i", "spiking", "neural", "activation", "surprise", "regret", "softmax"]):
|
||
return "neural_signal"
|
||
if any(term in lower for term in ["route", "routing", "warden", "promotion gate", "claim_boundary", "validation cap", "passes:", "fails:"]):
|
||
return "routing_control"
|
||
if any(term in lower for term in ["metadata_first", "preview", "learned shortcut", "king context", "retrieval", "adr-like", "search metadata"]):
|
||
return "retrieval_control"
|
||
if any(term in lower for term in ["qwen", "gemini", "kimi", "model indexing", "derived the equation", "requested synthesis"]):
|
||
return "external_model_workbench"
|
||
if any(term in lower for term in ["shannon", "entropy", "kolmogorov", "mdl", "zipf", "bwt", "hutter", "compression", "bits", "character"]):
|
||
return "information_theory"
|
||
if any(term in text for term in ["C_{ij}", "Λ", "UΛU", "lambda", "\\lambda", "eigen", "spectral", "zeta"]):
|
||
return "spectral"
|
||
if any(term in lower for term in ["ρ(", "density", "potential", "field", "manifold"]):
|
||
return "field"
|
||
if any(term in lower for term in ["g = a", "metric", "distance", "deformation", "shear", "a_{ij}"]):
|
||
return "shear"
|
||
if any(term in lower for term in ["γ", "packet", "codec", "encoding", "ans", "bitpack", "gcl"]):
|
||
return "packet"
|
||
if any(term in lower for term in ["basis", "qubo", "argmin"]):
|
||
return "spectral"
|
||
if any(term in lower for term in ["lean", "def ", "theorem", "lemma", "native_decide"]):
|
||
return "formal"
|
||
if "=" in text and any(marker in text for marker in ["\\", "_", "^", "sum", "Σ", "∑", "(", ")", "{", "}"]):
|
||
return "math_kernel"
|
||
return "sniffer_candidate"
|
||
|
||
|
||
def boundary_for(path: Path, text: str) -> str:
|
||
lower = text.lower()
|
||
if "hold" in lower or "blocked_usage" in lower or "blocked claim" in lower:
|
||
return "hold-or-routing-prior"
|
||
if path.suffix == ".lean":
|
||
return "lean-source-prior; build required before proof promotion"
|
||
if "conjecture" in lower:
|
||
return "conjecture-prior-only"
|
||
return "equation-awareness-prior-only"
|
||
|
||
|
||
def add_record(records: list[dict[str, Any]], *, source_path: Path, source_hash: str, kind: str, name: str, equation: str, locator: str, metadata: dict[str, Any] | None = None) -> None:
|
||
equation = " ".join(str(equation).split())
|
||
if not equation:
|
||
return
|
||
primitive_hint = classify_text(equation + " " + json.dumps(metadata or {}, ensure_ascii=False))
|
||
record = {
|
||
"id": f"{source_path}:{locator}:{name}",
|
||
"source_path": str(source_path),
|
||
"source_hash": source_hash,
|
||
"kind": kind,
|
||
"name": name[:160],
|
||
"equation": equation[:2000],
|
||
"equation_hash": hashlib.sha256(equation.encode("utf-8")).hexdigest(),
|
||
"locator": locator,
|
||
"primitive_hint": primitive_hint,
|
||
"claim_boundary": boundary_for(source_path, equation + " " + json.dumps(metadata or {}, ensure_ascii=False)),
|
||
}
|
||
if metadata:
|
||
record["metadata"] = metadata
|
||
records.append(record)
|
||
|
||
|
||
def walk_json_equations(value: Any, path: list[str] | None = None) -> Iterable[tuple[list[str], str, Any]]:
|
||
path = path or []
|
||
if isinstance(value, dict):
|
||
for key, child in value.items():
|
||
lower = str(key).lower()
|
||
if lower in {"equation", "formula", "display", "display_equation", "statement"} and isinstance(child, (str, int, float)):
|
||
yield path + [str(key)], str(key), child
|
||
elif lower in {"axioms", "unified_equations", "scientific_equations", "system_equations", "erdos_problems", "kernels", "primitives"}:
|
||
yield from walk_json_equations(child, path + [str(key)])
|
||
else:
|
||
yield from walk_json_equations(child, path + [str(key)])
|
||
elif isinstance(value, list):
|
||
for idx, child in enumerate(value):
|
||
yield from walk_json_equations(child, path + [str(idx)])
|
||
|
||
|
||
def extract_json(path: Path, source_hash: str, records: list[dict[str, Any]]) -> None:
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return
|
||
for key_path, key, equation in walk_json_equations(data):
|
||
parent = data
|
||
for part in key_path[:-1]:
|
||
try:
|
||
parent = parent[int(part)] if isinstance(parent, list) else parent[part]
|
||
except Exception:
|
||
parent = {}
|
||
break
|
||
name = (
|
||
parent.get("name")
|
||
if isinstance(parent, dict)
|
||
else None
|
||
) or (
|
||
parent.get("kernel_id")
|
||
if isinstance(parent, dict)
|
||
else None
|
||
) or ".".join(key_path[-4:])
|
||
metadata = {}
|
||
if isinstance(parent, dict):
|
||
for meta_key in (
|
||
"primitive",
|
||
"mapping",
|
||
"domain",
|
||
"domain_class",
|
||
"bucket",
|
||
"hyper_term",
|
||
"claim_state",
|
||
"authority_scope",
|
||
"blocked_usage",
|
||
"blocked_usages",
|
||
"functional_role",
|
||
"feasibility",
|
||
"approach",
|
||
):
|
||
if meta_key in parent:
|
||
metadata[meta_key] = parent[meta_key]
|
||
add_record(
|
||
records,
|
||
source_path=path,
|
||
source_hash=source_hash,
|
||
kind="json_equation",
|
||
name=str(name),
|
||
equation=str(equation),
|
||
locator=".".join(key_path),
|
||
metadata=metadata,
|
||
)
|
||
|
||
|
||
def extract_text(path: Path, source_hash: str, records: list[dict[str, Any]]) -> None:
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
for line_no, line in enumerate(text.splitlines(), start=1):
|
||
stripped = line.strip()
|
||
if not stripped or len(stripped) < 6:
|
||
continue
|
||
if path.suffix == ".tex" and (
|
||
stripped.startswith("\\begin{")
|
||
or stripped.startswith("\\end{")
|
||
or stripped.startswith("\\label{")
|
||
or stripped.startswith("\\title{")
|
||
or stripped.startswith("\\section{")
|
||
or stripped.startswith("\\subsection{")
|
||
):
|
||
continue
|
||
if not LINE_RE.search(stripped) and not any(marker in stripped for marker in SYMBOLIC_LINE_MARKERS):
|
||
continue
|
||
if stripped.startswith(("import ", "open ", "namespace ", "end ")):
|
||
continue
|
||
name = f"line_{line_no}"
|
||
lean_match = re.match(r"(def|structure|inductive|abbrev|theorem|lemma)\s+([A-Za-z0-9_'.]+)", stripped)
|
||
if lean_match:
|
||
name = f"{lean_match.group(1)}_{lean_match.group(2)}"
|
||
heading = re.match(r"^#+\s+(.+)$", stripped)
|
||
if heading:
|
||
name = heading.group(1)
|
||
add_record(
|
||
records,
|
||
source_path=path,
|
||
source_hash=source_hash,
|
||
kind="text_equation_line",
|
||
name=name,
|
||
equation=stripped,
|
||
locator=f"line:{line_no}",
|
||
metadata={"line": line_no, "suffix": path.suffix},
|
||
)
|
||
|
||
|
||
def curriculum_records(receipt: dict[str, Any], per_bucket_limit: int) -> list[dict[str, Any]]:
|
||
system = "You are a custom-equation-aware routing model. Return compact JSON with source and claim boundaries."
|
||
by_primitive: dict[str, list[dict[str, Any]]] = {}
|
||
for record in receipt["equations"]:
|
||
by_primitive.setdefault(record["primitive_hint"], []).append(record)
|
||
selected: list[dict[str, Any]] = []
|
||
low_value_limits = {"heading_or_doc": 12, "registry_schema": 16}
|
||
for primitive, items in sorted(by_primitive.items()):
|
||
selected.extend(items[: low_value_limits.get(primitive, per_bucket_limit)])
|
||
records = []
|
||
for item in selected:
|
||
prompt = {
|
||
"task": "route_custom_equation",
|
||
"source_path": item["source_path"],
|
||
"name": item["name"],
|
||
"equation": item["equation"],
|
||
"primitive_hint": item["primitive_hint"],
|
||
"claim_boundary": item["claim_boundary"],
|
||
"instruction": "Make the LLM aware of this local equation without overclaiming proof.",
|
||
}
|
||
answer = {
|
||
"selected": True,
|
||
"use_as": "custom_equation_awareness",
|
||
"primitive_hint": item["primitive_hint"],
|
||
"claim_boundary": item["claim_boundary"],
|
||
"source_path": item["source_path"],
|
||
"equation_hash": item["equation_hash"],
|
||
"route_rule": "Use the equation as a local routing/canonicalization prior; require source/build/prover receipts before truth promotion.",
|
||
}
|
||
records.append(
|
||
{
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
|
||
{"role": "assistant", "content": json.dumps(answer, ensure_ascii=False)},
|
||
]
|
||
}
|
||
)
|
||
return records
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--root", type=Path, action="append")
|
||
parser.add_argument("--receipt", type=Path, default=Path("4-Infrastructure/shim/custom_equation_awareness_manifest_receipt.json"))
|
||
parser.add_argument("--manifest", type=Path, default=Path("4-Infrastructure/shim/custom_equation_awareness_manifest.jsonl"))
|
||
parser.add_argument("--curriculum", type=Path, default=Path("4-Infrastructure/shim/custom_equation_awareness_curriculum.jsonl"))
|
||
parser.add_argument("--per-bucket-limit", type=int, default=80)
|
||
args = parser.parse_args()
|
||
|
||
roots = args.root or DEFAULT_ROOTS
|
||
files = iter_candidate_files(roots)
|
||
equations: list[dict[str, Any]] = []
|
||
source_summaries = []
|
||
for path in files:
|
||
try:
|
||
raw = safe_read(path)
|
||
except Exception:
|
||
continue
|
||
source_hash = sha256_bytes(raw)
|
||
before = len(equations)
|
||
if path.suffix in JSON_SUFFIXES:
|
||
extract_json(path, source_hash, equations)
|
||
elif path.suffix in TEXT_SUFFIXES:
|
||
extract_text(path, source_hash, equations)
|
||
count = len(equations) - before
|
||
source_summaries.append(
|
||
{
|
||
"path": str(path),
|
||
"sha256": source_hash,
|
||
"suffix": path.suffix,
|
||
"equations_extracted": count,
|
||
}
|
||
)
|
||
|
||
primitive_counts: dict[str, int] = {}
|
||
boundary_counts: dict[str, int] = {}
|
||
for equation in equations:
|
||
primitive_counts[equation["primitive_hint"]] = primitive_counts.get(equation["primitive_hint"], 0) + 1
|
||
boundary_counts[equation["claim_boundary"]] = boundary_counts.get(equation["claim_boundary"], 0) + 1
|
||
|
||
receipt = {
|
||
"schema": "custom_equation_awareness_manifest_v1",
|
||
"claim_boundary": "Equation awareness teaches local routing and recall; it does not prove or validate equations.",
|
||
"roots": [str(root) for root in roots],
|
||
"source_count": len(source_summaries),
|
||
"equation_count": len(equations),
|
||
"primitive_counts": primitive_counts,
|
||
"boundary_counts": boundary_counts,
|
||
"sources": source_summaries,
|
||
"equations": equations,
|
||
"lawful": bool(equations),
|
||
}
|
||
args.receipt.parent.mkdir(parents=True, exist_ok=True)
|
||
args.receipt.write_text(json.dumps(receipt, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||
with args.manifest.open("w", encoding="utf-8") as handle:
|
||
for equation in equations:
|
||
handle.write(json.dumps(equation, ensure_ascii=False) + "\n")
|
||
with args.curriculum.open("w", encoding="utf-8") as handle:
|
||
for record in curriculum_records(receipt, args.per_bucket_limit):
|
||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
print(json.dumps({k: receipt[k] for k in ("schema", "source_count", "equation_count", "primitive_counts", "boundary_counts", "lawful")}, indent=2, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|