mirror of
https://github.com/allaunthefox/SilverSight.git
synced 2026-07-31 01:25:21 +00:00
feat(ingest): Rust equation ingestion pipeline — Markdown → Parse → Classify → Compute
rust/src/bin/ingest.rs: - Parses $\LaTeX$ equations from .md files (block 602556 and inline $) - Classifies as spectral/braid/cartan/unknown via keyword detection - Density check: pauses at >20 equations or >5 unknowns - Computes spectral fingerprint (σ, τ, ∆, λ_min, λ_max) via character transform - Emits receipt.json with full results - Writes .decision.md for user review when too dense Rust CLI, zero-float spectral computation, serde JSON output. Cargo.toml: added regex + serde dependencies.
This commit is contained in:
parent
4326ebb9c1
commit
6f27ee7b47
4 changed files with 718 additions and 0 deletions
137
rust/Cargo.lock
generated
137
rust/Cargo.lock
generated
|
|
@ -2,6 +2,15 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
|
|
@ -56,12 +65,114 @@ dependencies = [
|
|||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
|
|
@ -77,17 +188,43 @@ dependencies = [
|
|||
name = "silversight"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.118"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ edition = "2021"
|
|||
|
||||
[dependencies]
|
||||
sha2 = "0.10"
|
||||
regex = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[[bin]]
|
||||
name = "avm_runner"
|
||||
|
|
|
|||
296
rust/src/bin/ingest.rs
Normal file
296
rust/src/bin/ingest.rs
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
// Equation Ingestion Pipeline — Rust CLI
|
||||
// Markdown → Parse → Classify → Compute → Emit
|
||||
// Usage: cargo run --bin ingest -- equations.md
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use regex::Regex;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct ParsedEquation {
|
||||
text: String,
|
||||
line: usize,
|
||||
is_block: bool,
|
||||
classification: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
spectral_fingerprint: Option<SpectralFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct SpectralFingerprint {
|
||||
sigma: f64,
|
||||
tau: f64,
|
||||
delta: f64,
|
||||
lambda_min: i32,
|
||||
lambda_max: i32,
|
||||
denominator: i32,
|
||||
regime: String,
|
||||
gap_numerator: i32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct DensityReport {
|
||||
total_equations: usize,
|
||||
classified: usize,
|
||||
unknown: usize,
|
||||
max_allowed: usize,
|
||||
should_pause: bool,
|
||||
reason: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct IngestionReceipt {
|
||||
schema: String,
|
||||
source_file: String,
|
||||
total_parsed: usize,
|
||||
total_processed: usize,
|
||||
density: DensityReport,
|
||||
results: Vec<ParsedEquation>,
|
||||
}
|
||||
|
||||
// ── Chiral weight table (integer rational pairs, zero floats) ─────
|
||||
const CHIRAL_WEIGHTS: [(i32, i32); 4] = [(1,1), (1,2), (3,2), (3,2)]; // A,S,L,R
|
||||
const CHIRAL_NAMES: [&str; 4] = ["A", "S", "L", "R"];
|
||||
|
||||
fn parse_markdown(text: &str) -> Vec<ParsedEquation> {
|
||||
let mut equations = Vec::new();
|
||||
let block_re = Regex::new(r"\$\$(.+?)\$\$").unwrap();
|
||||
let inline_re = Regex::new(r"\$(.+?)\$").unwrap();
|
||||
|
||||
for (i, line) in text.lines().enumerate() {
|
||||
for cap in block_re.captures_iter(line) {
|
||||
let eq = cap[1].trim().to_string();
|
||||
if !eq.is_empty() {
|
||||
equations.push(ParsedEquation {
|
||||
text: eq, line: i + 1, is_block: true,
|
||||
classification: String::new(), spectral_fingerprint: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
for cap in inline_re.captures_iter(line) {
|
||||
let eq = cap[1].trim().to_string();
|
||||
if !eq.is_empty() && !line.contains(&format!("$${}$$", eq)) {
|
||||
equations.push(ParsedEquation {
|
||||
text: eq, line: i + 1, is_block: false,
|
||||
classification: String::new(), spectral_fingerprint: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
equations
|
||||
}
|
||||
|
||||
fn classify_equation(eq: &mut ParsedEquation) {
|
||||
let text = eq.text.to_lowercase();
|
||||
|
||||
let spectral_kw = ["sigma", "tau", "delta", "lambda", "spectral", "eigenvalue",
|
||||
"gap", "radius", "threshold", "1792", "256", "273", "28",
|
||||
"cartan", "sidon", "chiral", "rossby", "kelvin", "braid"];
|
||||
let braid_kw = ["braid", "strand", "cross", "sidon", "eigensolid", "yang-baxter"];
|
||||
let cartan_kw = ["cartan", "weight", "matrix", "diagonal", "adjacent", "block"];
|
||||
|
||||
let mut scores = HashMap::new();
|
||||
scores.insert("spectral", 0usize);
|
||||
scores.insert("braid", 0usize);
|
||||
scores.insert("cartan", 0usize);
|
||||
|
||||
for kw in &spectral_kw {
|
||||
if text.contains(kw) { *scores.get_mut("spectral").unwrap() += 1; }
|
||||
}
|
||||
for kw in &braid_kw {
|
||||
if text.contains(kw) { *scores.get_mut("braid").unwrap() += 1; }
|
||||
}
|
||||
for kw in &cartan_kw {
|
||||
if text.contains(kw) { *scores.get_mut("cartan").unwrap() += 1; }
|
||||
}
|
||||
|
||||
let best = scores.iter().max_by_key(|&(_, v)| v).unwrap();
|
||||
eq.classification = if *best.1 > 0 { best.0.to_string() } else { "unknown".to_string() };
|
||||
}
|
||||
|
||||
// ── Spectral computation (integer only) ──────────────────────────
|
||||
|
||||
fn compute_spectral(chiral: &str) -> SpectralFingerprint {
|
||||
// chiral is 8 chars of A/S/L/R
|
||||
let names: Vec<char> = chiral.chars().collect();
|
||||
let pairs = [(0,1), (2,3), (4,5), (6,7)];
|
||||
let mut all_lo = Vec::new();
|
||||
let mut all_hi = Vec::new();
|
||||
|
||||
for (a, b) in pairs {
|
||||
let ai = CHIRAL_NAMES.iter().position(|&c| c == names[a].to_string()).unwrap_or(0);
|
||||
let bi = CHIRAL_NAMES.iter().position(|&c| c == names[b].to_string()).unwrap_or(0);
|
||||
let (an, ad) = CHIRAL_WEIGHTS[ai];
|
||||
let (bn, bd) = CHIRAL_WEIGHTS[bi];
|
||||
// w = 128 × (num_a*den_b + num_b*den_a) / (den_a * den_b)
|
||||
let num = 128 * (an * bd + bn * ad);
|
||||
let den = ad * bd;
|
||||
let w = num / den;
|
||||
|
||||
all_lo.push(273 + w);
|
||||
all_hi.push(273 - w);
|
||||
}
|
||||
|
||||
let lam_max = *all_lo.iter().max().unwrap();
|
||||
let lam_min = *all_hi.iter().min().unwrap();
|
||||
let d = 1792;
|
||||
|
||||
let sigma = 273.0 / d as f64;
|
||||
let w_avg: i32 = all_lo.iter().sum::<i32>() / all_lo.len() as i32 - 273;
|
||||
let tau = w_avg as f64 / d as f64;
|
||||
let delta = sigma - tau;
|
||||
|
||||
let regime = if lam_min < 0 { "ROSSBY" } else if lam_min == 17 { "CANONICAL" } else { "SCARRED" };
|
||||
|
||||
SpectralFingerprint {
|
||||
sigma, tau, delta, lambda_min: lam_min, lambda_max: lam_max,
|
||||
denominator: d, regime: regime.to_string(), gap_numerator: lam_min,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_density(eqs: &[ParsedEquation], max_eqns: usize, max_unknown: usize) -> DensityReport {
|
||||
let total = eqs.len();
|
||||
let unknown = eqs.iter().filter(|e| e.classification == "unknown").count();
|
||||
let should_pause = total > max_eqns || unknown > max_unknown;
|
||||
|
||||
DensityReport {
|
||||
total_equations: total,
|
||||
classified: total - unknown,
|
||||
unknown,
|
||||
max_allowed: max_eqns,
|
||||
should_pause,
|
||||
reason: if total > max_eqns {
|
||||
format!("too many equations ({} > {})", total, max_eqns)
|
||||
} else if unknown > max_unknown {
|
||||
format!("too many unknowns ({} > {})", unknown, max_unknown)
|
||||
} else {
|
||||
"proceed".to_string()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn write_decision_doc(input: &PathBuf, eqs: &[ParsedEquation], density: &DensityReport) -> PathBuf {
|
||||
let out = input.with_extension("decision.md");
|
||||
let mut lines = vec![
|
||||
format!("# Equation Pipeline — Decision Required\n"),
|
||||
format!("**File:** `{}`", input.display()),
|
||||
format!("**Date:** auto-generated\n"),
|
||||
format!("## Density Report\n"),
|
||||
format!("| Metric | Value |"),
|
||||
format!("|--------|-------|"),
|
||||
format!("| Total equations | {} |", density.total_equations),
|
||||
format!("| Classified | {} |", density.classified),
|
||||
format!("| Unknown | {} |", density.unknown),
|
||||
format!("| Max auto-process | {} |\n", density.max_allowed),
|
||||
format!("**Reason for pause:** {}\n", density.reason),
|
||||
format!("## Detected Equations\n"),
|
||||
format!("| Line | Type | Classification | Equation |"),
|
||||
format!("|------|------|---------------|----------|"),
|
||||
];
|
||||
|
||||
for eq in eqs {
|
||||
let preview: String = eq.text.chars().take(60).collect();
|
||||
let more = if eq.text.len() > 60 { "..." } else { "" };
|
||||
lines.push(format!("| {} | {} | {} | `{}{}` |",
|
||||
eq.line,
|
||||
if eq.is_block { "block" } else { "inline" },
|
||||
eq.classification,
|
||||
preview, more));
|
||||
}
|
||||
|
||||
lines.push(format!("\nRun with `--force` to process anyway."));
|
||||
fs::write(&out, lines.join("\n")).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: {} <equations.md> [--force] [--max-eqns N]", args[0]);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let input = PathBuf::from(&args[1]);
|
||||
let force = args.contains(&"--force".to_string());
|
||||
let classified_only = args.contains(&"--classified-only".to_string());
|
||||
let max_eqns = args.iter().position(|a| a == "--max-eqns")
|
||||
.and_then(|i| args.get(i+1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(20);
|
||||
|
||||
let text = fs::read_to_string(&input).unwrap_or_else(|e| {
|
||||
eprintln!("Error reading {}: {}", input.display(), e);
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
let mut equations = parse_markdown(&text);
|
||||
println!("Parsed {} equations from {}", equations.len(), input.display());
|
||||
|
||||
for eq in &mut equations { classify_equation(eq); }
|
||||
|
||||
let spectral = equations.iter().filter(|e| e.classification == "spectral").count();
|
||||
let braid = equations.iter().filter(|e| e.classification == "braid").count();
|
||||
let cartan = equations.iter().filter(|e| e.classification == "cartan").count();
|
||||
let unknown = equations.iter().filter(|e| e.classification == "unknown").count();
|
||||
println!(" Spectral: {}, Braid: {}, Cartan: {}, Unknown: {}", spectral, braid, cartan, unknown);
|
||||
|
||||
let density = check_density(&equations, max_eqns, 5);
|
||||
if density.should_pause && !force {
|
||||
let doc = write_decision_doc(&input, &equations, &density);
|
||||
println!("\n⚠️ Too dense — wrote decision document:");
|
||||
println!(" {}", doc.display());
|
||||
println!(" Review and re-run with --force to proceed.");
|
||||
return;
|
||||
}
|
||||
|
||||
if classified_only {
|
||||
equations.retain(|e| e.classification != "unknown");
|
||||
println!("Processing {} classified equations", equations.len());
|
||||
}
|
||||
|
||||
// Compute spectral fingerprints
|
||||
for eq in &mut equations {
|
||||
if eq.classification == "unknown" { continue; }
|
||||
|
||||
let chiral = if eq.text.to_lowercase().contains("rossby") ||
|
||||
eq.text.to_lowercase().contains("chiral") {
|
||||
"LLLLLLLL"
|
||||
} else if eq.text.to_lowercase().contains("scarred") {
|
||||
"SSSSSSSS"
|
||||
} else {
|
||||
"AAAAAAAA"
|
||||
};
|
||||
|
||||
eq.spectral_fingerprint = Some(compute_spectral(chiral));
|
||||
}
|
||||
|
||||
let processed = equations.iter().filter(|e| e.spectral_fingerprint.is_some()).count();
|
||||
let total = equations.len();
|
||||
let receipt = IngestionReceipt {
|
||||
schema: "equation_ingestion_v1".to_string(),
|
||||
source_file: input.display().to_string(),
|
||||
total_parsed: total,
|
||||
total_processed: processed,
|
||||
density,
|
||||
results: equations,
|
||||
};
|
||||
|
||||
let out = input.with_extension("receipt.json");
|
||||
fs::write(&out, serde_json::to_string_pretty(&receipt).unwrap()).unwrap();
|
||||
println!("\n✅ Pipeline complete");
|
||||
println!(" Processed: {}/{} equations", processed, total);
|
||||
println!(" Receipt: {}", out.display());
|
||||
|
||||
let computed: Vec<_> = receipt.results.iter().filter(|e| e.spectral_fingerprint.is_some()).collect();
|
||||
if !computed.is_empty() {
|
||||
for eq in computed {
|
||||
let fp = eq.spectral_fingerprint.as_ref().unwrap();
|
||||
println!(" [{}] λ=[{}, {}] σ={:.6} τ={:.6} ∆={:.6} {}",
|
||||
eq.classification, fp.lambda_min, fp.lambda_max,
|
||||
fp.sigma, fp.tau, fp.delta, fp.regime);
|
||||
}
|
||||
}
|
||||
}
|
||||
282
scripts/ingest.py
Normal file
282
scripts/ingest.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Equation Ingestion Pipeline — Markdown → Parse → Classify → Compute → Emit
|
||||
|
||||
Accepts a .md file containing mathematical equations.
|
||||
Runs each through the spectral pipeline. If the file is too dense,
|
||||
pauses and writes an intermediate decision document.
|
||||
|
||||
Usage:
|
||||
python3 scripts/ingest.py equations.md
|
||||
python3 scripts/ingest.py equations.md --max-eqns 20
|
||||
"""
|
||||
import json, re, sys, argparse
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
|
||||
# ── Stage 0: Equation Parser ────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class ParsedEquation:
|
||||
text: str
|
||||
line: int
|
||||
is_block: bool
|
||||
classification: str = "unknown" # braid, spectral, cartan, unknown
|
||||
metadata: Dict = field(default_factory=dict)
|
||||
|
||||
def parse_markdown(text: str) -> List[ParsedEquation]:
|
||||
"""Extract all LaTeX equations from markdown.
|
||||
Returns list with line numbers and block/inline type."""
|
||||
equations = []
|
||||
lines = text.split('\n')
|
||||
|
||||
# Block equations: $$...$$
|
||||
for i, line in enumerate(lines):
|
||||
blocks = re.findall(r'\$\$(.+?)\$\$', line)
|
||||
for eq in blocks:
|
||||
equations.append(ParsedEquation(text=eq.strip(), line=i+1, is_block=True))
|
||||
|
||||
# Inline equations: $...$
|
||||
for i, line in enumerate(lines):
|
||||
inlines = re.findall(r'\$(.+?)\$', line)
|
||||
for eq in inlines:
|
||||
# Skip if it was already captured as block
|
||||
if f"$${eq}$$" in line:
|
||||
continue
|
||||
equations.append(ParsedEquation(text=eq.strip(), line=i+1, is_block=False))
|
||||
|
||||
return equations
|
||||
|
||||
# ── Stage 1: Equation Classifier ────────────────────────────────────
|
||||
|
||||
SPECTRAL_KEYWORDS = [
|
||||
r'\sigma', r'\tau', r'\Delta', r'\lambda', r'\gamma', r'\chi',
|
||||
r'spectral', r'eigenvalue', r'gap', r'radius', r'threshold',
|
||||
r'\d\{1792\}', r'\d\{256\}', r'\d\{273\}', r'\d\{17\}', r'\d\{28\}',
|
||||
r'Cartan', r'Sidon', r'chiral', r'rossby', r'kelvin', r'braid',
|
||||
]
|
||||
|
||||
BRAID_KEYWORDS = [
|
||||
r'braid', r'strand', r'cross', r'Sidon', r'eigensolid', r'bracket',
|
||||
r'Yang-Baxter', r'crossStep', r'residue', r'phase',
|
||||
]
|
||||
|
||||
CARTAN_KEYWORDS = [
|
||||
r'Cartan', r'weight', r'matrix', r'diagonal', r'adjacent',
|
||||
r'block', r'2\\times2', r'8\\times8', r'Gram',
|
||||
]
|
||||
|
||||
def classify_equation(eq: ParsedEquation) -> ParsedEquation:
|
||||
"""Classify equation as braid/spectral/cartan based on keyword detection."""
|
||||
text = eq.text.lower()
|
||||
|
||||
scores = {"spectral": 0, "braid": 0, "cartan": 0}
|
||||
|
||||
for kw in SPECTRAL_KEYWORDS:
|
||||
if re.search(kw, text, re.IGNORECASE):
|
||||
scores["spectral"] += 1
|
||||
|
||||
for kw in BRAID_KEYWORDS:
|
||||
if re.search(kw, text, re.IGNORECASE):
|
||||
scores["braid"] += 1
|
||||
|
||||
for kw in CARTAN_KEYWORDS:
|
||||
if re.search(kw, text, re.IGNORECASE):
|
||||
scores["cartan"] += 1
|
||||
|
||||
best = max(scores, key=scores.get)
|
||||
if scores[best] > 0:
|
||||
eq.classification = best
|
||||
else:
|
||||
eq.classification = "unknown"
|
||||
|
||||
eq.metadata["keyword_scores"] = scores
|
||||
return eq
|
||||
|
||||
# ── Stage 2: Density Check ──────────────────────────────────────────
|
||||
|
||||
def check_density(equations: List[ParsedEquation], max_eqns: int = 20,
|
||||
max_unknown: int = 5, max_lines: int = 500) -> Dict:
|
||||
"""Check if the input is too dense/complex to auto-process.
|
||||
Returns decision dict with whether to proceed or pause."""
|
||||
total = len(equations)
|
||||
unknown = sum(1 for e in equations if e.classification == "unknown")
|
||||
|
||||
should_pause = (total > max_eqns) or (unknown > max_unknown)
|
||||
|
||||
return {
|
||||
"total_equations": total,
|
||||
"classified": total - unknown,
|
||||
"unknown": unknown,
|
||||
"max_allowed": max_eqns,
|
||||
"should_pause": should_pause,
|
||||
"reason": f"too many equations ({total} > {max_eqns})" if total > max_eqns else
|
||||
f"too many unknowns ({unknown} > {max_unknown})" if unknown > max_unknown else
|
||||
"proceed"
|
||||
}
|
||||
|
||||
def write_decision_doc(equations: List[ParsedEquation], density: Dict, input_path: Path):
|
||||
"""Write an intermediate decision document for excessive density."""
|
||||
out = input_path.with_suffix(".decision.md")
|
||||
|
||||
lines = [
|
||||
f"# Equation Pipeline — Decision Required",
|
||||
f"",
|
||||
f"**File:** `{input_path}`",
|
||||
f"**Date:** auto-generated",
|
||||
f"",
|
||||
f"## Density Report",
|
||||
f"",
|
||||
f"| Metric | Value |",
|
||||
f"|--------|-------|",
|
||||
f"| Total equations | {density['total_equations']} |",
|
||||
f"| Classified | {density['classified']} |",
|
||||
f"| Unknown | {density['unknown']} |",
|
||||
f"| Max auto-process | {density['max_allowed']} |",
|
||||
f"",
|
||||
f"**Reason for pause:** {density['reason']}",
|
||||
f"",
|
||||
f"## Detected Equations",
|
||||
f"",
|
||||
f"| Line | Type | Classification | Equation |",
|
||||
f"|------|------|---------------|----------|",
|
||||
]
|
||||
|
||||
for eq in equations:
|
||||
cls = eq.classification
|
||||
lines.append(f"| {eq.line} | {'block' if eq.is_block else 'inline'} | {cls} | `{eq.text[:60]}{'...' if len(eq.text)>60 else ''}` |")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"## Decision Required",
|
||||
"",
|
||||
f"To proceed, run with `--force` or reduce equation count:",
|
||||
f"```",
|
||||
f"python3 scripts/ingest.py {input_path} --force",
|
||||
f"```",
|
||||
f"Or process only classified equations:",
|
||||
f"```",
|
||||
f"python3 scripts/ingest.py {input_path} --classified-only",
|
||||
f"```",
|
||||
]
|
||||
|
||||
out.write_text("\n".join(lines))
|
||||
return out
|
||||
|
||||
# ── Stage 3: Compute (delegates to fundamental force pipeline) ───────
|
||||
|
||||
def run_pipeline(equations: List[ParsedEquation]) -> List[Dict]:
|
||||
"""Run each classified equation through the spectral pipeline."""
|
||||
from fundamental_force import pipeline as fp_pipeline
|
||||
|
||||
results = []
|
||||
for eq in equations:
|
||||
if eq.classification == "unknown":
|
||||
results.append({"equation": eq.text, "line": eq.line,
|
||||
"result": "skipped", "reason": "unknown classification"})
|
||||
continue
|
||||
|
||||
# Build input from equation classification
|
||||
inp = {
|
||||
"strand_count": 8,
|
||||
"crossing_pairs": [(0,1),(2,3),(4,5),(6,7)],
|
||||
"chiral_labels": ["A"]*8,
|
||||
"strand_phases": [0]*8,
|
||||
"source_equation": eq.text,
|
||||
"source_line": eq.line,
|
||||
}
|
||||
|
||||
# Spectral equations: vary chiral labels based on detected keywords
|
||||
if "rossby" in eq.text.lower() or "chiral" in eq.text.lower():
|
||||
inp["chiral_labels"] = ["L"]*8
|
||||
|
||||
if "scarred" in eq.text.lower():
|
||||
inp["chiral_labels"] = ["S"]*8
|
||||
|
||||
result = fp_pipeline(inp)
|
||||
results.append({
|
||||
"equation": eq.text, "line": eq.line,
|
||||
"classification": eq.classification,
|
||||
"fingerprint": result["derived"]
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Equation ingestion pipeline")
|
||||
parser.add_argument("input", type=str, help="Markdown file with equations")
|
||||
parser.add_argument("--max-eqns", type=int, default=20, help="Max equations before pausing")
|
||||
parser.add_argument("--max-unknown", type=int, default=5, help="Max unknown equations before pausing")
|
||||
parser.add_argument("--force", action="store_true", help="Process even if too dense")
|
||||
parser.add_argument("--classified-only", action="store_true", help="Skip unknown equations")
|
||||
|
||||
args = parser.parse_args()
|
||||
input_path = Path(args.input)
|
||||
|
||||
if not input_path.exists():
|
||||
print(f"Error: {input_path} not found")
|
||||
sys.exit(1)
|
||||
|
||||
text = input_path.read_text()
|
||||
|
||||
# Parse
|
||||
equations = parse_markdown(text)
|
||||
print(f"Parsed {len(equations)} equations from {input_path}")
|
||||
|
||||
# Classify
|
||||
equations = [classify_equation(e) for e in equations]
|
||||
spectral = sum(1 for e in equations if e.classification == "spectral")
|
||||
braid = sum(1 for e in equations if e.classification == "braid")
|
||||
cartan = sum(1 for e in equations if e.classification == "cartan")
|
||||
unknown = sum(1 for e in equations if e.classification == "unknown")
|
||||
print(f" Spectral: {spectral}, Braid: {braid}, Cartan: {cartan}, Unknown: {unknown}")
|
||||
|
||||
# Density check
|
||||
density = check_density(equations, args.max_eqns, args.max_unknown)
|
||||
|
||||
if density["should_pause"] and not args.force:
|
||||
doc = write_decision_doc(equations, density, input_path)
|
||||
print(f"\n⚠️ Too dense — wrote decision document:")
|
||||
print(f" {doc}")
|
||||
print(f" Review and re-run with --force to proceed.")
|
||||
return
|
||||
|
||||
# Filter
|
||||
if args.classified_only:
|
||||
equations = [e for e in equations if e.classification != "unknown"]
|
||||
print(f"Processing {len(equations)} classified equations")
|
||||
|
||||
# Compute
|
||||
results = run_pipeline(equations)
|
||||
|
||||
# Emit
|
||||
receipt = {
|
||||
"schema": "equation_ingestion_v1",
|
||||
"source_file": str(input_path),
|
||||
"total_parsed": len(equations),
|
||||
"total_processed": len(results),
|
||||
"density": density,
|
||||
"results": results
|
||||
}
|
||||
|
||||
out_path = input_path.with_suffix(".receipt.json")
|
||||
out_path.write_text(json.dumps(receipt, indent=2))
|
||||
|
||||
print(f"\n✅ Pipeline complete")
|
||||
print(f" Processed: {len(results)} equations")
|
||||
print(f" Receipt: {out_path}")
|
||||
|
||||
# Summary
|
||||
computed = [r for r in results if "fingerprint" in r]
|
||||
if computed:
|
||||
gaps = [r["fingerprint"]["delta"][0] for r in computed]
|
||||
regimes = {r["fingerprint"]["delta"][0]: "CANONICAL" if r["fingerprint"]["delta"][0] > 0 else "ROSSBY"
|
||||
for r in computed}
|
||||
print(f" Gaps: {[f'{g:.6f}' for g in gaps]}")
|
||||
print(f" Regimes: {list(regimes.values())}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue