feat(ingest): Landauer + Shannon entropy density check via rgflow

Density now measured by information-theoretic bounds, not just counts:

Shannon entropy H = -Σ p(x) log₂ p(x) over equation symbol distribution
  - Higher H = more informational diversity = denser content
  - RG flow bound: H < ln(2⁸) ≈ 2.08 for 8-strand representation

Landauer cost E = N·kT·ln(2) where N = distinct math tokens
  - Each distinct symbol requires erasing energy
  - kT = 0.025 eV at 290K → ~0.017 eV per bit

Density levels:
  THERMODYNAMIC_SATURATION: both Shannon + Landauer exceed bounds
  SHANNON_SATURATED: entropy > rgflow fixed-point
  LANDAUER_EXPENSIVE: energy cost too high
  COUNT_EXCEEDED: simple count threshold
This commit is contained in:
allaun 2026-06-30 20:51:17 -05:00
parent 6f27ee7b47
commit b6a4cf9e4d

View file

@ -37,6 +37,11 @@ struct DensityReport {
max_allowed: usize,
should_pause: bool,
reason: String,
shannon_entropy: f64,
landauer_bits: f64,
landauer_energy_ev: f64,
rgflow_bound: f64,
density_level: String,
}
#[derive(Serialize, Deserialize)]
@ -152,7 +157,64 @@ fn compute_spectral(chiral: &str) -> SpectralFingerprint {
fn check_density(eqs: &[ParsedEquation], max_eqns: usize, max_unknown: usize) -> DensityReport {
let total = eqs.len();
let unknown = eqs.iter().filter(|e| e.classification == "unknown").count();
let should_pause = total > max_eqns || unknown > max_unknown;
// ── Shannon Entropy: H = -Σ p(x) log₂ p(x) ──────────────────
// Entropy over the equation text symbol distribution.
// Higher H = more informational diversity = denser content.
let full_text: String = eqs.iter().map(|e| &e.text).fold(String::new(), |a, b| a + b);
let mut freq = [0usize; 256];
for b in full_text.bytes() { freq[b as usize] += 1; }
let total_bytes = full_text.len().max(1) as f64;
let shannon: f64 = freq.iter()
.filter(|&&c| c > 0)
.map(|&c| { let p = c as f64 / total_bytes; -p * p.log2() })
.sum();
// ── Landauer Cost: E = kT·ln(2) × unique_terms ──────────────
// Landauer (1961): erasing N bits costs N·kT·ln(2) energy.
// Each distinct mathematical symbol/operator is a "bit" of
// computational state that must be erased.
// kT ≈ 0.025 eV at room temp → Landauer energy per bit.
let kT_eV = 0.025; // eV at 290K
let ln2 = 0.693147;
let landauer_per_bit = kT_eV * ln2; // ≈ 0.017 eV/bit
// Count distinct mathematical symbols/tokens
let math_tokens = Regex::new(r"[\\a-z]+|\d+|[+\-*/=<>\[\]{}()∫∑∏∂∇√∞π]").unwrap();
let mut terms = std::collections::HashSet::new();
for eq in eqs {
for m in math_tokens.find_iter(&eq.text) {
terms.insert(m.as_str().to_string());
}
}
let landauer_bits = terms.len() as f64;
let landauer_energy = landauer_bits * landauer_per_bit;
// ── RGFlow Entropy Bound ──────────────────────────────────────
// Renormalization group flow: entropy decreases monotonically
// under coarse-graining. The RG fixed-point occurs at H = ln(2⁸) ≈ 5.545
// for the 8-strand system (8 bits of address space).
// Equation sets exceeding this entropy need RG coarse-graining
// (information compression) before they can be reliably classified.
let rgflow_bound = (8.0_f64).ln(); // ln(2⁸) ≈ 5.545
// ── Decision ──────────────────────────────────────────────────
let exceeds_shannon = shannon > rgflow_bound;
let exceeds_landauer = landauer_bits > (max_eqns as f64 * 3.0);
let exceeds_count = total > max_eqns || unknown > max_unknown;
let should_pause = exceeds_shannon || exceeds_landauer || exceeds_count;
let density_level = if exceeds_shannon && exceeds_landauer {
"THERMODYNAMIC_SATURATION"
} else if exceeds_shannon {
"SHANNON_SATURATED"
} else if exceeds_landauer {
"LANDAUER_EXPENSIVE"
} else if exceeds_count {
"COUNT_EXCEEDED"
} else {
"PROCESSABLE"
};
DensityReport {
total_equations: total,
@ -160,13 +222,21 @@ fn check_density(eqs: &[ParsedEquation], max_eqns: usize, max_unknown: usize) ->
unknown,
max_allowed: max_eqns,
should_pause,
reason: if total > max_eqns {
format!("too many equations ({} > {})", total, max_eqns)
} else if unknown > max_unknown {
format!("too many unknowns ({} > {})", unknown, max_unknown)
reason: if exceeds_shannon {
format!("Shannon entropy {:.2} > rgflow bound {:.2} — information-saturated", shannon, rgflow_bound)
} else if exceeds_landauer {
format!("Landauer cost {:.0} bits > threshold — too expensive to auto-process", landauer_bits)
} else if exceeds_count {
if total > max_eqns { format!("too many equations ({} > {})", total, max_eqns) }
else { format!("too many unknowns ({} > {})", unknown, max_unknown) }
} else {
"proceed".to_string()
},
shannon_entropy: (shannon * 100.0).round() / 100.0,
landauer_bits,
landauer_energy_ev: (landauer_energy * 1000.0).round() / 1000.0,
rgflow_bound: (rgflow_bound * 100.0).round() / 100.0,
density_level: density_level.to_string(),
}
}