#!/usr/bin/env python3 """ Virtual FPGA System Test — Prover Orchestration Layers ======================================================= Runs the prover-integrated orchestration engine through comprehensive system tests on simulated Tang Nano 9K FPGAs. Tests: 1. Normal operation — 1000 state transitions 2. Invariant violations — Q16.16 overflow, skew, PDN 3. Agent drift — BFS detects non-determinism 4. Invalid manifold reshape — bf4 blocks bad configs 5. Load test — sustained throughput measurement 6. Recovery — watchdog blocks then system recovers """ import json, time, random, statistics from pathlib import Path from dataclasses import dataclass, field from typing import List, Dict, Tuple from collections import defaultdict RESEARCH_STACK = Path("/home/allaun/Documents/Research Stack") # Import the orchestration engine import sys sys.path.insert(0, str(RESEARCH_STACK / "4-Infrastructure/shim")) from prover_orchestration_layer import ( ProverOrchestrationEngine, ProverWatchdog, SwarmConsensus, TopologicalAdaptation ) @dataclass class VirtualFPGA: """Simulated Tang Nano 9K FPGA instance.""" id: int luts_used: int = 0 bram_used: int = 0 dsp_used: int = 0 temperature_c: float = 35.0 state: Dict = field(default_factory=dict) errors: List[str] = field(default_factory=list) MAX_LUTS = 8640 MAX_BRAM = 26 # 18Kb blocks MAX_DSP = 20 def utilization(self) -> float: return max(self.luts_used / self.MAX_LUTS, self.bram_used / self.MAX_BRAM, self.dsp_used / self.MAX_DSP) def healthy(self) -> bool: return self.temperature_c < 85 and self.utilization() < 0.95 class VirtualFPGACluster: """Cluster of virtual FPGAs for system testing.""" def __init__(self, num_fpgas: int = 5): self.fpgas = [VirtualFPGA(i) for i in range(num_fpgas)] self.engine = ProverOrchestrationEngine() self.test_results: List[Dict] = [] def _make_state(self, fpga_id: int, skew_ps: float = 30, pdn_z: float = 8, q16_vals: List[int] = None, delays: List[int] = None) -> Dict: """Build a state dict for a virtual FPGA.""" return { "fpga_id": fpga_id, "q16": q16_vals or [random.randint(-1000, 1000) for _ in range(4)], "delays": delays or [random.randint(40, 80) for _ in range(4)], "pdn_z": pdn_z, "pdn_target": 10, "skew_ps": skew_ps, "nodes": 5, "edges": 8, "findings": [ {"agent_id": i, "in": {"fpga": fpga_id}, "out": {"ok": True}} for i in range(3) ], "manifold": { "dim": 4, "shape": "flat", "ev": [1.77, 2.51, 3.07, 3.54] } } def run_test_suite(self): """Execute all system tests.""" print("=" * 70) print("Virtual FPGA System Test — Prover Orchestration Layers") print("=" * 70) tests = [ ("Normal Operation", self.test_normal_operation), ("Invariant Violations", self.test_invariant_violations), ("Agent Drift Detection", self.test_agent_drift), ("Invalid Manifold Reshape", self.test_invalid_reshape), ("Sustained Load Test", self.test_sustained_load), ("Recovery After Violation", self.test_recovery), ] passed = 0 for name, test_fn in tests: print(f"\n{'─' * 70}") print(f"TEST: {name}") print(f"{'─' * 70}") try: result = test_fn() self.test_results.append({"test": name, "result": result}) if result.get("passed", False): passed += 1 print(f" ✓ PASSED") else: print(f" ✗ FAILED: {result.get('error', 'unknown')}") except Exception as e: self.test_results.append({"test": name, "result": {"passed": False, "error": str(e)}}) print(f" ✗ EXCEPTION: {e}") print(f"\n{'=' * 70}") print(f"RESULTS: {passed}/{len(tests)} tests passed") print(f"{'=' * 70}") return passed == len(tests) def test_normal_operation(self) -> Dict: """1000 normal state transitions — all should pass.""" print(" Running 1000 normal transitions...") latencies = [] violations_total = 0 for i in range(1000): fpga = self.fpgas[i % len(self.fpgas)] from_st = self._make_state(fpga.id) to_st = self._make_state(fpga.id) t0 = time.time() ok, report = self.engine.process(from_st, to_st) latencies.append((time.time() - t0) * 1000) if not ok: violations_total += 1 metrics = self.engine.metrics() result = { "passed": violations_total == 0, "transitions": 1000, "violations": violations_total, "mean_latency_ms": statistics.mean(latencies), "p50_latency_ms": statistics.median(latencies), "p99_latency_ms": sorted(latencies)[int(len(latencies) * 0.99)], "max_latency_ms": max(latencies), "throughput_ops_s": 1000 / (sum(latencies) / 1000), } print(f" Violations: {violations_total}/1000") print(f" Mean latency: {result['mean_latency_ms']:.3f}ms") print(f" P99 latency: {result['p99_latency_ms']:.3f}ms") print(f" Throughput: {result['throughput_ops_s']:.0f} ops/s") return result def test_invariant_violations(self) -> Dict: """Test that each invariant violation is caught.""" print(" Testing invariant violation detection...") violations_found = [] # Q16.16 overflow from_st = self._make_state(0) to_st = self._make_state(0, q16_vals=[40000, 0, 0, 0]) # overflow ok, report = self.engine.process(from_st, to_st) if not ok: violations_found.append("Q16_16_overflow") print(f" Q16.16 overflow: {'CAUGHT' if not ok else 'MISSED'}") # PDN impedance violation from_st = self._make_state(0) to_st = self._make_state(0, pdn_z=25) # exceeds target 10 ok, report = self.engine.process(from_st, to_st) if not ok: violations_found.append("PDN_impedance") print(f" PDN impedance: {'CAUGHT' if not ok else 'MISSED'}") # Trace skew violation from_st = self._make_state(0) to_st = self._make_state(0, skew_ps=80) # exceeds 50ps ok, report = self.engine.process(from_st, to_st) if not ok: violations_found.append("trace_skew") print(f" Trace skew: {'CAUGHT' if not ok else 'MISSED'}") # FAMM delay negative from_st = self._make_state(0) to_st = self._make_state(0, delays=[-5, 50, 50, 50]) ok, report = self.engine.process(from_st, to_st) if not ok: violations_found.append("FAMM_delay") print(f" FAMM negative delay: {'CAUGHT' if not ok else 'MISSED'}") # Topology disconnected from_st = self._make_state(0) to_st = self._make_state(0) to_st["nodes"] = 10 to_st["edges"] = 2 # 2 edges for 10 nodes = disconnected ok, report = self.engine.process(from_st, to_st) if not ok: violations_found.append("topology_disconnected") print(f" Topology disconnected: {'CAUGHT' if not ok else 'MISSED'}") return { "passed": len(violations_found) == 5, "violations_detected": len(violations_found), "expected": 5, "details": violations_found, } def test_agent_drift(self) -> Dict: """Test BFS-Prover-V2 drift detection.""" print(" Testing agent drift detection...") # First, establish baseline from_st = self._make_state(0) to_st = self._make_state(0) self.engine.process(from_st, to_st) # Now inject drift: same agent, different output for same input from_st2 = self._make_state(0) to_st2 = self._make_state(0) to_st2["findings"][0]["out"] = {"ok": False, "drift": True} ok, report = self.engine.process(from_st2, to_st2) drift_detected = self.engine.swarm.drift print(f" Drift detected: {drift_detected}") return { "passed": drift_detected, "drift_detected": drift_detected, } def test_invalid_reshape(self) -> Dict: """Test bf4prover blocks invalid manifold reshapes.""" print(" Testing invalid manifold reshape rejection...") # Negative eigenvalue from_st = self._make_state(0) to_st = self._make_state(0) to_st["manifold"] = {"dim": 4, "shape": "invalid", "ev": [-1.0, 2.0, 3.0, 4.0]} ok, report = self.engine.process(from_st, to_st) print(f" Negative eigenvalue: {'BLOCKED' if not ok else 'ALLOWED'}") # Zero dimension from_st2 = self._make_state(0) to_st2 = self._make_state(0) to_st2["manifold"] = {"dim": 0, "shape": "point", "ev": []} ok2, report2 = self.engine.process(from_st2, to_st2) print(f" Zero dimension: {'BLOCKED' if not ok2 else 'ALLOWED'}") return { "passed": not ok and not ok2, "negative_ev_blocked": not ok, "zero_dim_blocked": not ok2, } def test_sustained_load(self) -> Dict: """Sustained throughput test — 10,000 transitions.""" print(" Running 10,000 sustained load transitions...") latencies = [] batch_size = 1000 for batch in range(10): batch_start = time.time() for i in range(batch_size): fpga = self.fpgas[i % len(self.fpgas)] from_st = self._make_state(fpga.id) to_st = self._make_state(fpga.id) t0 = time.time() self.engine.process(from_st, to_st) latencies.append((time.time() - t0) * 1000) batch_time = time.time() - batch_start print(f" Batch {batch+1}/10: {batch_size} ops in {batch_time:.2f}s " f"({batch_size/batch_time:.0f} ops/s)") result = { "passed": True, "total_ops": 10000, "mean_latency_ms": statistics.mean(latencies), "p50_latency_ms": statistics.median(latencies), "p99_latency_ms": sorted(latencies)[int(len(latencies) * 0.99)], "throughput_ops_s": 10000 / (sum(latencies) / 1000), } print(f" Mean: {result['mean_latency_ms']:.3f}ms, " f"P99: {result['p99_latency_ms']:.3f}ms, " f"Throughput: {result['throughput_ops_s']:.0f} ops/s") return result def test_recovery(self) -> Dict: """Test system recovers after watchdog blocks a violation.""" print(" Testing recovery after violation...") # Cause a violation from_st = self._make_state(0) to_st_bad = self._make_state(0, q16_vals=[50000, 0, 0, 0]) ok_bad, _ = self.engine.process(from_st, to_st_bad) # Now try a valid transition to_st_good = self._make_state(0) ok_good, report = self.engine.process(from_st, to_st_good) recovered = ok_good and not ok_bad print(f" Bad transition blocked: {not ok_bad}") print(f" Good transition allowed: {ok_good}") print(f" Recovery successful: {recovered}") return { "passed": recovered, "bad_blocked": not ok_bad, "good_allowed": ok_good, "recovered": recovered, } def main(): cluster = VirtualFPGACluster(num_fpgas=5) all_passed = cluster.run_test_suite() # Save detailed report report_path = RESEARCH_STACK / "4-Infrastructure/shim/virtual_fpga_system_test_report.json" final_metrics = cluster.engine.metrics() report = { "timestamp": time.time(), "all_passed": all_passed, "tests": cluster.test_results, "final_metrics": final_metrics, "fpgas": [ {"id": f.id, "healthy": f.healthy(), "utilization": f.utilization()} for f in cluster.fpgas ], "summary": { "total_tests": len(cluster.test_results), "passed": sum(1 for t in cluster.test_results if t["result"].get("passed", False)), "total_transitions": 11000, "watchdog_violations": final_metrics["watchdog_violations"], "swarm_drift_events": final_metrics["swarm_drift"], "topology_adaptations": final_metrics["topology_adaptations"], "proved_configs": final_metrics["proved_configs"], } } with open(report_path, 'w') as f: json.dump(report, f, indent=2, default=str) print(f"\nDetailed report: {report_path}") if all_passed: print("\n✓ ALL SYSTEM TESTS PASSED — Virtual FPGA cluster verified") else: print("\n✗ SOME TESTS FAILED — Check report for details") return 0 if all_passed else 1 if __name__ == "__main__": exit(main())