"""NoDupeLabs Error Test Utilities Helper functions for error condition simulation and testing. """ import contextlib from typing import Dict, Any, List, Optional, Union, Callable, Type from unittest.mock import MagicMock, patch import random import os import tempfile from pathlib import Path def simulate_file_system_errors( error_type: str = "permission", operation: str = "read" ) -> Callable: """ Create a context manager to simulate file system errors. Args: error_type: Type of error to simulate operation: File operation to fail Returns: Context manager for error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating file system errors.""" error_map = { "permission": PermissionError("Operation not permitted"), "not_found": FileNotFoundError("File not found"), "disk_full": OSError("No space left on device"), "io_error": IOError("Input/output error"), "access_denied": PermissionError("Access denied") } error = error_map.get(error_type, IOError("File system error")) original_open = open original_path = Path class MockPath: """Mock Path class to simulate file system errors.""" def __init__(self, *args): """Initialize MockPath with path arguments.""" self.path = original_path(*args) def __truediv__(self, other): """Support path division operation.""" return MockPath(str(self.path) + "/" + str(other)) def __str__(self): """Return string representation of path.""" return str(self.path) def read_text(self, *args, **kwargs): """Simulate read_text error if operation is 'read'.""" if operation == "read": raise error return self.path.read_text(*args, **kwargs) def write_text(self, *args, **kwargs): """Simulate write_text error if operation is 'write'.""" if operation == "write": raise error return self.path.write_text(*args, **kwargs) def exists(self): """Simulate exists error if operation is 'exists'.""" if operation == "exists": raise error return self.path.exists() def unlink(self): """Simulate unlink error if operation is 'delete'.""" if operation == "delete": raise error return self.path.unlink() def mock_open(*args, **kwargs): """Mock open function to simulate file open errors.""" if operation == "open": raise error return original_open(*args, **kwargs) with patch('builtins.open', side_effect=mock_open): with patch('pathlib.Path', side_effect=MockPath): yield return error_context def create_error_test_scenarios() -> List[Dict[str, Any]]: """ Create error test scenarios. Returns: List of error test scenarios """ return [ { "name": "file_not_found", "error_type": "FileNotFoundError", "expected_behavior": "graceful_failure" }, { "name": "permission_denied", "error_type": "PermissionError", "expected_behavior": "retry_or_fail" }, { "name": "disk_full", "error_type": "OSError", "expected_behavior": "cleanup_and_fail" }, { "name": "network_timeout", "error_type": "TimeoutError", "expected_behavior": "retry_with_backoff" }, { "name": "invalid_input", "error_type": "ValueError", "expected_behavior": "validate_and_fail" } ] def simulate_network_errors( error_type: str = "timeout", failure_rate: float = 1.0 ) -> Callable: """ Create a context manager to simulate network errors. Args: error_type: Type of network error to simulate failure_rate: Probability of failure (0.0 to 1.0) Returns: Context manager for network error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating network errors.""" error_map = { "timeout": TimeoutError("Connection timed out"), "connection_refused": ConnectionRefusedError("Connection refused"), "dns_failure": OSError("Name or service not known"), "ssl_error": OSError("SSL handshake failed"), "network_unreachable": OSError("Network is unreachable") } error = error_map.get(error_type, OSError("Network error")) original_requests = __import__('requests') class MockRequests: """Mock requests class to simulate network errors.""" @staticmethod def get(*args, **kwargs): """Simulate GET request with potential error.""" if random.random() < failure_rate: raise error return original_requests.get(*args, **kwargs) @staticmethod def post(*args, **kwargs): """Simulate POST request with potential error.""" if random.random() < failure_rate: raise error return original_requests.post(*args, **kwargs) with patch('requests', MockRequests): with patch('urllib.request.urlopen') as mock_urlopen: if random.random() < failure_rate: mock_urlopen.side_effect = error yield return error_context def create_exception_test_cases() -> List[Dict[str, Any]]: """ Create exception test cases. Returns: List of exception test cases """ return [ { "name": "value_error", "exception": ValueError, "message": "Invalid value provided", "test_function": lambda: int("invalid") }, { "name": "type_error", "exception": TypeError, "message": "Invalid type provided", "test_function": lambda: "string" + 123 }, { "name": "index_error", "exception": IndexError, "message": "Index out of range", "test_function": lambda: [1, 2, 3][10] }, { "name": "key_error", "exception": KeyError, "message": "Key not found", "test_function": lambda: {"a": 1}["b"] }, { "name": "attribute_error", "exception": AttributeError, "message": "Attribute not found", "test_function": lambda: "string".nonexistent_method() } ] def simulate_memory_errors( error_type: str = "out_of_memory" ) -> Callable: """ Create a context manager to simulate memory errors. Args: error_type: Type of memory error to simulate Returns: Context manager for memory error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating memory errors.""" error_map = { "out_of_memory": MemoryError("Out of memory"), "memory_leak": MemoryError("Memory allocation failed"), "stack_overflow": RecursionError("Maximum recursion depth exceeded") } error = error_map.get(error_type, MemoryError("Memory error")) original_alloc = __import__('builtins').__dict__['object'].__new__ def mock_alloc(cls, *args, **kwargs): """Mock object allocation to simulate memory errors.""" if random.random() > 0.5: # 50% chance of failure raise error return original_alloc(cls, *args, **kwargs) with patch('builtins.object.__new__', side_effect=mock_alloc): yield return error_context def create_error_recovery_test_scenarios() -> List[Dict[str, Any]]: """ Create error recovery test scenarios. Returns: List of error recovery test scenarios """ return [ { "name": "automatic_retry", "error_type": "temporary_failure", "recovery_strategy": "retry", "max_retries": 3, "expected_result": "success" }, { "name": "fallback_mechanism", "error_type": "permanent_failure", "recovery_strategy": "fallback", "expected_result": "degraded_functionality" }, { "name": "graceful_degradation", "error_type": "resource_exhaustion", "recovery_strategy": "degrade", "expected_result": "reduced_performance" }, { "name": "manual_intervention", "error_type": "critical_failure", "recovery_strategy": "alert", "expected_result": "admin_notification" } ] def simulate_database_errors( error_type: str = "connection_failed" ) -> Callable: """ Create a context manager to simulate database errors. Args: error_type: Type of database error to simulate Returns: Context manager for database error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating database errors.""" import sqlite3 error_map = { "connection_failed": sqlite3.OperationalError("Unable to connect to database"), "query_failed": sqlite3.ProgrammingError("SQL syntax error"), "constraint_violation": sqlite3.IntegrityError("Constraint violation"), "timeout": sqlite3.OperationalError("Database locked"), "disk_full": sqlite3.OperationalError("Database or disk is full") } error = error_map.get(error_type, sqlite3.Error("Database error")) with patch('sqlite3.connect') as mock_connect: mock_conn = MagicMock() mock_cursor = MagicMock() if error_type == "connection_failed": mock_connect.side_effect = error else: mock_conn.cursor.return_value = mock_cursor if error_type == "query_failed": mock_cursor.execute.side_effect = error elif error_type == "constraint_violation": mock_cursor.execute.side_effect = error elif error_type == "timeout": mock_conn.commit.side_effect = error elif error_type == "disk_full": mock_cursor.execute.side_effect = error mock_connect.return_value = mock_conn yield return error_context def create_error_injection_test_scenarios() -> List[Dict[str, Any]]: """ Create error injection test scenarios. Returns: List of error injection test scenarios """ return [ { "name": "random_failures", "failure_rate": 0.1, "target_components": ["network", "database", "file_system"], "expected_behavior": "resilient_operation" }, { "name": "cascading_failures", "failure_sequence": ["database", "cache", "api"], "expected_behavior": "failure_containment" }, { "name": "intermittent_failures", "failure_pattern": "on_off", "expected_behavior": "automatic_recovery" } ] def simulate_tool_errors( error_type: str = "loading_failed" ) -> Callable: """ Create a context manager to simulate tool errors. Args: error_type: Type of tool error to simulate Returns: Context manager for tool error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating tool errors.""" error_map = { "loading_failed": ImportError("Cannot load tool"), "initialization_failed": RuntimeError("Tool initialization failed"), "execution_failed": ValueError("Tool execution error"), "compatibility_error": RuntimeError("Tool compatibility issue"), "security_violation": RuntimeError("Tool security violation") } error = error_map.get(error_type, RuntimeError("Tool error")) with patch('importlib.import_module') as mock_import: if error_type == "loading_failed": mock_import.side_effect = error else: mock_tool = MagicMock() if error_type == "initialization_failed": mock_tool.initialize.side_effect = error elif error_type == "execution_failed": mock_tool.execute.side_effect = error elif error_type == "compatibility_error": mock_tool.metadata = {"version": "incompatible"} elif error_type == "security_violation": mock_tool.execute.side_effect = error mock_import.return_value = mock_tool yield return error_context def create_error_handling_test_scenarios() -> List[Dict[str, Any]]: """ Create error handling test scenarios. Returns: List of error handling test scenarios """ return [ { "name": "exception_handling", "error_type": "ValueError", "handling_strategy": "catch_and_log", "expected_result": "logged_error" }, { "name": "resource_cleanup", "error_type": "IOError", "handling_strategy": "cleanup_and_rethrow", "expected_result": "cleaned_up_resources" }, { "name": "fallback_operation", "error_type": "TimeoutError", "handling_strategy": "use_fallback", "expected_result": "fallback_used" }, { "name": "retry_operation", "error_type": "TemporaryError", "handling_strategy": "retry_with_backoff", "expected_result": "operation_retry" } ] def simulate_concurrency_errors( error_type: str = "race_condition" ) -> Callable: """ Create a context manager to simulate concurrency errors. Args: error_type: Type of concurrency error to simulate Returns: Context manager for concurrency error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating concurrency errors.""" import threading error_map = { "race_condition": RuntimeError("Race condition detected"), "deadlock": RuntimeError("Deadlock detected"), "thread_failure": RuntimeError("Thread execution failed"), "resource_contention": RuntimeError("Resource contention detected") } error = error_map.get(error_type, RuntimeError("Concurrency error")) original_thread = threading.Thread class MockThread(threading.Thread): """Mock Thread class to simulate concurrency errors.""" def __init__(self, *args, **kwargs): """Initialize MockThread with error injection.""" super().__init__(*args, **kwargs) self._error = error if random.random() > 0.7 else None def run(self): """Run method that may inject concurrency errors.""" if self._error: raise self._error super().run() with patch('threading.Thread', MockThread): yield return error_context def create_error_validation_test_scenarios() -> List[Dict[str, Any]]: """ Create error validation test scenarios. Returns: List of error validation test scenarios """ return [ { "name": "input_validation", "test_cases": [ {"input": None, "expected_error": "ValueError"}, {"input": -1, "expected_error": "ValueError"}, {"input": "invalid", "expected_error": "TypeError"} ] }, { "name": "state_validation", "test_cases": [ {"state": "invalid_state", "expected_error": "RuntimeError"}, {"state": "corrupted_data", "expected_error": "DataError"} ] }, { "name": "security_validation", "test_cases": [ {"permission": "denied", "expected_error": "PermissionError"}, {"access": "unauthorized", "expected_error": "SecurityError"} ] } ] def simulate_resource_exhaustion_errors( resource_type: str = "memory" ) -> Callable: """ Create a context manager to simulate resource exhaustion errors. Args: resource_type: Type of resource to exhaust Returns: Context manager for resource exhaustion simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating resource exhaustion errors.""" error_map = { "memory": MemoryError("Out of memory"), "cpu": RuntimeError("CPU resources exhausted"), "disk": OSError("No space left on device"), "handles": OSError("Too many open files"), "threads": RuntimeError("Too many threads") } error = error_map.get(resource_type, RuntimeError("Resource exhausted")) if resource_type == "memory": original_alloc = __import__('builtins').__dict__['object'].__new__ def mock_alloc(cls, *args, **kwargs): """Mock object allocation for memory exhaustion simulation.""" if random.random() > 0.3: # 70% chance of failure raise error return original_alloc(cls, *args, **kwargs) with patch('builtins.object.__new__', side_effect=mock_alloc): yield elif resource_type == "disk": def mock_write(*args, **kwargs): """Mock write operation for disk exhaustion simulation.""" if random.random() > 0.5: # 50% chance of failure raise error original_write(*args, **kwargs) original_write = open with patch('builtins.open', side_effect=mock_write): yield else: # For other resource types, use a simpler approach def resource_check(): """Check resource availability for other resource types.""" if random.random() > 0.7: # 30% chance of failure raise error with patch('resource.getrlimit', side_effect=resource_check): yield return error_context def create_error_monitoring_test_scenarios() -> List[Dict[str, Any]]: """ Create error monitoring test scenarios. Returns: List of error monitoring test scenarios """ return [ { "name": "error_logging", "monitoring_type": "logging", "expected_behavior": "error_logged", "verification": "check_log_files" }, { "name": "error_metrics", "monitoring_type": "metrics", "expected_behavior": "metrics_updated", "verification": "check_metrics_endpoint" }, { "name": "error_alerting", "monitoring_type": "alerting", "expected_behavior": "alert_sent", "verification": "check_alert_system" }, { "name": "error_tracing", "monitoring_type": "tracing", "expected_behavior": "trace_recorded", "verification": "check_tracing_system" } ] def simulate_timeout_errors( timeout_type: str = "operation_timeout" ) -> Callable: """ Create a context manager to simulate timeout errors. Args: timeout_type: Type of timeout error to simulate Returns: Context manager for timeout error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating timeout errors.""" import time error_map = { "operation_timeout": TimeoutError("Operation timed out"), "connection_timeout": TimeoutError("Connection timed out"), "read_timeout": TimeoutError("Read operation timed out"), "write_timeout": TimeoutError("Write operation timed out") } error = error_map.get(timeout_type, TimeoutError("Timeout error")) original_monotonic = time.monotonic original_sleep = time.sleep start_time = original_monotonic() def slow_monotonic(): """Mock monotonic time that appears to run slowly.""" elapsed = original_monotonic() - start_time if elapsed > 1.0: # After 1 second, start causing timeouts raise error return elapsed def slow_sleep(seconds): """Mock sleep that causes timeouts for long durations.""" if seconds > 0.1: # Long sleeps cause timeouts raise error original_sleep(seconds) with patch('time.monotonic', side_effect=slow_monotonic): with patch('time.sleep', side_effect=slow_sleep): yield return error_context def create_error_recovery_validation_scenarios() -> List[Dict[str, Any]]: """ Create error recovery validation scenarios. Returns: List of error recovery validation scenarios """ return [ { "name": "data_consistency_after_error", "error_type": "database_error", "recovery_method": "transaction_rollback", "validation": "verify_data_integrity" }, { "name": "resource_cleanup_after_error", "error_type": "file_error", "recovery_method": "resource_release", "validation": "verify_no_resource_leaks" }, { "name": "state_consistency_after_error", "error_type": "state_error", "recovery_method": "state_reset", "validation": "verify_consistent_state" } ] import time import tempfile from pathlib import Path from typing import Dict, Any, List, Optional, Union, Callable import contextlib import os import sys from unittest.mock import MagicMock, patch try: import resource RESOURCE_AVAILABLE = True except ImportError: RESOURCE_AVAILABLE = False try: import psutil PSUTIL_AVAILABLE = True except ImportError: PSUTIL_AVAILABLE = False def benchmark_function_performance( func: Callable, iterations: int = 100, warmup_iterations: int = 10, *args, **kwargs ) -> Dict[str, float]: """ Benchmark a function's performance. Args: func: Function to benchmark iterations: Number of benchmark iterations warmup_iterations: Number of warmup iterations *args: Positional arguments for the function **kwargs: Keyword arguments for the function Returns: Dictionary of performance metrics """ # Warmup for _ in range(warmup_iterations): func(*args, **kwargs) # Benchmark start_time = time.time() for _ in range(iterations): func(*args, **kwargs) end_time = time.time() total_time = end_time - start_time avg_time = total_time / iterations ops_per_sec = iterations / total_time return { "total_time": total_time, "average_time": avg_time, "operations_per_second": ops_per_sec, "iterations": iterations } def measure_memory_usage( func: Callable, iterations: int = 10, *args, **kwargs ) -> Dict[str, float]: """ Measure memory usage of a function. Args: func: Function to measure iterations: Number of iterations *args: Positional arguments for the function **kwargs: Keyword arguments for the function Returns: Dictionary of memory usage metrics """ if not PSUTIL_AVAILABLE: # Fallback implementation when psutil is not available # Run the function but return mock memory values for _ in range(iterations): func(*args, **kwargs) return { "initial_memory": 0, "final_memory": 0, "total_memory_used": 0, "average_memory_per_call": 0, "iterations": iterations, "warning": "psutil not available, using fallback implementation" } # Get initial memory usage process = psutil.Process(os.getpid()) initial_mem = process.memory_info().rss # Run function multiple times for _ in range(iterations): func(*args, **kwargs) # Get final memory usage final_mem = process.memory_info().rss # Calculate memory usage total_memory_used = final_mem - initial_mem avg_memory_per_call = total_memory_used / iterations return { "initial_memory": initial_mem, "final_memory": final_mem, "total_memory_used": total_memory_used, "average_memory_per_call": avg_memory_per_call, "iterations": iterations } def create_performance_test_scenarios() -> List[Dict[str, Any]]: """ Create performance test scenarios. Returns: List of performance test scenarios """ return [ { "name": "small_dataset", "data_size": 100, "expected_max_time": 0.1, "expected_max_memory": 1024 * 1024 # 1MB }, { "name": "medium_dataset", "data_size": 10000, "expected_max_time": 1.0, "expected_max_memory": 10 * 1024 * 1024 # 10MB }, { "name": "large_dataset", "data_size": 1000000, "expected_max_time": 10.0, "expected_max_memory": 100 * 1024 * 1024 # 100MB } ] def simulate_resource_constraints( cpu_limit: Optional[float] = None, memory_limit: Optional[int] = None ) -> Callable: """ Create a context manager to simulate resource constraints. Args: cpu_limit: CPU limit (percentage) memory_limit: Memory limit in bytes Returns: Context manager for resource constraints """ @contextlib.contextmanager def resource_context(): """Inner context manager for simulating resource constraints.""" original_limits = {} try: if cpu_limit is not None: # Simulate CPU limit by adding artificial delay original_limits['cpu'] = cpu_limit if memory_limit is not None and RESOURCE_AVAILABLE: # Set memory limit using resource module if hasattr(resource, 'RLIMIT_AS'): original_limits['memory'] = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit)) else: # Resource module available but RLIMIT_AS not supported print("Warning: RLIMIT_AS not available on this platform") elif memory_limit is not None and not RESOURCE_AVAILABLE: print("Warning: resource module not available, memory limits will not be enforced") yield finally: # Restore original limits if 'memory' in original_limits and RESOURCE_AVAILABLE and hasattr(resource, 'RLIMIT_AS'): resource.setrlimit(resource.RLIMIT_AS, original_limits['memory']) return resource_context def create_load_test_scenarios() -> List[Dict[str, Any]]: """ Create load test scenarios. Returns: List of load test scenarios """ return [ { "name": "low_load", "concurrent_users": 10, "request_rate": 100, "duration": 60, "expected_response_time": 0.1 }, { "name": "medium_load", "concurrent_users": 100, "request_rate": 1000, "duration": 300, "expected_response_time": 0.5 }, { "name": "high_load", "concurrent_users": 1000, "request_rate": 10000, "duration": 600, "expected_response_time": 1.0 } ] def benchmark_file_operations( file_size: int = 1024 * 1024, # 1MB operations: List[str] = None, iterations: int = 100 ) -> Dict[str, float]: """ Benchmark file operations performance. Args: file_size: Size of test file in bytes operations: List of operations to benchmark iterations: Number of iterations Returns: Dictionary of file operation timings """ if operations is None: operations = ["read", "write", "copy", "delete"] results = {} temp_dir = Path(tempfile.mkdtemp()) try: test_file = temp_dir / "test_file.dat" test_file_copy = temp_dir / "test_file_copy.dat" # Create test file with open(test_file, "wb") as f: f.write(os.urandom(file_size)) for op in operations: if op == "read": def read_operation(): """Read operation for benchmarking.""" with open(test_file, "rb") as f: f.read() results[op] = benchmark_function_performance( read_operation, iterations )["average_time"] elif op == "write": def write_operation(): """Write operation for benchmarking.""" with open(test_file, "wb") as f: f.write(os.urandom(file_size)) results[op] = benchmark_function_performance( write_operation, iterations )["average_time"] elif op == "copy": def copy_operation(): """Copy operation for benchmarking.""" import shutil shutil.copy2(test_file, test_file_copy) if test_file_copy.exists(): test_file_copy.unlink() results[op] = benchmark_function_performance( copy_operation, iterations )["average_time"] elif op == "delete": def delete_operation(): """Delete operation for benchmarking.""" test_file.unlink() with open(test_file, "wb") as f: f.write(os.urandom(file_size)) results[op] = benchmark_function_performance( delete_operation, iterations )["average_time"] finally: # Cleanup import shutil shutil.rmtree(temp_dir) return results def create_performance_monitor() -> Callable: """ Create a performance monitor context manager. Returns: Context manager for performance monitoring """ @contextlib.contextmanager def monitor_context(): """Inner context manager for performance monitoring.""" start_time = time.time() if PSUTIL_AVAILABLE: start_mem = psutil.Process(os.getpid()).memory_info().rss yield end_time = time.time() if PSUTIL_AVAILABLE: end_mem = psutil.Process(os.getpid()).memory_info().rss memory_used = end_mem - start_mem cpu_usage = psutil.cpu_percent(interval=0.1) else: memory_used = 0 cpu_usage = 0 elapsed_time = end_time - start_time print(f"Performance Monitor Results:") print(f" Execution Time: {elapsed_time:.4f} seconds") if PSUTIL_AVAILABLE: print(f" Memory Used: {memory_used / 1024 / 1024:.2f} MB") print(f" CPU Usage: {cpu_usage}%") else: print(f" Memory Used: N/A (psutil not available)") print(f" CPU Usage: N/A (psutil not available)") return monitor_context def simulate_slow_operations( delay: float = 0.1, variability: float = 0.05 ) -> Callable: """ Create a context manager to simulate slow operations. Args: delay: Base delay in seconds variability: Random variability in delay Returns: Context manager for slow operation simulation """ import random @contextlib.contextmanager def slow_context(): """Inner context manager for simulating slow operations.""" original_monotonic = time.monotonic start_time = original_monotonic() def slow_monotonic(): """Mock monotonic time that adds artificial delay.""" elapsed = original_monotonic() - start_time variability_factor = 1.0 + random.uniform(-variability, variability) return elapsed + (delay * variability_factor) # Patch time functions with patch('time.monotonic', side_effect=slow_monotonic): with patch('time.sleep', side_effect=lambda x: time.sleep(x * 10)): yield return slow_context def create_stress_test_scenarios() -> List[Dict[str, Any]]: """ Create stress test scenarios. Returns: List of stress test scenarios """ return [ { "name": "memory_stress", "type": "memory", "target": "high_memory_usage", "duration": 300, "expected_behavior": "graceful_degradation" }, { "name": "cpu_stress", "type": "cpu", "target": "high_cpu_usage", "duration": 180, "expected_behavior": "resource_throttling" }, { "name": "io_stress", "type": "io", "target": "high_disk_usage", "duration": 240, "expected_behavior": "queue_management" } ] def benchmark_database_operations( db_connection: Any, queries: List[str], iterations: int = 100 ) -> Dict[str, float]: """ Benchmark database operations performance. Args: db_connection: Database connection queries: List of SQL queries to benchmark iterations: Number of iterations Returns: Dictionary of database operation timings """ results = {} for i, query in enumerate(queries): def query_operation(): """Execute a database query for benchmarking.""" cursor = db_connection.cursor() cursor.execute(query) cursor.fetchall() cursor.close() timing = benchmark_function_performance( query_operation, iterations )["average_time"] results[f"query_{i}"] = timing return results def create_network_performance_test_scenarios() -> List[Dict[str, Any]]: """ Create network performance test scenarios. Returns: List of network performance test scenarios """ return [ { "name": "low_latency", "latency": 10, # ms "bandwidth": 100, # Mbps "packet_loss": 0.0 # 0% }, { "name": "medium_latency", "latency": 100, # ms "bandwidth": 10, # Mbps "packet_loss": 0.1 # 1% }, { "name": "high_latency", "latency": 500, # ms "bandwidth": 1, # Mbps "packet_loss": 0.5 # 5% } ] def measure_concurrency_performance( func: Callable, worker_counts: List[int] = [1, 2, 4, 8, 16], iterations: int = 100, *args, **kwargs ) -> Dict[int, float]: """ Measure performance with different levels of concurrency. Args: func: Function to test worker_counts: List of worker counts to test iterations: Number of iterations per worker count *args: Positional arguments for the function **kwargs: Keyword arguments for the function Returns: Dictionary mapping worker counts to performance metrics """ import concurrent.futures results = {} for workers in worker_counts: def concurrent_operation(): """Execute function with multiple workers.""" with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: futures = [executor.submit(func, *args, **kwargs) for _ in range(iterations)] concurrent.futures.wait(futures) timing = benchmark_function_performance( concurrent_operation, 1 )["total_time"] results[workers] = { "total_time": timing, "throughput": iterations / timing } return results def create_performance_regression_test_scenarios() -> List[Dict[str, Any]]: """ Create performance regression test scenarios. Returns: List of performance regression test scenarios """ return [ { "name": "baseline_performance", "description": "Establish baseline performance metrics", "metrics": { "max_response_time": 0.5, "max_memory_usage": 50 * 1024 * 1024, # 50MB "min_throughput": 100 # operations per second } }, { "name": "regression_detection", "description": "Detect performance regressions", "thresholds": { "response_time_increase": 0.2, # 20% increase "memory_increase": 0.1, # 10% increase "throughput_decrease": 0.15 # 15% decrease } } ] def simulate_performance_degradation( degradation_factor: float = 0.1, degradation_type: str = "linear" ) -> Callable: """ Create a context manager to simulate performance degradation. Args: degradation_factor: Performance degradation factor degradation_type: Type of degradation (linear, exponential) Returns: Context manager for performance degradation simulation """ @contextlib.contextmanager def degradation_context(): """Inner context manager for simulating performance degradation.""" call_count = 0 original_monotonic = time.monotonic start_time = original_monotonic() def degraded_monotonic(): """Mock monotonic time with performance degradation.""" nonlocal call_count call_count += 1 elapsed = original_monotonic() - start_time if degradation_type == "linear": degradation = degradation_factor * call_count else: # exponential degradation = degradation_factor ** call_count return elapsed + degradation def degraded_sleep(seconds): """Mock sleep with performance degradation.""" original_sleep = time.sleep original_sleep(seconds * (1 + degradation_factor)) with patch('time.monotonic', side_effect=degraded_monotonic): with patch('time.sleep', side_effect=lambda x: time.sleep(x * (1 + degradation_factor))): yield return degradation_context def create_resource_monitoring_scenarios() -> List[Dict[str, Any]]: """ Create resource monitoring test scenarios. Returns: List of resource monitoring test scenarios """ return [ { "name": "normal_operation", "expected_cpu_usage": 0.3, # 30% "expected_memory_usage": 100 * 1024 * 1024, # 100MB "expected_disk_io": 1024 * 1024 # 1MB/s }, { "name": "high_load_operation", "expected_cpu_usage": 0.8, # 80% "expected_memory_usage": 500 * 1024 * 1024, # 500MB "expected_disk_io": 10 * 1024 * 1024 # 10MB/s }, { "name": "resource_leak_detection", "monitoring_duration": 300, # 5 minutes "leak_threshold": 0.05 # 5% increase } ] from typing import Dict, Any, List, Optional, Union, Callable from pathlib import Path import re import hashlib import json import tempfile from unittest.mock import MagicMock def validate_test_data_structure( data: Any, schema: Dict[str, Any] ) -> bool: """ Validate test data structure against a schema. Args: data: Data to validate schema: Schema definition Returns: True if data matches schema, False otherwise """ def _validate(item, schema_part): """Internal validation function for nested data structures.""" if "type" in schema_part: expected_type = schema_part["type"] if expected_type == "dict" and not isinstance(item, dict): return False elif expected_type == "list" and not isinstance(item, list): return False elif expected_type == "str" and not isinstance(item, str): return False elif expected_type == "int" and not isinstance(item, int): return False elif expected_type == "float" and not isinstance(item, float): return False elif expected_type == "bool" and not isinstance(item, bool): return False if "required" in schema_part and not all(key in item for key in schema_part["required"]): return False if "properties" in schema_part: for key, prop_schema in schema_part["properties"].items(): if key in item: if not _validate(item[key], prop_schema): return False if "items" in schema_part and isinstance(item, list): for list_item in item: if not _validate(list_item, schema_part["items"]): return False if "pattern" in schema_part and isinstance(item, str): if not re.match(schema_part["pattern"], item): return False if "min" in schema_part and isinstance(item, (int, float)): if item < schema_part["min"]: return False if "max" in schema_part and isinstance(item, (int, float)): if item > schema_part["max"]: return False return True return _validate(data, schema) def create_data_validation_test_cases() -> List[Dict[str, Any]]: """ Create data validation test cases. Returns: List of data validation test cases """ return [ { "name": "valid_config_data", "data": { "database": { "host": "localhost", "port": 5432, "username": "admin", "PASSWORD_REMOVED": "SECRET_REMOVED" }, "logging": { "level": "INFO", "file": "/var/log/app.log" } }, "schema": { "type": "dict", "properties": { "database": { "type": "dict", "required": ["host", "port", "username", "PASSWORD_REMOVED"], "properties": { "host": {"type": "str"}, "port": {"type": "int", "min": 1, "max": 65535}, "username": {"type": "str"}, "PASSWORD_REMOVED": {"type": "str"} } }, "logging": { "type": "dict", "required": ["level", "file"], "properties": { "level": {"type": "str", "pattern": "^(DEBUG|INFO|WARNING|ERROR|CRITICAL)$"}, "file": {"type": "str"} } } } }, "expected_result": True }, { "name": "invalid_config_data", "data": { "database": { "host": "localhost", "port": 70000, # Invalid port "username": "admin" # Missing PASSWORD_REMOVED } }, "schema": { "type": "dict", "properties": { "database": { "type": "dict", "required": ["host", "port", "username", "PASSWORD_REMOVED"], "properties": { "host": {"type": "str"}, "port": {"type": "int", "min": 1, "max": 65535}, "username": {"type": "str"}, "PASSWORD_REMOVED": {"type": "str"} } } } }, "expected_result": False } ] def validate_file_integrity( file_path: Path, expected_hash: str, algorithm: str = "sha256" ) -> bool: """ Validate file integrity using hash comparison. Args: file_path: Path to file expected_hash: Expected hash value algorithm: Hash algorithm to use Returns: True if file integrity is valid, False otherwise """ hash_func = hashlib.new(algorithm) with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_func.update(chunk) actual_hash = hash_func.hexdigest() return actual_hash == expected_hash def create_file_validation_test_scenarios() -> List[Dict[str, Any]]: """ Create file validation test scenarios. Returns: List of file validation test scenarios """ return [ { "name": "valid_file_integrity", "file_content": "test content for integrity check", "expected_hash": "a1b2c3d4e5f6", # Placeholder - would be actual hash in real test "expected_result": True }, { "name": "corrupted_file", "file_content": "corrupted content", "expected_hash": "a1b2c3d4e5f6", # Different from actual hash "expected_result": False }, { "name": "missing_file", "file_content": None, "expected_hash": "a1b2c3d4e5f6", "expected_result": False } ] def validate_json_schema( json_data: Union[str, Dict], schema: Dict[str, Any] ) -> bool: """ Validate JSON data against a schema. Args: json_data: JSON data to validate schema: JSON schema definition Returns: True if JSON is valid, False otherwise """ if isinstance(json_data, str): try: data = json.loads(json_data) except json.JSONDecodeError: return False else: data = json_data return validate_test_data_structure(data, schema) def create_json_validation_test_cases() -> List[Dict[str, Any]]: """ Create JSON validation test cases. Returns: List of JSON validation test cases """ return [ { "name": "valid_json_api_response", "json_data": """ { "status": "success", "data": { "users": [ {"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"} ] }, "timestamp": "2023-01-01T00:00:00Z" } """, "schema": { "type": "dict", "required": ["status", "data", "timestamp"], "properties": { "status": {"type": "str", "pattern": "^(success|error|warning)$"}, "data": { "type": "dict", "required": ["users"], "properties": { "users": { "type": "list", "items": { "type": "dict", "required": ["id", "name"], "properties": { "id": {"type": "int", "min": 1}, "name": {"type": "str"} } } } } }, "timestamp": {"type": "str", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"} } }, "expected_result": True }, { "name": "invalid_json_api_response", "json_data": """ { "status": "invalid_status", "data": { "users": [ {"id": "not_an_int", "name": "Alice"} ] } } """, "schema": { "type": "dict", "required": ["status", "data", "timestamp"], "properties": { "status": {"type": "str", "pattern": "^(success|error|warning)$"}, "data": { "type": "dict", "required": ["users"], "properties": { "users": { "type": "list", "items": { "type": "dict", "required": ["id", "name"], "properties": { "id": {"type": "int", "min": 1}, "name": {"type": "str"} } } } } }, "timestamp": {"type": "str", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"} } }, "expected_result": False } ] def validate_database_schema( database_schema: Dict[str, Any], expected_schema: Dict[str, Any] ) -> bool: """ Validate database schema structure. Args: database_schema: Actual database schema expected_schema: Expected database schema Returns: True if schemas match, False otherwise """ # Compare tables if set(database_schema.keys()) != set(expected_schema.keys()): return False # Compare table structures for table_name, table_def in expected_schema.items(): if table_name not in database_schema: return False actual_table = database_schema[table_name] # Compare columns if set(table_def["columns"].keys()) != set(actual_table["columns"].keys()): return False # Compare column definitions for col_name, col_def in table_def["columns"].items(): if col_name not in actual_table["columns"]: return False actual_col = actual_table["columns"][col_name] if col_def["type"] != actual_col["type"]: return False if "constraints" in col_def and col_def["constraints"] != actual_col.get("constraints", []): return False return True def create_database_validation_test_scenarios() -> List[Dict[str, Any]]: """ Create database validation test scenarios. Returns: List of database validation test scenarios """ return [ { "name": "valid_database_schema", "database_schema": { "users": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "name": {"type": "TEXT", "constraints": ["NOT NULL"]}, "email": {"type": "TEXT", "constraints": ["UNIQUE"]} } }, "posts": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "user_id": {"type": "INTEGER", "constraints": ["FOREIGN KEY"]}, "title": {"type": "TEXT"}, "content": {"type": "TEXT"} } } }, "expected_schema": { "users": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "name": {"type": "TEXT", "constraints": ["NOT NULL"]}, "email": {"type": "TEXT", "constraints": ["UNIQUE"]} } }, "posts": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "user_id": {"type": "INTEGER", "constraints": ["FOREIGN KEY"]}, "title": {"type": "TEXT"}, "content": {"type": "TEXT"} } } }, "expected_result": True }, { "name": "invalid_database_schema", "database_schema": { "users": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "name": {"type": "TEXT"} # Missing email column } } }, "expected_schema": { "users": { "columns": { "id": {"type": "INTEGER", "constraints": ["PRIMARY KEY"]}, "name": {"type": "TEXT", "constraints": ["NOT NULL"]}, "email": {"type": "TEXT", "constraints": ["UNIQUE"]} } } }, "expected_result": False } ] def validate_tool_structure( tool_definition: Dict[str, Any], expected_structure: Dict[str, Any] ) -> bool: """ Validate tool structure and metadata. Args: tool_definition: Tool definition to validate expected_structure: Expected tool structure Returns: True if tool structure is valid, False otherwise """ # Check required fields required_fields = expected_structure.get("required_fields", []) if not all(field in tool_definition for field in required_fields): return False # Check metadata structure if "metadata" in expected_structure: metadata_schema = expected_structure["metadata"] if not validate_test_data_structure(tool_definition.get("metadata", {}), metadata_schema): return False # Check function signatures if "functions" in expected_structure: for func_name, func_schema in expected_structure["functions"].items(): if func_name not in tool_definition.get("functions", {}): return False # Check function parameters actual_func = tool_definition["functions"][func_name] expected_params = func_schema.get("parameters", []) # Simple parameter count check if "parameters" in func_schema: try: import inspect sig = inspect.signature(actual_func) if len(sig.parameters) != len(expected_params): return False except: pass return True def create_tool_validation_test_cases() -> List[Dict[str, Any]]: """ Create tool validation test cases. Returns: List of tool validation test cases """ return [ { "name": "valid_tool_structure", "tool_definition": { "name": "test_tool", "version": "1.0.0", "author": "Test Author", "description": "Test tool", "metadata": { "category": "utility", "compatibility": ["1.0", "2.0"] }, "functions": { "initialize": lambda: True, "execute": lambda x: x * 2, "cleanup": lambda: None } }, "expected_structure": { "required_fields": ["name", "version", "author", "description"], "metadata": { "type": "dict", "required": ["category", "compatibility"], "properties": { "category": {"type": "str"}, "compatibility": {"type": "list", "items": {"type": "str"}} } }, "functions": { "initialize": {"parameters": []}, "execute": {"parameters": ["x"]}, "cleanup": {"parameters": []} } }, "expected_result": True }, { "name": "invalid_tool_structure", "tool_definition": { "name": "test_tool", # Missing required fields "functions": { "initialize": lambda: True # Missing required functions } }, "expected_structure": { "required_fields": ["name", "version", "author", "description"], "functions": { "initialize": {"parameters": []}, "execute": {"parameters": ["x"]}, "cleanup": {"parameters": []} } }, "expected_result": False } ] def validate_api_response( response: Dict[str, Any], expected_schema: Dict[str, Any] ) -> bool: """ Validate API response structure. Args: response: API response to validate expected_schema: Expected response schema Returns: True if response is valid, False otherwise """ return validate_test_data_structure(response, expected_schema) def create_api_validation_test_scenarios() -> List[Dict[str, Any]]: """ Create API validation test scenarios. Returns: List of API validation test scenarios """ return [ { "name": "valid_api_response", "response": { "status": "success", "code": 200, "data": { "items": [ {"id": 1, "name": "Item 1"}, {"id": 2, "name": "Item 2"} ], "pagination": { "page": 1, "page_size": 10, "total": 2 } }, "timestamp": "2023-01-01T00:00:00Z" }, "expected_schema": { "type": "dict", "required": ["status", "code", "data", "timestamp"], "properties": { "status": {"type": "str", "pattern": "^(success|error|warning)$"}, "code": {"type": "int", "min": 200, "max": 599}, "data": { "type": "dict", "required": ["items", "pagination"], "properties": { "items": { "type": "list", "items": { "type": "dict", "required": ["id", "name"], "properties": { "id": {"type": "int", "min": 1}, "name": {"type": "str"} } } }, "pagination": { "type": "dict", "required": ["page", "page_size", "total"], "properties": { "page": {"type": "int", "min": 1}, "page_size": {"type": "int", "min": 1, "max": 100}, "total": {"type": "int", "min": 0} } } } }, "timestamp": {"type": "str", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"} } }, "expected_result": True }, { "name": "invalid_api_response", "response": { "status": "invalid_status", "code": 999, # Invalid code "data": { "items": [ {"id": "not_an_int", "name": "Item 1"} # Invalid ID type ] } }, "expected_schema": { "type": "dict", "required": ["status", "code", "data", "timestamp"], "properties": { "status": {"type": "str", "pattern": "^(success|error|warning)$"}, "code": {"type": "int", "min": 200, "max": 599}, "data": { "type": "dict", "required": ["items", "pagination"], "properties": { "items": { "type": "list", "items": { "type": "dict", "required": ["id", "name"], "properties": { "id": {"type": "int", "min": 1}, "name": {"type": "str"} } } } } } } }, "expected_result": False } ] def validate_configuration_files( config_files: List[Path], expected_structure: Dict[str, Any] ) -> Dict[str, bool]: """ Validate multiple configuration files. Args: config_files: List of configuration file paths expected_structure: Expected configuration structure Returns: Dictionary mapping file paths to validation results """ results = {} for config_file in config_files: try: with open(config_file, "r") as f: config_data = json.load(f) results[str(config_file)] = validate_test_data_structure(config_data, expected_structure) except (json.JSONDecodeError, IOError): results[str(config_file)] = False return results def create_configuration_validation_test_cases() -> List[Dict[str, Any]]: """ Create configuration validation test cases. Returns: List of configuration validation test cases """ return [ { "name": "valid_configuration_files", "config_files": [ { "content": """ { "database": { "host": "localhost", "port": 5432, "username": "admin", "PASSWORD_REMOVED": "SECRET_REMOVED" }, "logging": { "level": "INFO", "file": "/var/log/app.log" } } """, "expected_result": True }, { "content": """ { "database": { "host": "localhost", "port": 5432, "username": "admin", "PASSWORD_REMOVED": "SECRET_REMOVED" } } """, "expected_result": True } ], "expected_structure": { "type": "dict", "properties": { "database": { "type": "dict", "required": ["host", "port", "username", "PASSWORD_REMOVED"], "properties": { "host": {"type": "str"}, "port": {"type": "int", "min": 1, "max": 65535}, "username": {"type": "str"}, "PASSWORD_REMOVED": {"type": "str"} } }, "logging": { "type": "dict", "properties": { "level": {"type": "str", "pattern": "^(DEBUG|INFO|WARNING|ERROR|CRITICAL)$"}, "file": {"type": "str"} } } } } }, { "name": "invalid_configuration_files", "config_files": [ { "content": """ { "database": { "host": "localhost", "port": 70000, "username": "admin" } } """, "expected_result": False }, { "content": "invalid json content", "expected_result": False } ], "expected_structure": { "type": "dict", "properties": { "database": { "type": "dict", "required": ["host", "port", "username", "PASSWORD_REMOVED"], "properties": { "host": {"type": "str"}, "port": {"type": "int", "min": 1, "max": 65535}, "username": {"type": "str"}, "PASSWORD_REMOVED": {"type": "str"} } } } } } ] def validate_data_consistency( data_source: Any, validation_rules: List[Dict[str, Any]] ) -> bool: """ Validate data consistency against validation rules. Args: data_source: Data to validate validation_rules: List of validation rules Returns: True if data is consistent, False otherwise """ def get_nested_value(data, field_path): """Get value from nested data structure using dot notation or direct field name""" if not isinstance(data, dict): return None # Try direct field access first if field_path in data: return data[field_path] # Try nested access using dot notation if '.' in field_path: parts = field_path.split('.') current = data for part in parts: if isinstance(current, dict) and part in current: current = current[part] else: return None return current return None for rule in validation_rules: field = rule["field"] validation_type = rule["type"] # Get field value using nested access value = get_nested_value(data_source, field) if value is None and rule.get("required", False): return False # Apply validation if validation_type == "range": min_val = rule.get("min") max_val = rule.get("max") if not (min_val <= value <= max_val): return False elif validation_type == "pattern": pattern = rule["pattern"] if not re.match(pattern, str(value)): return False elif validation_type == "enum": allowed_values = rule["values"] if value not in allowed_values: return False elif validation_type == "custom": validator = rule["validator"] if not validator(value): return False return True def create_data_consistency_test_scenarios() -> List[Dict[str, Any]]: """ Create data consistency test scenarios. Returns: List of data consistency test scenarios """ return [ { "name": "valid_data_consistency", "data": { "user": { "id": 123, "username": "test_user", "email": "test@example.com", "status": "active", "age": 25 } }, "validation_rules": [ {"field": "id", "type": "range", "min": 1, "max": 1000, "required": True}, {"field": "username", "type": "pattern", "pattern": "^[a-z_]+$", "required": True}, {"field": "email", "type": "pattern", "pattern": "^[^@]+@[^@]+\\.[^@]+$", "required": True}, {"field": "status", "type": "enum", "values": ["active", "inactive", "suspended"], "required": True}, {"field": "age", "type": "range", "min": 18, "max": 120} ], "expected_result": True }, { "name": "invalid_data_consistency", "data": { "user": { "id": 0, # Invalid ID "username": "Invalid User", # Invalid username "email": "not-an-email", # Invalid email "status": "unknown", # Invalid status "age": 15 # Invalid age } }, "validation_rules": [ {"field": "id", "type": "range", "min": 1, "max": 1000, "required": True}, {"field": "username", "type": "pattern", "pattern": "^[a-z_]+$", "required": True}, {"field": "email", "type": "pattern", "pattern": "^[^@]+@[^@]+\\.[^@]+$", "required": True}, {"field": "status", "type": "enum", "values": ["active", "inactive", "suspended"], "required": True}, {"field": "age", "type": "range", "min": 18, "max": 120} ], "expected_result": False } ] import sqlite3 import tempfile from pathlib import Path from typing import Dict, Any, List, Optional, Union, Callable from unittest.mock import MagicMock, patch import contextlib import time def create_test_database( schema: Optional[str] = None, data: Optional[List[Dict[str, Any]]] = None, db_name: str = "test_db", use_memory: bool = True ) -> Union[str, Path]: """ Create a test database with optional schema and data. Args: schema: SQL schema definition data: List of data dictionaries to insert db_name: Database name use_memory: Use in-memory database if True Returns: Database path or connection string """ if use_memory: conn = sqlite3.connect(":memory:") return conn else: db_path = Path(tempfile.gettempdir()) / f"{db_name}.db" conn = sqlite3.connect(str(db_path)) conn.close() return str(db_path) def setup_test_database_schema( conn: sqlite3.Connection, schema: str ) -> None: """ Set up database schema for testing. Args: conn: Database connection schema: SQL schema definition """ cursor = conn.cursor() cursor.executescript(schema) conn.commit() def insert_test_data( conn: sqlite3.Connection, table: str, data: List[Dict[str, Any]] ) -> None: """ Insert test data into a database table. Args: conn: Database connection table: Table name data: List of data dictionaries """ if not data: return cursor = conn.cursor() # Get column names from first data item columns = list(data[0].keys()) placeholders = ", ".join(["?"] * len(columns)) columns_str = ", ".join(columns) # Prepare and execute insert statements for item in data: values = [item[col] for col in columns] cursor.execute( f"INSERT INTO {table} ({columns_str}) VALUES ({placeholders})", values ) conn.commit() def verify_database_state( conn: sqlite3.Connection, expected_state: Dict[str, Any], tolerance: float = 0.0 ) -> bool: """ Verify database state matches expected state. Args: conn: Database connection expected_state: Expected database state tolerance: Numeric tolerance for floating point comparisons Returns: True if state matches, False otherwise """ cursor = conn.cursor() for table, expected_data in expected_state.items(): # Query all data from table cursor.execute(f"SELECT * FROM {table}") actual_data = cursor.fetchall() # Get column names cursor.execute(f"PRAGMA table_info({table})") columns = [col[1] for col in cursor.fetchall()] # Convert to list of dictionaries for comparison actual_records = [] for row in actual_data: record = dict(zip(columns, row)) actual_records.append(record) # Compare with expected data if len(actual_records) != len(expected_data): return False for actual, expected in zip(actual_records, expected_data): for key, expected_value in expected.items(): actual_value = actual[key] if isinstance(expected_value, (int, str, bool)): if actual_value != expected_value: return False elif isinstance(expected_value, float): if abs(actual_value - expected_value) > tolerance: return False else: if actual_value != expected_value: return False return True def create_database_mock() -> MagicMock: """ Create a mock database connection for testing. Returns: Mock database connection object """ mock_conn = MagicMock(spec=sqlite3.Connection) mock_cursor = MagicMock() # Set up mock behavior mock_conn.cursor.return_value = mock_cursor mock_cursor.fetchone.return_value = (1,) mock_cursor.fetchall.return_value = [(1, "test"), (2, "data")] mock_cursor.description = [("id",), ("name",)] return mock_conn def create_database_fixture( schema: str, initial_data: Optional[List[Dict[str, Any]]] = None ) -> Callable: """ Create a pytest fixture for database testing. Args: schema: Database schema initial_data: Initial data to populate Returns: Fixture function """ def database_fixture(): """Inner fixture function for database testing.""" # Create in-memory database conn = sqlite3.connect(":memory:") # Set up schema setup_test_database_schema(conn, schema) # Insert initial data if provided if initial_data: for table_data in initial_data: table_name = list(table_data.keys())[0] insert_test_data(conn, table_name, table_data[table_name]) yield conn # Cleanup conn.close() return database_fixture def simulate_database_errors( error_type: str = "connection", operation: str = "execute" ) -> Callable: """ Create a context manager to simulate database errors. Args: error_type: Type of error to simulate operation: Database operation to fail Returns: Context manager for error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating database errors.""" error_map = { "connection": sqlite3.OperationalError("Unable to connect"), "integrity": sqlite3.IntegrityError("Constraint violation"), "programming": sqlite3.ProgrammingError("SQL syntax error"), "timeout": sqlite3.OperationalError("Database locked") } error = error_map.get(error_type, sqlite3.Error("Database error")) with patch('sqlite3.Connection') as mock_conn_class: mock_conn = MagicMock() mock_cursor = MagicMock() if operation == "execute": mock_cursor.execute.side_effect = error elif operation == "commit": mock_conn.commit.side_effect = error elif operation == "fetch": mock_cursor.fetchall.side_effect = error mock_conn.cursor.return_value = mock_cursor mock_conn_class.return_value = mock_conn yield mock_conn return error_context def benchmark_database_operations( conn: sqlite3.Connection, operations: List[Callable], iterations: int = 100 ) -> Dict[str, float]: """ Benchmark database operations performance. Args: conn: Database connection operations: List of operation functions iterations: Number of iterations per operation Returns: Dictionary of operation timings """ results = {} for i, operation in enumerate(operations): start_time = time.time() for _ in range(iterations): operation(conn) end_time = time.time() avg_time = (end_time - start_time) / iterations results[f"operation_{i}"] = avg_time return results def create_transaction_test_scenarios() -> List[Dict[str, Any]]: """ Create test scenarios for transaction testing. Returns: List of transaction test scenarios """ return [ { "name": "successful_transaction", "operations": [ "BEGIN", "INSERT INTO test VALUES (1, 'data')", "COMMIT" ], "expected_result": "success" }, { "name": "failed_transaction", "operations": [ "BEGIN", "INSERT INTO test VALUES (1, 'data')", "ROLLBACK" ], "expected_result": "rollback" }, { "name": "nested_transaction", "operations": [ "BEGIN", "SAVEPOINT sp1", "INSERT INTO test VALUES (1, 'data')", "RELEASE SAVEPOINT sp1", "COMMIT" ], "expected_result": "success" } ] def verify_database_performance( conn: sqlite3.Connection, query: str, max_execution_time: float = 1.0, iterations: int = 10 ) -> bool: """ Verify database query performance meets requirements. Args: conn: Database connection query: SQL query to test max_execution_time: Maximum allowed execution time iterations: Number of test iterations Returns: True if performance is acceptable, False otherwise """ cursor = conn.cursor() total_time = 0.0 for _ in range(iterations): start_time = time.time() cursor.execute(query) cursor.fetchall() end_time = time.time() total_time += (end_time - start_time) avg_time = total_time / iterations return avg_time <= max_execution_time def create_database_snapshot( conn: sqlite3.Connection ) -> Dict[str, List[Dict[str, Any]]]: """ Create a snapshot of current database state. Args: conn: Database connection Returns: Dictionary representing database state """ cursor = conn.cursor() snapshot = {} # Get all tables cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") tables = [table[0] for table in cursor.fetchall()] for table in tables: # Get table data cursor.execute(f"SELECT * FROM {table}") rows = cursor.fetchall() # Get column names cursor.execute(f"PRAGMA table_info({table})") columns = [col[1] for col in cursor.fetchall()] # Convert to list of dictionaries table_data = [] for row in rows: record = dict(zip(columns, row)) table_data.append(record) snapshot[table] = table_data return snapshot def restore_database_snapshot( conn: sqlite3.Connection, snapshot: Dict[str, List[Dict[str, Any]]] ) -> None: """ Restore database to a previous snapshot state. Args: conn: Database connection snapshot: Database snapshot to restore """ cursor = conn.cursor() for table, data in snapshot.items(): # Clear existing data cursor.execute(f"DELETE FROM {table}") # Re-insert data if data: columns = list(data[0].keys()) placeholders = ", ".join(["?"] * len(columns)) columns_str = ", ".join(columns) for item in data: values = [item[col] for col in columns] cursor.execute( f"INSERT INTO {table} ({columns_str}) VALUES ({placeholders})", values ) conn.commit() import os import stat import hashlib from pathlib import Path from typing import Dict, Any, List, Optional, Union import tempfile import shutil from unittest.mock import MagicMock def create_test_file_structure( base_path: Path, structure: Dict[str, Union[Dict, str, bytes]], file_permissions: int = 0o644, dir_permissions: int = 0o755 ) -> Dict[str, Path]: """ Create a complex file structure for testing. Args: base_path: Base directory path structure: Dictionary describing the file structure file_permissions: Permissions for created files dir_permissions: Permissions for created directories Returns: Dictionary mapping file names to their paths """ created_files = {} for name, content in structure.items(): full_path = base_path / name if isinstance(content, dict): # It's a directory - create it and recurse full_path.mkdir(exist_ok=True, mode=dir_permissions) created_files.update(create_test_file_structure( full_path, content, file_permissions, dir_permissions )) else: # It's a file - create it with content if isinstance(content, str): full_path.write_text(content) else: full_path.write_bytes(content) # Set file permissions full_path.chmod(file_permissions) created_files[name] = full_path return created_files def create_duplicate_files( base_path: Path, original_content: str, num_duplicates: int = 3, file_size: int = 1024 ) -> List[Path]: """ Create multiple duplicate files for testing. Args: base_path: Directory to create files in original_content: Content for original file num_duplicates: Number of duplicate files to create file_size: Target file size Returns: List of created file paths """ if not base_path.exists(): base_path.mkdir(parents=True) files = [] # Create original file original_file = base_path / "original.txt" if len(original_content) < file_size: # Pad content to reach desired size multiplier = (file_size // len(original_content)) + 1 padded_content = original_content * multiplier original_file.write_text(padded_content[:file_size]) else: original_file.write_text(original_content[:file_size]) files.append(original_file) # Create duplicates for i in range(1, num_duplicates + 1): duplicate_file = base_path / f"duplicate_{i}.txt" shutil.copy2(original_file, duplicate_file) files.append(duplicate_file) return files def create_files_with_varying_sizes( base_path: Path, sizes: List[int], content_pattern: str = "Test content " ) -> List[Path]: """ Create files with varying sizes for testing. Args: base_path: Directory to create files in sizes: List of target file sizes in bytes content_pattern: Pattern to use for file content Returns: List of created file paths """ if not base_path.exists(): base_path.mkdir(parents=True) files = [] for i, size in enumerate(sizes): file_path = base_path / f"file_{size}.txt" # Create content that matches the desired size content = (content_pattern * ((size // len(content_pattern)) + 1))[:size] file_path.write_text(content) files.append(file_path) return files def create_symlinks_and_hardlinks( base_path: Path, target_file: Path ) -> Dict[str, List[Path]]: """ Create symlinks and hardlinks for testing. Args: base_path: Directory to create links in target_file: Target file for links Returns: Dictionary with 'symlinks' and 'hardlinks' lists """ if not base_path.exists(): base_path.mkdir(parents=True) result = { 'symlinks': [], 'hardlinks': [] } # Create symlinks for i in range(3): symlink = base_path / f"symlink_{i}.txt" try: symlink.symlink_to(target_file) result['symlinks'].append(symlink) except OSError: # Symlinks not supported on this system break # Create hardlinks for i in range(3): hardlink = base_path / f"hardlink_{i}.txt" try: hardlink.link_to(target_file) result['hardlinks'].append(hardlink) except OSError: # Hardlinks not supported for this file type break return result def calculate_file_hash(file_path: Path, algorithm: str = "sha256") -> str: """ Calculate hash of a file using specified algorithm. Args: file_path: Path to file algorithm: Hash algorithm to use Returns: Hexadecimal hash string """ hash_func = hashlib.new(algorithm) with open(file_path, "rb") as f: # Read file in chunks to handle large files for chunk in iter(lambda: f.read(4096), b""): hash_func.update(chunk) return hash_func.hexdigest() def compare_files(file1: Path, file2: Path) -> bool: """ Compare two files for identical content. Args: file1: First file path file2: Second file path Returns: True if files have identical content, False otherwise """ if file1.stat().st_size != file2.stat().st_size: return False with open(file1, "rb") as f1, open(file2, "rb") as f2: while True: chunk1 = f1.read(4096) chunk2 = f2.read(4096) if chunk1 != chunk2: return False if not chunk1: # End of file break return True def create_files_with_different_permissions( base_path: Path, permissions: List[int] ) -> List[Path]: """ Create files with different permissions for testing. Args: base_path: Directory to create files in permissions: List of permission modes (e.g., 0o644, 0o755) Returns: List of created file paths """ if not base_path.exists(): base_path.mkdir(parents=True) files = [] for i, perm in enumerate(permissions): file_path = base_path / f"perm_{oct(perm)}.txt" file_path.write_text(f"File with permissions {oct(perm)}") file_path.chmod(perm) files.append(file_path) return files def create_nested_directory_structure( base_path: Path, depth: int = 3, files_per_dir: int = 2 ) -> Dict[str, List[Path]]: """ Create a nested directory structure for testing. Args: base_path: Base directory path depth: Depth of nesting files_per_dir: Number of files per directory Returns: Dictionary mapping directory paths to their file lists """ structure = {} def _create_nested(current_path: Path, current_depth: int): """Internal function to recursively create nested directory structure.""" if current_depth > depth: return current_files = [] for i in range(files_per_dir): file_path = current_path / f"file_{current_depth}_{i}.txt" file_path.write_text(f"Content at depth {current_depth}") current_files.append(file_path) structure[str(current_path)] = current_files # Create subdirectories for i in range(2): # Create 2 subdirectories per level subdir = current_path / f"subdir_{i}" subdir.mkdir(exist_ok=True) _create_nested(subdir, current_depth + 1) _create_nested(base_path, 0) return structure def create_files_with_timestamps( base_path: Path, timestamps: List[float] ) -> List[Path]: """ Create files with specific timestamps for testing. Args: base_path: Directory to create files in timestamps: List of timestamps (seconds since epoch) Returns: List of created file paths """ if not base_path.exists(): base_path.mkdir(parents=True) files = [] import time for i, timestamp in enumerate(timestamps): file_path = base_path / f"timestamp_{i}.txt" file_path.write_text(f"File created at {timestamp}") # Set file timestamps os.utime(file_path, (timestamp, timestamp)) files.append(file_path) return files def verify_file_structure( base_path: Path, expected_structure: Dict[str, Union[Dict, str]] ) -> bool: """ Verify that a file structure matches expected structure. Args: base_path: Base directory path expected_structure: Expected structure dictionary Returns: True if structure matches, False otherwise """ for name, expected in expected_structure.items(): full_path = base_path / name if isinstance(expected, dict): # Should be a directory if not full_path.is_dir(): return False if not verify_file_structure(full_path, expected): return False else: # Should be a file if not full_path.is_file(): return False if isinstance(expected, str): # Check text content if full_path.read_text() != expected: return False else: # Check binary content if full_path.read_bytes() != expected: return False return True def mock_file_operations() -> MagicMock: """ Create a mock for file system operations. Returns: Mock object for file operations """ mock = MagicMock() # Mock common file operations mock.exists.return_value = True mock.is_file.return_value = True mock.is_dir.return_value = False mock.read_text.return_value = "Mock file content" mock.read_bytes.return_value = b"Mock binary content" mock.write_text.return_value = None mock.write_bytes.return_value = None mock.unlink.return_value = None mock.rename.return_value = None mock.stat.return_value = os.stat_result( (0o100644, 0, 0, 0, 0, 0, 1024, 0, 0, 0) ) return mock def create_large_file( base_path: Path, size_mb: int = 10, chunk_size: int = 1024 * 1024 ) -> Path: """ Create a large file for performance testing. Args: base_path: Directory to create file in size_mb: Size of file in megabytes chunk_size: Chunk size for writing Returns: Path to created file """ if not base_path.exists(): base_path.mkdir(parents=True) file_path = base_path / f"large_{size_mb}mb.dat" total_size = size_mb * 1024 * 1024 with open(file_path, "wb") as f: remaining = total_size while remaining > 0: write_size = min(chunk_size, remaining) f.write(os.urandom(write_size)) remaining -= write_size return file_path import tempfile from pathlib import Path from typing import Dict, Any, List, Optional, Union, Callable from unittest.mock import MagicMock, patch, Mock import importlib import sys from types import ModuleType import contextlib def create_mock_tool( name: str = "test_tool", functions: Optional[Dict[str, Callable]] = None, metadata: Optional[Dict[str, Any]] = None ) -> Mock: """ Create a mock tool for testing. Args: name: Tool name functions: Dictionary of tool functions metadata: Tool metadata Returns: Mock tool object """ mock_tool = Mock() mock_tool.name = name # Set up tool metadata if metadata is None: metadata = { "name": name, "version": "1.0.0", "author": "Test Author", "description": "Test tool for testing" } mock_tool.metadata = metadata # Set up tool functions if functions is None: functions = { "initialize": lambda: True, "execute": lambda *args, **kwargs: {"result": "success"}, "cleanup": lambda: None } for func_name, func_impl in functions.items(): setattr(mock_tool, func_name, func_impl) return mock_tool def create_tool_directory_structure( base_path: Path, tools: List[Dict[str, Any]] ) -> Dict[str, Path]: """ Create a tool directory structure for testing. Args: base_path: Base directory path tools: List of tool definitions Returns: Dictionary mapping tool names to their paths """ if not base_path.exists(): base_path.mkdir(parents=True) tool_paths = {} for tool_def in tools: tool_name = tool_def["name"] tool_dir = base_path / tool_name if not tool_dir.exists(): tool_dir.mkdir() # Create __init__.py init_file = tool_dir / "__init__.py" init_file.write_text(f"# {tool_name} tool\n") # Create main tool file tool_file = tool_dir / f"{tool_name}.py" tool_content = f""" def initialize(): '''Initialize the tool''' return True def execute(*args, **kwargs): '''Execute tool functionality''' return {{"tool": "{tool_name}", "status": "success"}} def cleanup(): '''Clean up tool resources''' pass metadata = {{ "name": "{tool_name}", "version": "{tool_def.get("version", "1.0.0")}", "author": "{tool_def.get("author", "Test Author")}", "description": "{tool_def.get("description", "Test tool")}" }} """ tool_file.write_text(tool_content.strip()) tool_paths[tool_name] = tool_dir return tool_paths def mock_tool_loader( tools: Optional[List[Mock]] = None ) -> MagicMock: """ Create a mock tool loader for testing. Args: tools: List of mock tools to load Returns: Mock tool loader """ mock_loader = MagicMock() if tools is None: tools = [ create_mock_tool("tool1"), create_mock_tool("tool2") ] mock_loader.load_tools.return_value = tools mock_loader.get_tool_by_name.side_effect = lambda name: next( (p for p in tools if p.name == name), None ) return mock_loader def create_tool_test_scenarios() -> List[Dict[str, Any]]: """ Create test scenarios for tool testing. Returns: List of tool test scenarios """ return [ { "name": "successful_tool_loading", "tools": [ {"name": "valid_tool1", "version": "1.0.0"}, {"name": "valid_tool2", "version": "2.0.0"} ], "expected_result": "success" }, { "name": "tool_loading_failure", "tools": [ {"name": "invalid_tool", "version": "1.0.0", "has_error": True} ], "expected_result": "failure" }, { "name": "tool_compatibility_issue", "tools": [ {"name": "old_tool", "version": "0.5.0"}, {"name": "new_tool", "version": "3.0.0"} ], "expected_result": "compatibility_warning" } ] def simulate_tool_errors( error_type: str = "loading", tool_name: str = "test_tool" ) -> Callable: """ Create a context manager to simulate tool errors. Args: error_type: Type of error to simulate tool_name: Name of tool to fail Returns: Context manager for error simulation """ @contextlib.contextmanager def error_context(): """Inner context manager for simulating tool errors.""" error_map = { "loading": ImportError(f"Cannot load tool {tool_name}"), "initialization": RuntimeError(f"Tool {tool_name} initialization failed"), "execution": ValueError(f"Tool {tool_name} execution error"), "compatibility": RuntimeError(f"Tool {tool_name} compatibility issue") } error = error_map.get(error_type, RuntimeError("Tool error")) with patch('importlib.import_module') as mock_import: if error_type == "loading": mock_import.side_effect = error else: mock_tool = create_mock_tool(tool_name) if error_type == "initialization": mock_tool.initialize.side_effect = error elif error_type == "execution": mock_tool.execute.side_effect = error mock_import.return_value = mock_tool yield return error_context def verify_tool_functionality( tool: Union[Mock, ModuleType], test_cases: List[Dict[str, Any]] ) -> Dict[str, bool]: """ Verify tool functionality against test cases. Args: tool: Tool to test test_cases: List of test cases Returns: Dictionary of test results """ results = {} for test_case in test_cases: test_name = test_case["name"] try: # Call the appropriate tool function if test_case["function"] == "initialize": result = tool.initialize() elif test_case["function"] == "execute": result = tool.execute(*test_case.get("args", []), **test_case.get("kwargs", {})) elif test_case["function"] == "cleanup": result = tool.cleanup() else: results[test_name] = False continue # Verify result expected = test_case.get("expected", True) if result == expected: results[test_name] = True else: results[test_name] = False except Exception: results[test_name] = False return results def create_tool_dependency_graph( tools: List[Dict[str, Any]] ) -> Dict[str, List[str]]: """ Create a tool dependency graph for testing. Args: tools: List of tool definitions with dependencies Returns: Dictionary representing dependency graph """ graph = {} for tool in tools: tool_name = tool["name"] dependencies = tool.get("dependencies", []) graph[tool_name] = dependencies return graph def test_tool_dependency_resolution( dependency_graph: Dict[str, List[str]], resolution_order: List[str] ) -> bool: """ Test tool dependency resolution. Args: dependency_graph: Tool dependency graph resolution_order: Proposed resolution order Returns: True if dependencies are satisfied, False otherwise """ resolved = set() for tool in resolution_order: # Check if all dependencies are resolved dependencies = dependency_graph.get(tool, []) for dep in dependencies: if dep not in resolved: return False resolved.add(tool) return True def create_tool_sandbox_environment() -> Dict[str, Any]: """ Create a sandbox environment for tool testing. Returns: Dictionary representing sandbox environment """ return { "temp_dir": tempfile.mkdtemp(), "allowed_modules": ["os", "sys", "pathlib", "json"], "resource_limits": { "memory": 1024 * 1024, # 1MB "cpu": 1.0, # 1 CPU core "timeout": 30 # 30 seconds }, "permissions": { "file_access": "read_only", "network_access": False, "process_creation": False } } def mock_tool_registry( tools: Optional[List[Mock]] = None ) -> MagicMock: """ Create a mock tool registry for testing. Args: tools: List of tools to register Returns: Mock tool registry """ mock_registry = MagicMock() if tools is None: tools = [ create_mock_tool("registered_tool1"), create_mock_tool("registered_tool2") ] # Mock registry methods mock_registry.get_all_tools.return_value = tools mock_registry.get_tool.return_value = tools[0] if tools else None mock_registry.register_tool.return_value = True mock_registry.unregister_tool.return_value = True return mock_registry def create_tool_lifecycle_test_scenarios() -> List[Dict[str, Any]]: """ Create test scenarios for tool lifecycle testing. Returns: List of tool lifecycle test scenarios """ return [ { "name": "normal_lifecycle", "steps": [ {"action": "load", "expected": "success"}, {"action": "initialize", "expected": "success"}, {"action": "execute", "expected": "success"}, {"action": "cleanup", "expected": "success"}, {"action": "unload", "expected": "success"} ] }, { "name": "initialization_failure", "steps": [ {"action": "load", "expected": "success"}, {"action": "initialize", "expected": "failure"}, {"action": "cleanup", "expected": "success"}, {"action": "unload", "expected": "success"} ] }, { "name": "execution_failure", "steps": [ {"action": "load", "expected": "success"}, {"action": "initialize", "expected": "success"}, {"action": "execute", "expected": "failure"}, {"action": "cleanup", "expected": "success"}, {"action": "unload", "expected": "success"} ] } ] def benchmark_tool_performance( tool: Union[Mock, ModuleType], test_data: List[Dict[str, Any]], iterations: int = 100 ) -> Dict[str, float]: """ Benchmark tool performance. Args: tool: Tool to benchmark test_data: List of test data inputs iterations: Number of iterations Returns: Dictionary of performance metrics """ import time results = { "initialize": 0.0, "execute": 0.0, "cleanup": 0.0 } # Benchmark initialize start_time = time.time() for _ in range(iterations): tool.initialize() end_time = time.time() results["initialize"] = (end_time - start_time) / iterations # Benchmark execute start_time = time.time() for _ in range(iterations): for data in test_data: tool.execute(**data) end_time = time.time() results["execute"] = (end_time - start_time) / (iterations * len(test_data)) # Benchmark cleanup start_time = time.time() for _ in range(iterations): tool.cleanup() end_time = time.time() results["cleanup"] = (end_time - start_time) / iterations return results def create_tool_security_test_scenarios() -> List[Dict[str, Any]]: """ Create test scenarios for tool security testing. Returns: List of tool security test scenarios """ return [ { "name": "safe_tool", "tool_code": """ def execute(): return {"result": "success"} """, "expected_result": "allowed" }, { "name": "dangerous_tool", "tool_code": """ import os def execute(): os.system("rm -rf /") return {"result": "success"} """, "expected_result": "blocked" }, { "name": "resource_intensive_tool", "tool_code": """ def execute(): while True: pass return {"result": "success"} """, "expected_result": "timeout" } ]