mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
625 lines
23 KiB
Python
625 lines
23 KiB
Python
"""
|
|
TimeSync Failure Handling Rules
|
|
|
|
This module defines comprehensive failure handling and retry strategies for the TimeSync plugin,
|
|
including NTP server connection rules, fallback hierarchies, and graceful degradation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
import logging
|
|
from enum import Enum
|
|
from dataclasses import dataclass
|
|
from typing import List, Dict, Optional, Tuple, Any
|
|
from collections import defaultdict, deque
|
|
from datetime import datetime
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ServerPriority(Enum):
|
|
"""Priority levels for NTP servers."""
|
|
PRIMARY = 1 # Google, Cloudflare - preferred
|
|
SECONDARY = 2 # Apple, Microsoft - backup
|
|
TERTIARY = 3 # Pool servers - last resort
|
|
FALLBACK = 4 # User-defined - custom fallback
|
|
|
|
|
|
class FailureReason(Enum):
|
|
"""Reasons for NTP server connection failures."""
|
|
TIMEOUT = "timeout"
|
|
NETWORK_ERROR = "network_error"
|
|
INVALID_RESPONSE = "invalid_response"
|
|
HIGH_DELAY = "high_delay"
|
|
DNS_FAILURE = "dns_failure"
|
|
SOCKET_ERROR = "socket_error"
|
|
|
|
|
|
@dataclass
|
|
class ServerStats:
|
|
"""Statistics for an NTP server."""
|
|
host: str
|
|
priority: ServerPriority
|
|
success_count: int = 0
|
|
failure_count: int = 0
|
|
total_attempts: int = 0
|
|
last_success: Optional[float] = None
|
|
last_failure: Optional[float] = None
|
|
failure_reasons: Dict[FailureReason, int] = None
|
|
recent_delays: deque = None # Rolling window of recent delays
|
|
|
|
def __post_init__(self):
|
|
"""Initialize default fields."""
|
|
if self.failure_reasons is None:
|
|
self.failure_reasons = defaultdict(int)
|
|
if self.recent_delays is None:
|
|
self.recent_delays = deque(maxlen=10) # Keep last 10 delays
|
|
|
|
@property
|
|
def success_rate(self) -> float:
|
|
"""Calculate success rate as percentage."""
|
|
if self.total_attempts == 0:
|
|
return 0.0
|
|
return (self.success_count / self.total_attempts) * 100
|
|
|
|
@property
|
|
def avg_delay(self) -> float:
|
|
"""Calculate average delay from recent measurements."""
|
|
if not self.recent_delays:
|
|
return 0.0
|
|
return sum(self.recent_delays) / len(self.recent_delays)
|
|
|
|
@property
|
|
def is_healthy(self) -> bool:
|
|
"""Determine if server is considered healthy."""
|
|
if self.total_attempts < 3:
|
|
return True # Not enough data yet
|
|
|
|
# Server is unhealthy if success rate is below 50%
|
|
return self.success_rate >= 50.0
|
|
|
|
def record_success(self, delay: float):
|
|
"""Record a successful connection."""
|
|
self.success_count += 1
|
|
self.total_attempts += 1
|
|
self.last_success = time.time()
|
|
self.recent_delays.append(delay)
|
|
|
|
def record_failure(self, reason: FailureReason):
|
|
"""Record a failed connection."""
|
|
self.failure_count += 1
|
|
self.total_attempts += 1
|
|
self.last_failure = time.time()
|
|
self.failure_reasons[reason] += 1
|
|
|
|
|
|
@dataclass
|
|
class ConnectionAttempt:
|
|
"""Record of a single connection attempt."""
|
|
host: str
|
|
attempt_time: float
|
|
success: bool
|
|
delay: Optional[float] = None
|
|
failure_reason: Optional[FailureReason] = None
|
|
response_time: Optional[float] = None
|
|
|
|
|
|
class FailureRuleEngine:
|
|
"""
|
|
Rule engine for handling NTP server connection failures and retries.
|
|
|
|
Implements intelligent retry strategies, server health monitoring,
|
|
and graceful fallback hierarchies.
|
|
"""
|
|
|
|
def __init__(self,
|
|
max_retries: int = 3,
|
|
base_retry_delay: float = 1.0,
|
|
max_retry_delay: float = 30.0,
|
|
health_check_interval: float = 300.0, # 5 minutes
|
|
failure_decay_hours: float = 24.0):
|
|
"""
|
|
Initialize the failure rule engine.
|
|
|
|
Args:
|
|
max_retries: Maximum number of retries per server
|
|
base_retry_delay: Base delay between retries (exponential backoff)
|
|
max_retry_delay: Maximum retry delay
|
|
health_check_interval: Interval for health checks in seconds
|
|
failure_decay_hours: Hours after which failure counts decay
|
|
"""
|
|
self.max_retries = max_retries
|
|
self.base_retry_delay = base_retry_delay
|
|
self.max_retry_delay = max_retry_delay
|
|
self.health_check_interval = health_check_interval
|
|
self.failure_decay_hours = failure_decay_hours
|
|
|
|
self.server_stats: Dict[str, ServerStats] = {}
|
|
self.connection_history: List[ConnectionAttempt] = []
|
|
self._last_health_check = time.time()
|
|
|
|
def get_server_priority(self, host: str) -> ServerPriority:
|
|
"""Determine server priority based on hostname."""
|
|
host_lower = host.lower()
|
|
|
|
if any(provider in host_lower for provider in ['google', 'cloudflare']):
|
|
return ServerPriority.PRIMARY
|
|
elif any(provider in host_lower for provider in ['apple', 'microsoft', 'windows']):
|
|
return ServerPriority.SECONDARY
|
|
elif 'pool' in host_lower:
|
|
return ServerPriority.TERTIARY
|
|
else:
|
|
return ServerPriority.FALLBACK
|
|
|
|
def should_retry_server(self, host: str, attempt_count: int,
|
|
last_failure_reason: Optional[FailureReason] = None) -> Tuple[bool, float]:
|
|
"""
|
|
Determine if a server should be retried and when.
|
|
|
|
Args:
|
|
host: Server hostname
|
|
attempt_count: Number of previous attempts
|
|
last_failure_reason: Reason for last failure
|
|
|
|
Returns:
|
|
Tuple of (should_retry, delay_seconds)
|
|
"""
|
|
if attempt_count >= self.max_retries:
|
|
logger.debug(f"Server {host} reached max retries ({self.max_retries})")
|
|
return False, 0.0
|
|
|
|
# Check server health
|
|
stats = self.server_stats.get(host)
|
|
if stats and not stats.is_healthy:
|
|
# Unhealthy server - longer delay
|
|
delay = min(self.max_retry_delay, self.base_retry_delay * (2 ** attempt_count))
|
|
logger.warning(f"Server {host} is unhealthy, retrying in {delay:.1f}s")
|
|
return True, delay
|
|
|
|
# Standard exponential backoff
|
|
delay = self.base_retry_delay * (2 ** attempt_count)
|
|
delay = min(delay, self.max_retry_delay)
|
|
|
|
# Additional delay for certain failure types
|
|
if last_failure_reason == FailureReason.TIMEOUT:
|
|
delay *= 1.5
|
|
elif last_failure_reason == FailureReason.NETWORK_ERROR:
|
|
delay *= 1.2
|
|
|
|
return True, delay
|
|
|
|
def select_best_servers(self, available_hosts: List[str],
|
|
max_selections: int = 4) -> List[str]:
|
|
"""
|
|
Select the best servers based on health and priority.
|
|
|
|
Args:
|
|
available_hosts: List of available server hosts
|
|
max_selections: Maximum number of servers to select
|
|
|
|
Returns:
|
|
List of selected server hosts in priority order
|
|
"""
|
|
# Update server stats if needed
|
|
self._decay_old_failures()
|
|
|
|
# Create or update server stats
|
|
servers_with_stats = []
|
|
for host in available_hosts:
|
|
if host not in self.server_stats:
|
|
self.server_stats[host] = ServerStats(
|
|
host=host,
|
|
priority=self.get_server_priority(host)
|
|
)
|
|
|
|
stats = self.server_stats[host]
|
|
servers_with_stats.append((host, stats))
|
|
|
|
# Sort by: health status, priority, success rate, average delay
|
|
def sort_key(item):
|
|
"""Sort key for server selection."""
|
|
stats = item[1]
|
|
health_score = 1 if stats.is_healthy else 0
|
|
priority_score = -stats.priority.value # Lower is better
|
|
success_score = stats.success_rate / 100.0
|
|
delay_score = -stats.avg_delay if stats.avg_delay > 0 else 0
|
|
|
|
return (health_score, priority_score, success_score, delay_score)
|
|
|
|
sorted_servers = sorted(servers_with_stats, key=sort_key, reverse=True)
|
|
selected = [host for host, _ in sorted_servers[:max_selections]]
|
|
|
|
logger.info(f"Selected servers: {selected}")
|
|
return selected
|
|
|
|
def should_fallback_to_rtc(self) -> bool:
|
|
"""
|
|
Determine if we should fallback to RTC based on recent failure patterns.
|
|
|
|
Returns:
|
|
True if RTC fallback is warranted
|
|
"""
|
|
if len(self.connection_history) < 5:
|
|
return False
|
|
|
|
# Check last 10 attempts
|
|
recent_attempts = self.connection_history[-10:]
|
|
total_attempts = len(recent_attempts)
|
|
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
|
|
|
|
# Fallback if 80% or more recent attempts failed
|
|
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
|
|
|
|
if failure_rate >= 0.8:
|
|
logger.warning(f"High failure rate ({failure_rate:.1%}), falling back to RTC")
|
|
return True
|
|
|
|
return False
|
|
|
|
def should_use_file_fallback(self) -> bool:
|
|
"""
|
|
Determine if we should use file timestamp fallback.
|
|
|
|
Returns:
|
|
True if file fallback is warranted
|
|
"""
|
|
if len(self.connection_history) < 10:
|
|
return False
|
|
|
|
# Check last 20 attempts
|
|
recent_attempts = self.connection_history[-20:]
|
|
total_attempts = len(recent_attempts)
|
|
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
|
|
|
|
# File fallback if 90% or more recent attempts failed
|
|
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
|
|
|
|
if failure_rate >= 0.9:
|
|
logger.warning(f"Very high failure rate ({failure_rate:.1%}), using file fallback")
|
|
return True
|
|
|
|
return False
|
|
|
|
def should_use_monotonic_only(self) -> bool:
|
|
"""
|
|
Determine if we should use pure monotonic time only.
|
|
|
|
Returns:
|
|
True if only monotonic time should be used
|
|
"""
|
|
if len(self.connection_history) < 20:
|
|
return False
|
|
|
|
# Check last 50 attempts
|
|
recent_attempts = self.connection_history[-50:]
|
|
total_attempts = len(recent_attempts)
|
|
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
|
|
|
|
# Pure monotonic if 95% or more recent attempts failed
|
|
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
|
|
|
|
if failure_rate >= 0.95:
|
|
logger.critical(f"Critical failure rate ({failure_rate:.1%}), using monotonic only")
|
|
return True
|
|
|
|
return False
|
|
|
|
def get_connection_strategy(self, available_hosts: List[str]) -> ConnectionStrategy:
|
|
"""
|
|
Determine the optimal connection strategy based on current conditions.
|
|
|
|
Args:
|
|
available_hosts: List of available server hosts
|
|
|
|
Returns:
|
|
ConnectionStrategy with optimal parameters
|
|
"""
|
|
# Health check if needed
|
|
if time.time() - self._last_health_check > self.health_check_interval:
|
|
self._perform_health_check()
|
|
self._last_health_check = time.time()
|
|
|
|
# Determine fallback level
|
|
if self.should_use_monotonic_only():
|
|
fallback_level = FallbackLevel.MONOTONIC_ONLY
|
|
elif self.should_use_file_fallback():
|
|
fallback_level = FallbackLevel.FILE_FALLBACK
|
|
elif self.should_fallback_to_rtc():
|
|
fallback_level = FallbackLevel.RTC_FALLBACK
|
|
else:
|
|
fallback_level = FallbackLevel.NTP_ONLY
|
|
|
|
# Select servers based on health
|
|
selected_servers = self.select_best_servers(available_hosts)
|
|
|
|
# Adjust retry strategy based on failure patterns
|
|
avg_success_rate = self._calculate_average_success_rate()
|
|
if avg_success_rate < 30:
|
|
retry_strategy = RetryStrategy.CONSERVATIVE
|
|
elif avg_success_rate < 70:
|
|
retry_strategy = RetryStrategy.MODERATE
|
|
else:
|
|
retry_strategy = RetryStrategy.AGGRESSIVE
|
|
|
|
return ConnectionStrategy(
|
|
servers=selected_servers,
|
|
max_retries=self._get_adaptive_retries(retry_strategy),
|
|
timeout=self._get_adaptive_timeout(retry_strategy),
|
|
parallel_queries=self._get_adaptive_parallelism(retry_strategy),
|
|
fallback_level=fallback_level,
|
|
retry_strategy=retry_strategy
|
|
)
|
|
|
|
def record_attempt(self, attempt: ConnectionAttempt):
|
|
"""Record a connection attempt for analysis."""
|
|
self.connection_history.append(attempt)
|
|
|
|
# Keep only last 1000 attempts
|
|
if len(self.connection_history) > 1000:
|
|
self.connection_history.pop(0)
|
|
|
|
# Update server stats
|
|
stats = self.server_stats.get(attempt.host)
|
|
if not stats:
|
|
stats = ServerStats(
|
|
host=attempt.host,
|
|
priority=self.get_server_priority(attempt.host)
|
|
)
|
|
self.server_stats[attempt.host] = stats
|
|
|
|
if attempt.success:
|
|
stats.record_success(attempt.delay or 0.0)
|
|
else:
|
|
stats.record_failure(attempt.failure_reason or FailureReason.NETWORK_ERROR)
|
|
|
|
def get_server_health_report(self) -> Dict[str, Dict]:
|
|
"""Get health report for all servers."""
|
|
self._decay_old_failures()
|
|
|
|
report = {}
|
|
for host, stats in self.server_stats.items():
|
|
report[host] = {
|
|
'priority': stats.priority.name,
|
|
'success_rate': round(stats.success_rate, 2),
|
|
'avg_delay': round(stats.avg_delay, 3),
|
|
'total_attempts': stats.total_attempts,
|
|
'is_healthy': stats.is_healthy,
|
|
'last_success': stats.last_success,
|
|
'last_failure': stats.last_failure,
|
|
'failure_reasons': dict(stats.failure_reasons)
|
|
}
|
|
|
|
return report
|
|
|
|
def _decay_old_failures(self):
|
|
"""Decay old failure counts based on time."""
|
|
cutoff_time = time.time() - (self.failure_decay_hours * 3600)
|
|
|
|
for stats in self.server_stats.values():
|
|
if stats.last_success and stats.last_success > cutoff_time:
|
|
# Reduce failure count for servers with recent successes
|
|
stats.failure_count = max(0, stats.failure_count - 1)
|
|
stats.total_attempts = stats.success_count + stats.failure_count
|
|
|
|
def _perform_health_check(self):
|
|
"""Perform periodic health checks on servers."""
|
|
for stats in self.server_stats.values():
|
|
# Consider server unhealthy if no success in last 30 minutes
|
|
if stats.last_success:
|
|
time_since_success = time.time() - stats.last_success
|
|
if time_since_success > 1800: # 30 minutes
|
|
logger.warning(f"Server {stats.host} has no success in {time_since_success/60:.1f} minutes")
|
|
|
|
def _calculate_average_success_rate(self) -> float:
|
|
"""Calculate average success rate across all servers."""
|
|
if not self.server_stats:
|
|
return 50.0 # Default
|
|
|
|
total_success = sum(stats.success_count for stats in self.server_stats.values())
|
|
total_attempts = sum(stats.total_attempts for stats in self.server_stats.values())
|
|
|
|
if total_attempts == 0:
|
|
return 50.0
|
|
|
|
return (total_success / total_attempts) * 100
|
|
|
|
def _get_adaptive_retries(self, strategy: 'RetryStrategy') -> int:
|
|
"""Get adaptive retry count based on strategy."""
|
|
base_retries = self.max_retries
|
|
|
|
if strategy == RetryStrategy.CONSERVATIVE:
|
|
return max(2, base_retries - 1)
|
|
elif strategy == RetryStrategy.AGGRESSIVE:
|
|
return min(5, base_retries + 1)
|
|
else:
|
|
return base_retries
|
|
|
|
def _get_adaptive_timeout(self, strategy: 'RetryStrategy') -> float:
|
|
"""Get adaptive timeout based on strategy."""
|
|
if strategy == RetryStrategy.CONSERVATIVE:
|
|
return 5.0
|
|
elif strategy == RetryStrategy.AGGRESSIVE:
|
|
return 2.0
|
|
else:
|
|
return 3.0
|
|
|
|
def _get_adaptive_parallelism(self, strategy: 'RetryStrategy') -> bool:
|
|
"""Get adaptive parallelism setting based on strategy."""
|
|
if strategy == RetryStrategy.CONSERVATIVE:
|
|
return False # Sequential for conservative
|
|
else:
|
|
return True # Parallel for moderate/aggressive
|
|
|
|
|
|
class FallbackLevel(Enum):
|
|
"""Levels of fallback when NTP fails."""
|
|
NTP_ONLY = 1 # Only use NTP servers
|
|
RTC_FALLBACK = 2 # Fallback to RTC
|
|
FILE_FALLBACK = 3 # Fallback to file timestamps
|
|
MONOTONIC_ONLY = 4 # Use only monotonic time
|
|
|
|
|
|
class RetryStrategy(Enum):
|
|
"""Retry strategies based on network conditions."""
|
|
CONSERVATIVE = 1 # Fewer retries, longer delays, sequential
|
|
MODERATE = 2 # Balanced approach
|
|
AGGRESSIVE = 3 # More retries, shorter delays, parallel
|
|
|
|
|
|
@dataclass
|
|
class ConnectionStrategy:
|
|
"""Optimal connection strategy for current conditions."""
|
|
servers: List[str]
|
|
max_retries: int
|
|
timeout: float
|
|
parallel_queries: bool
|
|
fallback_level: FallbackLevel
|
|
retry_strategy: RetryStrategy
|
|
|
|
|
|
class AdaptiveFailureHandler:
|
|
"""
|
|
Adaptive failure handler that learns from network patterns
|
|
and adjusts behavior accordingly.
|
|
"""
|
|
|
|
def __init__(self, rule_engine: FailureRuleEngine):
|
|
"""Initialize adaptive failure handler."""
|
|
self.rule_engine = rule_engine
|
|
self._network_patterns = defaultdict(list)
|
|
self._last_pattern_update = time.time()
|
|
|
|
def analyze_network_pattern(self) -> Dict[str, Any]:
|
|
"""Analyze current network patterns and return recommendations."""
|
|
if time.time() - self._last_pattern_update < 60: # Update every minute
|
|
return self._get_cached_pattern()
|
|
|
|
# Analyze recent connection history
|
|
if len(self.rule_engine.connection_history) < 10:
|
|
return {'pattern': 'insufficient_data', 'recommendation': 'continue_monitoring'}
|
|
|
|
recent_attempts = self.rule_engine.connection_history[-50:]
|
|
|
|
# Calculate failure patterns
|
|
hourly_failures = self._calculate_hourly_failures(recent_attempts)
|
|
failure_by_reason = self._calculate_failure_reasons(recent_attempts)
|
|
success_by_server = self._calculate_success_by_server(recent_attempts)
|
|
|
|
# Determine pattern
|
|
avg_hourly_failures = sum(hourly_failures.values()) / max(1, len(hourly_failures))
|
|
|
|
if avg_hourly_failures > 20:
|
|
pattern = 'high_failure_network'
|
|
elif avg_hourly_failures > 5:
|
|
pattern = 'moderate_failure_network'
|
|
else:
|
|
pattern = 'healthy_network'
|
|
|
|
# Generate recommendations
|
|
recommendations = self._generate_recommendations(pattern, failure_by_reason, success_by_server)
|
|
|
|
result = {
|
|
'pattern': pattern,
|
|
'recommendations': recommendations,
|
|
'metrics': {
|
|
'avg_hourly_failures': avg_hourly_failures,
|
|
'failure_by_reason': failure_by_reason,
|
|
'success_by_server': success_by_server
|
|
}
|
|
}
|
|
|
|
self._network_patterns[pattern].append(result)
|
|
self._last_pattern_update = time.time()
|
|
|
|
return result
|
|
|
|
def _calculate_hourly_failures(self, attempts: List[ConnectionAttempt]) -> Dict[int, int]:
|
|
"""Calculate failures per hour."""
|
|
hourly_failures = defaultdict(int)
|
|
for attempt in attempts:
|
|
if not attempt.success:
|
|
hour = datetime.fromtimestamp(attempt.attempt_time).hour
|
|
hourly_failures[hour] += 1
|
|
return hourly_failures
|
|
|
|
def _calculate_failure_reasons(self, attempts: List[ConnectionAttempt]) -> Dict[str, int]:
|
|
"""Calculate failure reasons distribution."""
|
|
reasons = defaultdict(int)
|
|
for attempt in attempts:
|
|
if not attempt.success and attempt.failure_reason:
|
|
reasons[attempt.failure_reason.value] += 1
|
|
return reasons
|
|
|
|
def _calculate_success_by_server(self, attempts: List[ConnectionAttempt]) -> Dict[str, float]:
|
|
"""Calculate success rate by server."""
|
|
server_stats = defaultdict(lambda: {'success': 0, 'total': 0})
|
|
|
|
for attempt in attempts:
|
|
server_stats[attempt.host]['total'] += 1
|
|
if attempt.success:
|
|
server_stats[attempt.host]['success'] += 1
|
|
|
|
success_rates = {}
|
|
for host, stats in server_stats.items():
|
|
if stats['total'] > 0:
|
|
success_rates[host] = (stats['success'] / stats['total']) * 100
|
|
|
|
return success_rates
|
|
|
|
def _generate_recommendations(self, pattern: str, failure_reasons: Dict, success_rates: Dict) -> List[str]:
|
|
"""Generate recommendations based on pattern analysis."""
|
|
recommendations = []
|
|
|
|
if pattern == 'high_failure_network':
|
|
recommendations.append('Switch to conservative retry strategy')
|
|
recommendations.append('Reduce parallel query count')
|
|
recommendations.append('Increase timeout values')
|
|
recommendations.append('Prioritize primary servers only')
|
|
|
|
elif pattern == 'moderate_failure_network':
|
|
recommendations.append('Use moderate retry strategy')
|
|
recommendations.append('Monitor server health closely')
|
|
recommendations.append('Consider geographic server distribution')
|
|
|
|
# Check for specific failure reasons
|
|
if failure_reasons.get('timeout', 0) > 10:
|
|
recommendations.append('Increase timeout due to network latency')
|
|
|
|
if failure_reasons.get('dns_failure', 0) > 5:
|
|
recommendations.append('Check DNS configuration or use IP addresses')
|
|
|
|
# Check server-specific issues
|
|
for host, rate in success_rates.items():
|
|
if rate < 30:
|
|
recommendations.append(f'Exclude {host} due to poor performance')
|
|
|
|
return recommendations
|
|
|
|
def _get_cached_pattern(self) -> Dict[str, Any]:
|
|
"""Get the most recent cached pattern analysis."""
|
|
if not self._network_patterns:
|
|
return {'pattern': 'unknown', 'recommendation': 'insufficient_data'}
|
|
|
|
# Return the most recent pattern
|
|
latest_pattern = max(self._network_patterns.keys(),
|
|
key=lambda p: len(self._network_patterns[p]))
|
|
return self._network_patterns[latest_pattern][-1] if self._network_patterns[latest_pattern] else {}
|
|
|
|
|
|
# Global failure rule engine instance
|
|
GLOBAL_FAILURE_RULES = None
|
|
|
|
|
|
def get_failure_rules() -> FailureRuleEngine:
|
|
"""Get the global failure rule engine instance."""
|
|
global GLOBAL_FAILURE_RULES
|
|
if GLOBAL_FAILURE_RULES is None:
|
|
GLOBAL_FAILURE_RULES = FailureRuleEngine()
|
|
return GLOBAL_FAILURE_RULES
|
|
|
|
|
|
def reset_failure_rules():
|
|
"""Reset the global failure rule engine."""
|
|
global GLOBAL_FAILURE_RULES
|
|
GLOBAL_FAILURE_RULES = FailureRuleEngine()
|