Research-Stack/4-Infrastructure/NoDupeLabs/nodupe/tools/time_sync/failure_rules.py

625 lines
23 KiB
Python

"""
TimeSync Failure Handling Rules
This module defines comprehensive failure handling and retry strategies for the TimeSync plugin,
including NTP server connection rules, fallback hierarchies, and graceful degradation.
"""
from __future__ import annotations
import time
import logging
from enum import Enum
from dataclasses import dataclass
from typing import List, Dict, Optional, Tuple, Any
from collections import defaultdict, deque
from datetime import datetime
logger = logging.getLogger(__name__)
class ServerPriority(Enum):
"""Priority levels for NTP servers."""
PRIMARY = 1 # Google, Cloudflare - preferred
SECONDARY = 2 # Apple, Microsoft - backup
TERTIARY = 3 # Pool servers - last resort
FALLBACK = 4 # User-defined - custom fallback
class FailureReason(Enum):
"""Reasons for NTP server connection failures."""
TIMEOUT = "timeout"
NETWORK_ERROR = "network_error"
INVALID_RESPONSE = "invalid_response"
HIGH_DELAY = "high_delay"
DNS_FAILURE = "dns_failure"
SOCKET_ERROR = "socket_error"
@dataclass
class ServerStats:
"""Statistics for an NTP server."""
host: str
priority: ServerPriority
success_count: int = 0
failure_count: int = 0
total_attempts: int = 0
last_success: Optional[float] = None
last_failure: Optional[float] = None
failure_reasons: Dict[FailureReason, int] = None
recent_delays: deque = None # Rolling window of recent delays
def __post_init__(self):
"""Initialize default fields."""
if self.failure_reasons is None:
self.failure_reasons = defaultdict(int)
if self.recent_delays is None:
self.recent_delays = deque(maxlen=10) # Keep last 10 delays
@property
def success_rate(self) -> float:
"""Calculate success rate as percentage."""
if self.total_attempts == 0:
return 0.0
return (self.success_count / self.total_attempts) * 100
@property
def avg_delay(self) -> float:
"""Calculate average delay from recent measurements."""
if not self.recent_delays:
return 0.0
return sum(self.recent_delays) / len(self.recent_delays)
@property
def is_healthy(self) -> bool:
"""Determine if server is considered healthy."""
if self.total_attempts < 3:
return True # Not enough data yet
# Server is unhealthy if success rate is below 50%
return self.success_rate >= 50.0
def record_success(self, delay: float):
"""Record a successful connection."""
self.success_count += 1
self.total_attempts += 1
self.last_success = time.time()
self.recent_delays.append(delay)
def record_failure(self, reason: FailureReason):
"""Record a failed connection."""
self.failure_count += 1
self.total_attempts += 1
self.last_failure = time.time()
self.failure_reasons[reason] += 1
@dataclass
class ConnectionAttempt:
"""Record of a single connection attempt."""
host: str
attempt_time: float
success: bool
delay: Optional[float] = None
failure_reason: Optional[FailureReason] = None
response_time: Optional[float] = None
class FailureRuleEngine:
"""
Rule engine for handling NTP server connection failures and retries.
Implements intelligent retry strategies, server health monitoring,
and graceful fallback hierarchies.
"""
def __init__(self,
max_retries: int = 3,
base_retry_delay: float = 1.0,
max_retry_delay: float = 30.0,
health_check_interval: float = 300.0, # 5 minutes
failure_decay_hours: float = 24.0):
"""
Initialize the failure rule engine.
Args:
max_retries: Maximum number of retries per server
base_retry_delay: Base delay between retries (exponential backoff)
max_retry_delay: Maximum retry delay
health_check_interval: Interval for health checks in seconds
failure_decay_hours: Hours after which failure counts decay
"""
self.max_retries = max_retries
self.base_retry_delay = base_retry_delay
self.max_retry_delay = max_retry_delay
self.health_check_interval = health_check_interval
self.failure_decay_hours = failure_decay_hours
self.server_stats: Dict[str, ServerStats] = {}
self.connection_history: List[ConnectionAttempt] = []
self._last_health_check = time.time()
def get_server_priority(self, host: str) -> ServerPriority:
"""Determine server priority based on hostname."""
host_lower = host.lower()
if any(provider in host_lower for provider in ['google', 'cloudflare']):
return ServerPriority.PRIMARY
elif any(provider in host_lower for provider in ['apple', 'microsoft', 'windows']):
return ServerPriority.SECONDARY
elif 'pool' in host_lower:
return ServerPriority.TERTIARY
else:
return ServerPriority.FALLBACK
def should_retry_server(self, host: str, attempt_count: int,
last_failure_reason: Optional[FailureReason] = None) -> Tuple[bool, float]:
"""
Determine if a server should be retried and when.
Args:
host: Server hostname
attempt_count: Number of previous attempts
last_failure_reason: Reason for last failure
Returns:
Tuple of (should_retry, delay_seconds)
"""
if attempt_count >= self.max_retries:
logger.debug(f"Server {host} reached max retries ({self.max_retries})")
return False, 0.0
# Check server health
stats = self.server_stats.get(host)
if stats and not stats.is_healthy:
# Unhealthy server - longer delay
delay = min(self.max_retry_delay, self.base_retry_delay * (2 ** attempt_count))
logger.warning(f"Server {host} is unhealthy, retrying in {delay:.1f}s")
return True, delay
# Standard exponential backoff
delay = self.base_retry_delay * (2 ** attempt_count)
delay = min(delay, self.max_retry_delay)
# Additional delay for certain failure types
if last_failure_reason == FailureReason.TIMEOUT:
delay *= 1.5
elif last_failure_reason == FailureReason.NETWORK_ERROR:
delay *= 1.2
return True, delay
def select_best_servers(self, available_hosts: List[str],
max_selections: int = 4) -> List[str]:
"""
Select the best servers based on health and priority.
Args:
available_hosts: List of available server hosts
max_selections: Maximum number of servers to select
Returns:
List of selected server hosts in priority order
"""
# Update server stats if needed
self._decay_old_failures()
# Create or update server stats
servers_with_stats = []
for host in available_hosts:
if host not in self.server_stats:
self.server_stats[host] = ServerStats(
host=host,
priority=self.get_server_priority(host)
)
stats = self.server_stats[host]
servers_with_stats.append((host, stats))
# Sort by: health status, priority, success rate, average delay
def sort_key(item):
"""Sort key for server selection."""
stats = item[1]
health_score = 1 if stats.is_healthy else 0
priority_score = -stats.priority.value # Lower is better
success_score = stats.success_rate / 100.0
delay_score = -stats.avg_delay if stats.avg_delay > 0 else 0
return (health_score, priority_score, success_score, delay_score)
sorted_servers = sorted(servers_with_stats, key=sort_key, reverse=True)
selected = [host for host, _ in sorted_servers[:max_selections]]
logger.info(f"Selected servers: {selected}")
return selected
def should_fallback_to_rtc(self) -> bool:
"""
Determine if we should fallback to RTC based on recent failure patterns.
Returns:
True if RTC fallback is warranted
"""
if len(self.connection_history) < 5:
return False
# Check last 10 attempts
recent_attempts = self.connection_history[-10:]
total_attempts = len(recent_attempts)
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
# Fallback if 80% or more recent attempts failed
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
if failure_rate >= 0.8:
logger.warning(f"High failure rate ({failure_rate:.1%}), falling back to RTC")
return True
return False
def should_use_file_fallback(self) -> bool:
"""
Determine if we should use file timestamp fallback.
Returns:
True if file fallback is warranted
"""
if len(self.connection_history) < 10:
return False
# Check last 20 attempts
recent_attempts = self.connection_history[-20:]
total_attempts = len(recent_attempts)
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
# File fallback if 90% or more recent attempts failed
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
if failure_rate >= 0.9:
logger.warning(f"Very high failure rate ({failure_rate:.1%}), using file fallback")
return True
return False
def should_use_monotonic_only(self) -> bool:
"""
Determine if we should use pure monotonic time only.
Returns:
True if only monotonic time should be used
"""
if len(self.connection_history) < 20:
return False
# Check last 50 attempts
recent_attempts = self.connection_history[-50:]
total_attempts = len(recent_attempts)
failed_attempts = sum(1 for attempt in recent_attempts if not attempt.success)
# Pure monotonic if 95% or more recent attempts failed
failure_rate = failed_attempts / total_attempts if total_attempts > 0 else 0
if failure_rate >= 0.95:
logger.critical(f"Critical failure rate ({failure_rate:.1%}), using monotonic only")
return True
return False
def get_connection_strategy(self, available_hosts: List[str]) -> ConnectionStrategy:
"""
Determine the optimal connection strategy based on current conditions.
Args:
available_hosts: List of available server hosts
Returns:
ConnectionStrategy with optimal parameters
"""
# Health check if needed
if time.time() - self._last_health_check > self.health_check_interval:
self._perform_health_check()
self._last_health_check = time.time()
# Determine fallback level
if self.should_use_monotonic_only():
fallback_level = FallbackLevel.MONOTONIC_ONLY
elif self.should_use_file_fallback():
fallback_level = FallbackLevel.FILE_FALLBACK
elif self.should_fallback_to_rtc():
fallback_level = FallbackLevel.RTC_FALLBACK
else:
fallback_level = FallbackLevel.NTP_ONLY
# Select servers based on health
selected_servers = self.select_best_servers(available_hosts)
# Adjust retry strategy based on failure patterns
avg_success_rate = self._calculate_average_success_rate()
if avg_success_rate < 30:
retry_strategy = RetryStrategy.CONSERVATIVE
elif avg_success_rate < 70:
retry_strategy = RetryStrategy.MODERATE
else:
retry_strategy = RetryStrategy.AGGRESSIVE
return ConnectionStrategy(
servers=selected_servers,
max_retries=self._get_adaptive_retries(retry_strategy),
timeout=self._get_adaptive_timeout(retry_strategy),
parallel_queries=self._get_adaptive_parallelism(retry_strategy),
fallback_level=fallback_level,
retry_strategy=retry_strategy
)
def record_attempt(self, attempt: ConnectionAttempt):
"""Record a connection attempt for analysis."""
self.connection_history.append(attempt)
# Keep only last 1000 attempts
if len(self.connection_history) > 1000:
self.connection_history.pop(0)
# Update server stats
stats = self.server_stats.get(attempt.host)
if not stats:
stats = ServerStats(
host=attempt.host,
priority=self.get_server_priority(attempt.host)
)
self.server_stats[attempt.host] = stats
if attempt.success:
stats.record_success(attempt.delay or 0.0)
else:
stats.record_failure(attempt.failure_reason or FailureReason.NETWORK_ERROR)
def get_server_health_report(self) -> Dict[str, Dict]:
"""Get health report for all servers."""
self._decay_old_failures()
report = {}
for host, stats in self.server_stats.items():
report[host] = {
'priority': stats.priority.name,
'success_rate': round(stats.success_rate, 2),
'avg_delay': round(stats.avg_delay, 3),
'total_attempts': stats.total_attempts,
'is_healthy': stats.is_healthy,
'last_success': stats.last_success,
'last_failure': stats.last_failure,
'failure_reasons': dict(stats.failure_reasons)
}
return report
def _decay_old_failures(self):
"""Decay old failure counts based on time."""
cutoff_time = time.time() - (self.failure_decay_hours * 3600)
for stats in self.server_stats.values():
if stats.last_success and stats.last_success > cutoff_time:
# Reduce failure count for servers with recent successes
stats.failure_count = max(0, stats.failure_count - 1)
stats.total_attempts = stats.success_count + stats.failure_count
def _perform_health_check(self):
"""Perform periodic health checks on servers."""
for stats in self.server_stats.values():
# Consider server unhealthy if no success in last 30 minutes
if stats.last_success:
time_since_success = time.time() - stats.last_success
if time_since_success > 1800: # 30 minutes
logger.warning(f"Server {stats.host} has no success in {time_since_success/60:.1f} minutes")
def _calculate_average_success_rate(self) -> float:
"""Calculate average success rate across all servers."""
if not self.server_stats:
return 50.0 # Default
total_success = sum(stats.success_count for stats in self.server_stats.values())
total_attempts = sum(stats.total_attempts for stats in self.server_stats.values())
if total_attempts == 0:
return 50.0
return (total_success / total_attempts) * 100
def _get_adaptive_retries(self, strategy: 'RetryStrategy') -> int:
"""Get adaptive retry count based on strategy."""
base_retries = self.max_retries
if strategy == RetryStrategy.CONSERVATIVE:
return max(2, base_retries - 1)
elif strategy == RetryStrategy.AGGRESSIVE:
return min(5, base_retries + 1)
else:
return base_retries
def _get_adaptive_timeout(self, strategy: 'RetryStrategy') -> float:
"""Get adaptive timeout based on strategy."""
if strategy == RetryStrategy.CONSERVATIVE:
return 5.0
elif strategy == RetryStrategy.AGGRESSIVE:
return 2.0
else:
return 3.0
def _get_adaptive_parallelism(self, strategy: 'RetryStrategy') -> bool:
"""Get adaptive parallelism setting based on strategy."""
if strategy == RetryStrategy.CONSERVATIVE:
return False # Sequential for conservative
else:
return True # Parallel for moderate/aggressive
class FallbackLevel(Enum):
"""Levels of fallback when NTP fails."""
NTP_ONLY = 1 # Only use NTP servers
RTC_FALLBACK = 2 # Fallback to RTC
FILE_FALLBACK = 3 # Fallback to file timestamps
MONOTONIC_ONLY = 4 # Use only monotonic time
class RetryStrategy(Enum):
"""Retry strategies based on network conditions."""
CONSERVATIVE = 1 # Fewer retries, longer delays, sequential
MODERATE = 2 # Balanced approach
AGGRESSIVE = 3 # More retries, shorter delays, parallel
@dataclass
class ConnectionStrategy:
"""Optimal connection strategy for current conditions."""
servers: List[str]
max_retries: int
timeout: float
parallel_queries: bool
fallback_level: FallbackLevel
retry_strategy: RetryStrategy
class AdaptiveFailureHandler:
"""
Adaptive failure handler that learns from network patterns
and adjusts behavior accordingly.
"""
def __init__(self, rule_engine: FailureRuleEngine):
"""Initialize adaptive failure handler."""
self.rule_engine = rule_engine
self._network_patterns = defaultdict(list)
self._last_pattern_update = time.time()
def analyze_network_pattern(self) -> Dict[str, Any]:
"""Analyze current network patterns and return recommendations."""
if time.time() - self._last_pattern_update < 60: # Update every minute
return self._get_cached_pattern()
# Analyze recent connection history
if len(self.rule_engine.connection_history) < 10:
return {'pattern': 'insufficient_data', 'recommendation': 'continue_monitoring'}
recent_attempts = self.rule_engine.connection_history[-50:]
# Calculate failure patterns
hourly_failures = self._calculate_hourly_failures(recent_attempts)
failure_by_reason = self._calculate_failure_reasons(recent_attempts)
success_by_server = self._calculate_success_by_server(recent_attempts)
# Determine pattern
avg_hourly_failures = sum(hourly_failures.values()) / max(1, len(hourly_failures))
if avg_hourly_failures > 20:
pattern = 'high_failure_network'
elif avg_hourly_failures > 5:
pattern = 'moderate_failure_network'
else:
pattern = 'healthy_network'
# Generate recommendations
recommendations = self._generate_recommendations(pattern, failure_by_reason, success_by_server)
result = {
'pattern': pattern,
'recommendations': recommendations,
'metrics': {
'avg_hourly_failures': avg_hourly_failures,
'failure_by_reason': failure_by_reason,
'success_by_server': success_by_server
}
}
self._network_patterns[pattern].append(result)
self._last_pattern_update = time.time()
return result
def _calculate_hourly_failures(self, attempts: List[ConnectionAttempt]) -> Dict[int, int]:
"""Calculate failures per hour."""
hourly_failures = defaultdict(int)
for attempt in attempts:
if not attempt.success:
hour = datetime.fromtimestamp(attempt.attempt_time).hour
hourly_failures[hour] += 1
return hourly_failures
def _calculate_failure_reasons(self, attempts: List[ConnectionAttempt]) -> Dict[str, int]:
"""Calculate failure reasons distribution."""
reasons = defaultdict(int)
for attempt in attempts:
if not attempt.success and attempt.failure_reason:
reasons[attempt.failure_reason.value] += 1
return reasons
def _calculate_success_by_server(self, attempts: List[ConnectionAttempt]) -> Dict[str, float]:
"""Calculate success rate by server."""
server_stats = defaultdict(lambda: {'success': 0, 'total': 0})
for attempt in attempts:
server_stats[attempt.host]['total'] += 1
if attempt.success:
server_stats[attempt.host]['success'] += 1
success_rates = {}
for host, stats in server_stats.items():
if stats['total'] > 0:
success_rates[host] = (stats['success'] / stats['total']) * 100
return success_rates
def _generate_recommendations(self, pattern: str, failure_reasons: Dict, success_rates: Dict) -> List[str]:
"""Generate recommendations based on pattern analysis."""
recommendations = []
if pattern == 'high_failure_network':
recommendations.append('Switch to conservative retry strategy')
recommendations.append('Reduce parallel query count')
recommendations.append('Increase timeout values')
recommendations.append('Prioritize primary servers only')
elif pattern == 'moderate_failure_network':
recommendations.append('Use moderate retry strategy')
recommendations.append('Monitor server health closely')
recommendations.append('Consider geographic server distribution')
# Check for specific failure reasons
if failure_reasons.get('timeout', 0) > 10:
recommendations.append('Increase timeout due to network latency')
if failure_reasons.get('dns_failure', 0) > 5:
recommendations.append('Check DNS configuration or use IP addresses')
# Check server-specific issues
for host, rate in success_rates.items():
if rate < 30:
recommendations.append(f'Exclude {host} due to poor performance')
return recommendations
def _get_cached_pattern(self) -> Dict[str, Any]:
"""Get the most recent cached pattern analysis."""
if not self._network_patterns:
return {'pattern': 'unknown', 'recommendation': 'insufficient_data'}
# Return the most recent pattern
latest_pattern = max(self._network_patterns.keys(),
key=lambda p: len(self._network_patterns[p]))
return self._network_patterns[latest_pattern][-1] if self._network_patterns[latest_pattern] else {}
# Global failure rule engine instance
GLOBAL_FAILURE_RULES = None
def get_failure_rules() -> FailureRuleEngine:
"""Get the global failure rule engine instance."""
global GLOBAL_FAILURE_RULES
if GLOBAL_FAILURE_RULES is None:
GLOBAL_FAILURE_RULES = FailureRuleEngine()
return GLOBAL_FAILURE_RULES
def reset_failure_rules():
"""Reset the global failure rule engine."""
global GLOBAL_FAILURE_RULES
GLOBAL_FAILURE_RULES = FailureRuleEngine()