# SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2025 Allaun """Verify tool for NoDupeLabs. This tool provides integrity verification functionality to check the consistency and integrity of processed files and database state. It ensures that file operations were completed successfully and that no corruption occurred during processing. Key Features: - File integrity checking - Database consistency verification - Checksum validation - Rollback safety verification - Progress tracking - Tool integration Dependencies: - Core modules """ import argparse import hashlib from pathlib import Path from typing import Any, Dict, List from nodupe.core.tool_system.base import Tool from nodupe.tools.databases.files import FileRepository from nodupe.tools.databases.connection import DatabaseConnection class VerifyTool(Tool): """Verify tool implementation. This tool provides integrity verification functionality to check the consistency and integrity of processed files and database state. It ensures that file operations were completed successfully and that no corruption occurred during processing. The tool offers three verification modes: - Integrity: Checks file existence and basic properties - Consistency: Verifies database relationships and constraints - Checksums: Validates file hashes against stored values - All: Runs all verification modes Key Features: - Multi-mode verification (integrity, consistency, checksums) - Fast verification option for quick checks - Repair functionality for detected issues - Detailed output and logging - Progress tracking """ def __init__(self) -> None: """Initialize verify tool.""" self.description = "Verify file integrity and database consistency" @property def name(self) -> str: """Tool name.""" return "verify" @property def version(self) -> str: """Tool version.""" return "1.0.0" @property def dependencies(self) -> List[str]: """List of tool dependencies.""" return ["database"] def initialize(self, container: Any) -> None: """Initialize the tool.""" def shutdown(self) -> None: """Shutdown the tool.""" def get_capabilities(self) -> Dict[str, Any]: """Get tool capabilities.""" return {'commands': ['verify']} @property def api_methods(self) -> Dict[str, Any]: """Dictionary of methods exposed via programmatic API.""" return { 'verify': self.execute_verify, 'verify_integrity': self._verify_integrity, 'verify_consistency': self._verify_consistency, 'verify_checksums': self._verify_checksums, } def run_standalone(self, args: List[str]) -> int: """Execute the tool in stand-alone mode. Args: args: List of command line arguments Returns: Exit code (0 for success, 1 for failure) """ parser = argparse.ArgumentParser(description=self.describe_usage()) parser.add_argument( '--mode', choices=['integrity', 'consistency', 'checksums', 'all'], default='all', help='Verification mode to run' ) parser.add_argument( '--fast', action='store_true', help='Perform fast verification (skip heavy checks)' ) parser.add_argument( '--verbose', '-v', action='store_true', help='Verbose output' ) parser.add_argument( '--repair', action='store_true', help='Attempt to repair detected issues' ) parser.add_argument( '--output', help='Output results to file' ) parsed_args = parser.parse_args(args) # Create a namespace object similar to the command parser class ArgsNamespace: pass args_namespace = ArgsNamespace() args_namespace.mode = parsed_args.mode args_namespace.fast = parsed_args.fast args_namespace.verbose = parsed_args.verbose args_namespace.repair = parsed_args.repair args_namespace.output = parsed_args.output return self.execute_verify(args_namespace) # type: ignore def describe_usage(self) -> str: """Return human-readable usage description. Returns: Usage description string """ return """Verify Tool - File Integrity and Database Consistency Verification This tool checks the consistency and integrity of processed files and database state. It ensures that file operations were completed successfully and that no corruption occurred during processing. Usage: nodupe verify [OPTIONS] Options: --mode MODE Verification mode: integrity, consistency, checksums, or all (default: all) --fast Perform fast verification (skip heavy checks) --verbose, -v Show detailed output --repair Attempt to repair detected issues --output FILE Save detailed results to file Examples: Run all verification checks: nodupe verify --mode all Quick integrity check: nodupe verify --mode integrity --fast Full verification with verbose output: nodupe verify --mode all --verbose Verify and save results to file: nodupe verify --mode checksums --output results.json Verification Modes: integrity - Checks file existence and basic properties consistency - Verifies database relationships and constraints checksums - Validates file hashes against stored values all - Runs all verification modes (default) """ def _on_verify_start(self, **kwargs: Any) -> None: """Handle verify start event.""" print(f"[TOOL] Verify started: {kwargs.get('mode', 'unknown')}") def _on_verify_complete(self, **kwargs: Any) -> None: """Handle verify complete event.""" print(f"[TOOL] Verify completed: {kwargs.get('checks_performed', 0)} checks, " f"{kwargs.get('errors_found', 0)} errors") def register_commands(self, subparsers: Any) -> None: """Register verify command with argument parser.""" verify_parser = subparsers.add_parser( 'verify', help='Verify file integrity and database consistency') verify_parser.add_argument( '--mode', choices=['integrity', 'consistency', 'checksums', 'all'], default='all', help='Verification mode to run' ) verify_parser.add_argument( '--fast', action='store_true', help='Perform fast verification (skip heavy checks)' ) verify_parser.add_argument( '--verbose', '-v', action='store_true', help='Verbose output' ) verify_parser.add_argument( '--repair', action='store_true', help='Attempt to repair detected issues' ) verify_parser.add_argument( '--output', help='Output results to file' ) verify_parser.set_defaults(func=self.execute_verify) def execute_verify(self, args: argparse.Namespace) -> int: """Execute verify command. Args: args: Command arguments including injected 'container' """ from typing import TypedDict class VerificationResult(TypedDict): """Typed dictionary for verification results. Contains the results of a verification operation including counts of checks performed, errors found, warnings, and detailed error information. """ checks: int errors: int warnings: int error_details: List[Any] try: print(f"[TOOL] Executing verify command: {args.mode} mode") print(f"[TOOL] Fast mode: {args.fast}, Repair: {args.repair}") # 1. Get services container = getattr(args, 'container', None) if not container: print("[ERROR] Dependency container not available") return 1 db_connection = container.get_service('database') if not db_connection: print("[ERROR] Database service not available") db_connection = DatabaseConnection.get_instance() file_repo = FileRepository(db_connection) # 2. Determine verification mode modes = [] if args.mode == 'all': modes = ['integrity', 'consistency', 'checksums'] else: modes = [args.mode] results: Dict[str, VerificationResult] = { 'integrity': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []}, 'consistency': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []}, 'checksums': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []} } total_errors = 0 total_warnings = 0 # 3. Run verification modes for mode in modes: print(f"\n[TOOL] Running {mode} verification...") if mode == 'integrity': mode_results = self._verify_integrity(file_repo, args) results['integrity'] = { 'checks': mode_results['checks'], 'errors': mode_results['errors'], 'warnings': mode_results['warnings'], 'error_details': [] } total_errors += mode_results['errors'] total_warnings += mode_results['warnings'] elif mode == 'consistency': mode_results = self._verify_consistency(file_repo, args) results['consistency'] = { 'checks': mode_results['checks'], 'errors': mode_results['errors'], 'warnings': mode_results['warnings'], 'error_details': [] } total_errors += mode_results['errors'] total_warnings += mode_results['warnings'] elif mode == 'checksums': mode_results = self._verify_checksums(file_repo, args) results['checksums'] = { 'checks': mode_results['checks'], 'errors': mode_results['errors'], 'warnings': mode_results['warnings'], 'error_details': [] } total_errors += mode_results['errors'] total_warnings += mode_results['warnings'] # 4. Report results print("\n[TOOL] Verification Summary:") for mode, stats in results.items(): if stats['checks'] > 0: print(f" {mode.title()}: {stats['checks']} checks, " f"{stats['errors']} errors, {stats['warnings']} warnings") print(f"\n[TOOL] Total: {sum(r['checks'] for r in results.values())} checks, " f"{total_errors} errors, {total_warnings} warnings") if total_errors > 0: print(f"[TOOL] ❌ {total_errors} integrity issues detected!") if args.repair: print("[TOOL] Repair mode enabled - this would attempt to fix issues") # TODO: Implement repair functionality to automatically fix integrity issues # Potential repairs: regenerate missing checksums, rebuild file index, fix metadata else: print("[TOOL] ✅ All verification checks passed!") # Output detailed results to file if requested if args.output: self._output_findings_to_file(results, args.output, args) print(f"[TOOL] Detailed findings saved to: {args.output}") self._on_verify_complete( checks_performed=sum(r['checks'] for r in results.values()), errors_found=total_errors ) return 0 if total_errors == 0 else 1 except Exception as e: print(f"[TOOL ERROR] Verify failed: {e}") if args.verbose: import traceback traceback.print_exc() return 1 def _output_findings_to_file(self, results: Dict[str, Any], output_file: str, args: argparse.Namespace) -> None: """Output detailed verification findings to a file. Args: results: Verification results dictionary output_file: Path to output file args: Command arguments """ import json from datetime import datetime findings: Dict[str, Any] = { 'timestamp': datetime.now().isoformat(), 'command_args': { 'mode': args.mode, 'fast': args.fast, 'verbose': args.verbose, 'repair': args.repair }, 'summary': { 'total_checks': sum(r['checks'] for r in results.values()), 'total_errors': sum(r['errors'] for r in results.values()), 'total_warnings': sum(r['warnings'] for r in results.values()) }, 'details': {} } for mode, stats in results.items(): findings['details'][mode] = { 'checks': stats['checks'], 'errors': stats['errors'], 'warnings': stats['warnings'], 'error_details': stats.get('error_details', []) } with open(output_file, 'w', encoding='utf-8') as f: json.dump(findings, f, indent=2, ensure_ascii=False) def _verify_integrity(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]: """Verify file integrity by checking file existence and basic properties.""" results = {'checks': 0, 'errors': 0, 'warnings': 0} try: files = file_repo.get_all_files() print(f"[TOOL] Checking integrity of {len(files)} files...") for file_data in files: results['checks'] += 1 file_path = Path(file_data['path']) # Check if file exists if not file_path.exists(): results['errors'] += 1 if args.verbose: print(f"[ERROR] File not found: {file_path}") continue # Check file size matches database try: actual_size = file_path.stat().st_size if actual_size != file_data['size']: results['errors'] += 1 if args.verbose: print(f"[ERROR] Size mismatch for {file_path}: " f"expected {file_data['size']}, got {actual_size}") except OSError: results['errors'] += 1 if args.verbose: print(f"[ERROR] Cannot access file: {file_path}") # Check file readability (unless in fast mode) if not args.fast: try: with open(file_path, 'rb') as f: f.read(1) # Test if file is readable except (OSError, PermissionError): results['errors'] += 1 if args.verbose: print(f"[ERROR] Cannot read file: {file_path}") print(f"[TOOL] Integrity check: {results['checks']} files, " f"{results['errors']} errors, {results['warnings']} warnings") except Exception as e: print(f"[TOOL ERROR] Integrity verification failed: {e}") results['errors'] += 1 return results def _verify_consistency(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]: """Verify database consistency and relationships.""" results = {'checks': 0, 'errors': 0, 'warnings': 0} try: files = file_repo.get_all_files() print(f"[TOOL] Checking consistency of {len(files)} files...") for file_data in files: results['checks'] += 1 # Check duplicate relationships if file_data['is_duplicate']: if not file_data['duplicate_of']: results['errors'] += 1 if args.verbose: print( f"[ERROR] Duplicate file {file_data['path']} has no duplicate_of reference") else: # Verify the referenced original file exists original = file_repo.get_file(file_data['duplicate_of']) if not original: results['errors'] += 1 if args.verbose: print(f"[ERROR] Duplicate file {file_data['path']} references " f"non-existent original ID {file_data['duplicate_of']}") # Check for circular references or invalid relationships if file_data['duplicate_of'] == file_data['id']: results['errors'] += 1 if args.verbose: print(f"[ERROR] File {file_data['path']} references itself as duplicate") # Check for orphaned duplicate references duplicates = file_repo.get_duplicate_files() for dup in duplicates: if dup['duplicate_of']: original = file_repo.get_file(dup['duplicate_of']) if not original: results['errors'] += 1 if args.verbose: print( f"[ERROR] Orphaned duplicate reference: {dup['path']} -> {dup['duplicate_of']}") print(f"[TOOL] Consistency check: {results['checks']} files, " f"{results['errors']} errors, {results['warnings']} warnings") except Exception as e: print(f"[TOOL ERROR] Consistency verification failed: {e}") results['errors'] += 1 return results def _verify_checksums(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]: """Verify file checksums by recalculating hashes.""" results = {'checks': 0, 'errors': 0, 'warnings': 0} if args.fast: print("[TOOL] Skipping checksum verification in fast mode") return results try: files = file_repo.get_all_files() print(f"[TOOL] Verifying checksums for {len(files)} files...") for file_data in files: if not file_data['hash']: results['warnings'] += 1 if args.verbose: print(f"[WARN] No hash stored for: {file_data['path']}") continue results['checks'] += 1 file_path = Path(file_data['path']) # Skip if file doesn't exist (already caught in integrity check) if not file_path.exists(): results['errors'] += 1 continue # Recalculate hash and compare try: with open(file_path, 'rb') as f: file_hash = hashlib.sha256(f.read()).hexdigest() if file_hash != file_data['hash']: results['errors'] += 1 if args.verbose: print(f"[ERROR] Hash mismatch for {file_path}: " f"stored {file_data['hash'][:8]}..., calculated {file_hash[:8]}...") except (OSError, MemoryError) as e: results['errors'] += 1 if args.verbose: print(f"[ERROR] Cannot calculate hash for {file_path}: {e}") print(f"[TOOL] Checksum check: {results['checks']} files, " f"{results['errors']} errors, {results['warnings']} warnings") except Exception as e: print(f"[TOOL ERROR] Checksum verification failed: {e}") results['errors'] += 1 return results # Create tool instance when module is loaded verify_tool = VerifyTool() # Register tool with core system def register_tool() -> VerifyTool: """Register tool with core system.""" return verify_tool # Export tool interface __all__ = ['verify_tool', 'register_tool']