mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
561 lines
21 KiB
Python
561 lines
21 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright (c) 2025 Allaun
|
|
|
|
"""Verify tool for NoDupeLabs.
|
|
|
|
This tool provides integrity verification functionality to check
|
|
the consistency and integrity of processed files and database state.
|
|
It ensures that file operations were completed successfully and
|
|
that no corruption occurred during processing.
|
|
|
|
Key Features:
|
|
- File integrity checking
|
|
- Database consistency verification
|
|
- Checksum validation
|
|
- Rollback safety verification
|
|
- Progress tracking
|
|
- Tool integration
|
|
|
|
Dependencies:
|
|
- Core modules
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List
|
|
from nodupe.core.tool_system.base import Tool
|
|
from nodupe.tools.databases.files import FileRepository
|
|
from nodupe.tools.databases.connection import DatabaseConnection
|
|
|
|
|
|
class VerifyTool(Tool):
|
|
"""Verify tool implementation.
|
|
|
|
This tool provides integrity verification functionality to check
|
|
the consistency and integrity of processed files and database state.
|
|
It ensures that file operations were completed successfully and
|
|
that no corruption occurred during processing.
|
|
|
|
The tool offers three verification modes:
|
|
- Integrity: Checks file existence and basic properties
|
|
- Consistency: Verifies database relationships and constraints
|
|
- Checksums: Validates file hashes against stored values
|
|
- All: Runs all verification modes
|
|
|
|
Key Features:
|
|
- Multi-mode verification (integrity, consistency, checksums)
|
|
- Fast verification option for quick checks
|
|
- Repair functionality for detected issues
|
|
- Detailed output and logging
|
|
- Progress tracking
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
"""Initialize verify tool."""
|
|
self.description = "Verify file integrity and database consistency"
|
|
|
|
@property
|
|
def name(self) -> str:
|
|
"""Tool name."""
|
|
return "verify"
|
|
|
|
@property
|
|
def version(self) -> str:
|
|
"""Tool version."""
|
|
return "1.0.0"
|
|
|
|
@property
|
|
def dependencies(self) -> List[str]:
|
|
"""List of tool dependencies."""
|
|
return ["database"]
|
|
|
|
def initialize(self, container: Any) -> None:
|
|
"""Initialize the tool."""
|
|
|
|
def shutdown(self) -> None:
|
|
"""Shutdown the tool."""
|
|
|
|
def get_capabilities(self) -> Dict[str, Any]:
|
|
"""Get tool capabilities."""
|
|
return {'commands': ['verify']}
|
|
|
|
@property
|
|
def api_methods(self) -> Dict[str, Any]:
|
|
"""Dictionary of methods exposed via programmatic API."""
|
|
return {
|
|
'verify': self.execute_verify,
|
|
'verify_integrity': self._verify_integrity,
|
|
'verify_consistency': self._verify_consistency,
|
|
'verify_checksums': self._verify_checksums,
|
|
}
|
|
|
|
def run_standalone(self, args: List[str]) -> int:
|
|
"""Execute the tool in stand-alone mode.
|
|
|
|
Args:
|
|
args: List of command line arguments
|
|
|
|
Returns:
|
|
Exit code (0 for success, 1 for failure)
|
|
"""
|
|
parser = argparse.ArgumentParser(description=self.describe_usage())
|
|
parser.add_argument(
|
|
'--mode',
|
|
choices=['integrity', 'consistency', 'checksums', 'all'],
|
|
default='all',
|
|
help='Verification mode to run'
|
|
)
|
|
parser.add_argument(
|
|
'--fast',
|
|
action='store_true',
|
|
help='Perform fast verification (skip heavy checks)'
|
|
)
|
|
parser.add_argument(
|
|
'--verbose', '-v',
|
|
action='store_true',
|
|
help='Verbose output'
|
|
)
|
|
parser.add_argument(
|
|
'--repair',
|
|
action='store_true',
|
|
help='Attempt to repair detected issues'
|
|
)
|
|
parser.add_argument(
|
|
'--output',
|
|
help='Output results to file'
|
|
)
|
|
|
|
parsed_args = parser.parse_args(args)
|
|
|
|
# Create a namespace object similar to the command parser
|
|
class ArgsNamespace:
|
|
pass
|
|
|
|
args_namespace = ArgsNamespace()
|
|
args_namespace.mode = parsed_args.mode
|
|
args_namespace.fast = parsed_args.fast
|
|
args_namespace.verbose = parsed_args.verbose
|
|
args_namespace.repair = parsed_args.repair
|
|
args_namespace.output = parsed_args.output
|
|
|
|
return self.execute_verify(args_namespace) # type: ignore
|
|
|
|
def describe_usage(self) -> str:
|
|
"""Return human-readable usage description.
|
|
|
|
Returns:
|
|
Usage description string
|
|
"""
|
|
return """Verify Tool - File Integrity and Database Consistency Verification
|
|
|
|
This tool checks the consistency and integrity of processed files and database state.
|
|
It ensures that file operations were completed successfully and that no corruption
|
|
occurred during processing.
|
|
|
|
Usage:
|
|
nodupe verify [OPTIONS]
|
|
|
|
Options:
|
|
--mode MODE Verification mode: integrity, consistency, checksums, or all (default: all)
|
|
--fast Perform fast verification (skip heavy checks)
|
|
--verbose, -v Show detailed output
|
|
--repair Attempt to repair detected issues
|
|
--output FILE Save detailed results to file
|
|
|
|
Examples:
|
|
Run all verification checks:
|
|
nodupe verify --mode all
|
|
|
|
Quick integrity check:
|
|
nodupe verify --mode integrity --fast
|
|
|
|
Full verification with verbose output:
|
|
nodupe verify --mode all --verbose
|
|
|
|
Verify and save results to file:
|
|
nodupe verify --mode checksums --output results.json
|
|
|
|
Verification Modes:
|
|
integrity - Checks file existence and basic properties
|
|
consistency - Verifies database relationships and constraints
|
|
checksums - Validates file hashes against stored values
|
|
all - Runs all verification modes (default)
|
|
"""
|
|
|
|
def _on_verify_start(self, **kwargs: Any) -> None:
|
|
"""Handle verify start event."""
|
|
print(f"[TOOL] Verify started: {kwargs.get('mode', 'unknown')}")
|
|
|
|
def _on_verify_complete(self, **kwargs: Any) -> None:
|
|
"""Handle verify complete event."""
|
|
print(f"[TOOL] Verify completed: {kwargs.get('checks_performed', 0)} checks, "
|
|
f"{kwargs.get('errors_found', 0)} errors")
|
|
|
|
def register_commands(self, subparsers: Any) -> None:
|
|
"""Register verify command with argument parser."""
|
|
verify_parser = subparsers.add_parser(
|
|
'verify', help='Verify file integrity and database consistency')
|
|
verify_parser.add_argument(
|
|
'--mode',
|
|
choices=['integrity', 'consistency', 'checksums', 'all'],
|
|
default='all',
|
|
help='Verification mode to run'
|
|
)
|
|
verify_parser.add_argument(
|
|
'--fast',
|
|
action='store_true',
|
|
help='Perform fast verification (skip heavy checks)'
|
|
)
|
|
verify_parser.add_argument(
|
|
'--verbose', '-v',
|
|
action='store_true',
|
|
help='Verbose output'
|
|
)
|
|
verify_parser.add_argument(
|
|
'--repair',
|
|
action='store_true',
|
|
help='Attempt to repair detected issues'
|
|
)
|
|
verify_parser.add_argument(
|
|
'--output',
|
|
help='Output results to file'
|
|
)
|
|
verify_parser.set_defaults(func=self.execute_verify)
|
|
|
|
def execute_verify(self, args: argparse.Namespace) -> int:
|
|
"""Execute verify command.
|
|
|
|
Args:
|
|
args: Command arguments including injected 'container'
|
|
"""
|
|
from typing import TypedDict
|
|
|
|
class VerificationResult(TypedDict):
|
|
"""Typed dictionary for verification results.
|
|
|
|
Contains the results of a verification operation including
|
|
counts of checks performed, errors found, warnings, and
|
|
detailed error information.
|
|
"""
|
|
checks: int
|
|
errors: int
|
|
warnings: int
|
|
error_details: List[Any]
|
|
|
|
try:
|
|
print(f"[TOOL] Executing verify command: {args.mode} mode")
|
|
print(f"[TOOL] Fast mode: {args.fast}, Repair: {args.repair}")
|
|
|
|
# 1. Get services
|
|
container = getattr(args, 'container', None)
|
|
if not container:
|
|
print("[ERROR] Dependency container not available")
|
|
return 1
|
|
|
|
db_connection = container.get_service('database')
|
|
if not db_connection:
|
|
print("[ERROR] Database service not available")
|
|
db_connection = DatabaseConnection.get_instance()
|
|
|
|
file_repo = FileRepository(db_connection)
|
|
|
|
# 2. Determine verification mode
|
|
modes = []
|
|
if args.mode == 'all':
|
|
modes = ['integrity', 'consistency', 'checksums']
|
|
else:
|
|
modes = [args.mode]
|
|
|
|
results: Dict[str, VerificationResult] = {
|
|
'integrity': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []},
|
|
'consistency': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []},
|
|
'checksums': {'checks': 0, 'errors': 0, 'warnings': 0, 'error_details': []}
|
|
}
|
|
|
|
total_errors = 0
|
|
total_warnings = 0
|
|
|
|
# 3. Run verification modes
|
|
for mode in modes:
|
|
print(f"\n[TOOL] Running {mode} verification...")
|
|
|
|
if mode == 'integrity':
|
|
mode_results = self._verify_integrity(file_repo, args)
|
|
results['integrity'] = {
|
|
'checks': mode_results['checks'],
|
|
'errors': mode_results['errors'],
|
|
'warnings': mode_results['warnings'],
|
|
'error_details': []
|
|
}
|
|
total_errors += mode_results['errors']
|
|
total_warnings += mode_results['warnings']
|
|
|
|
elif mode == 'consistency':
|
|
mode_results = self._verify_consistency(file_repo, args)
|
|
results['consistency'] = {
|
|
'checks': mode_results['checks'],
|
|
'errors': mode_results['errors'],
|
|
'warnings': mode_results['warnings'],
|
|
'error_details': []
|
|
}
|
|
total_errors += mode_results['errors']
|
|
total_warnings += mode_results['warnings']
|
|
|
|
elif mode == 'checksums':
|
|
mode_results = self._verify_checksums(file_repo, args)
|
|
results['checksums'] = {
|
|
'checks': mode_results['checks'],
|
|
'errors': mode_results['errors'],
|
|
'warnings': mode_results['warnings'],
|
|
'error_details': []
|
|
}
|
|
total_errors += mode_results['errors']
|
|
total_warnings += mode_results['warnings']
|
|
|
|
# 4. Report results
|
|
print("\n[TOOL] Verification Summary:")
|
|
for mode, stats in results.items():
|
|
if stats['checks'] > 0:
|
|
print(f" {mode.title()}: {stats['checks']} checks, "
|
|
f"{stats['errors']} errors, {stats['warnings']} warnings")
|
|
|
|
print(f"\n[TOOL] Total: {sum(r['checks'] for r in results.values())} checks, "
|
|
f"{total_errors} errors, {total_warnings} warnings")
|
|
|
|
if total_errors > 0:
|
|
print(f"[TOOL] ❌ {total_errors} integrity issues detected!")
|
|
if args.repair:
|
|
print("[TOOL] Repair mode enabled - this would attempt to fix issues")
|
|
# TODO: Implement repair functionality to automatically fix integrity issues
|
|
# Potential repairs: regenerate missing checksums, rebuild file index, fix metadata
|
|
else:
|
|
print("[TOOL] ✅ All verification checks passed!")
|
|
|
|
# Output detailed results to file if requested
|
|
if args.output:
|
|
self._output_findings_to_file(results, args.output, args)
|
|
print(f"[TOOL] Detailed findings saved to: {args.output}")
|
|
|
|
self._on_verify_complete(
|
|
checks_performed=sum(r['checks'] for r in results.values()),
|
|
errors_found=total_errors
|
|
)
|
|
|
|
return 0 if total_errors == 0 else 1
|
|
|
|
except Exception as e:
|
|
print(f"[TOOL ERROR] Verify failed: {e}")
|
|
if args.verbose:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
def _output_findings_to_file(self, results: Dict[str, Any], output_file: str, args: argparse.Namespace) -> None:
|
|
"""Output detailed verification findings to a file.
|
|
|
|
Args:
|
|
results: Verification results dictionary
|
|
output_file: Path to output file
|
|
args: Command arguments
|
|
"""
|
|
import json
|
|
from datetime import datetime
|
|
|
|
findings: Dict[str, Any] = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'command_args': {
|
|
'mode': args.mode,
|
|
'fast': args.fast,
|
|
'verbose': args.verbose,
|
|
'repair': args.repair
|
|
},
|
|
'summary': {
|
|
'total_checks': sum(r['checks'] for r in results.values()),
|
|
'total_errors': sum(r['errors'] for r in results.values()),
|
|
'total_warnings': sum(r['warnings'] for r in results.values())
|
|
},
|
|
'details': {}
|
|
}
|
|
|
|
for mode, stats in results.items():
|
|
findings['details'][mode] = {
|
|
'checks': stats['checks'],
|
|
'errors': stats['errors'],
|
|
'warnings': stats['warnings'],
|
|
'error_details': stats.get('error_details', [])
|
|
}
|
|
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
json.dump(findings, f, indent=2, ensure_ascii=False)
|
|
|
|
def _verify_integrity(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]:
|
|
"""Verify file integrity by checking file existence and basic properties."""
|
|
results = {'checks': 0, 'errors': 0, 'warnings': 0}
|
|
|
|
try:
|
|
files = file_repo.get_all_files()
|
|
print(f"[TOOL] Checking integrity of {len(files)} files...")
|
|
|
|
for file_data in files:
|
|
results['checks'] += 1
|
|
file_path = Path(file_data['path'])
|
|
|
|
# Check if file exists
|
|
if not file_path.exists():
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] File not found: {file_path}")
|
|
continue
|
|
|
|
# Check file size matches database
|
|
try:
|
|
actual_size = file_path.stat().st_size
|
|
if actual_size != file_data['size']:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Size mismatch for {file_path}: "
|
|
f"expected {file_data['size']}, got {actual_size}")
|
|
except OSError:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Cannot access file: {file_path}")
|
|
|
|
# Check file readability (unless in fast mode)
|
|
if not args.fast:
|
|
try:
|
|
with open(file_path, 'rb') as f:
|
|
f.read(1) # Test if file is readable
|
|
except (OSError, PermissionError):
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Cannot read file: {file_path}")
|
|
|
|
print(f"[TOOL] Integrity check: {results['checks']} files, "
|
|
f"{results['errors']} errors, {results['warnings']} warnings")
|
|
|
|
except Exception as e:
|
|
print(f"[TOOL ERROR] Integrity verification failed: {e}")
|
|
results['errors'] += 1
|
|
|
|
return results
|
|
|
|
def _verify_consistency(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]:
|
|
"""Verify database consistency and relationships."""
|
|
results = {'checks': 0, 'errors': 0, 'warnings': 0}
|
|
|
|
try:
|
|
files = file_repo.get_all_files()
|
|
print(f"[TOOL] Checking consistency of {len(files)} files...")
|
|
|
|
for file_data in files:
|
|
results['checks'] += 1
|
|
|
|
# Check duplicate relationships
|
|
if file_data['is_duplicate']:
|
|
if not file_data['duplicate_of']:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(
|
|
f"[ERROR] Duplicate file {file_data['path']} has no duplicate_of reference")
|
|
else:
|
|
# Verify the referenced original file exists
|
|
original = file_repo.get_file(file_data['duplicate_of'])
|
|
if not original:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Duplicate file {file_data['path']} references "
|
|
f"non-existent original ID {file_data['duplicate_of']}")
|
|
|
|
# Check for circular references or invalid relationships
|
|
if file_data['duplicate_of'] == file_data['id']:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] File {file_data['path']} references itself as duplicate")
|
|
|
|
# Check for orphaned duplicate references
|
|
duplicates = file_repo.get_duplicate_files()
|
|
for dup in duplicates:
|
|
if dup['duplicate_of']:
|
|
original = file_repo.get_file(dup['duplicate_of'])
|
|
if not original:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(
|
|
f"[ERROR] Orphaned duplicate reference: {dup['path']} -> {dup['duplicate_of']}")
|
|
|
|
print(f"[TOOL] Consistency check: {results['checks']} files, "
|
|
f"{results['errors']} errors, {results['warnings']} warnings")
|
|
|
|
except Exception as e:
|
|
print(f"[TOOL ERROR] Consistency verification failed: {e}")
|
|
results['errors'] += 1
|
|
|
|
return results
|
|
|
|
def _verify_checksums(self, file_repo: FileRepository, args: argparse.Namespace) -> Dict[str, int]:
|
|
"""Verify file checksums by recalculating hashes."""
|
|
results = {'checks': 0, 'errors': 0, 'warnings': 0}
|
|
|
|
if args.fast:
|
|
print("[TOOL] Skipping checksum verification in fast mode")
|
|
return results
|
|
|
|
try:
|
|
files = file_repo.get_all_files()
|
|
print(f"[TOOL] Verifying checksums for {len(files)} files...")
|
|
|
|
for file_data in files:
|
|
if not file_data['hash']:
|
|
results['warnings'] += 1
|
|
if args.verbose:
|
|
print(f"[WARN] No hash stored for: {file_data['path']}")
|
|
continue
|
|
|
|
results['checks'] += 1
|
|
file_path = Path(file_data['path'])
|
|
|
|
# Skip if file doesn't exist (already caught in integrity check)
|
|
if not file_path.exists():
|
|
results['errors'] += 1
|
|
continue
|
|
|
|
# Recalculate hash and compare
|
|
try:
|
|
with open(file_path, 'rb') as f:
|
|
file_hash = hashlib.sha256(f.read()).hexdigest()
|
|
|
|
if file_hash != file_data['hash']:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Hash mismatch for {file_path}: "
|
|
f"stored {file_data['hash'][:8]}..., calculated {file_hash[:8]}...")
|
|
|
|
except (OSError, MemoryError) as e:
|
|
results['errors'] += 1
|
|
if args.verbose:
|
|
print(f"[ERROR] Cannot calculate hash for {file_path}: {e}")
|
|
|
|
print(f"[TOOL] Checksum check: {results['checks']} files, "
|
|
f"{results['errors']} errors, {results['warnings']} warnings")
|
|
|
|
except Exception as e:
|
|
print(f"[TOOL ERROR] Checksum verification failed: {e}")
|
|
results['errors'] += 1
|
|
|
|
return results
|
|
|
|
|
|
# Create tool instance when module is loaded
|
|
verify_tool = VerifyTool()
|
|
|
|
# Register tool with core system
|
|
|
|
|
|
def register_tool() -> VerifyTool:
|
|
"""Register tool with core system."""
|
|
return verify_tool
|
|
|
|
|
|
# Export tool interface
|
|
__all__ = ['verify_tool', 'register_tool']
|