mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
321 lines
10 KiB
Python
321 lines
10 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright (c) 2025 Allaun
|
|
|
|
"""File processor for metadata extraction and duplicate detection.
|
|
|
|
This module provides file processing functionality including metadata extraction,
|
|
hashing, and duplicate detection using only standard library.
|
|
|
|
Key Features:
|
|
- File metadata extraction
|
|
- Cryptographic hashing
|
|
- Duplicate detection
|
|
- Batch processing
|
|
- Error handling
|
|
|
|
Dependencies:
|
|
- hashlib (standard library)
|
|
- os (standard library)
|
|
- typing (standard library)
|
|
"""
|
|
|
|
import os
|
|
import hashlib
|
|
import logging
|
|
from typing import List, Dict, Any, Optional, Callable
|
|
from .walker import FileWalker
|
|
from ..container import container as global_container
|
|
from ..hasher_interface import HasherInterface
|
|
from ..api.codes import ActionCode
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class FileProcessor:
|
|
"""File processor for metadata extraction and duplicate detection.
|
|
|
|
Responsibilities:
|
|
- Process files and extract metadata
|
|
- Calculate file hashes
|
|
- Detect duplicates
|
|
- Handle processing errors
|
|
- Support batch operations
|
|
"""
|
|
|
|
def __init__(self, file_walker: Optional[FileWalker] = None, hasher: Optional[HasherInterface] = None):
|
|
"""Initialize file processor.
|
|
|
|
Args:
|
|
file_walker: Optional FileWalker instance
|
|
hasher: Optional HasherInterface implementation.
|
|
If None, attempts to resolve from global_container.
|
|
"""
|
|
self.logger = logger
|
|
self.file_walker = file_walker or FileWalker()
|
|
|
|
# Dependency Injection (Constructor Injection preferred)
|
|
if hasher:
|
|
self._hasher = hasher
|
|
else:
|
|
# Service Location fallback for backward compatibility
|
|
self._hasher = global_container.get_service('hasher_service')
|
|
|
|
# Create default hasher if service not available
|
|
if self._hasher is None:
|
|
from ..hashing.hasher_logic import FileHasher
|
|
self._hasher = FileHasher()
|
|
|
|
self._hash_algorithm = 'sha256'
|
|
self._hash_buffer_size = 65536 # 64KB buffer
|
|
|
|
def process_files(self, root_path: str, file_filter: Optional[Callable[[Any], bool]] = None,
|
|
on_progress: Optional[Callable[[Any], None]] = None) -> List[Dict[str, Any]]:
|
|
"""Process files in directory and return processed file information.
|
|
|
|
Args:
|
|
root_path: Root directory to process
|
|
file_filter: Optional function to filter files
|
|
on_progress: Optional callback for progress updates
|
|
|
|
Returns:
|
|
List of processed file information
|
|
"""
|
|
# First, walk the directory to get file list
|
|
files = self.file_walker.walk(root_path, file_filter, on_progress)
|
|
|
|
# Then process each file to add hashes and other metadata
|
|
processed_files = []
|
|
for i, file_info in enumerate(files):
|
|
try:
|
|
processed_file = self._process_single_file(file_info)
|
|
|
|
if processed_file:
|
|
processed_files.append(processed_file)
|
|
|
|
# Update progress
|
|
if on_progress and (i % 10 == 0 or i == len(files) - 1):
|
|
progress = {
|
|
'files_processed': i + 1,
|
|
'total_files': len(files),
|
|
'current_file': file_info['path']
|
|
}
|
|
on_progress(progress)
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f"[{ActionCode.FPT_FLS_FAIL}] Error processing file {file_info['path']}: {e}")
|
|
continue
|
|
|
|
return processed_files
|
|
|
|
def _process_single_file(self, file_info: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
"""Process a single file and return enhanced file information.
|
|
|
|
Args:
|
|
file_info: Basic file information
|
|
|
|
Returns:
|
|
Enhanced file information with hash and metadata
|
|
"""
|
|
try:
|
|
# Calculate file hash
|
|
file_hash = self._calculate_file_hash(file_info['path'])
|
|
|
|
# Create enhanced file info
|
|
processed_file = {
|
|
**file_info,
|
|
'hash': file_hash,
|
|
'hash_algorithm': self._hash_algorithm,
|
|
'is_duplicate': False,
|
|
'duplicate_of': None
|
|
}
|
|
|
|
return processed_file
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f"[{ActionCode.FPT_FLS_FAIL}] Error processing file {file_info['path']}: {e}")
|
|
return None
|
|
|
|
def _calculate_file_hash(self, file_path: str) -> str:
|
|
"""Calculate cryptographic hash of file.
|
|
|
|
Args:
|
|
file_path: Path to file
|
|
|
|
Returns:
|
|
Hexadecimal hash string
|
|
"""
|
|
try:
|
|
# Sync internal state if needed (backward compatibility for direct attribute changes)
|
|
if hasattr(self._hasher, 'set_algorithm'):
|
|
self._hasher.set_algorithm(self._hash_algorithm)
|
|
|
|
return self._hasher.hash_file(file_path)
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f"[{ActionCode.FPT_FLS_FAIL}] Error calculating hash for {file_path}: {e}")
|
|
raise
|
|
|
|
def detect_duplicates(self, files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
"""Detect duplicates in list of files.
|
|
|
|
Args:
|
|
files: List of file information
|
|
|
|
Returns:
|
|
List of files with duplicate information updated
|
|
"""
|
|
if not files:
|
|
return files
|
|
|
|
# Group files by hash
|
|
hash_groups = {}
|
|
for file_info in files:
|
|
if 'hash' not in file_info or not file_info['hash']:
|
|
continue
|
|
|
|
file_hash = file_info['hash']
|
|
if file_hash not in hash_groups:
|
|
hash_groups[file_hash] = []
|
|
hash_groups[file_hash].append(file_info)
|
|
|
|
# Mark duplicates
|
|
for file_hash, group in hash_groups.items():
|
|
if len(group) > 1:
|
|
# Sort by file size (largest is likely original)
|
|
group.sort(key=lambda x: x['size'], reverse=True)
|
|
|
|
# First file is original, rest are duplicates
|
|
original_file = group[0]
|
|
for duplicate_file in group[1:]:
|
|
duplicate_file['is_duplicate'] = True
|
|
duplicate_file['duplicate_of'] = original_file['path']
|
|
|
|
return files
|
|
|
|
def batch_process_files(self, file_paths: List[str],
|
|
on_progress: Optional[Callable[[Any], None]] = None) -> List[Dict[str, Any]]:
|
|
"""Process multiple files in batch.
|
|
|
|
Args:
|
|
file_paths: List of file paths to process
|
|
on_progress: Optional progress callback
|
|
|
|
Returns:
|
|
List of processed file information
|
|
"""
|
|
processed_files = []
|
|
|
|
for i, file_path in enumerate(file_paths):
|
|
try:
|
|
if not os.path.isfile(file_path):
|
|
continue
|
|
|
|
file_info = self._get_basic_file_info(file_path)
|
|
processed_file = self._process_single_file(file_info)
|
|
|
|
if processed_file:
|
|
processed_files.append(processed_file)
|
|
|
|
# Update progress
|
|
if on_progress and (i % 10 == 0 or i == len(file_paths) - 1):
|
|
progress = {
|
|
'files_processed': i + 1,
|
|
'total_files': len(file_paths),
|
|
'current_file': file_path
|
|
}
|
|
on_progress(progress)
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f"[{ActionCode.FPT_FLS_FAIL}] Error processing file {file_path}: {e}")
|
|
continue
|
|
|
|
return processed_files
|
|
|
|
def _get_basic_file_info(self, file_path: str) -> Dict[str, Any]:
|
|
"""Get basic file information for a single file.
|
|
|
|
Args:
|
|
file_path: Path to file
|
|
|
|
Returns:
|
|
Basic file information dictionary
|
|
"""
|
|
try:
|
|
stat = os.stat(file_path)
|
|
|
|
return {
|
|
'path': file_path,
|
|
'relative_path': os.path.basename(file_path),
|
|
'name': os.path.basename(file_path),
|
|
'extension': os.path.splitext(file_path)[1].lower(),
|
|
'size': stat.st_size,
|
|
'modified_time': int(stat.st_mtime),
|
|
'created_time': int(stat.st_ctime),
|
|
'is_directory': False,
|
|
'is_file': True,
|
|
'is_symlink': os.path.islink(file_path)
|
|
}
|
|
except Exception as e:
|
|
self.logger.warning(f"[{ActionCode.FPT_FLS_FAIL}] Error getting file info for {file_path}: {e}")
|
|
raise
|
|
|
|
def set_hash_algorithm(self, algorithm: str) -> None:
|
|
"""Set hash algorithm to use.
|
|
|
|
Args:
|
|
algorithm: Hash algorithm name (e.g., 'sha256', 'md5')
|
|
"""
|
|
if algorithm.lower() not in hashlib.algorithms_available:
|
|
raise ValueError(f"Hash algorithm {algorithm} not available")
|
|
|
|
self._hash_algorithm = algorithm.lower()
|
|
|
|
def get_hash_algorithm(self) -> str:
|
|
"""Get current hash algorithm.
|
|
|
|
Returns:
|
|
Current hash algorithm name
|
|
"""
|
|
return self._hash_algorithm
|
|
|
|
def set_hash_buffer_size(self, buffer_size: int) -> None:
|
|
"""Set buffer size for hash calculation.
|
|
|
|
Args:
|
|
buffer_size: Buffer size in bytes
|
|
"""
|
|
if buffer_size <= 0:
|
|
raise ValueError("Buffer size must be positive")
|
|
|
|
self._hash_buffer_size = buffer_size
|
|
|
|
def get_hash_buffer_size(self) -> int:
|
|
"""Get current hash buffer size.
|
|
|
|
Returns:
|
|
Current buffer size in bytes
|
|
"""
|
|
return self._hash_buffer_size
|
|
|
|
|
|
def create_file_processor(file_walker: Optional[FileWalker] = None) -> FileProcessor:
|
|
"""Create and return a FileProcessor instance.
|
|
|
|
Args:
|
|
file_walker: Optional FileWalker instance
|
|
|
|
Returns:
|
|
FileProcessor instance
|
|
"""
|
|
return FileProcessor(file_walker)
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="This tool creates digital fingerprints for files to find duplicates.")
|
|
parser.add_argument("path", help="The file or folder you want to process")
|
|
args = parser.parse_args()
|
|
processor = FileProcessor()
|
|
results = processor.process_files(args.path)
|
|
for r in results:
|
|
print(f"{r["hash"]} {r["path"]}")
|