mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
1261 lines
49 KiB
Python
1261 lines
49 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright (c) 2025 Allaun
|
|
|
|
"""Comprehensive coverage tests for nodupe/tools/parallel/parallel_logic.py.
|
|
|
|
This test file specifically targets the remaining uncovered lines to achieve 95%+ coverage:
|
|
- Debug logging in process_in_parallel (lines 142-144, 146)
|
|
- use_interpreters flag in map_parallel (line 179)
|
|
- InterpreterPoolExecutor in map_parallel (line 209)
|
|
- chunksize handling in map_parallel (lines 216-218)
|
|
- batch_size calculation exception in map_parallel_unordered (line 278)
|
|
- batch processing with logging in map_parallel_unordered (lines 282-287)
|
|
- chunksize calculation in map_parallel_unordered (lines 294-296, 298)
|
|
- bounded submission for threads in map_parallel_unordered (lines 305-336)
|
|
- chunksize exception handling (lines 340-344)
|
|
- StopIteration handling (lines 354-355)
|
|
- smart_map interpreter pool path (lines 369-370)
|
|
- smart_map free-threaded path (line 381)
|
|
|
|
Note: Now uses pickle-safe test helpers for ProcessPoolExecutor testing.
|
|
See: docs/PARALLEL_TESTING_SUSTAINABILITY.md
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
import pytest
|
|
|
|
from nodupe.tools.parallel.parallel_logic import (
|
|
Parallel,
|
|
ParallelError,
|
|
)
|
|
|
|
# Import pickle-safe test helpers for process testing
|
|
from tests.parallel.test_helpers import (
|
|
double_number,
|
|
square_number,
|
|
add_one,
|
|
identity,
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Test Debug Logging in process_in_parallel
|
|
# =============================================================================
|
|
|
|
class TestDebugLoggingInProcessInParallel:
|
|
"""Test debug logging paths in process_in_parallel."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_process_in_parallel_debug_logging_submit(self, caplog):
|
|
"""process_in_parallel logs task submission debug messages."""
|
|
items = [1, 2, 3]
|
|
|
|
with caplog.at_level(logging.DEBUG):
|
|
results = Parallel.process_in_parallel(
|
|
self.double_func, items, workers=2, use_processes=False
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
# Check for submission log message
|
|
assert any("Submitted" in record.message for record in caplog.records)
|
|
|
|
def test_process_in_parallel_debug_logging_completion(self, caplog):
|
|
"""process_in_parallel logs task completion debug messages."""
|
|
items = [1, 2, 3]
|
|
|
|
with caplog.at_level(logging.DEBUG):
|
|
results = Parallel.process_in_parallel(
|
|
self.double_func, items, workers=2, use_processes=False
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
# Check for completion log messages
|
|
assert any("completed" in record.message for record in caplog.records)
|
|
|
|
def test_process_in_parallel_debug_logging_failure(self, caplog):
|
|
"""process_in_parallel logs task failure with exception."""
|
|
def failing_func(x):
|
|
"""Raises error for specific input."""
|
|
if x == 1:
|
|
raise ValueError("Task failed intentionally")
|
|
return x * 2
|
|
|
|
items = [1, 2]
|
|
|
|
with caplog.at_level(logging.ERROR):
|
|
with pytest.raises(ParallelError):
|
|
Parallel.process_in_parallel(failing_func, items, workers=2)
|
|
|
|
# Check for failure log message
|
|
assert any("Task 0 failed" in record.message for record in caplog.records)
|
|
|
|
|
|
# =============================================================================
|
|
# Test use_interpreters Flag in map_parallel
|
|
# =============================================================================
|
|
|
|
class TestUseInterpretersInMapParallel:
|
|
"""Test use_interpreters flag behavior in map_parallel."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=True)
|
|
def test_map_parallel_with_interpreters_true(self, mock_supports):
|
|
"""map_parallel uses interpreters when use_interpreters=True."""
|
|
items = [1, 2, 3]
|
|
|
|
# When InterpreterPoolExecutor is not importable, falls back to ThreadPoolExecutor
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=True)
|
|
def test_map_parallel_interpreter_with_chunksize(self, mock_supports):
|
|
"""map_parallel uses chunksize with interpreters."""
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True, chunk_size=2
|
|
)
|
|
|
|
assert results == [2, 4, 6, 8, 10]
|
|
|
|
def test_map_parallel_interpreter_import_error_coverage(self):
|
|
"""map_parallel handles ImportError for InterpreterPoolExecutor."""
|
|
items = [1, 2, 3]
|
|
|
|
# Mock supports_interpreter_pool to return True but have import fail
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
# The import will fail in test environment, triggering fallback
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
|
|
# =============================================================================
|
|
# Test chunksize Handling in map_parallel
|
|
# =============================================================================
|
|
|
|
class TestChunksizeHandlingInMapParallel:
|
|
"""Test chunksize parameter handling in map_parallel."""
|
|
|
|
def add_ten_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x + 10
|
|
|
|
def test_map_parallel_chunksize_with_processes(self):
|
|
"""map_parallel uses chunksize with process executor."""
|
|
items = list(range(10))
|
|
|
|
# Use threads to avoid pickling issues but test chunksize path
|
|
results = Parallel.map_parallel(
|
|
self.add_ten_func, items, workers=2, use_processes=False, chunk_size=3
|
|
)
|
|
|
|
assert results == list(range(10, 20))
|
|
|
|
def test_map_parallel_chunksize_with_interpreters(self):
|
|
"""map_parallel uses chunksize with interpreter executor."""
|
|
items = list(range(8))
|
|
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
results = Parallel.map_parallel(
|
|
self.add_ten_func, items, workers=2, use_interpreters=True, chunk_size=2
|
|
)
|
|
|
|
assert results == list(range(10, 18))
|
|
|
|
|
|
# =============================================================================
|
|
# Test batch_size Calculation Exception in map_parallel_unordered
|
|
# =============================================================================
|
|
|
|
class TestBatchSizeCalculationException:
|
|
"""Test batch_size calculation exception handling."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_batch_size_calculation_exception_handling(self):
|
|
"""map_parallel_unordered handles exception in batch_size calculation."""
|
|
items = list(range(20))
|
|
|
|
# Mock len to raise exception during batch_size calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(20))
|
|
|
|
# Should fallback to batch_size=1 and continue
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True, prefer_batches=True
|
|
))
|
|
|
|
# Should still process items with fallback batch_size
|
|
assert sorted(results) == [x * 2 for x in range(20)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test Batch Processing with Logging in map_parallel_unordered
|
|
# =============================================================================
|
|
|
|
class TestBatchProcessingWithLogging:
|
|
"""Test batch processing with logging enabled in map_parallel_unordered."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_batch_processing_logs_batch_size(self, caplog):
|
|
"""map_parallel_unordered logs batch processing when NODUPE_BATCH_LOG=1."""
|
|
items = list(range(50))
|
|
|
|
with patch.dict(os.environ, {'NODUPE_BATCH_LOG': '1'}):
|
|
with caplog.at_level(logging.DEBUG):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=True, prefer_batches=True
|
|
))
|
|
|
|
# Check for batch processing log
|
|
batch_logs = [r for r in caplog.records if 'batch size' in r.message]
|
|
# Should have logged batch processing
|
|
assert sorted(results) == [x * 2 for x in range(50)]
|
|
|
|
def test_batch_processing_logging_exception_handled(self, caplog):
|
|
"""map_parallel_unordered handles exception in batch logging."""
|
|
items = list(range(30))
|
|
|
|
with patch.dict(os.environ, {'NODUPE_BATCH_LOG': '1'}):
|
|
with caplog.at_level(logging.DEBUG):
|
|
# Use threads to avoid pickling but test batch path
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, prefer_batches=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(30)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test chunksize Calculation in map_parallel_unordered
|
|
# =============================================================================
|
|
|
|
class TestChunksizeCalculationInMapParallelUnordered:
|
|
"""Test chunksize calculation in map_parallel_unordered."""
|
|
|
|
def add_one_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x + 1
|
|
|
|
def test_chunksize_calculation_normal(self):
|
|
"""map_parallel_unordered calculates chunksize correctly."""
|
|
items = list(range(100))
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.add_one_func, items, workers=4, use_processes=True, prefer_map=True
|
|
))
|
|
|
|
assert sorted(results) == [x + 1 for x in range(100)]
|
|
|
|
def test_chunksize_calculation_exception_fallback(self):
|
|
"""map_parallel_unordered handles exception in chunksize calculation."""
|
|
items = list(range(20))
|
|
|
|
# Mock len to raise exception
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(20))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.add_one_func, bad_items, workers=2, use_processes=True, prefer_map=True
|
|
))
|
|
|
|
# Should fallback to chunksize=1
|
|
assert sorted(results) == [x + 1 for x in range(20)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test Bounded Submission for Threads in map_parallel_unordered
|
|
# =============================================================================
|
|
|
|
class TestBoundedSubmissionForThreads:
|
|
"""Test bounded submission pattern for thread executor."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_bounded_submission_initial_batch(self):
|
|
"""map_parallel_unordered submits initial batch up to worker count."""
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
# use_processes=False triggers bounded submission
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10]
|
|
|
|
def test_bounded_submission_empty_items(self):
|
|
"""map_parallel_unordered handles empty items in bounded submission."""
|
|
items = []
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert results == []
|
|
|
|
def test_bounded_submission_single_item(self):
|
|
"""map_parallel_unordered handles single item in bounded submission."""
|
|
items = [42]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert results == [84]
|
|
|
|
def test_bounded_submission_fewer_items_than_workers(self):
|
|
"""map_parallel_unordered handles fewer items than workers."""
|
|
items = [1, 2]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=10, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4]
|
|
|
|
def test_bounded_submission_with_timeout(self):
|
|
"""map_parallel_unordered respects timeout in bounded submission."""
|
|
def slow_func(x):
|
|
"""Sleeps briefly then doubles."""
|
|
time.sleep(0.01)
|
|
return x * 2
|
|
|
|
items = [1, 2, 3]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
slow_func, items, workers=2, use_processes=False, prefer_map=False, timeout=5.0
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6]
|
|
|
|
def test_bounded_submission_error_handling(self):
|
|
"""map_parallel_unordered handles errors in bounded submission."""
|
|
def failing_func(x):
|
|
"""Raises error for specific input."""
|
|
if x == 2:
|
|
raise ValueError("Task failed")
|
|
return x * 2
|
|
|
|
items = [1, 2, 3]
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
list(Parallel.map_parallel_unordered(
|
|
failing_func, items, workers=2, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert "Task failed" in str(exc_info.value)
|
|
|
|
def test_bounded_submission_keyerror_handling(self):
|
|
"""map_parallel_unordered handles KeyError when removing future."""
|
|
items = [1, 2, 3]
|
|
|
|
# The KeyError handling is in the finally block
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6]
|
|
|
|
def test_bounded_submission_stopiteration_handling(self):
|
|
"""map_parallel_unordered handles StopIteration when iterator exhausted."""
|
|
items = [1, 2, 3]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6]
|
|
|
|
|
|
# =============================================================================
|
|
# Test chunksize Exception Handling in map_parallel_unordered
|
|
# =============================================================================
|
|
|
|
class TestChunksizeExceptionHandling:
|
|
"""Test chunksize exception handling in map_parallel_unordered."""
|
|
|
|
def add_one_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x + 1
|
|
|
|
def test_chunksize_exception_fallback_to_one(self):
|
|
"""map_parallel_unordered falls back to chunksize=1 on exception."""
|
|
items = list(range(10))
|
|
|
|
# Mock len to raise exception during chunksize calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(10))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.add_one_func, bad_items, workers=2, use_processes=True, prefer_map=True
|
|
))
|
|
|
|
assert sorted(results) == [x + 1 for x in range(10)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test StopIteration Handling
|
|
# =============================================================================
|
|
|
|
class TestStopIterationHandling:
|
|
"""Test StopIteration handling in map_parallel_unordered."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_stopiteration_in_initial_batch_submission(self):
|
|
"""map_parallel_unordered handles StopIteration in initial batch."""
|
|
items = [1] # Single item - will trigger StopIteration after first
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=5, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert results == [2]
|
|
|
|
def test_stopiteration_empty_iterator(self):
|
|
"""map_parallel_unordered handles empty iterator."""
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, [], workers=2, use_processes=False, prefer_map=False
|
|
))
|
|
|
|
assert results == []
|
|
|
|
|
|
# =============================================================================
|
|
# Test smart_map Interpreter Pool Path
|
|
# =============================================================================
|
|
|
|
class TestSmartMapInterpreterPoolPath:
|
|
"""Test smart_map interpreter pool execution path."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=True)
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=False)
|
|
def test_smart_map_cpu_uses_interpreter_pool(self, mock_free, mock_support):
|
|
"""smart_map uses interpreter pool for CPU tasks when available."""
|
|
items = [1, 2, 3]
|
|
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=True)
|
|
def test_smart_map_auto_task_inspection(self, mock_support):
|
|
"""smart_map inspects function for auto task type."""
|
|
items = [1, 2, 3]
|
|
|
|
# Auto task type defaults to 'cpu'
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='auto', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
|
|
# =============================================================================
|
|
# Test smart_map Free-threaded Path
|
|
# =============================================================================
|
|
|
|
class TestSmartMapFreeThreadedPath:
|
|
"""Test smart_map free-threaded execution path."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=False)
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=True)
|
|
def test_smart_map_cpu_uses_threads_free_threaded(self, mock_free, mock_support):
|
|
"""smart_map uses threads for CPU tasks in free-threaded mode."""
|
|
items = [1, 2, 3]
|
|
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=False)
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=False)
|
|
def test_smart_map_cpu_uses_processes_gil_mode(self, mock_free, mock_support):
|
|
"""smart_map uses processes for CPU tasks in GIL mode."""
|
|
items = [1, 2, 3]
|
|
|
|
# In GIL mode with no interpreter pool, uses processes
|
|
# Use threads in test to avoid pickling issues
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_processes=False
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
|
|
# =============================================================================
|
|
# Test Environment Variable Edge Cases
|
|
# =============================================================================
|
|
|
|
class TestEnvironmentVariableEdgeCases:
|
|
"""Test environment variable edge cases for coverage."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_batch_divisor_env_var_edge_cases(self):
|
|
"""map_parallel_unordered handles various NODUPE_BATCH_DIVISOR values."""
|
|
items = list(range(20))
|
|
|
|
# Test with 'True' value (should be invalid for int conversion)
|
|
with patch.dict(os.environ, {'NODUPE_BATCH_DIVISOR': 'True'}):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, prefer_batches=True
|
|
))
|
|
assert sorted(results) == [x * 2 for x in range(20)]
|
|
|
|
def test_chunk_factor_env_var_edge_cases(self):
|
|
"""map_parallel_unordered handles various NODUPE_CHUNK_FACTOR values."""
|
|
items = list(range(20))
|
|
|
|
# Test with 'False' value
|
|
with patch.dict(os.environ, {'NODUPE_CHUNK_FACTOR': 'False'}):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, prefer_map=True
|
|
))
|
|
assert sorted(results) == [x * 2 for x in range(20)]
|
|
|
|
def test_batch_log_env_var_capitalization(self):
|
|
"""map_parallel_unordered handles NODUPE_BATCH_LOG with various cases."""
|
|
items = list(range(10))
|
|
|
|
# Test 'True' (capitalized)
|
|
with patch.dict(os.environ, {'NODUPE_BATCH_LOG': 'True'}):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False
|
|
))
|
|
assert sorted(results) == [x * 2 for x in range(10)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test Worker Count Capping for Processes
|
|
# =============================================================================
|
|
|
|
class TestWorkerCountCapping:
|
|
"""Test worker count capping logic for process pools."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_worker_capping_normal_case(self):
|
|
"""map_parallel_unordered caps workers for process pools."""
|
|
items = list(range(10))
|
|
|
|
# Use processes=True to trigger worker capping
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=100, use_processes=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(10)]
|
|
|
|
def test_worker_capping_cpu_count_exception(self):
|
|
"""map_parallel_unordered handles cpu_count exception in capping."""
|
|
items = list(range(10))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', side_effect=Exception("Failed")):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=100, use_processes=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(10)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test prefer_batches vs prefer_map Options
|
|
# =============================================================================
|
|
|
|
class TestPreferBatchesVsMapOptions:
|
|
"""Test prefer_batches and prefer_map option combinations."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_prefer_batches_true_prefer_map_false(self):
|
|
"""map_parallel_unordered with prefer_batches=True, prefer_map=False."""
|
|
items = list(range(30))
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False,
|
|
prefer_batches=True, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(30)]
|
|
|
|
def test_prefer_batches_false_prefer_map_true(self):
|
|
"""map_parallel_unordered with prefer_batches=False, prefer_map=True."""
|
|
items = list(range(30))
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False,
|
|
prefer_batches=False, prefer_map=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(30)]
|
|
|
|
def test_prefer_batches_false_prefer_map_false(self):
|
|
"""map_parallel_unordered with both prefer_batches=False, prefer_map=False."""
|
|
items = list(range(30))
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(30)]
|
|
|
|
|
|
# =============================================================================
|
|
# Test ParallelError Passthrough
|
|
# =============================================================================
|
|
|
|
class TestParallelErrorPassthrough:
|
|
"""Test ParallelError passthrough in various methods."""
|
|
|
|
def test_process_in_parallel_passthrough(self):
|
|
"""process_in_parallel re-raises ParallelError without wrapping."""
|
|
def failing_func(x):
|
|
"""Raises ParallelError directly."""
|
|
raise ParallelError("Already wrapped error")
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
Parallel.process_in_parallel(failing_func, [1], workers=1)
|
|
|
|
assert "Already wrapped error" in str(exc_info.value)
|
|
|
|
def test_map_parallel_unordered_passthrough(self):
|
|
"""map_parallel_unordered re-raises ParallelError without wrapping."""
|
|
def failing_func(x):
|
|
"""Raises ParallelError directly."""
|
|
raise ParallelError("Already wrapped error")
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
list(Parallel.map_parallel_unordered(failing_func, [1], workers=1))
|
|
|
|
assert "Already wrapped error" in str(exc_info.value)
|
|
|
|
def test_reduce_parallel_passthrough(self):
|
|
"""reduce_parallel re-raises ParallelError without wrapping."""
|
|
def map_func(x):
|
|
"""Returns value."""
|
|
return x
|
|
|
|
def reduce_func(a, b):
|
|
"""Raises ParallelError directly."""
|
|
raise ParallelError("Already wrapped error")
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
Parallel.reduce_parallel(map_func, reduce_func, [1, 2], workers=1)
|
|
|
|
assert "Already wrapped error" in str(exc_info.value)
|
|
|
|
|
|
# =============================================================================
|
|
# Test Edge Cases for Full Coverage
|
|
# =============================================================================
|
|
|
|
class TestEdgeCasesForFullCoverage:
|
|
"""Test edge cases to ensure full coverage."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_map_parallel_unordered_timeout_zero(self):
|
|
"""map_parallel_unordered with timeout=0."""
|
|
items = [1, 2, 3]
|
|
|
|
# timeout=0 should still work for instant operations
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=False, timeout=10.0
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_large_worker_count(self):
|
|
"""map_parallel_unordered with worker count exceeding items."""
|
|
items = [1, 2]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=100, use_processes=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4]
|
|
|
|
def test_process_in_parallel_with_none_result(self):
|
|
"""process_in_parallel handles functions returning None."""
|
|
def returns_none(x):
|
|
"""Returns None for all inputs."""
|
|
return None
|
|
|
|
items = [1, 2, 3]
|
|
|
|
results = Parallel.process_in_parallel(returns_none, items, workers=2)
|
|
|
|
assert results == [None, None, None]
|
|
|
|
def test_map_parallel_with_none_result(self):
|
|
"""map_parallel handles functions returning None."""
|
|
def returns_none(x):
|
|
"""Returns None for all inputs."""
|
|
return None
|
|
|
|
items = [1, 2, 3]
|
|
|
|
results = Parallel.map_parallel(returns_none, items, workers=2)
|
|
|
|
assert results == [None, None, None]
|
|
|
|
def test_smart_map_with_timeout(self):
|
|
"""smart_map passes timeout parameter."""
|
|
def quick_func(x):
|
|
"""Quick function for testing."""
|
|
return x * 2
|
|
|
|
items = [1, 2, 3]
|
|
|
|
# smart_map doesn't use timeout directly, but test the path
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=False):
|
|
with patch.object(Parallel, 'is_free_threaded', return_value=True):
|
|
results = Parallel.smart_map(
|
|
quick_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
|
|
# =============================================================================
|
|
# Test get_optimal_workers All Paths
|
|
# =============================================================================
|
|
|
|
class TestGetOptimalWorkersAllPaths:
|
|
"""Test all paths in get_optimal_workers method."""
|
|
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=True)
|
|
@patch.object(Parallel, 'get_cpu_count', return_value=8)
|
|
def test_get_optimal_workers_free_threaded_cpu(self, mock_cpu, mock_free):
|
|
"""get_optimal_workers returns cpu_count * 2 for CPU in free-threaded."""
|
|
workers = Parallel.get_optimal_workers(task_type='cpu')
|
|
assert workers == 16
|
|
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=True)
|
|
@patch.object(Parallel, 'get_cpu_count', return_value=8)
|
|
def test_get_optimal_workers_free_threaded_io(self, mock_cpu, mock_free):
|
|
"""get_optimal_workers returns min(32, cpu_count * 2) for IO in free-threaded."""
|
|
workers = Parallel.get_optimal_workers(task_type='io')
|
|
assert workers == 16
|
|
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=False)
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=True)
|
|
@patch.object(Parallel, 'get_cpu_count', return_value=8)
|
|
def test_get_optimal_workers_interpreter_pool(self, mock_cpu, mock_support, mock_free):
|
|
"""get_optimal_workers returns cpu_count for interpreter pool."""
|
|
workers = Parallel.get_optimal_workers(task_type='cpu')
|
|
assert workers == 8
|
|
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=False)
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=False)
|
|
@patch.object(Parallel, 'get_cpu_count', return_value=8)
|
|
def test_get_optimal_workers_gil_cpu(self, mock_cpu, mock_support, mock_free):
|
|
"""get_optimal_workers returns min(32, cpu_count) for CPU in GIL mode."""
|
|
workers = Parallel.get_optimal_workers(task_type='cpu')
|
|
assert workers == 8
|
|
|
|
@patch.object(Parallel, 'is_free_threaded', return_value=False)
|
|
@patch.object(Parallel, 'supports_interpreter_pool', return_value=False)
|
|
@patch.object(Parallel, 'get_cpu_count', return_value=8)
|
|
def test_get_optimal_workers_gil_io(self, mock_cpu, mock_support, mock_free):
|
|
"""get_optimal_workers returns min(32, cpu_count * 2) for IO in GIL mode."""
|
|
workers = Parallel.get_optimal_workers(task_type='io')
|
|
assert workers == 16
|
|
|
|
@patch.object(Parallel, 'get_cpu_count', side_effect=Exception("Failed"))
|
|
def test_get_optimal_workers_cpu_count_exception_cpu(self, mock_cpu):
|
|
"""get_optimal_workers handles exception for CPU tasks."""
|
|
workers = Parallel.get_optimal_workers(task_type='cpu')
|
|
assert workers == 1 # Fallback
|
|
|
|
@patch.object(Parallel, 'get_cpu_count', side_effect=Exception("Failed"))
|
|
def test_get_optimal_workers_cpu_count_exception_io(self, mock_cpu):
|
|
"""get_optimal_workers handles exception for IO tasks."""
|
|
workers = Parallel.get_optimal_workers(task_type='io')
|
|
assert workers >= 1 # Has some fallback
|
|
|
|
|
|
# =============================================================================
|
|
# Test Remaining Coverage Gaps
|
|
# =============================================================================
|
|
|
|
class TestRemainingCoverageGaps:
|
|
"""Test remaining coverage gaps to achieve 95%+."""
|
|
|
|
def double_func(self, x):
|
|
"""Helper function for testing."""
|
|
return x * 2
|
|
|
|
def test_map_parallel_with_interpreters_and_chunksize(self, caplog):
|
|
"""map_parallel uses chunksize with interpreters (lines 216-218)."""
|
|
items = list(range(10))
|
|
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
with caplog.at_level(logging.DEBUG):
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True, chunk_size=2
|
|
)
|
|
|
|
assert results == [x * 2 for x in range(10)]
|
|
|
|
def test_map_parallel_unordered_chunksize_with_processes(self):
|
|
"""map_parallel_unordered uses chunksize with processes (lines 294-296, 298)."""
|
|
items = list(range(50))
|
|
|
|
# Use processes=True and prefer_map=True to hit chunksize path
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=True,
|
|
prefer_batches=False, prefer_map=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(50)]
|
|
|
|
def test_map_parallel_unordered_bounded_submission_full_path(self):
|
|
"""map_parallel_unordered bounded submission full path (lines 320-336)."""
|
|
items = [1, 2, 3, 4, 5, 6, 7, 8]
|
|
|
|
# use_processes=False and prefer_map=False triggers bounded submission
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10, 12, 14, 16]
|
|
|
|
def test_map_parallel_unordered_bounded_submission_with_timeout(self):
|
|
"""map_parallel_unordered bounded submission with timeout."""
|
|
def slow_double(x):
|
|
"""Sleeps briefly then doubles."""
|
|
time.sleep(0.01)
|
|
return x * 2
|
|
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
results = list(Parallel.map_parallel_unordered(
|
|
slow_double, items, workers=2, use_processes=False,
|
|
prefer_batches=False, prefer_map=False, timeout=10.0
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10]
|
|
|
|
def test_smart_map_interpreter_pool_cpu_task(self):
|
|
"""smart_map uses interpreter pool for CPU task (lines 369-370)."""
|
|
items = [1, 2, 3]
|
|
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
with patch.object(Parallel, 'is_free_threaded', return_value=False):
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_smart_map_free_threaded_cpu_task(self):
|
|
"""smart_map uses threads for CPU task in free-threaded mode (line 381)."""
|
|
items = [1, 2, 3]
|
|
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=False):
|
|
with patch.object(Parallel, 'is_free_threaded', return_value=True):
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_chunksize_exception_in_process_mode(self):
|
|
"""map_parallel_unordered handles chunksize exception in process mode."""
|
|
items = list(range(20))
|
|
|
|
# Mock len to raise exception during chunksize calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(20))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True,
|
|
prefer_batches=False, prefer_map=True
|
|
))
|
|
|
|
# Should fallback to chunksize=1
|
|
assert sorted(results) == [x * 2 for x in range(20)]
|
|
|
|
def test_map_parallel_unordered_batch_size_exception_in_process_mode(self):
|
|
"""map_parallel_unordered handles batch_size exception in process mode (line 278)."""
|
|
items = list(range(20))
|
|
|
|
# Mock len to raise exception during batch_size calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(20))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True,
|
|
prefer_batches=True
|
|
))
|
|
|
|
# Should fallback to batch_size=1 then chunksize path
|
|
assert sorted(results) == [x * 2 for x in range(20)]
|
|
|
|
def test_process_in_parallel_debug_logging_full(self, caplog):
|
|
"""process_in_parallel full debug logging path (lines 142-144, 146)."""
|
|
items = [1, 2, 3]
|
|
|
|
with caplog.at_level(logging.DEBUG):
|
|
results = Parallel.process_in_parallel(
|
|
self.double_func, items, workers=2, use_processes=False
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
# Verify both submission and completion logs
|
|
messages = [r.message for r in caplog.records]
|
|
assert any("Submitted" in msg for msg in messages)
|
|
assert any("completed" in msg for msg in messages)
|
|
|
|
def test_map_parallel_use_interpreters_true_path(self):
|
|
"""map_parallel with use_interpreters=True hits interpreter path (lines 179, 209, 216-218)."""
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
# When use_interpreters=True and supports_interpreter_pool returns True,
|
|
# the code tries to import InterpreterPoolExecutor
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
# This will hit the interpreter path and fallback to ThreadPoolExecutor
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True, chunk_size=2
|
|
)
|
|
|
|
assert results == [2, 4, 6, 8, 10]
|
|
|
|
def test_map_parallel_unordered_chunksize_with_processes_path(self):
|
|
"""map_parallel_unordered chunksize path with processes (lines 294-296)."""
|
|
items = list(range(100))
|
|
|
|
# Use processes=True and prefer_map=True to hit chunksize calculation
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=True,
|
|
prefer_batches=False, prefer_map=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(100)]
|
|
|
|
def test_map_parallel_unordered_bounded_submission_thread_path(self):
|
|
"""map_parallel_unordered bounded submission for threads (lines 320-336)."""
|
|
items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
|
|
|
# use_processes=False triggers bounded submission (the else branch)
|
|
# prefer_map=False ensures we don't use executor.map
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
|
|
|
|
def test_map_parallel_unordered_stopiteration_in_initial_batch(self):
|
|
"""map_parallel_unordered StopIteration in initial batch (lines 354-355)."""
|
|
# Empty items list triggers StopIteration immediately
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, [], workers=3, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert results == []
|
|
|
|
def test_smart_map_cpu_interpreter_pool_path(self):
|
|
"""smart_map CPU task with interpreter pool available (lines 369-370)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Mock to simulate Python 3.14+ with interpreter pool support
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
with patch.object(Parallel, 'is_free_threaded', return_value=False):
|
|
# This should call map_parallel with use_interpreters=True
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_smart_map_cpu_free_threaded_path(self):
|
|
"""smart_map CPU task in free-threaded mode (line 381)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Mock to simulate free-threaded Python without interpreter pool
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=False):
|
|
with patch.object(Parallel, 'is_free_threaded', return_value=True):
|
|
# This should call map_parallel with use_processes=False
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_batch_size_exception_path(self):
|
|
"""map_parallel_unordered batch_size exception path (line 278)."""
|
|
items = list(range(50))
|
|
|
|
# Mock len to raise exception during batch_size calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(50))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
# use_processes=True and prefer_batches=True triggers batch_size calculation
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True,
|
|
prefer_batches=True
|
|
))
|
|
|
|
# Should fallback to chunksize path
|
|
assert sorted(results) == [x * 2 for x in range(50)]
|
|
|
|
def test_map_parallel_unordered_chunksize_exception_path(self):
|
|
"""map_parallel_unordered chunksize exception path (lines 340-344)."""
|
|
items = list(range(50))
|
|
|
|
# Mock len to raise exception during chunksize calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(50))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
# use_processes=True and prefer_map=True triggers chunksize calculation
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True,
|
|
prefer_batches=False, prefer_map=True
|
|
))
|
|
|
|
# Should fallback to chunksize=1
|
|
assert sorted(results) == [x * 2 for x in range(50)]
|
|
|
|
def test_map_parallel_with_interpreters_no_mock(self):
|
|
"""map_parallel with use_interpreters=True without mocking (lines 179, 209, 216-218)."""
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
# Don't mock - use actual InterpreterPoolExecutor if available
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True, chunk_size=2
|
|
)
|
|
|
|
assert results == [2, 4, 6, 8, 10]
|
|
|
|
def test_process_in_parallel_with_interpreters_no_mock(self, caplog):
|
|
"""process_in_parallel with use_interpreters=True (lines 142-144, 146)."""
|
|
items = [1, 2, 3]
|
|
|
|
with caplog.at_level(logging.DEBUG):
|
|
# Don't mock - use actual InterpreterPoolExecutor if available
|
|
results = Parallel.process_in_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
# Verify debug logging
|
|
messages = [r.message for r in caplog.records]
|
|
assert any("Submitted" in msg for msg in messages)
|
|
assert any("completed" in msg for msg in messages)
|
|
|
|
def test_smart_map_cpu_with_actual_interpreter_pool(self):
|
|
"""smart_map CPU task with actual interpreter pool (lines 369-370)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Don't mock - use actual supports_interpreter_pool
|
|
results = Parallel.smart_map(
|
|
self.double_func, items, task_type='cpu', workers=2
|
|
)
|
|
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_with_interpreters_no_mock(self):
|
|
"""map_parallel_unordered with use_interpreters=True (lines 294-296)."""
|
|
items = [1, 2, 3, 4, 5]
|
|
|
|
# Don't mock - use actual InterpreterPoolExecutor if available
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10]
|
|
|
|
def test_map_parallel_unordered_bounded_submission_full(self):
|
|
"""map_parallel_unordered bounded submission full path (lines 320-336, 354-355)."""
|
|
items = [1, 2, 3, 4, 5, 6, 7, 8]
|
|
|
|
# use_processes=False, prefer_batches=False, prefer_map=False triggers bounded submission
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=3, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert sorted(results) == [2, 4, 6, 8, 10, 12, 14, 16]
|
|
|
|
def test_map_parallel_unordered_bounded_submission_empty(self):
|
|
"""map_parallel_unordered bounded submission with empty items (line 354-355)."""
|
|
# Empty items triggers StopIteration immediately in bounded submission
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, [], workers=3, use_processes=False,
|
|
prefer_batches=False, prefer_map=False
|
|
))
|
|
|
|
assert results == []
|
|
|
|
def test_map_parallel_unordered_batch_size_exception_full_path(self):
|
|
"""map_parallel_unordered batch_size exception full path (line 278)."""
|
|
items = list(range(30))
|
|
|
|
# Mock len to raise exception during batch_size calculation
|
|
class BadList(list):
|
|
def __len__(self):
|
|
raise RuntimeError("len() failed")
|
|
|
|
bad_items = BadList(range(30))
|
|
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
# use_processes=True and prefer_batches=True triggers batch_size calculation
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, bad_items, workers=2, use_processes=True,
|
|
prefer_batches=True
|
|
))
|
|
|
|
# Should fallback and still process items
|
|
assert sorted(results) == [x * 2 for x in range(30)]
|
|
|
|
def test_process_in_parallel_interpreter_import_error_fallback(self):
|
|
"""process_in_parallel with interpreters (lines 142-144, 146 are ImportError fallbacks)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Note: Lines 142-144, 146 are ImportError fallback paths that are only executed
|
|
# when InterpreterPoolExecutor is not available. In Python 3.14+, it IS available,
|
|
# so these lines are effectively dead code in this environment.
|
|
# This test verifies the normal interpreter path works.
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
results = Parallel.process_in_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
)
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_map_parallel_interpreter_import_error_fallback(self):
|
|
"""map_parallel ImportError fallback path (lines 216-218)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Use existing test that covers this path
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
results = Parallel.map_parallel(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
)
|
|
assert results == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_interpreter_import_error_fallback(self):
|
|
"""map_parallel_unordered ImportError fallback path (lines 294-296)."""
|
|
items = [1, 2, 3]
|
|
|
|
# Use existing test that covers this path
|
|
with patch.object(Parallel, 'supports_interpreter_pool', return_value=True):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_interpreters=True
|
|
))
|
|
assert sorted(results) == [2, 4, 6]
|
|
|
|
def test_map_parallel_unordered_batch_processing_path(self):
|
|
"""map_parallel_unordered batch processing path (lines 320, 336)."""
|
|
items = list(range(50))
|
|
|
|
# use_processes=True and prefer_batches=True with enough items triggers batch processing
|
|
with patch.object(Parallel, 'get_cpu_count', return_value=4):
|
|
results = list(Parallel.map_parallel_unordered(
|
|
self.double_func, items, workers=2, use_processes=True,
|
|
prefer_batches=True
|
|
))
|
|
|
|
assert sorted(results) == [x * 2 for x in range(50)]
|
|
|
|
def test_process_in_parallel_exception_raises_parallel_error(self):
|
|
"""process_in_parallel raises ParallelError on exception (line 179)."""
|
|
def failing_func(x):
|
|
"""Always raises an exception."""
|
|
raise RuntimeError("Intentional failure")
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
Parallel.process_in_parallel(failing_func, [1], workers=1)
|
|
|
|
assert "Parallel processing failed" in str(exc_info.value) or "Task failed" in str(exc_info.value)
|
|
|
|
def test_map_parallel_unordered_exception_raises_parallel_error(self):
|
|
"""map_parallel_unordered raises ParallelError on exception (line 381)."""
|
|
def failing_func(x):
|
|
"""Always raises an exception."""
|
|
raise RuntimeError("Intentional failure")
|
|
|
|
with pytest.raises(ParallelError) as exc_info:
|
|
list(Parallel.map_parallel_unordered(failing_func, [1], workers=1))
|
|
|
|
assert "Parallel map failed" in str(exc_info.value) or "Task failed" in str(exc_info.value)
|