Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Preserve parallel executor task types with ty (#10688)

Isaac Miller661b2f1a2521
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.py
index 7afcebd5e..1115ea99a 100644
--- a/dspy/utils/parallelizer.py
+++ b/dspy/utils/parallelizer.py
@@ -6,29 +6,43 @@ import sys
import threading
import time
import traceback
-from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from collections.abc import Callable, Collection, Generator
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+from enum import Enum
+from types import FrameType
+from typing import Any, Literal, TypeVar, cast
import tqdm
from dspy.dsp.utils.settings import settings, thread_local_overrides
from dspy.utils.callback_context import _bind_active_call_id
+T = TypeVar("T")
+R = TypeVar("R")
+_WorkerResult = tuple[int, R | Exception | Literal["cancelled"] | None]
+
logger = logging.getLogger(__name__)
-_UNSET = object()
+
+class _ResultState(Enum):
+ # A distinct sentinel type lets ty separate unfinished slots from task results.
+ UNSET = "unset"
+
+
+_UNSET = _ResultState.UNSET
class ParallelExecutor:
def __init__(
self,
- num_threads=None,
- max_errors=None,
- disable_progress_bar=False,
- provide_traceback=None,
- compare_results=False,
- timeout=120,
- straggler_limit=3,
- ):
+ num_threads: int | None = None,
+ max_errors: int | None = None,
+ disable_progress_bar: bool = False,
+ provide_traceback: bool | None = None,
+ compare_results: bool = False,
+ timeout: float = 120,
+ straggler_limit: int = 3,
+ ) -> None:
"""
Propagates DSPy settings and callback ancestry into each task while isolating task-local changes,
irrespective of whether num_threads == 1 or > 1. Handles also straggler timeouts.
@@ -44,18 +58,24 @@ class ParallelExecutor:
self.error_count = 0
self.error_lock = threading.Lock()
self.cancel_jobs = threading.Event()
- self.failed_indices = []
- self.exceptions_map = {}
+ self.failed_indices: list[int] = []
+ self.exceptions_map: dict[int, Exception] = {}
+
+ def execute(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:
+ """Return results in input iteration order, with None for failed or unfinished items.
- def execute(self, function, data):
+ Successful tasks may also return None. Use failed_indices and exceptions_map
+ to identify task exceptions. With compare_results=True, each non-None result
+ must support indexing its final element as a numeric score.
+ """
tqdm.tqdm._instances.clear()
- wrapped = self._wrap_function(_bind_active_call_id(function))
+ function = _bind_active_call_id(function)
if self.num_threads == 1:
- return self._execute_sequential(wrapped, data)
- return self._execute_parallel(wrapped, data)
+ return self._execute_sequential(function, data)
+ return self._execute_parallel(function, data)
- def _wrap_function(self, user_function):
- def safe_func(item):
+ def _wrap_function(self, user_function: Callable[[T], R]) -> Callable[[T], R | Exception | None]:
+ def safe_func(item: T) -> R | Exception | None:
if self.cancel_jobs.is_set():
return None
try:
@@ -73,9 +93,10 @@ class ParallelExecutor:
return safe_func
- def _execute_sequential(self, function, data):
+ def _execute_sequential(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:
"""Execute items sequentially on the main thread."""
- results = [_UNSET] * len(data)
+ safe_function = self._wrap_function(function)
+ results: list[R | _ResultState | None] = [_UNSET] * len(data)
pbar = tqdm.tqdm(
total=len(data),
@@ -89,7 +110,7 @@ class ParallelExecutor:
if self.cancel_jobs.is_set():
break
- outcome = function(item)
+ outcome = safe_function(item)
self._process_outcome(results, idx, outcome)
self._report_progress(pbar, results, len(data))
except KeyboardInterrupt:
@@ -105,17 +126,20 @@ class ParallelExecutor:
return [None if result is _UNSET else result for result in results]
- def _execute_parallel(self, function, data):
- results = [_UNSET] * len(data)
+ def _execute_parallel(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:
+ safe_function = self._wrap_function(function)
+ results: list[R | _ResultState | None] = [_UNSET] * len(data)
job_cancelled = "cancelled"
# We resubmit at most once per item.
- start_time_map = {}
+ start_time_map: dict[int, float] = {}
start_time_lock = threading.Lock()
- resubmitted = set()
+ resubmitted: set[Future[_WorkerResult[R]]] = set()
# This is the worker function each thread will run.
- def worker(parent_overrides, submission_id, index, item):
+ def worker(
+ parent_overrides: dict[str, Any], submission_id: int, index: int, item: T
+ ) -> _WorkerResult[R]:
if self.cancel_jobs.is_set():
return index, job_cancelled
# Record actual start time
@@ -131,17 +155,17 @@ class ParallelExecutor:
token = thread_local_overrides.set(new_overrides)
try:
- return index, function(item)
+ return index, safe_function(item)
finally:
thread_local_overrides.reset(token)
# Handle Ctrl-C in the main thread
@contextlib.contextmanager
- def interrupt_manager():
+ def interrupt_manager() -> Generator[None, None, None]:
if threading.current_thread() is threading.main_thread():
orig_handler = signal.getsignal(signal.SIGINT)
- def handler(sig, frame):
+ def handler(sig: int, frame: FrameType | None) -> None:
self.cancel_jobs.set()
logger.warning("SIGINT received. Cancelling.")
if callable(orig_handler):
@@ -165,8 +189,8 @@ class ParallelExecutor:
with interrupt_manager():
parent_overrides = thread_local_overrides.get().copy()
- futures_map = {}
- futures_set = set()
+ futures_map: dict[Future[_WorkerResult[R]], tuple[int, int, T]] = {}
+ futures_set: set[Future[_WorkerResult[R]]] = set()
submission_counter = 0
for idx, item in enumerate(data):
@@ -182,7 +206,7 @@ class ParallelExecutor:
file=sys.stdout,
)
- def all_done():
+ def all_done() -> bool:
return all(result is not _UNSET for result in results)
while futures_set and not self.cancel_jobs.is_set():
@@ -197,7 +221,9 @@ class ParallelExecutor:
pass
else:
if outcome != job_cancelled and results[index] is _UNSET:
- self._process_outcome(results, index, outcome)
+ # Restore R after excluding the cancellation marker; ty otherwise
+ # infers a narrower R incompatible with the invariant results list.
+ self._process_outcome(results, index, cast(R | Exception | None, outcome))
self._report_progress(pbar, results, len(data))
@@ -237,7 +263,9 @@ class ParallelExecutor:
return [None if result is _UNSET else result for result in results]
- def _process_outcome(self, results, idx, outcome):
+ def _process_outcome(
+ self, results: list[R | _ResultState | None], idx: int, outcome: R | Exception | None
+ ) -> None:
"""Store a single outcome and track errors."""
if isinstance(outcome, Exception):
with self.error_lock:
@@ -246,8 +274,8 @@ class ParallelExecutor:
else:
results[idx] = outcome
- def _report_progress(self, pbar, results, total):
- """Compute metrics and update the progress bar."""
+ def _report_progress(self, pbar: tqdm.tqdm, results: Collection[Any], total: int) -> None:
+ """Compute progress; compare_results opts into dynamically shaped, score-bearing results."""
if self.compare_results:
vals = [result[-1] for result in results if result is not _UNSET and result is not None]
self._update_progress(pbar, sum(vals), len(vals))
@@ -258,7 +286,7 @@ class ParallelExecutor:
total,
)
- def _update_progress(self, pbar, nresults, ntotal):
+ def _update_progress(self, pbar: tqdm.tqdm, nresults: float, ntotal: int) -> None:
if self.compare_results:
pct = round(100 * nresults / ntotal, 1) if ntotal else 0
pbar.set_description(f"Average Metric: {nresults:.2f} / {ntotal} ({pct}%)")
diff --git a/pyproject.toml b/pyproject.toml
index b48d1fbd7..0db231fed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -142,6 +142,7 @@ include = [
"dspy/utils/asyncify.py",
"dspy/utils/callback.py",
"dspy/utils/callback_context.py",
+ "dspy/utils/parallelizer.py",
"dspy/utils/usage_tracker.py",
"tests/typing",
]
diff --git a/tests/typing/parallelizer.py b/tests/typing/parallelizer.py
new file mode 100644
index 000000000..7343d1caa
--- /dev/null
+++ b/tests/typing/parallelizer.py
@@ -0,0 +1,37 @@
+"""Executor contracts checked by ty, not executed by pytest.
+
+Ignores on deliberately invalid calls assert expected diagnostics: unused-ignore
+errors make these checks fail if invalid calls become accepted.
+"""
+
+from typing_extensions import assert_type
+
+from dspy.utils.parallelizer import ParallelExecutor
+
+
+def format_count(count: int) -> str:
+ return str(count)
+
+
+def measure(value: str) -> float:
+ return len(value) / 2
+
+
+def optional_result(count: int) -> str | None:
+ return str(count) if count else None
+
+
+def check_executor() -> None:
+ executor = ParallelExecutor(num_threads=3)
+ assert_type(executor.execute(format_count, range(3)), list[str | None])
+ assert_type(executor.execute(measure, ("one", "three")), list[float | None])
+ assert_type(executor.execute(optional_result, [0, 1]), list[str | None])
+ assert_type(executor.failed_indices, list[int])
+ assert_type(executor.exceptions_map, dict[int, Exception])
+
+ sequential = ParallelExecutor(num_threads=1)
+ assert_type(sequential.execute(format_count, {1, 3}), list[str | None])
+
+ executor.execute(format_count, ["one"]) # ty: ignore[invalid-argument-type]
+ executor.execute(format_count, iter([1, 3])) # ty: ignore[invalid-argument-type]
+ ParallelExecutor(num_threads="three") # ty: ignore[invalid-argument-type]