DSPy: The framework for programming—not prompting—language models
Python
Preserve parallel executor task types with ty (#10688)
Isaac Miller661b2f1a2521
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.pyindex 7afcebd5e..1115ea99a 100644--- a/dspy/utils/parallelizer.py+++ b/dspy/utils/parallelizer.py@@ -6,29 +6,43 @@ import sysimport threadingimport timeimport traceback-from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait+from collections.abc import Callable, Collection, Generator+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait+from enum import Enum+from types import FrameType+from typing import Any, Literal, TypeVar, castimport tqdmfrom dspy.dsp.utils.settings import settings, thread_local_overridesfrom dspy.utils.callback_context import _bind_active_call_id+T = TypeVar("T")+R = TypeVar("R")+_WorkerResult = tuple[int, R | Exception | Literal["cancelled"] | None]+logger = logging.getLogger(__name__)-_UNSET = object()++class _ResultState(Enum):+ # A distinct sentinel type lets ty separate unfinished slots from task results.+ UNSET = "unset"+++_UNSET = _ResultState.UNSETclass ParallelExecutor:def __init__(self,- num_threads=None,- max_errors=None,- disable_progress_bar=False,- provide_traceback=None,- compare_results=False,- timeout=120,- straggler_limit=3,- ):+ num_threads: int | None = None,+ max_errors: int | None = None,+ disable_progress_bar: bool = False,+ provide_traceback: bool | None = None,+ compare_results: bool = False,+ timeout: float = 120,+ straggler_limit: int = 3,+ ) -> None:"""Propagates DSPy settings and callback ancestry into each task while isolating task-local changes,irrespective of whether num_threads == 1 or > 1. Handles also straggler timeouts.@@ -44,18 +58,24 @@ class ParallelExecutor:self.error_count = 0self.error_lock = threading.Lock()self.cancel_jobs = threading.Event()- self.failed_indices = []- self.exceptions_map = {}+ self.failed_indices: list[int] = []+ self.exceptions_map: dict[int, Exception] = {}++ def execute(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:+ """Return results in input iteration order, with None for failed or unfinished items.- def execute(self, function, data):+ Successful tasks may also return None. Use failed_indices and exceptions_map+ to identify task exceptions. With compare_results=True, each non-None result+ must support indexing its final element as a numeric score.+ """tqdm.tqdm._instances.clear()- wrapped = self._wrap_function(_bind_active_call_id(function))+ function = _bind_active_call_id(function)if self.num_threads == 1:- return self._execute_sequential(wrapped, data)- return self._execute_parallel(wrapped, data)+ return self._execute_sequential(function, data)+ return self._execute_parallel(function, data)- def _wrap_function(self, user_function):- def safe_func(item):+ def _wrap_function(self, user_function: Callable[[T], R]) -> Callable[[T], R | Exception | None]:+ def safe_func(item: T) -> R | Exception | None:if self.cancel_jobs.is_set():return Nonetry:@@ -73,9 +93,10 @@ class ParallelExecutor:return safe_func- def _execute_sequential(self, function, data):+ def _execute_sequential(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:"""Execute items sequentially on the main thread."""- results = [_UNSET] * len(data)+ safe_function = self._wrap_function(function)+ results: list[R | _ResultState | None] = [_UNSET] * len(data)pbar = tqdm.tqdm(total=len(data),@@ -89,7 +110,7 @@ class ParallelExecutor:if self.cancel_jobs.is_set():break- outcome = function(item)+ outcome = safe_function(item)self._process_outcome(results, idx, outcome)self._report_progress(pbar, results, len(data))except KeyboardInterrupt:@@ -105,17 +126,20 @@ class ParallelExecutor:return [None if result is _UNSET else result for result in results]- def _execute_parallel(self, function, data):- results = [_UNSET] * len(data)+ def _execute_parallel(self, function: Callable[[T], R], data: Collection[T]) -> list[R | None]:+ safe_function = self._wrap_function(function)+ results: list[R | _ResultState | None] = [_UNSET] * len(data)job_cancelled = "cancelled"# We resubmit at most once per item.- start_time_map = {}+ start_time_map: dict[int, float] = {}start_time_lock = threading.Lock()- resubmitted = set()+ resubmitted: set[Future[_WorkerResult[R]]] = set()# This is the worker function each thread will run.- def worker(parent_overrides, submission_id, index, item):+ def worker(+ parent_overrides: dict[str, Any], submission_id: int, index: int, item: T+ ) -> _WorkerResult[R]:if self.cancel_jobs.is_set():return index, job_cancelled# Record actual start time@@ -131,17 +155,17 @@ class ParallelExecutor:token = thread_local_overrides.set(new_overrides)try:- return index, function(item)+ return index, safe_function(item)finally:thread_local_overrides.reset(token)# Handle Ctrl-C in the main thread@contextlib.contextmanager- def interrupt_manager():+ def interrupt_manager() -> Generator[None, None, None]:if threading.current_thread() is threading.main_thread():orig_handler = signal.getsignal(signal.SIGINT)- def handler(sig, frame):+ def handler(sig: int, frame: FrameType | None) -> None:self.cancel_jobs.set()logger.warning("SIGINT received. Cancelling.")if callable(orig_handler):@@ -165,8 +189,8 @@ class ParallelExecutor:with interrupt_manager():parent_overrides = thread_local_overrides.get().copy()- futures_map = {}- futures_set = set()+ futures_map: dict[Future[_WorkerResult[R]], tuple[int, int, T]] = {}+ futures_set: set[Future[_WorkerResult[R]]] = set()submission_counter = 0for idx, item in enumerate(data):@@ -182,7 +206,7 @@ class ParallelExecutor:file=sys.stdout,)- def all_done():+ def all_done() -> bool:return all(result is not _UNSET for result in results)while futures_set and not self.cancel_jobs.is_set():@@ -197,7 +221,9 @@ class ParallelExecutor:passelse:if outcome != job_cancelled and results[index] is _UNSET:- self._process_outcome(results, index, outcome)+ # Restore R after excluding the cancellation marker; ty otherwise+ # infers a narrower R incompatible with the invariant results list.+ self._process_outcome(results, index, cast(R | Exception | None, outcome))self._report_progress(pbar, results, len(data))@@ -237,7 +263,9 @@ class ParallelExecutor:return [None if result is _UNSET else result for result in results]- def _process_outcome(self, results, idx, outcome):+ def _process_outcome(+ self, results: list[R | _ResultState | None], idx: int, outcome: R | Exception | None+ ) -> None:"""Store a single outcome and track errors."""if isinstance(outcome, Exception):with self.error_lock:@@ -246,8 +274,8 @@ class ParallelExecutor:else:results[idx] = outcome- def _report_progress(self, pbar, results, total):- """Compute metrics and update the progress bar."""+ def _report_progress(self, pbar: tqdm.tqdm, results: Collection[Any], total: int) -> None:+ """Compute progress; compare_results opts into dynamically shaped, score-bearing results."""if self.compare_results:vals = [result[-1] for result in results if result is not _UNSET and result is not None]self._update_progress(pbar, sum(vals), len(vals))@@ -258,7 +286,7 @@ class ParallelExecutor:total,)- def _update_progress(self, pbar, nresults, ntotal):+ def _update_progress(self, pbar: tqdm.tqdm, nresults: float, ntotal: int) -> None:if self.compare_results:pct = round(100 * nresults / ntotal, 1) if ntotal else 0pbar.set_description(f"Average Metric: {nresults:.2f} / {ntotal} ({pct}%)")diff --git a/pyproject.toml b/pyproject.tomlindex b48d1fbd7..0db231fed 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -142,6 +142,7 @@ include = ["dspy/utils/asyncify.py","dspy/utils/callback.py","dspy/utils/callback_context.py",+ "dspy/utils/parallelizer.py","dspy/utils/usage_tracker.py","tests/typing",]diff --git a/tests/typing/parallelizer.py b/tests/typing/parallelizer.pynew file mode 100644index 000000000..7343d1caa--- /dev/null+++ b/tests/typing/parallelizer.py@@ -0,0 +1,37 @@+"""Executor contracts checked by ty, not executed by pytest.++Ignores on deliberately invalid calls assert expected diagnostics: unused-ignore+errors make these checks fail if invalid calls become accepted.+"""++from typing_extensions import assert_type++from dspy.utils.parallelizer import ParallelExecutor+++def format_count(count: int) -> str:+ return str(count)+++def measure(value: str) -> float:+ return len(value) / 2+++def optional_result(count: int) -> str | None:+ return str(count) if count else None+++def check_executor() -> None:+ executor = ParallelExecutor(num_threads=3)+ assert_type(executor.execute(format_count, range(3)), list[str | None])+ assert_type(executor.execute(measure, ("one", "three")), list[float | None])+ assert_type(executor.execute(optional_result, [0, 1]), list[str | None])+ assert_type(executor.failed_indices, list[int])+ assert_type(executor.exceptions_map, dict[int, Exception])++ sequential = ParallelExecutor(num_threads=1)+ assert_type(sequential.execute(format_count, {1, 3}), list[str | None])++ executor.execute(format_count, ["one"]) # ty: ignore[invalid-argument-type]+ executor.execute(format_count, iter([1, 3])) # ty: ignore[invalid-argument-type]+ ParallelExecutor(num_threads="three") # ty: ignore[invalid-argument-type]