DSPy: The framework for programming—not prompting—language models
Python
fix: run evaluation on main thread when num_threads=1 (#9328)
at384c3424073261f
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.pyindex 5b5544d3d..029a9773c 100644--- a/dspy/utils/parallelizer.py+++ b/dspy/utils/parallelizer.py@@ -47,6 +47,8 @@ class ParallelExecutor:def execute(self, function, data):tqdm.tqdm._instances.clear()wrapped = self._wrap_function(function)+ if self.num_threads == 1:+ return self._execute_sequential(wrapped, data)return self._execute_parallel(wrapped, data)def _wrap_function(self, user_function):@@ -68,6 +70,38 @@ class ParallelExecutor:return safe_func+ def _execute_sequential(self, function, data):+ """Execute items sequentially on the main thread."""+ results = [None] * len(data)++ pbar = tqdm.tqdm(+ total=len(data),+ dynamic_ncols=True,+ disable=self.disable_progress_bar,+ file=sys.stdout,+ )++ try:+ for idx, item in enumerate(data):+ if self.cancel_jobs.is_set():+ break++ outcome = function(item)+ self._process_outcome(results, idx, outcome)+ self._report_progress(pbar, results, len(data))+ except KeyboardInterrupt:+ self.cancel_jobs.set()+ logger.warning("SIGINT received. Cancelling.")+ raise+ finally:+ pbar.close()++ if self.cancel_jobs.is_set():+ logger.warning("Execution cancelled due to errors or interruption.")+ raise Exception("Execution cancelled due to errors or interruption.")++ return results+def _execute_parallel(self, function, data):results = [None] * len(data)job_cancelled = "cancelled"@@ -158,25 +192,9 @@ class ParallelExecutor:passelse:if outcome != job_cancelled and results[index] is None:- # Check if this is an exception- if isinstance(outcome, Exception):- with self.error_lock:- self.failed_indices.append(index)- self.exceptions_map[index] = outcome- results[index] = None # Keep None for failed examples- else:- results[index] = outcome-- # Update progress- if self.compare_results:- vals = [r[-1] for r in results if r is not None]- self._update_progress(pbar, sum(vals), len(vals))- else:- self._update_progress(- pbar,- len([r for r in results if r is not None]),- len(data),- )+ self._process_outcome(results, index, outcome)++ self._report_progress(pbar, results, len(data))if all_done():break@@ -214,6 +232,27 @@ class ParallelExecutor:return results+ def _process_outcome(self, results, idx, outcome):+ """Store a single outcome and track errors."""+ if isinstance(outcome, Exception):+ with self.error_lock:+ self.failed_indices.append(idx)+ self.exceptions_map[idx] = outcome+ else:+ results[idx] = outcome++ def _report_progress(self, pbar, results, total):+ """Compute metrics and update the progress bar."""+ if self.compare_results:+ vals = [r[-1] for r in results if r is not None]+ self._update_progress(pbar, sum(vals), len(vals))+ else:+ self._update_progress(+ pbar,+ len([r for r in results if r is not None]),+ total,+ )+def _update_progress(self, pbar, nresults, ntotal):if self.compare_results:pct = round(100 * nresults / ntotal, 1) if ntotal else 0diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.pyindex e35f08eba..3ac313be1 100644--- a/tests/evaluate/test_evaluate.py+++ b/tests/evaluate/test_evaluate.py@@ -56,6 +56,33 @@ def test_evaluate_call():assert score.score == 100.0+def test_evaluate_single_thread_runs_in_main_thread():+ """Evaluate with num_threads=1 should run in the main thread."""+ dspy.configure(+ lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}})+ )+ devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]++ execution_threads = []++ original_metric = answer_exact_match++ def tracking_metric(example, prediction, **kwargs):+ execution_threads.append(threading.current_thread())+ return original_metric(example, prediction, **kwargs)++ program = Predict("question -> answer")+ ev = Evaluate(+ devset=devset,+ metric=tracking_metric,+ display_progress=False,+ num_threads=1,+ )+ result = ev(program)+ assert result.score == 100.0+ assert all(t is threading.main_thread() for t in execution_threads)++@pytest.mark.extradef test_construct_result_df():import pandas as pddiff --git a/tests/utils/test_parallelizer.py b/tests/utils/test_parallelizer.pyindex 128614ffc..5903f26bf 100644--- a/tests/utils/test_parallelizer.py+++ b/tests/utils/test_parallelizer.py@@ -1,3 +1,4 @@+import threadingimport timeimport pytest@@ -83,3 +84,85 @@ def test_parallel_executor_tracks_failed_indices_and_exceptions():assert str(executor.exceptions_map[2]) == "test error for 3"assert isinstance(executor.exceptions_map[4], RuntimeError)assert str(executor.exceptions_map[4]) == "test error for 5"+++def test_sequential_execution_runs_on_main_thread():+ """With num_threads=1, all work should run on the main thread (not in a ThreadPoolExecutor)."""+ execution_threads = []++ def task(item):+ execution_threads.append(threading.current_thread())+ return item * 2++ data = [1, 2, 3, 4, 5]+ executor = ParallelExecutor(num_threads=1)+ results = executor.execute(task, data)++ assert results == [2, 4, 6, 8, 10]+ assert all(t is threading.main_thread() for t in execution_threads)+++def test_sequential_max_errors_not_met():+ """Sequential execution should handle errors without crashing when max_errors is not reached."""+ def task(item):+ if item == 3:+ raise ValueError("Intentional error")+ return item++ data = [1, 2, 3, 4, 5]+ executor = ParallelExecutor(num_threads=1, max_errors=2)++ results = executor.execute(task, data)++ assert results == [1, 2, None, 4, 5]+++def test_sequential_max_errors_exceeded():+ """Sequential execution should cancel when max_errors is reached."""+ def task(item):+ if item == 3:+ raise ValueError("Intentional error")+ return item++ data = [1, 2, 3, 4, 5]+ executor = ParallelExecutor(num_threads=1, max_errors=1)++ with pytest.raises(Exception, match="Execution cancelled due to errors or interruption."):+ executor.execute(task, data)+++def test_sequential_tracks_failed_indices_and_exceptions():+ """Sequential execution should track failed indices and exception objects."""+ def task(item):+ if item == 3:+ raise ValueError("test error for 3")+ if item == 5:+ raise RuntimeError("test error for 5")+ return item++ data = [1, 2, 3, 4, 5]+ executor = ParallelExecutor(num_threads=1, max_errors=3)++ results = executor.execute(task, data)++ assert results == [1, 2, None, 4, None]++ assert sorted(executor.failed_indices) == [2, 4]++ assert len(executor.exceptions_map) == 2+ assert isinstance(executor.exceptions_map[2], ValueError)+ assert str(executor.exceptions_map[2]) == "test error for 3"+ assert isinstance(executor.exceptions_map[4], RuntimeError)+ assert str(executor.exceptions_map[4]) == "test error for 5"+++def test_sequential_compare_results():+ """Sequential execution should track and display comparison metrics correctly."""+ def task(item):+ return item, item > 2 # (result, score)++ data = [1, 2, 3, 4, 5]+ executor = ParallelExecutor(num_threads=1, compare_results=True, disable_progress_bar=True)+ results = executor.execute(task, data)++ assert results == [(1, False), (2, False), (3, True), (4, True), (5, True)]