Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix: run evaluation on main thread when num_threads=1 (#9328)

at384c3424073261f
diff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.py
index 5b5544d3d..029a9773c 100644
--- a/dspy/utils/parallelizer.py
+++ b/dspy/utils/parallelizer.py
@@ -47,6 +47,8 @@ class ParallelExecutor:
def execute(self, function, data):
tqdm.tqdm._instances.clear()
wrapped = self._wrap_function(function)
+ if self.num_threads == 1:
+ return self._execute_sequential(wrapped, data)
return self._execute_parallel(wrapped, data)
def _wrap_function(self, user_function):
@@ -68,6 +70,38 @@ class ParallelExecutor:
return safe_func
+ def _execute_sequential(self, function, data):
+ """Execute items sequentially on the main thread."""
+ results = [None] * len(data)
+
+ pbar = tqdm.tqdm(
+ total=len(data),
+ dynamic_ncols=True,
+ disable=self.disable_progress_bar,
+ file=sys.stdout,
+ )
+
+ try:
+ for idx, item in enumerate(data):
+ if self.cancel_jobs.is_set():
+ break
+
+ outcome = function(item)
+ self._process_outcome(results, idx, outcome)
+ self._report_progress(pbar, results, len(data))
+ except KeyboardInterrupt:
+ self.cancel_jobs.set()
+ logger.warning("SIGINT received. Cancelling.")
+ raise
+ finally:
+ pbar.close()
+
+ if self.cancel_jobs.is_set():
+ logger.warning("Execution cancelled due to errors or interruption.")
+ raise Exception("Execution cancelled due to errors or interruption.")
+
+ return results
+
def _execute_parallel(self, function, data):
results = [None] * len(data)
job_cancelled = "cancelled"
@@ -158,25 +192,9 @@ class ParallelExecutor:
pass
else:
if outcome != job_cancelled and results[index] is None:
- # Check if this is an exception
- if isinstance(outcome, Exception):
- with self.error_lock:
- self.failed_indices.append(index)
- self.exceptions_map[index] = outcome
- results[index] = None # Keep None for failed examples
- else:
- results[index] = outcome
-
- # Update progress
- if self.compare_results:
- vals = [r[-1] for r in results if r is not None]
- self._update_progress(pbar, sum(vals), len(vals))
- else:
- self._update_progress(
- pbar,
- len([r for r in results if r is not None]),
- len(data),
- )
+ self._process_outcome(results, index, outcome)
+
+ self._report_progress(pbar, results, len(data))
if all_done():
break
@@ -214,6 +232,27 @@ class ParallelExecutor:
return results
+ def _process_outcome(self, results, idx, outcome):
+ """Store a single outcome and track errors."""
+ if isinstance(outcome, Exception):
+ with self.error_lock:
+ self.failed_indices.append(idx)
+ self.exceptions_map[idx] = outcome
+ else:
+ results[idx] = outcome
+
+ def _report_progress(self, pbar, results, total):
+ """Compute metrics and update the progress bar."""
+ if self.compare_results:
+ vals = [r[-1] for r in results if r is not None]
+ self._update_progress(pbar, sum(vals), len(vals))
+ else:
+ self._update_progress(
+ pbar,
+ len([r for r in results if r is not None]),
+ total,
+ )
+
def _update_progress(self, pbar, nresults, ntotal):
if self.compare_results:
pct = round(100 * nresults / ntotal, 1) if ntotal else 0
diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.py
index e35f08eba..3ac313be1 100644
--- a/tests/evaluate/test_evaluate.py
+++ b/tests/evaluate/test_evaluate.py
@@ -56,6 +56,33 @@ def test_evaluate_call():
assert score.score == 100.0
+def test_evaluate_single_thread_runs_in_main_thread():
+ """Evaluate with num_threads=1 should run in the main thread."""
+ dspy.configure(
+ lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}})
+ )
+ devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
+
+ execution_threads = []
+
+ original_metric = answer_exact_match
+
+ def tracking_metric(example, prediction, **kwargs):
+ execution_threads.append(threading.current_thread())
+ return original_metric(example, prediction, **kwargs)
+
+ program = Predict("question -> answer")
+ ev = Evaluate(
+ devset=devset,
+ metric=tracking_metric,
+ display_progress=False,
+ num_threads=1,
+ )
+ result = ev(program)
+ assert result.score == 100.0
+ assert all(t is threading.main_thread() for t in execution_threads)
+
+
@pytest.mark.extra
def test_construct_result_df():
import pandas as pd
diff --git a/tests/utils/test_parallelizer.py b/tests/utils/test_parallelizer.py
index 128614ffc..5903f26bf 100644
--- a/tests/utils/test_parallelizer.py
+++ b/tests/utils/test_parallelizer.py
@@ -1,3 +1,4 @@
+import threading
import time
import pytest
@@ -83,3 +84,85 @@ def test_parallel_executor_tracks_failed_indices_and_exceptions():
assert str(executor.exceptions_map[2]) == "test error for 3"
assert isinstance(executor.exceptions_map[4], RuntimeError)
assert str(executor.exceptions_map[4]) == "test error for 5"
+
+
+def test_sequential_execution_runs_on_main_thread():
+ """With num_threads=1, all work should run on the main thread (not in a ThreadPoolExecutor)."""
+ execution_threads = []
+
+ def task(item):
+ execution_threads.append(threading.current_thread())
+ return item * 2
+
+ data = [1, 2, 3, 4, 5]
+ executor = ParallelExecutor(num_threads=1)
+ results = executor.execute(task, data)
+
+ assert results == [2, 4, 6, 8, 10]
+ assert all(t is threading.main_thread() for t in execution_threads)
+
+
+def test_sequential_max_errors_not_met():
+ """Sequential execution should handle errors without crashing when max_errors is not reached."""
+ def task(item):
+ if item == 3:
+ raise ValueError("Intentional error")
+ return item
+
+ data = [1, 2, 3, 4, 5]
+ executor = ParallelExecutor(num_threads=1, max_errors=2)
+
+ results = executor.execute(task, data)
+
+ assert results == [1, 2, None, 4, 5]
+
+
+def test_sequential_max_errors_exceeded():
+ """Sequential execution should cancel when max_errors is reached."""
+ def task(item):
+ if item == 3:
+ raise ValueError("Intentional error")
+ return item
+
+ data = [1, 2, 3, 4, 5]
+ executor = ParallelExecutor(num_threads=1, max_errors=1)
+
+ with pytest.raises(Exception, match="Execution cancelled due to errors or interruption."):
+ executor.execute(task, data)
+
+
+def test_sequential_tracks_failed_indices_and_exceptions():
+ """Sequential execution should track failed indices and exception objects."""
+ def task(item):
+ if item == 3:
+ raise ValueError("test error for 3")
+ if item == 5:
+ raise RuntimeError("test error for 5")
+ return item
+
+ data = [1, 2, 3, 4, 5]
+ executor = ParallelExecutor(num_threads=1, max_errors=3)
+
+ results = executor.execute(task, data)
+
+ assert results == [1, 2, None, 4, None]
+
+ assert sorted(executor.failed_indices) == [2, 4]
+
+ assert len(executor.exceptions_map) == 2
+ assert isinstance(executor.exceptions_map[2], ValueError)
+ assert str(executor.exceptions_map[2]) == "test error for 3"
+ assert isinstance(executor.exceptions_map[4], RuntimeError)
+ assert str(executor.exceptions_map[4]) == "test error for 5"
+
+
+def test_sequential_compare_results():
+ """Sequential execution should track and display comparison metrics correctly."""
+ def task(item):
+ return item, item > 2 # (result, score)
+
+ data = [1, 2, 3, 4, 5]
+ executor = ParallelExecutor(num_threads=1, compare_results=True, disable_progress_bar=True)
+ results = executor.execute(task, data)
+
+ assert results == [(1, False), (2, False), (3, True), (4, True), (5, True)]