DSPy: The framework for programming—not prompting—language models
Python
fix(callbacks): preserve call lineage in parallel workers (#10043)
Isaac Miller96bae53d458d
diff --git a/dspy/utils/callback.py b/dspy/utils/callback.pyindex cd9094f7e..8c8dc0033 100644--- a/dspy/utils/callback.py+++ b/dspy/utils/callback.py@@ -2,12 +2,10 @@ import functoolsimport inspectimport loggingimport uuid-from contextvars import ContextVarfrom typing import Any, Callableimport dspy--ACTIVE_CALL_ID = ContextVar("active_call_id", default=None)+from dspy.utils.callback_context import ACTIVE_CALL_IDlogger = logging.getLogger(__name__)diff --git a/dspy/utils/callback_context.py b/dspy/utils/callback_context.pynew file mode 100644index 000000000..7aca4d701--- /dev/null+++ b/dspy/utils/callback_context.py@@ -0,0 +1,20 @@+import functools+from contextvars import ContextVar+from typing import Callable++ACTIVE_CALL_ID = ContextVar("active_call_id", default=None)+++def _bind_active_call_id(fn: Callable) -> Callable:+ """Bind the current callback call ID to each synchronous invocation of ``fn``."""+ parent_call_id = ACTIVE_CALL_ID.get()++ @functools.wraps(fn)+ def wrapper(*args, **kwargs):+ token = ACTIVE_CALL_ID.set(parent_call_id)+ try:+ return fn(*args, **kwargs)+ finally:+ ACTIVE_CALL_ID.reset(token)++ return wrapperdiff --git a/dspy/utils/parallelizer.py b/dspy/utils/parallelizer.pyindex 029a9773c..b94c18ca0 100644--- a/dspy/utils/parallelizer.py+++ b/dspy/utils/parallelizer.py@@ -10,6 +10,9 @@ from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, waitimport tqdm+from dspy.dsp.utils.settings import settings, thread_local_overrides+from dspy.utils.callback_context import _bind_active_call_id+logger = logging.getLogger(__name__)@@ -25,11 +28,9 @@ class ParallelExecutor:straggler_limit=3,):"""- Offers isolation between the tasks (dspy.settings) irrespective of whether num_threads == 1 or > 1.- Handles also straggler timeouts.+ Propagates DSPy settings and callback ancestry into each task while isolating task-local changes,+ irrespective of whether num_threads == 1 or > 1. Handles also straggler timeouts."""- from dspy.dsp.utils.settings import settings-self.num_threads = num_threads or settings.num_threadsself.max_errors = settings.max_errors if max_errors is None else max_errorsself.disable_progress_bar = disable_progress_bar@@ -46,7 +47,7 @@ class ParallelExecutor:def execute(self, function, data):tqdm.tqdm._instances.clear()- wrapped = self._wrap_function(function)+ wrapped = self._wrap_function(_bind_active_call_id(function))if self.num_threads == 1:return self._execute_sequential(wrapped, data)return self._execute_parallel(wrapped, data)@@ -120,8 +121,6 @@ class ParallelExecutor:start_time_map[submission_id] = time.time()# Apply parent's thread-local overrides- from dspy.dsp.utils.settings import thread_local_overrides-original = thread_local_overrides.get()new_overrides = {**original, **parent_overrides.copy()}if new_overrides.get("usage_tracker"):@@ -156,8 +155,6 @@ class ParallelExecutor:executor = ThreadPoolExecutor(max_workers=self.num_threads)try:with interrupt_manager():- from dspy.dsp.utils.settings import thread_local_overrides-parent_overrides = thread_local_overrides.get().copy()futures_map = {}diff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.pyindex d0c73a16d..3b4d3ee9a 100644--- a/tests/callback/test_callback.py+++ b/tests/callback/test_callback.py@@ -4,6 +4,7 @@ import pytestimport dspyfrom dspy.utils.callback import ACTIVE_CALL_ID, BaseCallback, with_callbacks+from dspy.utils.callback_context import _bind_active_call_idfrom dspy.utils.dummies import DummyLM@@ -54,6 +55,26 @@ class MyCallback(BaseCallback):self.calls.append({"handler": "on_tool_end", "outputs": outputs, "exception": exception})+def test_bind_active_call_id_restores_context_after_exception():+ observed_call_ids = []++ def fail():+ observed_call_ids.append(ACTIVE_CALL_ID.get())+ raise ValueError("boom")++ token = ACTIVE_CALL_ID.set("parent-call")+ try:+ bound = _bind_active_call_id(fail)+ finally:+ ACTIVE_CALL_ID.reset(token)++ with pytest.raises(ValueError, match="boom"):+ bound()++ assert observed_call_ids == ["parent-call"]+ assert ACTIVE_CALL_ID.get() is None++@pytest.mark.parametrize(("args", "kwargs"),[diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.pyindex 3ac313be1..4f35c65cd 100644--- a/tests/evaluate/test_evaluate.py+++ b/tests/evaluate/test_evaluate.py@@ -10,7 +10,7 @@ import dspyfrom dspy.evaluate.evaluate import Evaluate, EvaluationResultfrom dspy.evaluate.metrics import answer_exact_matchfrom dspy.predict import Predict-from dspy.utils.callback import BaseCallback+from dspy.utils.callback import ACTIVE_CALL_ID, BaseCallbackfrom dspy.utils.dummies import DummyLM@@ -287,6 +287,44 @@ def test_evaluate_callback():assert callback.end_call_outputs.score == 100.0assert callback.end_call_count == 1++def test_parallel_program_callbacks_are_children_of_evaluate_callback():+ class LineageCallback(BaseCallback):+ def __init__(self):+ self.evaluate_call_id = None+ self.module_parent_call_ids = []+ self.lock = threading.Lock()++ def on_evaluate_start(self, call_id, instance, inputs):+ self.evaluate_call_id = call_id++ def on_module_start(self, call_id, instance, inputs):+ with self.lock:+ self.module_parent_call_ids.append(ACTIVE_CALL_ID.get())++ callback = LineageCallback()+ dspy.configure(+ lm=DummyLM(+ {+ "What is 1+1?": {"answer": "2"},+ "What is 2+2?": {"answer": "4"},+ }+ ),+ callbacks=[callback],+ )+ evaluator = Evaluate(+ devset=[new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")],+ metric=answer_exact_match,+ num_threads=2,+ display_progress=False,+ )++ evaluator(Predict("question -> answer"))++ assert callback.evaluate_call_id is not None+ assert callback.module_parent_call_ids == [callback.evaluate_call_id, callback.evaluate_call_id]++def test_evaluation_result_repr():result = EvaluationResult(score=100.0, results=[(new_example("What is 1+1?", "2"), {"answer": "2"}, 100.0)])assert repr(result) == "EvaluationResult(score=100.0, results=<list of 1 results>)"diff --git a/tests/predict/test_parallel.py b/tests/predict/test_parallel.pyindex 2af61c722..00d023957 100644--- a/tests/predict/test_parallel.py+++ b/tests/predict/test_parallel.py@@ -1,4 +1,7 @@+import threading+import dspy+from dspy.utils.callback import ACTIVE_CALL_ID, BaseCallbackfrom dspy.utils.dummies import DummyLM@@ -39,6 +42,41 @@ def test_parallel_module():assert {r.output for r in output} == expected_outputs+def test_parallel_children_inherit_parent_callback_call_id():+ class Child(dspy.Module):+ def forward(self, value):+ return value++ class Parent(dspy.Module):+ def __init__(self, children):+ self.children = children+ self.parallel = dspy.Parallel(num_threads=2, disable_progress_bar=True)++ def forward(self):+ return self.parallel([(child, {"value": index}) for index, child in enumerate(self.children)])++ class LineageCallback(BaseCallback):+ def __init__(self):+ self.calls = []+ self.lock = threading.Lock()++ def on_module_start(self, call_id, instance, inputs):+ with self.lock:+ self.calls.append((instance, call_id, ACTIVE_CALL_ID.get()))++ children = [Child(), Child()]+ parent = Parent(children)+ callback = LineageCallback()+ dspy.configure(callbacks=[callback])++ assert parent() == [0, 1]++ parent_call = next(call for call in callback.calls if call[0] is parent)+ child_calls = [call for call in callback.calls if any(call[0] is child for child in children)]+ assert len(child_calls) == 2+ assert all(call[2] == parent_call[1] for call in child_calls)++def test_batch_module():lm = DummyLM([diff --git a/tests/utils/test_parallelizer.py b/tests/utils/test_parallelizer.pyindex 5903f26bf..b83b5e4a2 100644--- a/tests/utils/test_parallelizer.py+++ b/tests/utils/test_parallelizer.py@@ -3,6 +3,7 @@ import timeimport pytest+from dspy.utils.callback import ACTIVE_CALL_IDfrom dspy.utils.parallelizer import ParallelExecutor@@ -18,6 +19,21 @@ def test_worker_threads_independence():assert results == [2, 4, 6, 8, 10][email protected]("num_threads", [1, 3])+def test_workers_inherit_active_callback_call_id(num_threads):+ executor = ParallelExecutor(num_threads=num_threads)++ for parent_call_id in ["first-parent", "second-parent"]:+ token = ACTIVE_CALL_ID.set(parent_call_id)+ try:+ observed_call_ids = executor.execute(lambda _: ACTIVE_CALL_ID.get(), range(5))++ assert observed_call_ids == [parent_call_id] * 5+ assert ACTIVE_CALL_ID.get() == parent_call_id+ finally:+ ACTIVE_CALL_ID.reset(token)++def test_parallel_execution_speed():def task(item):time.sleep(0.1) # Simulate a time-consuming task