DSPy: The framework for programming—not prompting—language models
Python
fix(callbacks): report base exceptions to end handlers (#10194)
Isaac Miller970f7290038e
diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.pyindex 637f6bb46..0c58e19ec 100644--- a/dspy/streaming/messages.py+++ b/dspy/streaming/messages.py@@ -117,7 +117,7 @@ class StatusStreamingCallback(BaseCallback):self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):stream = settings.send_streamif stream is None or outputs == "Completed.":@@ -145,7 +145,7 @@ class StatusStreamingCallback(BaseCallback):self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):stream = settings.send_streamif stream is None:@@ -173,7 +173,7 @@ class StatusStreamingCallback(BaseCallback):self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):stream = settings.send_streamif stream is None:diff --git a/dspy/utils/callback.py b/dspy/utils/callback.pyindex 529287e83..901b20bcf 100644--- a/dspy/utils/callback.py+++ b/dspy/utils/callback.py@@ -89,7 +89,7 @@ class BaseCallback:self,call_id: str,outputs: Any | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after forward() method of a module (subclass of dspy.Module) is executed.@@ -121,7 +121,7 @@ class BaseCallback:self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after __call__ method of dspy.LM instance is executed.@@ -153,7 +153,7 @@ class BaseCallback:self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after format() method of an adapter (subclass of dspy.Adapter) is called..@@ -185,7 +185,7 @@ class BaseCallback:self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after parse() method of an adapter (subclass of dspy.Adapter) is called.@@ -217,7 +217,7 @@ class BaseCallback:self,call_id: str,outputs: dict[str, Any] | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after a tool is executed.@@ -249,7 +249,7 @@ class BaseCallback:self,call_id: str,outputs: Any | None,- exception: Exception | None = None,+ exception: BaseException | None = None,):"""A handler triggered after evaluation is executed.@@ -264,25 +264,25 @@ class BaseCallback:def on_interpreter_execute_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):pass- def on_interpreter_execute_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):+ def on_interpreter_execute_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):passdef on_interpreter_tool_call_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):pass- def on_interpreter_tool_call_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):+ def on_interpreter_tool_call_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):passdef on_interpreter_startup_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):pass- def on_interpreter_startup_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):+ def on_interpreter_startup_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):passdef on_interpreter_shutdown_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):pass- def on_interpreter_shutdown_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):+ def on_interpreter_shutdown_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):passdef on_compile_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):@@ -359,9 +359,9 @@ def with_callbacks(fn):try:results = await fn(instance, *args, **kwargs)return results- except Exception as e:+ except BaseException as e:exception = e- raise exception+ raisefinally:callback_context.ACTIVE_CALL_ID.set(parent_call_id)_execute_end_callbacks(instance, fn, call_id, results, exception, callbacks)@@ -389,9 +389,9 @@ def with_callbacks(fn):try:results = fn(instance, *args, **kwargs)return results- except Exception as e:+ except BaseException as e:exception = e- raise exception+ raisefinally:callback_context.ACTIVE_CALL_ID.set(parent_call_id)_execute_end_callbacks(instance, fn, call_id, results, exception, callbacks)diff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.pyindex d13392fa9..fb60e9546 100644--- a/tests/callback/test_callback.py+++ b/tests/callback/test_callback.py@@ -1,3 +1,4 @@+import asyncioimport timeimport cloudpickle@@ -149,24 +150,70 @@ def test_callback_injection_local():def test_callback_error_handling():+ expected_error = ValueError("Error")+class Target(dspy.Module):@with_callbacksdef forward(self, x: int, y: str, z: float) -> int:time.sleep(0.1)- raise ValueError("Error")+ raise expected_errorcallback = MyCallback()dspy.configure(callbacks=[callback])target = Target()- with pytest.raises(ValueError, match="Error"):+ with pytest.raises(ValueError, match="Error") as exc_info:target.forward(1, "2", 3.0)assert len(callback.calls) == 2assert callback.calls[0]["handler"] == "on_module_start"assert callback.calls[1]["handler"] == "on_module_end"- assert isinstance(callback.calls[1]["exception"], ValueError)+ assert callback.calls[1]["exception"] is exc_info.value is expected_error++[email protected]("expected_error", [KeyboardInterrupt(), SystemExit()])+def test_callback_reports_and_propagates_base_exception(expected_error):+ class Target(dspy.Module):+ @with_callbacks+ def forward(self):+ raise expected_error++ callback = MyCallback()+ target = Target(callbacks=[callback])++ with pytest.raises(BaseException) as exc_info:+ target.forward()++ assert callback.calls[1]["handler"] == "on_module_end"+ assert callback.calls[1]["outputs"] is None+ assert callback.calls[1]["exception"] is exc_info.value is expected_error+ assert ACTIVE_CALL_ID.get() is None+++async def test_callback_reports_and_propagates_async_cancellation():+ started = asyncio.Event()++ class Target(dspy.Module):+ @with_callbacks+ async def run(self):+ started.set()+ await asyncio.Event().wait()++ callback = MyCallback()+ target = Target(callbacks=[callback])+ task = asyncio.create_task(target.run())+ await started.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ assert callback.calls[1]["handler"] == "on_module_end"+ assert callback.calls[1]["outputs"] is None+ assert isinstance(callback.calls[1]["exception"], asyncio.CancelledError)+ assert ACTIVE_CALL_ID.get() is Nonedef test_multiple_callbacks():