Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(callbacks): report base exceptions to end handlers (#10194)

Isaac Miller970f7290038e
diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.py
index 637f6bb46..0c58e19ec 100644
--- a/dspy/streaming/messages.py
+++ b/dspy/streaming/messages.py
@@ -117,7 +117,7 @@ class StatusStreamingCallback(BaseCallback):
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
stream = settings.send_stream
if stream is None or outputs == "Completed.":
@@ -145,7 +145,7 @@ class StatusStreamingCallback(BaseCallback):
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
stream = settings.send_stream
if stream is None:
@@ -173,7 +173,7 @@ class StatusStreamingCallback(BaseCallback):
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
stream = settings.send_stream
if stream is None:
diff --git a/dspy/utils/callback.py b/dspy/utils/callback.py
index 529287e83..901b20bcf 100644
--- a/dspy/utils/callback.py
+++ b/dspy/utils/callback.py
@@ -89,7 +89,7 @@ class BaseCallback:
self,
call_id: str,
outputs: Any | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after forward() method of a module (subclass of dspy.Module) is executed.
@@ -121,7 +121,7 @@ class BaseCallback:
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after __call__ method of dspy.LM instance is executed.
@@ -153,7 +153,7 @@ class BaseCallback:
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after format() method of an adapter (subclass of dspy.Adapter) is called..
@@ -185,7 +185,7 @@ class BaseCallback:
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after parse() method of an adapter (subclass of dspy.Adapter) is called.
@@ -217,7 +217,7 @@ class BaseCallback:
self,
call_id: str,
outputs: dict[str, Any] | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after a tool is executed.
@@ -249,7 +249,7 @@ class BaseCallback:
self,
call_id: str,
outputs: Any | None,
- exception: Exception | None = None,
+ exception: BaseException | None = None,
):
"""A handler triggered after evaluation is executed.
@@ -264,25 +264,25 @@ class BaseCallback:
def on_interpreter_execute_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):
pass
- def on_interpreter_execute_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):
+ def on_interpreter_execute_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):
pass
def on_interpreter_tool_call_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):
pass
- def on_interpreter_tool_call_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):
+ def on_interpreter_tool_call_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):
pass
def on_interpreter_startup_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):
pass
- def on_interpreter_startup_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):
+ def on_interpreter_startup_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):
pass
def on_interpreter_shutdown_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):
pass
- def on_interpreter_shutdown_end(self, call_id: str, outputs: Any | None, exception: Exception | None = None):
+ def on_interpreter_shutdown_end(self, call_id: str, outputs: Any | None, exception: BaseException | None = None):
pass
def on_compile_start(self, call_id: str, instance: Any, inputs: dict[str, Any]):
@@ -359,9 +359,9 @@ def with_callbacks(fn):
try:
results = await fn(instance, *args, **kwargs)
return results
- except Exception as e:
+ except BaseException as e:
exception = e
- raise exception
+ raise
finally:
callback_context.ACTIVE_CALL_ID.set(parent_call_id)
_execute_end_callbacks(instance, fn, call_id, results, exception, callbacks)
@@ -389,9 +389,9 @@ def with_callbacks(fn):
try:
results = fn(instance, *args, **kwargs)
return results
- except Exception as e:
+ except BaseException as e:
exception = e
- raise exception
+ raise
finally:
callback_context.ACTIVE_CALL_ID.set(parent_call_id)
_execute_end_callbacks(instance, fn, call_id, results, exception, callbacks)
diff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.py
index d13392fa9..fb60e9546 100644
--- a/tests/callback/test_callback.py
+++ b/tests/callback/test_callback.py
@@ -1,3 +1,4 @@
+import asyncio
import time
import cloudpickle
@@ -149,24 +150,70 @@ def test_callback_injection_local():
def test_callback_error_handling():
+ expected_error = ValueError("Error")
+
class Target(dspy.Module):
@with_callbacks
def forward(self, x: int, y: str, z: float) -> int:
time.sleep(0.1)
- raise ValueError("Error")
+ raise expected_error
callback = MyCallback()
dspy.configure(callbacks=[callback])
target = Target()
- with pytest.raises(ValueError, match="Error"):
+ with pytest.raises(ValueError, match="Error") as exc_info:
target.forward(1, "2", 3.0)
assert len(callback.calls) == 2
assert callback.calls[0]["handler"] == "on_module_start"
assert callback.calls[1]["handler"] == "on_module_end"
- assert isinstance(callback.calls[1]["exception"], ValueError)
+ assert callback.calls[1]["exception"] is exc_info.value is expected_error
+
+
[email protected]("expected_error", [KeyboardInterrupt(), SystemExit()])
+def test_callback_reports_and_propagates_base_exception(expected_error):
+ class Target(dspy.Module):
+ @with_callbacks
+ def forward(self):
+ raise expected_error
+
+ callback = MyCallback()
+ target = Target(callbacks=[callback])
+
+ with pytest.raises(BaseException) as exc_info:
+ target.forward()
+
+ assert callback.calls[1]["handler"] == "on_module_end"
+ assert callback.calls[1]["outputs"] is None
+ assert callback.calls[1]["exception"] is exc_info.value is expected_error
+ assert ACTIVE_CALL_ID.get() is None
+
+
+async def test_callback_reports_and_propagates_async_cancellation():
+ started = asyncio.Event()
+
+ class Target(dspy.Module):
+ @with_callbacks
+ async def run(self):
+ started.set()
+ await asyncio.Event().wait()
+
+ callback = MyCallback()
+ target = Target(callbacks=[callback])
+ task = asyncio.create_task(target.run())
+ await started.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ assert callback.calls[1]["handler"] == "on_module_end"
+ assert callback.calls[1]["outputs"] is None
+ assert isinstance(callback.calls[1]["exception"], asyncio.CancelledError)
+ assert ACTIVE_CALL_ID.get() is None
def test_multiple_callbacks():