DSPy: The framework for programming—not prompting—language models
Python
Add on_tool_start/end callbacks (#1879)
Yuki Watanabe3edcd9ed3333
diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.pyindex d889d8549..0260ead37 100644--- a/dspy/predict/__init__.py+++ b/dspy/predict/__init__.py@@ -5,6 +5,6 @@ from .knn import KNNfrom .multi_chain_comparison import MultiChainComparisonfrom .predict import Predictfrom .program_of_thought import ProgramOfThought-from .react import ReAct+from .react import ReAct, Toolfrom .retry import Retryfrom .parallel import Parallel\ No newline at end of filediff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex f11446018..c6fcb6130 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -5,6 +5,7 @@ from pydantic import BaseModelfrom dspy.primitives.program import Modulefrom dspy.signatures.signature import ensure_signaturefrom dspy.adapters.json_adapter import get_annotation_name+from dspy.utils.callback import with_callbacksfrom typing import Callable, Any, get_type_hints, get_origin, Literalclass Tool:@@ -19,6 +20,7 @@ class Tool:for k, v in (args or get_type_hints(annotations_func)).items() if k != 'return'}+ @with_callbacksdef __call__(self, *args, **kwargs):return self.func(*args, **kwargs)diff --git a/dspy/utils/callback.py b/dspy/utils/callback.pyindex 5f9bcdc33..a1292ce52 100644--- a/dspy/utils/callback.py+++ b/dspy/utils/callback.py@@ -190,6 +190,38 @@ class BaseCallback:"""pass+ def on_tool_start(+ self,+ call_id: str,+ instance: Any,+ inputs: Dict[str, Any],+ ):+ """A handler triggered when a tool is called.++ Args:+ call_id: A unique identifier for the call. Can be used to connect start/end handlers.+ instance: The Tool instance.+ inputs: The inputs to the Tool's __call__ method. Each arguments is stored as+ a key-value pair in a dictionary.+ """+ pass++ def on_tool_end(+ self,+ call_id: str,+ outputs: Optional[Dict[str, Any]],+ exception: Optional[Exception] = None,+ ):+ """A handler triggered after a tool is executed.++ Args:+ call_id: A unique identifier for the call. Can be used to connect start/end handlers.+ outputs: The outputs of the Tool's __call__ method. If the method is interrupted by+ an exception, this will be None.+ exception: If an exception is raised during the execution, it will be stored here.+ """+ pass+def with_callbacks(fn):@functools.wraps(fn)@@ -256,6 +288,9 @@ def _get_on_start_handler(callback: BaseCallback, instance: Any, fn: Callable) -else:raise ValueError(f"Unsupported adapter method for using callback: {fn.__name__}.")+ if isinstance(instance, dspy.Tool):+ return callback.on_tool_start+# We treat everything else as a module.return callback.on_module_start@@ -272,5 +307,9 @@ def _get_on_end_handler(callback: BaseCallback, instance: Any, fn: Callable) ->return callback.on_adapter_parse_endelse:raise ValueError(f"Unsupported adapter method for using callback: {fn.__name__}.")++ if isinstance(instance, dspy.Tool):+ return callback.on_tool_end+# We treat everything else as a module.return callback.on_module_enddiff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.pyindex 50b04d0e3..ae8753889 100644--- a/tests/callback/test_callback.py+++ b/tests/callback/test_callback.py@@ -47,6 +47,12 @@ class MyCallback(BaseCallback):def on_adapter_parse_end(self, call_id, outputs, exception):self.calls.append({"handler": "on_adapter_parse_end", "outputs": outputs, "exception": exception})+ def on_tool_start(self, call_id, instance, inputs):+ self.calls.append({"handler": "on_tool_start", "instance": instance, "inputs": inputs})++ def on_tool_end(self, call_id, outputs, exception):+ self.calls.append({"handler": "on_tool_end", "outputs": outputs, "exception": exception})+@pytest.mark.parametrize(("args", "kwargs"),@@ -181,6 +187,41 @@ def test_callback_complex_module():]+def test_tool_calls():+ callback = MyCallback()+ dspy.settings.configure(callbacks=[callback])++ def tool_1(query: str) -> str:+ """A dummy tool function."""+ return "result 1"++ def tool_2(query: str) -> str:+ """Another dummy tool function."""+ return "result 2"++ class MyModule(dspy.Module):+ def __init__(self):+ self.tools = [dspy.Tool(tool_1), dspy.Tool(tool_2)]++ def forward(self, query: str) -> str:+ query = self.tools[0](query)+ return self.tools[1](query)++ module = MyModule()+ result = module("query")++ assert result == "result 2"+ assert len(callback.calls) == 6+ assert [call["handler"] for call in callback.calls] == [+ "on_module_start",+ "on_tool_start",+ "on_tool_end",+ "on_tool_start",+ "on_tool_end",+ "on_module_end",+ ]++def test_active_id():# Test the call ID is generated and handled properlyclass CustomCallback(BaseCallback):