DSPy: The framework for programming—not prompting—language models
Python
Some fixes to callback (#1696)
Chen Qian16ba98a44ff9
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 89f259f5a..523f8ffc1 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -1,23 +1,22 @@-from concurrent.futures import ThreadPoolExecutor-from datetime import datetimeimport functoolsimport os+import uuid+from concurrent.futures import ThreadPoolExecutor+from datetime import datetimefrom pathlib import Pathfrom typing import Any, Dict, List, Optional++import litellmimport ujson-import uuid+from litellm.caching import Cache-from dspy.utils.logging import loggerfrom dspy.clients.finetune import FinetuneJob, TrainingMethodfrom dspy.clients.lm_finetune_utils import (- get_provider_finetune_job_class,execute_finetune_job,+ get_provider_finetune_job_class,)-from dspy.utils.callback import with_callbacks-import litellm-from litellm.caching import Cache-+from dspy.utils.logging import loggerDISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.pyindex 1f8dfb4ce..bff180328 100644--- a/dspy/utils/__init__.py+++ b/dspy/utils/__init__.py@@ -1,3 +1,3 @@-from .callback import *-from .dummies import *-from .logging import *+from dspy.utils.callback import BaseCallback, with_callbacks+from dspy.utils.dummies import *+from dspy.utils.logging import *diff --git a/dspy/utils/callback.py b/dspy/utils/callback.pyindex b50bfd39a..5f9bcdc33 100644--- a/dspy/utils/callback.py+++ b/dspy/utils/callback.py@@ -13,14 +13,16 @@ logger = logging.getLogger(__name__)class BaseCallback:- """- A base class for defining callback handlers for DSPy components.+ """A base class for defining callback handlers for DSPy components.To use a callback, subclass this class and implement the desired handlers. Each handler- will be called at the appropriate time before/after the execution of the corresponding component.+ will be called at the appropriate time before/after the execution of the corresponding component. For example, if+ you want to print a message before and after an LM is called, implement `the on_llm_start` and `on_lm_end` handler.+ Users can set the callback globally using `dspy.settings.configure` or locally by passing it to the component+ constructor.+- For example, if you want to print a message before and after an LM is called, implement- the on_llm_start and on_lm_end handler and set the callback to the global settings using `dspy.settings.configure`.+ Example 1: Set a global callback using `dspy.settings.configure`.```import dspy@@ -45,19 +47,18 @@ class BaseCallback:# > LM is finished with outputs: {'answer': '42'}```- Another way to set the callback is to pass it directly to the component constructor.- In this case, the callback will only be triggered for that specific instance.+ Example 2: Set a local callback by passing it to the component constructor.```- lm = dspy.LM("gpt-3.5-turbo", callbacks=[LoggingCallback()])- lm(question="What is the meaning of life?")+ lm_1 = dspy.LM("gpt-3.5-turbo", callbacks=[LoggingCallback()])+ lm_1(question="What is the meaning of life?")# > LM is called with inputs: {'question': 'What is the meaning of life?'}# > LM is finished with outputs: {'answer': '42'}lm_2 = dspy.LM("gpt-3.5-turbo")lm_2(question="What is the meaning of life?")- # No logging here+ # No logging here because only `lm_1` has the callback set.```"""@@ -67,8 +68,7 @@ class BaseCallback:instance: Any,inputs: Dict[str, Any],):- """- A handler triggered when forward() method of a module (subclass of dspy.Module) is called.+ """A handler triggered when forward() method of a module (subclass of dspy.Module) is called.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -84,8 +84,7 @@ class BaseCallback:outputs: Optional[Any],exception: Optional[Exception] = None,):- """- A handler triggered after forward() method of a module (subclass of dspy.Module) is executed.+ """A handler triggered after forward() method of a module (subclass of dspy.Module) is executed.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -101,8 +100,7 @@ class BaseCallback:instance: Any,inputs: Dict[str, Any],):- """- A handler triggered when __call__ method of dspy.LM instance is called.+ """A handler triggered when __call__ method of dspy.LM instance is called.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -118,8 +116,7 @@ class BaseCallback:outputs: Optional[Dict[str, Any]],exception: Optional[Exception] = None,):- """- A handler triggered after __call__ method of dspy.LM instance is executed.+ """A handler triggered after __call__ method of dspy.LM instance is executed.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -129,14 +126,13 @@ class BaseCallback:"""pass- def on_format_start(+ def on_adapter_format_start(self,call_id: str,instance: Any,inputs: Dict[str, Any],):- """- A handler triggered when format() method of an adapter (subclass of dspy.Adapter) is called.+ """A handler triggered when format() method of an adapter (subclass of dspy.Adapter) is called.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -146,14 +142,13 @@ class BaseCallback:"""pass- def on_format_end(+ def on_adapter_format_end(self,call_id: str,outputs: Optional[Dict[str, Any]],exception: Optional[Exception] = None,):- """- A handler triggered after format() method of dspy.LM instance is executed.+ """A handler triggered after format() method of an adapter (subclass of dspy.Adapter) is called..Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -163,14 +158,13 @@ class BaseCallback:"""pass- def on_parse_start(+ def on_adapter_parse_start(self,call_id: str,instance: Any,inputs: Dict[str, Any],):- """- A handler triggered when parse() method of an adapter (subclass of dspy.Adapter) is called.+ """A handler triggered when parse() method of an adapter (subclass of dspy.Adapter) is called.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -180,14 +174,13 @@ class BaseCallback:"""pass- def on_parse_end(+ def on_adapter_parse_end(self,call_id: str,outputs: Optional[Dict[str, Any]],exception: Optional[Exception] = None,):- """- A handler triggered after parse() method of dspy.LM instance is executed.+ """A handler triggered after parse() method of an adapter (subclass of dspy.Adapter) is called.Args:call_id: A unique identifier for the call. Can be used to connect start/end handlers.@@ -200,23 +193,23 @@ class BaseCallback:def with_callbacks(fn):@functools.wraps(fn)- def wrapper(self, *args, **kwargs):- # Combine global and local (per-instance) callbacks- callbacks = dspy.settings.get("callbacks", []) + getattr(self, "callbacks", [])+ def wrapper(instance, *args, **kwargs):+ # Combine global and local (per-instance) callbacks.+ callbacks = dspy.settings.get("callbacks", []) + getattr(instance, "callbacks", [])- # if no callbacks are provided, just call the function+ # If no callbacks are provided, just call the functionif not callbacks:- return fn(self, *args, **kwargs)+ return fn(instance, *args, **kwargs)- # Generate call ID to connect start/end handlers if needed+ # Generate call ID as the unique identifier for the call, this is useful for instrumentation.call_id = uuid.uuid4().hex- inputs = inspect.getcallargs(fn, self, *args, **kwargs)+ inputs = inspect.getcallargs(fn, instance, *args, **kwargs)inputs.pop("self") # Not logging self as inputfor callback in callbacks:try:- _get_on_start_handler(callback, self, fn)(call_id=call_id, instance=self, inputs=inputs)+ _get_on_start_handler(callback, instance, fn)(call_id=call_id, instance=instance, inputs=inputs)except Exception as e:logger.warning(f"Error when calling callback {callback}: {e}")@@ -225,58 +218,59 @@ def with_callbacks(fn):exception = Nonetry:parent_call_id = ACTIVE_CALL_ID.get()- # Active ID must be set right before the function is called,- # not before calling the callbacks.+ # Active ID must be set right before the function is called, not before calling the callbacks.ACTIVE_CALL_ID.set(call_id)- results = fn(self, *args, **kwargs)+ results = fn(instance, *args, **kwargs)return resultsexcept Exception as e:exception = eraise exceptionfinally:+ # Execute the end handlers even if the function call raises an exception.ACTIVE_CALL_ID.set(parent_call_id)for callback in callbacks:try:- _get_on_end_handler(callback, self, fn)(+ _get_on_end_handler(callback, instance, fn)(call_id=call_id,outputs=results,exception=exception,)except Exception as e:- logger.warning(f"Error when calling callback {callback}: {e}")+ logger.warning(+ f"Error when applying callback {callback}'s end handler on function {fn.__name__}: {e}."+ )return wrapperdef _get_on_start_handler(callback: BaseCallback, instance: Any, fn: Callable) -> Callable:- """- Selects the appropriate on_start handler of the callback- based on the instance and function name.- """- if isinstance(instance, (dspy.LM)):+ """Selects the appropriate on_start handler of the callback based on the instance and function name."""+ if isinstance(instance, dspy.LM):return callback.on_lm_start- elif isinstance(instance, (dspy.Adapter)):++ if isinstance(instance, dspy.Adapter):if fn.__name__ == "format":- return callback.on_format_start+ return callback.on_adapter_format_startelif fn.__name__ == "parse":- return callback.on_parse_start+ return callback.on_adapter_parse_start+ else:+ raise ValueError(f"Unsupported adapter method for using callback: {fn.__name__}.")# We treat everything else as a module.return callback.on_module_startdef _get_on_end_handler(callback: BaseCallback, instance: Any, fn: Callable) -> Callable:- """- Selects the appropriate on_end handler of the callback- based on the instance and function name.- """+ """Selects the appropriate on_end handler of the callback based on the instance and function name."""if isinstance(instance, (dspy.LM)):return callback.on_lm_end- elif isinstance(instance, (dspy.Adapter)):++ if isinstance(instance, (dspy.Adapter)):if fn.__name__ == "format":- return callback.on_format_end+ return callback.on_adapter_format_endelif fn.__name__ == "parse":- return callback.on_parse_end-+ return callback.on_adapter_parse_end+ else:+ raise ValueError(f"Unsupported adapter method for using callback: {fn.__name__}.")# We treat everything else as a module.return callback.on_module_enddiff --git a/tests/callback/test_callback.py b/tests/callback/test_callback.pyindex a80e4060f..50b04d0e3 100644--- a/tests/callback/test_callback.py+++ b/tests/callback/test_callback.py@@ -18,6 +18,8 @@ def reset_settings():class MyCallback(BaseCallback):+ """A simple callback that records the calls."""+def __init__(self):self.calls = []@@ -33,17 +35,17 @@ class MyCallback(BaseCallback):def on_lm_end(self, call_id, outputs, exception):self.calls.append({"handler": "on_lm_end", "outputs": outputs, "exception": exception})- def on_format_start(self, call_id, instance, inputs):- self.calls.append({"handler": "on_format_start", "instance": instance, "inputs": inputs})+ def on_adapter_format_start(self, call_id, instance, inputs):+ self.calls.append({"handler": "on_adapter_format_start", "instance": instance, "inputs": inputs})- def on_format_end(self, call_id, outputs, exception):- self.calls.append({"handler": "on_format_end", "outputs": outputs, "exception": exception})+ def on_adapter_format_end(self, call_id, outputs, exception):+ self.calls.append({"handler": "on_adapter_format_end", "outputs": outputs, "exception": exception})- def on_parse_start(self, call_id, instance, inputs):- self.calls.append({"handler": "on_parse_start", "instance": instance, "inputs": inputs})+ def on_adapter_parse_start(self, call_id, instance, inputs):+ self.calls.append({"handler": "on_adapter_parse_start", "instance": instance, "inputs": inputs})- def on_parse_end(self, call_id, outputs, exception):- self.calls.append({"handler": "on_parse_end", "outputs": outputs, "exception": exception})+ def on_adapter_parse_end(self, call_id, outputs, exception):+ self.calls.append({"handler": "on_adapter_parse_end", "outputs": outputs, "exception": exception})@pytest.mark.parametrize(@@ -163,17 +165,17 @@ def test_callback_complex_module():assert [call["handler"] for call in callback.calls] == ["on_module_start","on_module_start",- "on_format_start",- "on_format_end",+ "on_adapter_format_start",+ "on_adapter_format_end","on_lm_start","on_lm_end",# Parsing will run per output (n=3)- "on_parse_start",- "on_parse_end",- "on_parse_start",- "on_parse_end",- "on_parse_start",- "on_parse_end",+ "on_adapter_parse_start",+ "on_adapter_parse_end",+ "on_adapter_parse_start",+ "on_adapter_parse_end",+ "on_adapter_parse_start",+ "on_adapter_parse_end","on_module_end","on_module_end",]