DSPy: The framework for programming—not prompting—language models
Python
Make BaseLM own shared LM runtime state (#9821)
Maxime Rivest77cb233156ed
diff --git a/docs/docs/learn/programming/language_models.md b/docs/docs/learn/programming/language_models.mdindex 971583564..9282c0887 100644--- a/docs/docs/learn/programming/language_models.md+++ b/docs/docs/learn/programming/language_models.md@@ -320,3 +320,9 @@ program.load("program.json", allow_unsafe_lm_state=True)Use this only for files you trust. The flag also preserves serialized LM endpoint configuration such as `api_base`, `base_url`, and `model_list`.+### Copying custom LMs++DSPy uses `lm.copy(...)` when it needs the same LM with different request parameters, such as a different `temperature` or `rollout_id`. The default `BaseLM.copy()` implementation makes a shallow runtime copy: provider clients, sessions, and local model handles are shared by reference, while DSPy-owned mutable state (`history`, the `callbacks` list, and `kwargs`) is isolated on the copy. The callback list is copied, but callback objects themselves are shared.++If your custom LM stores additional mutable DSPy-owned state that should not be shared across copies, override `copy()` and isolate that state explicitly. Runtime handles that are expensive or unsafe to duplicate should usually continue to be shared by reference.+diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.pyindex 78d1a4c49..80e3a5f75 100644--- a/dspy/clients/base_lm.py+++ b/dspy/clients/base_lm.py@@ -1,3 +1,4 @@+import copy as copy_moduleimport datetimeimport importlibimport inspect@@ -5,7 +6,7 @@ import uuidfrom typing import Any, TextIOfrom dspy.dsp.utils import settings-from dspy.utils.callback import with_callbacks+from dspy.utils.callback import BaseCallback, with_callbacksfrom dspy.utils.inspect_history import pretty_print_historyMAX_HISTORY_SIZE = 10_000@@ -99,18 +100,38 @@ class BaseLM:self,model,model_type="chat",- temperature=0.0,- max_tokens=1000,+ temperature=None,+ max_tokens=None,cache=True,- num_retries=0,+ callbacks: list[BaseCallback] | None = None,+ num_retries: int = 3,**kwargs,):+ """Initialize a base language model.++ Args:+ model: The model identifier.+ model_type: The LM API type, such as `"chat"`, `"text"`, or+ `"responses"`.+ temperature: The default sampling temperature.+ max_tokens: The default maximum number of output tokens.+ cache: Whether requests should use DSPy's cache by default.+ num_retries: The default number of provider request retries.+ callbacks: Optional instance-level callback handlers.+ **kwargs: Additional default request parameters stored in+ `self.kwargs`.+ """self.model = modelself.model_type = model_typeself.cache = cache+ self.callbacks = list(callbacks or [])self.num_retries = num_retries- self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)+ self.kwargs = self._get_initial_kwargs(temperature=temperature, max_tokens=max_tokens, **kwargs)self.history = []+ self._warned_zero_temp_rollout = False++ def _get_initial_kwargs(self, *, temperature, max_tokens, **kwargs) -> dict[str, Any]:+ return dict(temperature=temperature, max_tokens=max_tokens, **kwargs)@propertydef supports_function_calling(self) -> bool:@@ -140,6 +161,9 @@ class BaseLM:else:outputs = self._process_completion(response, merged_kwargs)+ if not getattr(response, "cache_hit", False) and settings.usage_tracker:+ settings.usage_tracker.add_usage(self.model, dict(getattr(response, "usage", {}) or {}))+if settings.disable_history:return outputs@@ -252,7 +276,7 @@ class BaseLM:"model": self.model,"model_type": self.model_type,"cache": self.cache,- "num_retries": getattr(self, "num_retries", 0),+ "num_retries": getattr(self, "num_retries", 3),**filtered_kwargs,}@@ -305,22 +329,34 @@ class BaseLM:return cls(**state)def copy(self, **kwargs):- """Returns a copy of the language model with possibly updated parameters.+ """Return a copy of the language model with updated parameters.- Any provided keyword arguments update the corresponding attributes or LM kwargs of- the copy. For example, ``lm.copy(rollout_id=1, temperature=1.0)`` returns an LM whose- requests use a different rollout ID at non-zero temperature to bypass cache collisions.- """+ The default implementation makes a shallow runtime copy. Provider+ clients, sessions, and local model handles are preserved by reference.+ DSPy-owned mutable state is isolated for `history`, the `callbacks`+ list, and the `kwargs` dict. Other attributes are shared by reference.+ Subclasses with additional mutable DSPy-owned state should override this+ method.++ Args:+ **kwargs: Attribute or request-parameter updates to apply to the+ copy. For example, `lm.copy(rollout_id=1, temperature=1.0)`+ returns an LM whose requests use a different rollout ID at+ non-zero temperature to bypass cache collisions.- import copy+ Returns:+ A copied LM instance.+ """- new_instance = copy.deepcopy(self)+ new_instance = copy_module.copy(self)new_instance.history = []+ new_instance.callbacks = list(getattr(self, "callbacks", []) or [])+ new_instance.kwargs = dict(getattr(self, "kwargs", {}) or {})for key, value in kwargs.items():- if hasattr(self, key):+ if hasattr(new_instance, key):setattr(new_instance, key, value)- if (key in self.kwargs) or (not hasattr(self, key)):+ if (key in new_instance.kwargs) or (not hasattr(self, key)):if value is None:new_instance.kwargs.pop(key, None)else:diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 599ffbdd4..ca67acfad 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -16,11 +16,10 @@ from dspy.clients.cache import request_cachefrom dspy.clients.openai import OpenAIProviderfrom dspy.clients.provider import Provider, ReinforceJob, TrainingJobfrom dspy.clients.utils_finetune import TrainDataFormat-from dspy.dsp.utils.settings import settingsfrom dspy.utils.callback import BaseCallbackfrom dspy.utils.exceptions import ContextWindowExceededError-from .base_lm import LM_CLASS_STATE_KEY, BaseLM+from .base_lm import BaseLMlogger = logging.getLogger(__name__)@@ -50,21 +49,22 @@ class LM(BaseLM):use_developer_role: bool = False,**kwargs,):- """- Create a new language model instance for use with DSPy modules and programs.+ """Create a new language model instance for use with DSPy modules and programs.Args:- model: The model to use. This should be a string of the form ``"llm_provider/llm_name"``- supported by LiteLLM. For example, ``"openai/gpt-4o"``.- model_type: The type of the model, either ``"chat"`` or ``"text"``.+ model: The model to use. This should be a string of the form+ `"llm_provider/llm_name"` supported by LiteLLM. For example,+ `"openai/gpt-4o"`.+ model_type: The type of the model, such as `"chat"`, `"text"`, or+ `"responses"`.temperature: The sampling temperature to use when generating responses.max_tokens: The maximum number of tokens to generate per response.cache: Whether to cache the model responses for reuse to improve performance- and reduce costs.+ and reduce costs.callbacks: A list of callback functions to run before and after each request.num_retries: The number of times to retry a request if it fails transiently due to- network error, rate limiting, etc. Requests are retried with exponential- backoff.+ network error, rate limiting, etc. Requests are retried with exponential+ backoff.provider: The provider to use. If not specified, the provider will be inferred from the model.finetuning_model: The model to finetune. In some providers, the models available for finetuning is differentfrom the models available for inference.@@ -74,22 +74,28 @@ class LM(BaseLM):only affects generation when `temperature` is non-zero. This argument isstripped before sending requests to the provider."""- # Remember to update LM.copy() if you modify the constructor!- self.model = model- self.model_type = model_type- self.cache = cache+ super().__init__(+ model=model,+ model_type=model_type,+ temperature=temperature,+ max_tokens=max_tokens,+ cache=cache,+ num_retries=num_retries,+ callbacks=callbacks,+ **kwargs,+ )+self.provider = provider or self.infer_provider()- self.callbacks = callbacks or []- self.history = []- self.num_retries = num_retriesself.finetuning_model = finetuning_modelself.launch_kwargs = launch_kwargs or {}self.train_kwargs = train_kwargs or {}self.use_developer_role = use_developer_role- self._warned_zero_temp_rollout = False- # Handle model-specific configuration for different model families- model_family = model.split("/")[-1].lower() if "/" in model else model.lower()+ self._warn_zero_temp_rollout(self.kwargs.get("temperature"), self.kwargs.get("rollout_id"))++ def _get_initial_kwargs(self, *, temperature, max_tokens, **kwargs) -> dict[str, Any]:+ # Override BaseLM's default kwargs shape for LiteLLM/model-family-specific token parameters.+ model_family = self.model.split("/")[-1].lower() if "/" in self.model else self.model.lower()# Recognize OpenAI reasoning models (o1, o3, o4, gpt-5 family)# Exclude non-reasoning variants like gpt-5-chat this is in azure ai foundry@@ -106,15 +112,13 @@ class LM(BaseLM):"OpenAI's reasoning models require passing temperature=1.0 or None and max_tokens >= 16000 or None to ""`dspy.LM(...)`, e.g., dspy.LM('openai/gpt-5', temperature=1.0, max_tokens=16000)")- self.kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)- if self.kwargs.get("rollout_id") is None:- self.kwargs.pop("rollout_id", None)+ initial_kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)else:- self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)- if self.kwargs.get("rollout_id") is None:- self.kwargs.pop("rollout_id", None)+ initial_kwargs = super()._get_initial_kwargs(temperature=temperature, max_tokens=max_tokens, **kwargs)- self._warn_zero_temp_rollout(self.kwargs.get("temperature"), self.kwargs.get("rollout_id"))+ if initial_kwargs.get("rollout_id") is None:+ initial_kwargs.pop("rollout_id", None)+ return initial_kwargs@propertydef _provider_name(self) -> str:@@ -199,8 +203,6 @@ class LM(BaseLM):self._check_truncation(results)- if not getattr(results, "cache_hit", False) and dspy.settings.usage_tracker:- settings.usage_tracker.add_usage(self.model, dict(getattr(results, "usage", {}) or {}))return resultsasync def aforward(@@ -242,8 +244,6 @@ class LM(BaseLM):self._check_truncation(results)- if not getattr(results, "cache_hit", False) and dspy.settings.usage_tracker:- settings.usage_tracker.add_usage(self.model, dict(getattr(results, "usage", {}) or {}))return resultsdef launch(self, launch_kwargs: dict[str, Any] | None = None):@@ -324,23 +324,17 @@ class LM(BaseLM):A dictionary that can be passed to `BaseLM.load_state` toreconstruct this `LM`. The state excludes API keys."""- state_keys = [- "model",- "model_type",- "cache",- "num_retries",- "finetuning_model",- "launch_kwargs",- "train_kwargs",- ]- # Exclude api_key from kwargs to prevent API keys from being saved in plain text, and exclude the internal- # class marker so callers cannot override the computed class path.- filtered_kwargs = {k: v for k, v in self.kwargs.items() if k not in ("api_key", LM_CLASS_STATE_KEY)}- return {- LM_CLASS_STATE_KEY: f"{type(self).__module__}.{type(self).__qualname__}",- **{key: getattr(self, key) for key in state_keys},- **filtered_kwargs,- }+ state = super().dump_state()+ state.update(+ {+ "finetuning_model": self.finetuning_model,+ "launch_kwargs": self.launch_kwargs,+ "train_kwargs": self.train_kwargs,+ }+ )+ if self.use_developer_role:+ state["use_developer_role"] = self.use_developer_role+ return statedef _check_truncation(self, results):if self.model_type != "responses" and any(c.finish_reason == "length" for c in results["choices"]):diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex d9c3b246d..b53430727 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -339,7 +339,7 @@ def test_reasoning_model_requirements(model_name):# Should raise assertion error if temperature or max_tokens requirements not metwith pytest.raises(ValueError,- match="reasoning models require passing temperature=1.0 or None and max_tokens >= 16000 or None",+ match=r"reasoning models require passing temperature=1\.0 or None and max_tokens >= 16000 or None",):dspy.LM(model=model_name,@@ -378,6 +378,60 @@ def test_gpt_5_chat_not_reasoning_model():assert lm.kwargs["temperature"] == 0.7+def test_base_lm_init_uses_lm_defaults_and_isolates_callback_list():+ callbacks = [object()]+ lm = dspy.BaseLM("custom-model", callbacks=callbacks)++ assert lm.kwargs == {"temperature": None, "max_tokens": None}+ assert lm.num_retries == 3+ assert lm.callbacks == callbacks+ assert lm.callbacks is not callbacks+++def test_base_lm_tracks_usage_for_custom_subclasses():+ class CustomLM(dspy.BaseLM):+ def forward(self, prompt=None, messages=None, **kwargs):+ return ModelResponse(+ choices=[Choices(message=Message(role="assistant", content="Hi!"))],+ usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ model="custom-model",+ )++ lm = CustomLM(model="custom-model")++ with track_usage() as usage_tracker:+ lm("Query")++ total_usage = usage_tracker.get_total_tokens()["custom-model"]+ assert total_usage["prompt_tokens"] == 1+ assert total_usage["completion_tokens"] == 1+ assert total_usage["total_tokens"] == 2+++def test_base_lm_copy_is_shallow_runtime_copy_with_isolated_dspy_state():+ class CustomLM(dspy.BaseLM):+ pass++ callback = object()+ client = object()+ lm = CustomLM(model="custom-model", callbacks=[callback], temperature=0.1)+ lm.client = client+ lm.extra_state = {"mutable": []}+ lm.history = [{"prompt": "original"}]++ copied_lm = lm.copy(temperature=0.2, rollout_id=1)++ assert copied_lm is not lm+ assert copied_lm.client is client+ assert copied_lm.extra_state is lm.extra_state+ assert copied_lm.history == []+ assert copied_lm.history is not lm.history+ assert copied_lm.callbacks == [callback]+ assert copied_lm.callbacks is not lm.callbacks+ assert copied_lm.kwargs == {"temperature": 0.2, "max_tokens": None, "rollout_id": 1}+ assert lm.kwargs == {"temperature": 0.1, "max_tokens": None}++def test_dump_state():lm = dspy.LM(model="openai/gpt-4o-mini",@@ -403,6 +457,13 @@ def test_dump_state():}+def test_dump_state_preserves_enabled_developer_role():+ lm = dspy.LM("openai/gpt-4o-mini", use_developer_role=True)++ assert lm.dump_state()["use_developer_role"] is True+ assert dspy.LM.load_state(lm.dump_state()).use_developer_role is True++def test_dump_state_ignores_internal_class_marker_kwarg():lm = dspy.LM(model="openai/gpt-4o-mini",