Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Make BaseLM own shared LM runtime state (#9821)

Maxime Rivest77cb233156ed
diff --git a/docs/docs/learn/programming/language_models.md b/docs/docs/learn/programming/language_models.md
index 971583564..9282c0887 100644
--- a/docs/docs/learn/programming/language_models.md
+++ b/docs/docs/learn/programming/language_models.md
@@ -320,3 +320,9 @@ program.load("program.json", allow_unsafe_lm_state=True)
Use this only for files you trust. The flag also preserves serialized LM endpoint configuration such as `api_base`, `base_url`, and `model_list`.
+### Copying custom LMs
+
+DSPy uses `lm.copy(...)` when it needs the same LM with different request parameters, such as a different `temperature` or `rollout_id`. The default `BaseLM.copy()` implementation makes a shallow runtime copy: provider clients, sessions, and local model handles are shared by reference, while DSPy-owned mutable state (`history`, the `callbacks` list, and `kwargs`) is isolated on the copy. The callback list is copied, but callback objects themselves are shared.
+
+If your custom LM stores additional mutable DSPy-owned state that should not be shared across copies, override `copy()` and isolate that state explicitly. Runtime handles that are expensive or unsafe to duplicate should usually continue to be shared by reference.
+
diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py
index 78d1a4c49..80e3a5f75 100644
--- a/dspy/clients/base_lm.py
+++ b/dspy/clients/base_lm.py
@@ -1,3 +1,4 @@
+import copy as copy_module
import datetime
import importlib
import inspect
@@ -5,7 +6,7 @@ import uuid
from typing import Any, TextIO
from dspy.dsp.utils import settings
-from dspy.utils.callback import with_callbacks
+from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.inspect_history import pretty_print_history
MAX_HISTORY_SIZE = 10_000
@@ -99,18 +100,38 @@ class BaseLM:
self,
model,
model_type="chat",
- temperature=0.0,
- max_tokens=1000,
+ temperature=None,
+ max_tokens=None,
cache=True,
- num_retries=0,
+ callbacks: list[BaseCallback] | None = None,
+ num_retries: int = 3,
**kwargs,
):
+ """Initialize a base language model.
+
+ Args:
+ model: The model identifier.
+ model_type: The LM API type, such as `"chat"`, `"text"`, or
+ `"responses"`.
+ temperature: The default sampling temperature.
+ max_tokens: The default maximum number of output tokens.
+ cache: Whether requests should use DSPy's cache by default.
+ num_retries: The default number of provider request retries.
+ callbacks: Optional instance-level callback handlers.
+ **kwargs: Additional default request parameters stored in
+ `self.kwargs`.
+ """
self.model = model
self.model_type = model_type
self.cache = cache
+ self.callbacks = list(callbacks or [])
self.num_retries = num_retries
- self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
+ self.kwargs = self._get_initial_kwargs(temperature=temperature, max_tokens=max_tokens, **kwargs)
self.history = []
+ self._warned_zero_temp_rollout = False
+
+ def _get_initial_kwargs(self, *, temperature, max_tokens, **kwargs) -> dict[str, Any]:
+ return dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
@property
def supports_function_calling(self) -> bool:
@@ -140,6 +161,9 @@ class BaseLM:
else:
outputs = self._process_completion(response, merged_kwargs)
+ if not getattr(response, "cache_hit", False) and settings.usage_tracker:
+ settings.usage_tracker.add_usage(self.model, dict(getattr(response, "usage", {}) or {}))
+
if settings.disable_history:
return outputs
@@ -252,7 +276,7 @@ class BaseLM:
"model": self.model,
"model_type": self.model_type,
"cache": self.cache,
- "num_retries": getattr(self, "num_retries", 0),
+ "num_retries": getattr(self, "num_retries", 3),
**filtered_kwargs,
}
@@ -305,22 +329,34 @@ class BaseLM:
return cls(**state)
def copy(self, **kwargs):
- """Returns a copy of the language model with possibly updated parameters.
+ """Return a copy of the language model with updated parameters.
- Any provided keyword arguments update the corresponding attributes or LM kwargs of
- the copy. For example, ``lm.copy(rollout_id=1, temperature=1.0)`` returns an LM whose
- requests use a different rollout ID at non-zero temperature to bypass cache collisions.
- """
+ The default implementation makes a shallow runtime copy. Provider
+ clients, sessions, and local model handles are preserved by reference.
+ DSPy-owned mutable state is isolated for `history`, the `callbacks`
+ list, and the `kwargs` dict. Other attributes are shared by reference.
+ Subclasses with additional mutable DSPy-owned state should override this
+ method.
+
+ Args:
+ **kwargs: Attribute or request-parameter updates to apply to the
+ copy. For example, `lm.copy(rollout_id=1, temperature=1.0)`
+ returns an LM whose requests use a different rollout ID at
+ non-zero temperature to bypass cache collisions.
- import copy
+ Returns:
+ A copied LM instance.
+ """
- new_instance = copy.deepcopy(self)
+ new_instance = copy_module.copy(self)
new_instance.history = []
+ new_instance.callbacks = list(getattr(self, "callbacks", []) or [])
+ new_instance.kwargs = dict(getattr(self, "kwargs", {}) or {})
for key, value in kwargs.items():
- if hasattr(self, key):
+ if hasattr(new_instance, key):
setattr(new_instance, key, value)
- if (key in self.kwargs) or (not hasattr(self, key)):
+ if (key in new_instance.kwargs) or (not hasattr(self, key)):
if value is None:
new_instance.kwargs.pop(key, None)
else:
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 599ffbdd4..ca67acfad 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -16,11 +16,10 @@ from dspy.clients.cache import request_cache
from dspy.clients.openai import OpenAIProvider
from dspy.clients.provider import Provider, ReinforceJob, TrainingJob
from dspy.clients.utils_finetune import TrainDataFormat
-from dspy.dsp.utils.settings import settings
from dspy.utils.callback import BaseCallback
from dspy.utils.exceptions import ContextWindowExceededError
-from .base_lm import LM_CLASS_STATE_KEY, BaseLM
+from .base_lm import BaseLM
logger = logging.getLogger(__name__)
@@ -50,21 +49,22 @@ class LM(BaseLM):
use_developer_role: bool = False,
**kwargs,
):
- """
- Create a new language model instance for use with DSPy modules and programs.
+ """Create a new language model instance for use with DSPy modules and programs.
Args:
- model: The model to use. This should be a string of the form ``"llm_provider/llm_name"``
- supported by LiteLLM. For example, ``"openai/gpt-4o"``.
- model_type: The type of the model, either ``"chat"`` or ``"text"``.
+ model: The model to use. This should be a string of the form
+ `"llm_provider/llm_name"` supported by LiteLLM. For example,
+ `"openai/gpt-4o"`.
+ model_type: The type of the model, such as `"chat"`, `"text"`, or
+ `"responses"`.
temperature: The sampling temperature to use when generating responses.
max_tokens: The maximum number of tokens to generate per response.
cache: Whether to cache the model responses for reuse to improve performance
- and reduce costs.
+ and reduce costs.
callbacks: A list of callback functions to run before and after each request.
num_retries: The number of times to retry a request if it fails transiently due to
- network error, rate limiting, etc. Requests are retried with exponential
- backoff.
+ network error, rate limiting, etc. Requests are retried with exponential
+ backoff.
provider: The provider to use. If not specified, the provider will be inferred from the model.
finetuning_model: The model to finetune. In some providers, the models available for finetuning is different
from the models available for inference.
@@ -74,22 +74,28 @@ class LM(BaseLM):
only affects generation when `temperature` is non-zero. This argument is
stripped before sending requests to the provider.
"""
- # Remember to update LM.copy() if you modify the constructor!
- self.model = model
- self.model_type = model_type
- self.cache = cache
+ super().__init__(
+ model=model,
+ model_type=model_type,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ cache=cache,
+ num_retries=num_retries,
+ callbacks=callbacks,
+ **kwargs,
+ )
+
self.provider = provider or self.infer_provider()
- self.callbacks = callbacks or []
- self.history = []
- self.num_retries = num_retries
self.finetuning_model = finetuning_model
self.launch_kwargs = launch_kwargs or {}
self.train_kwargs = train_kwargs or {}
self.use_developer_role = use_developer_role
- self._warned_zero_temp_rollout = False
- # Handle model-specific configuration for different model families
- model_family = model.split("/")[-1].lower() if "/" in model else model.lower()
+ self._warn_zero_temp_rollout(self.kwargs.get("temperature"), self.kwargs.get("rollout_id"))
+
+ def _get_initial_kwargs(self, *, temperature, max_tokens, **kwargs) -> dict[str, Any]:
+ # Override BaseLM's default kwargs shape for LiteLLM/model-family-specific token parameters.
+ model_family = self.model.split("/")[-1].lower() if "/" in self.model else self.model.lower()
# Recognize OpenAI reasoning models (o1, o3, o4, gpt-5 family)
# Exclude non-reasoning variants like gpt-5-chat this is in azure ai foundry
@@ -106,15 +112,13 @@ class LM(BaseLM):
"OpenAI's reasoning models require passing temperature=1.0 or None and max_tokens >= 16000 or None to "
"`dspy.LM(...)`, e.g., dspy.LM('openai/gpt-5', temperature=1.0, max_tokens=16000)"
)
- self.kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)
- if self.kwargs.get("rollout_id") is None:
- self.kwargs.pop("rollout_id", None)
+ initial_kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)
else:
- self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
- if self.kwargs.get("rollout_id") is None:
- self.kwargs.pop("rollout_id", None)
+ initial_kwargs = super()._get_initial_kwargs(temperature=temperature, max_tokens=max_tokens, **kwargs)
- self._warn_zero_temp_rollout(self.kwargs.get("temperature"), self.kwargs.get("rollout_id"))
+ if initial_kwargs.get("rollout_id") is None:
+ initial_kwargs.pop("rollout_id", None)
+ return initial_kwargs
@property
def _provider_name(self) -> str:
@@ -199,8 +203,6 @@ class LM(BaseLM):
self._check_truncation(results)
- if not getattr(results, "cache_hit", False) and dspy.settings.usage_tracker:
- settings.usage_tracker.add_usage(self.model, dict(getattr(results, "usage", {}) or {}))
return results
async def aforward(
@@ -242,8 +244,6 @@ class LM(BaseLM):
self._check_truncation(results)
- if not getattr(results, "cache_hit", False) and dspy.settings.usage_tracker:
- settings.usage_tracker.add_usage(self.model, dict(getattr(results, "usage", {}) or {}))
return results
def launch(self, launch_kwargs: dict[str, Any] | None = None):
@@ -324,23 +324,17 @@ class LM(BaseLM):
A dictionary that can be passed to `BaseLM.load_state` to
reconstruct this `LM`. The state excludes API keys.
"""
- state_keys = [
- "model",
- "model_type",
- "cache",
- "num_retries",
- "finetuning_model",
- "launch_kwargs",
- "train_kwargs",
- ]
- # Exclude api_key from kwargs to prevent API keys from being saved in plain text, and exclude the internal
- # class marker so callers cannot override the computed class path.
- filtered_kwargs = {k: v for k, v in self.kwargs.items() if k not in ("api_key", LM_CLASS_STATE_KEY)}
- return {
- LM_CLASS_STATE_KEY: f"{type(self).__module__}.{type(self).__qualname__}",
- **{key: getattr(self, key) for key in state_keys},
- **filtered_kwargs,
- }
+ state = super().dump_state()
+ state.update(
+ {
+ "finetuning_model": self.finetuning_model,
+ "launch_kwargs": self.launch_kwargs,
+ "train_kwargs": self.train_kwargs,
+ }
+ )
+ if self.use_developer_role:
+ state["use_developer_role"] = self.use_developer_role
+ return state
def _check_truncation(self, results):
if self.model_type != "responses" and any(c.finish_reason == "length" for c in results["choices"]):
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index d9c3b246d..b53430727 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -339,7 +339,7 @@ def test_reasoning_model_requirements(model_name):
# Should raise assertion error if temperature or max_tokens requirements not met
with pytest.raises(
ValueError,
- match="reasoning models require passing temperature=1.0 or None and max_tokens >= 16000 or None",
+ match=r"reasoning models require passing temperature=1\.0 or None and max_tokens >= 16000 or None",
):
dspy.LM(
model=model_name,
@@ -378,6 +378,60 @@ def test_gpt_5_chat_not_reasoning_model():
assert lm.kwargs["temperature"] == 0.7
+def test_base_lm_init_uses_lm_defaults_and_isolates_callback_list():
+ callbacks = [object()]
+ lm = dspy.BaseLM("custom-model", callbacks=callbacks)
+
+ assert lm.kwargs == {"temperature": None, "max_tokens": None}
+ assert lm.num_retries == 3
+ assert lm.callbacks == callbacks
+ assert lm.callbacks is not callbacks
+
+
+def test_base_lm_tracks_usage_for_custom_subclasses():
+ class CustomLM(dspy.BaseLM):
+ def forward(self, prompt=None, messages=None, **kwargs):
+ return ModelResponse(
+ choices=[Choices(message=Message(role="assistant", content="Hi!"))],
+ usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ model="custom-model",
+ )
+
+ lm = CustomLM(model="custom-model")
+
+ with track_usage() as usage_tracker:
+ lm("Query")
+
+ total_usage = usage_tracker.get_total_tokens()["custom-model"]
+ assert total_usage["prompt_tokens"] == 1
+ assert total_usage["completion_tokens"] == 1
+ assert total_usage["total_tokens"] == 2
+
+
+def test_base_lm_copy_is_shallow_runtime_copy_with_isolated_dspy_state():
+ class CustomLM(dspy.BaseLM):
+ pass
+
+ callback = object()
+ client = object()
+ lm = CustomLM(model="custom-model", callbacks=[callback], temperature=0.1)
+ lm.client = client
+ lm.extra_state = {"mutable": []}
+ lm.history = [{"prompt": "original"}]
+
+ copied_lm = lm.copy(temperature=0.2, rollout_id=1)
+
+ assert copied_lm is not lm
+ assert copied_lm.client is client
+ assert copied_lm.extra_state is lm.extra_state
+ assert copied_lm.history == []
+ assert copied_lm.history is not lm.history
+ assert copied_lm.callbacks == [callback]
+ assert copied_lm.callbacks is not lm.callbacks
+ assert copied_lm.kwargs == {"temperature": 0.2, "max_tokens": None, "rollout_id": 1}
+ assert lm.kwargs == {"temperature": 0.1, "max_tokens": None}
+
+
def test_dump_state():
lm = dspy.LM(
model="openai/gpt-4o-mini",
@@ -403,6 +457,13 @@ def test_dump_state():
}
+def test_dump_state_preserves_enabled_developer_role():
+ lm = dspy.LM("openai/gpt-4o-mini", use_developer_role=True)
+
+ assert lm.dump_state()["use_developer_role"] is True
+ assert dspy.LM.load_state(lm.dump_state()).use_developer_role is True
+
+
def test_dump_state_ignores_internal_class_marker_kwarg():
lm = dspy.LM(
model="openai/gpt-4o-mini",