Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add explicit BaseLM forward contract (#9843)

Maxime Rivest9ae8290fb8aa
diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py
index b08e02c0d..e3b842eef 100644
--- a/dspy/clients/base_lm.py
+++ b/dspy/clients/base_lm.py
@@ -3,8 +3,10 @@ import datetime
import importlib
import inspect
import uuid
-from typing import Any, TextIO
+import warnings
+from typing import Any, Literal, TextIO
+from dspy.core.types import LMResponse
from dspy.dsp.utils import settings
from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.inspect_history import pretty_print_history
@@ -13,6 +15,7 @@ MAX_HISTORY_SIZE = 10_000
GLOBAL_HISTORY = []
LM_CLASS_STATE_KEY = "_dspy_lm_class"
_BUILTIN_LM_CLASS_PATH = "dspy.clients.lm.LM"
+ForwardContract = Literal["legacy", "typed_lm"]
def _import_lm_class(class_path: str) -> type:
@@ -96,6 +99,14 @@ class BaseLM:
```
"""
+ forward_contract: ForwardContract = "legacy"
+ """The `forward()` implementation contract used by this LM.
+
+ `"legacy"` means `forward(prompt=None, messages=None, **kwargs)` returns an
+ OpenAI-like provider response. `"typed_lm"` means
+ `forward(request: dspy.LMRequest) -> dspy.LMResponse`.
+ """
+
def __init__(
self,
model,
@@ -133,6 +144,64 @@ class BaseLM:
def _get_initial_kwargs(self, *, temperature, max_tokens, **kwargs) -> dict[str, Any]:
return dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
+ def _declares_forward_contract(self) -> bool:
+ """Return whether this concrete LM class declares `forward_contract`."""
+ return "forward_contract" in type(self).__dict__
+
+ def _get_forward_contract(self) -> ForwardContract:
+ """Return the declared `forward()` contract for this LM.
+
+ DSPy deliberately does not inspect `forward()` signatures during the
+ migration to typed LM calls. Subclasses opt into the typed contract by
+ setting `forward_contract = "typed_lm"`; all existing custom LMs remain
+ on the legacy contract unless they say otherwise.
+ """
+ contract = getattr(type(self), "forward_contract", "legacy")
+ if contract not in {"legacy", "typed_lm"}:
+ raise ValueError(
+ f"{type(self).__name__}.forward_contract must be 'legacy' or 'typed_lm', "
+ f"but got {contract!r}."
+ )
+ return contract
+
+ def _validate_typed_lm_response(self, response: Any) -> LMResponse:
+ """Validate the result of a typed `forward(request)` implementation."""
+ if isinstance(response, LMResponse):
+ return response
+ raise TypeError(
+ f"{type(self).__name__}.forward_contract='typed_lm' requires forward(request) "
+ f"to return dspy.LMResponse, but got {type(response).__name__}."
+ )
+
+ def _validate_legacy_lm_response(
+ self,
+ response: Any,
+ *,
+ stacklevel: int = 2,
+ ) -> LMResponse | None:
+ """Validate a legacy `forward()` result that already looks typed.
+
+ During the 3.3 migration, custom LMs that inherit the default legacy
+ contract may accidentally return `LMResponse`; warn so authors can add
+ `forward_contract = "typed_lm"`. If a class explicitly declares
+ `forward_contract = "legacy"`, treat `LMResponse` as a contract
+ mismatch and fail fast.
+ """
+ if not isinstance(response, LMResponse):
+ return None
+ if self._declares_forward_contract():
+ raise TypeError(
+ f"{type(self).__name__}.forward_contract='legacy' requires forward() to return an "
+ "OpenAI-like provider response, but got dspy.LMResponse. Set forward_contract='typed_lm'."
+ )
+ warnings.warn(
+ f"{type(self).__name__}.forward() returned dspy.LMResponse while using the default legacy "
+ "forward_contract. Set forward_contract='typed_lm' before the typed LM API becomes the default.",
+ DeprecationWarning,
+ stacklevel=stacklevel,
+ )
+ return response
+
@property
def supports_function_calling(self) -> bool:
"""Whether the model supports function calling (tool use)."""
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 974b2568f..468b4dc0b 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -57,6 +57,8 @@ class LM(BaseLM):
A language model supporting chat or text completion requests for use with DSPy modules.
"""
+ forward_contract = "legacy"
+
def __init__(
self,
model: str,
diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py
index 21889b66a..a5ac8fc1e 100644
--- a/dspy/utils/dummies.py
+++ b/dspy/utils/dummies.py
@@ -69,6 +69,8 @@ class DummyLM(BaseLM):
"""
+ forward_contract = "legacy"
+
def __init__(
self,
answers: list[dict[str, Any]] | dict[str, dict[str, Any]],
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 71a5bf0cb..ebd437602 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -450,6 +450,71 @@ def test_base_lm_init_uses_lm_defaults_and_isolates_callback_list():
assert lm.callbacks is not callbacks
+def test_base_lm_forward_contract_defaults_to_legacy():
+ class CustomLM(dspy.BaseLM):
+ pass
+
+ lm = CustomLM("custom-model")
+
+ assert lm._get_forward_contract() == "legacy"
+ assert not lm._declares_forward_contract()
+
+
+def test_base_lm_forward_contract_accepts_explicit_values():
+ class LegacyLM(dspy.BaseLM):
+ forward_contract = "legacy"
+
+ class TypedLM(dspy.BaseLM):
+ forward_contract = "typed_lm"
+
+ assert LegacyLM("custom-model")._get_forward_contract() == "legacy"
+ assert LegacyLM("custom-model")._declares_forward_contract()
+ assert TypedLM("custom-model")._get_forward_contract() == "typed_lm"
+ assert TypedLM("custom-model")._declares_forward_contract()
+
+
+def test_base_lm_forward_contract_rejects_unknown_values():
+ class CustomLM(dspy.BaseLM):
+ forward_contract = "normalized"
+
+ with pytest.raises(ValueError, match="forward_contract must be 'legacy' or 'typed_lm'"):
+ CustomLM("custom-model")._get_forward_contract()
+
+
+def test_base_lm_validates_typed_lm_response():
+ lm = dspy.BaseLM("custom-model")
+ response = dspy.LMResponse.from_text("ok", model="custom-model")
+
+ assert lm._validate_typed_lm_response(response) is response
+
+ with pytest.raises(TypeError, match=r"requires forward\(request\).*dspy.LMResponse"):
+ lm._validate_typed_lm_response(["ok"])
+
+
+def test_base_lm_warns_when_inherited_legacy_forward_returns_lm_response():
+ class CustomLM(dspy.BaseLM):
+ pass
+
+ lm = CustomLM("custom-model")
+ response = dspy.LMResponse.from_text("ok", model="custom-model")
+
+ with pytest.warns(DeprecationWarning, match="default legacy forward_contract"):
+ assert lm._validate_legacy_lm_response(response) is response
+
+ assert lm._validate_legacy_lm_response(["ok"]) is None
+
+
+def test_base_lm_errors_when_explicit_legacy_forward_returns_lm_response():
+ class CustomLM(dspy.BaseLM):
+ forward_contract = "legacy"
+
+ lm = CustomLM("custom-model")
+ response = dspy.LMResponse.from_text("ok", model="custom-model")
+
+ with pytest.raises(TypeError, match=r"forward_contract='legacy'.*got dspy.LMResponse"):
+ lm._validate_legacy_lm_response(response)
+
+
def test_base_lm_tracks_usage_for_custom_subclasses():
class CustomLM(dspy.BaseLM):
def forward(self, prompt=None, messages=None, **kwargs):