DSPy: The framework for programming—not prompting—language models
Python
"""Offline checks for the coordinated LM engine path."""
import json
import zipfile
from pathlib import Path
import pytest
import dspy
from dspy._vendor.lm15.testing import FakeResponse, FakeTransport
from dspy.clients.engines import LiteLLMEngine
from dspy.lm15 import Config, Message, Request, Response, RouterConfig
from dspy.utils.callback import BaseCallback
from dspy.utils.usage_tracker import track_usage
class DualResponse(FakeResponse):
async def __aenter__(self):
source = self
class AsyncResponse:
def __getattr__(self, name):
return getattr(source, name)
async def read(self):
return source.read()
async def __aiter__(self):
for chunk in source:
yield chunk
return AsyncResponse()
async def __aexit__(self, *exc):
pass
def wire(text="hello", status=200):
body = {"id": "response-1", "model": "gpt-4o-mini", "choices": [
{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}
], "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}}
if status != 200:
body = {"error": {"message": "try later", "type": "authentication_error" if status == 401 else "rate_limit_error"}}
return DualResponse(status=status, body=json.dumps(body).encode())
def native_transport(monkeypatch, responses):
import dspy.clients.execution as execution
transport = FakeTransport(responses)
monkeypatch.setattr(execution, "RouterConfig", lambda **kwargs: RouterConfig(
**{**kwargs, "api_keys": {"openai-chat": "fake"}, "transport": transport},
))
return transport
def test_default_native_cache_and_typed_request(monkeypatch):
transport = native_transport(monkeypatch, [wire(), wire("typed")])
lm = dspy.LM("openai/gpt-4o-mini")
with track_usage() as usage:
assert lm("hello") == ["hello"]
assert lm("hello") == ["hello"]
assert len(transport.requests) == 1
assert usage.get_total_tokens()[lm.model]["total_tokens"] == 3
assert lm.history[-1]["usage"] == {}
request = Request(model=lm.model, messages=(Message.user("explicit"),), config=Config(max_tokens=12))
response = lm(request)
assert isinstance(response, Response)
assert response.text == "typed"
assert lm.history[-1]["request"] is request
assert len(transport.requests) == 2
@pytest.mark.asyncio
async def test_async_native_cache_and_history(monkeypatch):
transport = native_transport(monkeypatch, [wire("async")])
lm = dspy.LM("openai/gpt-4o-mini")
try:
with track_usage() as usage:
assert await lm.acall("hello") == ["async"]
assert await lm.acall("hello") == ["async"]
assert len(transport.requests) == 1
assert len(lm.history) == 2
assert usage.get_total_tokens()[lm.model]["total_tokens"] == 3
finally:
await lm.aclose()
def test_native_n_fans_out_but_bookkeeping_is_one_call(monkeypatch):
transport = native_transport(monkeypatch, [wire("first"), wire("second")])
class Trace(BaseCallback):
def __init__(self):
self.events = []
def on_lm_start(self, call_id, instance, inputs):
self.events.append(("start", call_id))
def on_lm_end(self, call_id, outputs, exception):
self.events.append(("end", call_id))
trace = Trace()
lm = dspy.LM("openai/gpt-4o-mini", callbacks=[trace])
with track_usage() as usage:
assert lm("hello", n=2) == ["first", "second"]
assert len(transport.requests) == 2
assert len(lm.history) == 1
assert trace.events[1] == ("end", trace.events[0][1])
assert len(trace.events) == 2
assert usage.get_total_tokens()[lm.model]["total_tokens"] == 6
assert lm("hello", n=2) == ["first", "second"]
assert len(transport.requests) == 2
def test_native_retries_without_backend_fallback(monkeypatch):
import dspy.clients.execution as execution
transport = native_transport(monkeypatch, [wire(status=429), wire("recovered")])
delays = []
monkeypatch.setattr(execution.time, "sleep", delays.append)
lm = dspy.LM("openai/gpt-4o-mini", num_retries=1, cache=False)
assert lm("hello") == ["recovered"]
assert len(transport.requests) == 2
assert delays == [1]
def test_provider_auth_error_does_not_switch_backend(monkeypatch):
native_transport(monkeypatch, [wire(status=401)])
def forbidden(*args, **kwargs):
pytest.fail("An authentication failure must not invoke LiteLLM")
monkeypatch.setattr(LiteLLMEngine, "complete_legacy", forbidden)
with pytest.raises(dspy.LMAuthError):
dspy.LM("openai/gpt-4o-mini", cache=False)("hello")
def test_unrepresentable_ordinary_options_choose_compatibility_before_io(monkeypatch):
transport = native_transport(monkeypatch, [])
from dspy.clients.call_result import CallResult
calls = []
def complete(engine, lm, request, **context):
calls.append(request)
return CallResult(outputs=["compatible"], response_model=lm.model)
monkeypatch.setattr(LiteLLMEngine, "complete_legacy", complete)
assert dspy.LM("openai/gpt-4o-mini")("hello", audio={"voice": "alloy", "format": "wav"}) == ["compatible"]
assert calls[0]["audio"]["voice"] == "alloy"
assert not transport.requests
@pytest.mark.asyncio
async def test_legacy_plugin_sync_async_preserve_forward_inputs():
from dspy.dsp.utils.utils import dotdict
class Plugin(dspy.BaseLM):
def forward(self, prompt=None, messages=None, **kwargs):
self.received = prompt, messages, kwargs
return dotdict(model=self.model, usage={"total_tokens": 1}, choices=[
dotdict(message=dotdict(content="ok"), finish_reason="stop")
])
async def aforward(self, **kwargs):
return self.forward(**kwargs)
plugin = Plugin("custom")
with pytest.warns(DeprecationWarning, match="scheduled for removal in 3.5"):
assert plugin("hello") == ["ok"]
assert plugin.received == ("hello", None, {})
with pytest.warns(DeprecationWarning, match="scheduled for removal in 3.5"):
assert await plugin.acall("hello") == ["ok"]
assert len(plugin.history) == 2
def test_new_native_cache_uses_restricted_plain_data(monkeypatch, tmp_path):
transport = native_transport(monkeypatch, [wire()])
dspy.configure_cache(enable_disk_cache=True, enable_memory_cache=False, disk_cache_dir=tmp_path, restrict_pickle=True)
try:
lm = dspy.LM("openai/gpt-4o-mini")
assert lm("hello") == ["hello"]
assert lm("hello") == ["hello"]
assert len(transport.requests) == 1
finally:
dspy.cache.disk_cache.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("case_index", range(6))
async def test_old_disk_cache_hits_before_native_routing(case_index, monkeypatch, tmp_path):
from tests.clients.test_lm_migration_compatibility import plain
fixtures = Path(__file__).parent / "fixtures" / "lm_3_3_0"
case = json.loads((fixtures / "manifest.json").read_text())["cases"][case_index]
with zipfile.ZipFile(fixtures / f"{case['name']}.zip") as archive:
archive.extractall(tmp_path)
dspy.configure_cache(enable_disk_cache=True, enable_memory_cache=False, disk_cache_dir=tmp_path)
import dspy.clients.execution as execution
def forbidden(*args, **kwargs):
pytest.fail("An old disk hit must not even initialize an engine")
monkeypatch.setattr(execution, "_engine", forbidden)
try:
lm = dspy.LM(**case["init"])
output = lm(**case["call"]) if case["mode"] == "sync" else await lm.acall(**case["call"])
assert plain(output) == case["outputs"]
assert lm.history[-1]["usage"] == {}
finally:
dspy.cache.disk_cache.close()
@pytest.mark.asyncio
async def test_native_streaming_reaches_existing_listener(monkeypatch):
payloads = [{"choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}]}
for text in ("[[ ## answer ## ]]\n", "native", "\n\n[[ ## completed ## ]]")]
payloads.append({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}})
chunks = [f"data: {json.dumps(payload)}\n\n".encode() for payload in payloads]
chunks.append(b"data: [DONE]\n\n")
transport = native_transport(monkeypatch, [FakeResponse(status=200, body=b"", chunks=chunks)])
lm = dspy.LM("openai/gpt-4o-mini")
predict = dspy.Predict("question -> answer")
streamer = dspy.streamify(predict, stream_listeners=[dspy.streaming.StreamListener("answer")])
with dspy.context(lm=lm):
values = [value async for value in streamer(question="hello")]
cached = [value async for value in streamer(question="hello")]
assert values[-1].answer == "native"
assert "".join(value.chunk for value in values if isinstance(value, dspy.streaming.StreamResponse)) == "native"
assert len(cached) == 1 and cached[0].answer == "native"
assert len(transport.requests) == 1
@pytest.mark.asyncio
async def test_partial_stream_is_not_retried_and_reports_callback_error():
from dspy.lm15 import StreamDeltaEvent, StreamStartEvent, TextDelta
class Engine:
def __init__(self):
self.calls = 0
def complete(self, request):
raise AssertionError("Expected a streaming call")
def stream(self, request):
self.calls += 1
yield StreamStartEvent(model=request.model)
yield StreamDeltaEvent(TextDelta("partial"))
raise dspy.LMTransportError("stream broke")
class Sink:
async def send(self, chunk):
pass
class Trace(BaseCallback):
error = None
def on_lm_end(self, call_id, outputs, exception):
self.error = exception
engine = Engine()
trace = Trace()
lm = dspy.LM("custom", engine=engine, callbacks=[trace], num_retries=3)
import anyio
with dspy.context(send_stream=Sink()):
with pytest.raises(dspy.LMTransportError, match="stream broke"):
await anyio.to_thread.run_sync(lambda: lm("hello"))
assert engine.calls == 1
assert isinstance(trace.error, dspy.LMTransportError)
assert lm.history == []
def test_dummy_uses_canonical_engine_without_changing_script_behavior():
from dspy.utils.dummies import DummyLM
lm = DummyLM([{"answer": "one"}, {"answer": "two"}])
with dspy.context(lm=lm):
predict = dspy.Predict("question -> answer")
assert predict(question="same").answer == "one"
assert predict(question="same").answer == "two"
assert len(lm.history) == 2