DSPy: The framework for programming—not prompting—language models
Python
import asyncio
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from unittest import mock
import pytest
from litellm import Choices, Message, ModelResponse
import dspy
def test_basic_dspy_settings():
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
assert dspy.settings.lm.model == "openai/gpt-4o"
assert isinstance(dspy.settings.adapter, dspy.JSONAdapter)
assert len(dspy.settings.callbacks) == 1
def test_forbid_configure_call_in_child_thread():
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
def worker():
with pytest.raises(RuntimeError, match="dspy.settings can only be changed by the thread"):
dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[])
with ThreadPoolExecutor(max_workers=1) as executor:
executor.submit(worker).result()
def test_dspy_context():
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), callbacks=[]):
assert dspy.settings.lm.model == "openai/gpt-4o-mini"
assert len(dspy.settings.callbacks) == 0
assert dspy.settings.lm.model == "openai/gpt-4o"
assert len(dspy.settings.callbacks) == 1
def test_dspy_context_parallel():
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
def worker(i):
with dspy.context(lm=dspy.LM("openai/gpt-4o-mini"), trace=[i], callbacks=[]):
assert dspy.settings.lm.model == "openai/gpt-4o-mini"
assert dspy.settings.trace == [i]
assert len(dspy.settings.callbacks) == 0
with ThreadPoolExecutor(max_workers=5) as executor:
list(executor.map(worker, range(3)))
assert dspy.settings.lm.model == "openai/gpt-4o"
assert len(dspy.settings.callbacks) == 1
def test_dspy_context_with_dspy_parallel():
dspy.configure(lm=dspy.LM("openai/gpt-4o", engine="litellm", cache=False), adapter=dspy.ChatAdapter())
class MyModule(dspy.Module):
def __init__(self):
self.predict = dspy.Predict("question -> answer")
def forward(self, question: str) -> str:
lm = dspy.LM("openai/gpt-4o-mini", engine="litellm", cache=False) if "France" in question else dspy.settings.lm
with dspy.context(lm=lm):
time.sleep(1)
assert dspy.settings.lm.model == lm.model
return self.predict(question=question)
with mock.patch("litellm.completion") as mock_completion:
mock_completion.return_value = ModelResponse(
choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],
model="openai/gpt-4o-mini",
)
module = MyModule()
parallelizer = dspy.Parallel()
input_pairs = [
(module, {"question": "What is the capital of France?"}),
(module, {"question": "What is the capital of Germany?"}),
]
parallelizer(input_pairs)
# Verify mock was called correctly
assert mock_completion.call_count == 2
for call_args in mock_completion.call_args_list:
if "France" in call_args.kwargs["messages"][-1]["content"]:
# France question uses gpt-4o-mini
assert call_args.kwargs["model"] == "openai/gpt-4o-mini"
else:
# Germany question uses gpt-4o
assert call_args.kwargs["model"] == "openai/gpt-4o"
# The main thread is not affected by the context
assert dspy.settings.lm.model == "openai/gpt-4o"
@pytest.mark.asyncio
async def test_dspy_context_with_async_task_group():
class MyModule(dspy.Module):
def __init__(self):
self.predict = dspy.Predict("question -> answer")
async def aforward(self, question: str) -> str:
lm = (
dspy.LM("openai/gpt-4o-mini", engine="litellm", cache=False)
if "France" in question
else dspy.LM("openai/gpt-4o", engine="litellm", cache=False)
)
with dspy.context(lm=lm, trace=[]):
await asyncio.sleep(1)
assert dspy.settings.lm.model == lm.model
result = await self.predict.acall(question=question)
assert len(dspy.settings.trace) == 1
return result
module = MyModule()
with dspy.context(lm=dspy.LM("openai/gpt-4.1", engine="litellm", cache=False), adapter=dspy.ChatAdapter()):
with mock.patch("litellm.acompletion") as mock_completion:
mock_completion.return_value = ModelResponse(
choices=[Choices(message=Message(content="[[ ## answer ## ]]\nParis"))],
model="openai/gpt-4o-mini",
)
# Define the coroutines to be run
coroutines = [
module.acall(question="What is the capital of France?"),
module.acall(question="What is the capital of France?"),
module.acall(question="What is the capital of Germany?"),
module.acall(question="What is the capital of Germany?"),
]
# Run them concurrently and gather results
results = await asyncio.gather(*coroutines)
assert results[0].answer == "Paris"
assert results[1].answer == "Paris"
assert results[2].answer == "Paris"
assert results[3].answer == "Paris"
# Concurrent calls can arrive in any order. Check the association,
# not just model counts, so leaked task-local settings still fail.
assert mock_completion.call_count == 4
seen = {"France": 0, "Germany": 0}
for call in mock_completion.call_args_list:
content = call.kwargs["messages"][-1]["content"]
country = "France" if "France" in content else "Germany"
assert country in content
expected = "openai/gpt-4o-mini" if country == "France" else "openai/gpt-4o"
assert call.kwargs["model"] == expected
seen[country] += 1
assert seen == {"France": 2, "Germany": 2}
# The main thread is not affected by the context
assert dspy.settings.lm.model == "openai/gpt-4.1"
assert dspy.settings.trace == []
@pytest.mark.asyncio
async def test_dspy_configure_allowance_async():
def bar1():
# `dspy.configure` is disallowed in different async tasks from the initial one.
# In this case, foo1 (async) calls bar1 (sync), and bar1 uses the async task from foo1.
with pytest.raises(RuntimeError) as e:
dspy.configure(lm=dspy.LM("openai/gpt-4o"))
assert "dspy.configure(...) can only be called from the same async" in str(e.value)
async def foo1():
bar1()
await asyncio.sleep(0.1)
async def foo2():
# `dspy.configure` is disallowed in different async tasks from the initial one.
with pytest.raises(RuntimeError) as e:
dspy.configure(lm=dspy.LM("openai/gpt-4o"))
assert "dspy.configure(...) can only be called from the same async" in str(e.value)
await asyncio.sleep(0.1)
async def foo3():
# `dspy.context` is allowed in different async tasks from the initial one.
with dspy.context(lm=dspy.LM("openai/gpt-4o")):
await asyncio.sleep(0.1)
async def foo4():
# foo4 is directly invoked by the entry task, so it has the same async task as the entry task.
dspy.configure(lm=dspy.LM("openai/gpt-4o"))
await asyncio.sleep(0.1)
# `dspy.configure` is allowed to be called multiple times in the same async task.
dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"))
dspy.configure(lm=dspy.LM("openai/gpt-4o"))
dspy.configure(adapter=dspy.JSONAdapter())
await asyncio.gather(foo1(), foo2(), foo3())
await foo4()
def test_dspy_settings_save_load(tmp_path):
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), callbacks=[lambda x: x])
dspy.settings.save(tmp_path / "settings.pkl")
dspy.configure(lm=None, adapter=None, callbacks=None)
loaded_settings = dspy.load_settings(tmp_path / "settings.pkl", allow_pickle=True)
dspy.configure(**loaded_settings)
assert dspy.settings.lm.model == "openai/gpt-4o"
assert isinstance(dspy.settings.adapter, dspy.JSONAdapter)
assert len(dspy.settings.callbacks) == 1
def test_dspy_settings_save_exclude_keys(tmp_path):
dspy.configure(lm=dspy.LM("openai/gpt-4o"), adapter=dspy.JSONAdapter(), track_usage=True)
dspy.settings.save(tmp_path / "settings.pkl", exclude_keys=["adapter", "track_usage"])
dspy.configure(lm=None, adapter=None, track_usage=False)
loaded_settings = dspy.load_settings(tmp_path / "settings.pkl", allow_pickle=True)
dspy.configure(**loaded_settings)
assert dspy.settings.lm.model == "openai/gpt-4o"
assert dspy.settings.adapter is None
assert not dspy.settings.track_usage
def test_settings_save_with_extra_modules(tmp_path):
# Create a temporary Python file with our custom module
custom_module_path = tmp_path / "custom_module.py"
with open(custom_module_path, "w") as f:
f.write(
"""
def callback(x):
return x + 1
"""
)
# Add the tmp_path to Python path so we can import the module
sys.path.insert(0, str(tmp_path))
try:
import custom_module
dspy.configure(callbacks=[custom_module.callback])
settings_path = tmp_path / "settings.pkl"
sys.path.insert(0, str(tmp_path))
dspy.configure(callbacks=[custom_module.callback])
dspy.settings.save(settings_path, modules_to_serialize=[custom_module])
# Remove the custom module again to simulate it not being available at load time
sys.modules.pop("custom_module", None)
sys.path.remove(str(tmp_path))
del custom_module
dspy.configure(callbacks=None)
# Loading should now succeed and preserve the adapter instance
loaded_settings = dspy.load_settings(settings_path, allow_pickle=True)
dspy.settings.configure(**loaded_settings)
assert dspy.settings.callbacks[0](3) == 4
finally:
# Only need to clean up sys.path
if str(tmp_path) in sys.path:
sys.path.remove(str(tmp_path))