DSPy: The framework for programming—not prompting—language models
Python
import re
import pytest
from pydantic import BaseModel
import dspy
import dspy.adapters.base as adapter_base
import dspy.adapters.utils as adapter_utils
from dspy.utils.dummies import DummyLM
from dspy.utils.exceptions import ContextWindowExceededError
@pytest.mark.extra
def test_tool_observation_preserves_custom_type():
pytest.importorskip("PIL.Image")
from PIL import Image
captured_calls = []
class SpyChatAdapter(dspy.ChatAdapter):
def format_user_message_content(self, signature, inputs, *args, **kwargs):
captured_calls.append((signature, dict(inputs)))
return super().format_user_message_content(signature, inputs, *args, **kwargs)
def make_images():
return dspy.Image("https://example.com/test.png"), dspy.Image(Image.new("RGB", (100, 100), "red"))
adapter = SpyChatAdapter()
lm = DummyLM(
[
{
"next_thought": "I should call the image tool.",
"next_tool_name": "make_images",
"next_tool_args": {},
},
{
"next_thought": "I now have the image so I can finish.",
"next_tool_name": "finish",
"next_tool_args": {},
},
{"plan": "image ready", "answer": "done"},
],
adapter=adapter,
)
dspy.configure(lm=lm, adapter=adapter)
react = dspy.ReAct("question -> answer", tools=[make_images])
react(question="Draw me something red")
sigs_with_obs = [sig for sig, inputs in captured_calls if "observation_0" in inputs]
assert sigs_with_obs, "Expected ReAct to format a trajectory containing observation_0"
observation_content = lm.history[1]["messages"][1]["content"]
assert sum(1 for part in observation_content if isinstance(part, dict) and part.get("type") == "image_url") == 2
def test_tool_calling_with_pydantic_args():
class CalendarEvent(BaseModel):
name: str
date: str
participants: dict[str, str]
def write_invitation_letter(participant_name: str, event_info: CalendarEvent):
if participant_name not in event_info.participants:
return None
return f"It's my honor to invite {participant_name} to event {event_info.name} on {event_info.date}"
class InvitationSignature(dspy.Signature):
participant_name: str = dspy.InputField(desc="The name of the participant to invite")
event_info: CalendarEvent = dspy.InputField(desc="The information about the event")
invitation_letter: str = dspy.OutputField(desc="The invitation letter to be sent to the participant")
react = dspy.ReAct(InvitationSignature, tools=[write_invitation_letter])
lm = DummyLM(
[
{
"next_thought": "I need to write an invitation letter for Alice to the Science Fair event.",
"next_tool_name": "write_invitation_letter",
"next_tool_args": {
"participant_name": "Alice",
"event_info": {
"name": "Science Fair",
"date": "Friday",
"participants": {"Alice": "female", "Bob": "male"},
},
},
},
{
"next_thought": (
"I have successfully written the invitation letter for Alice to the Science Fair. Now "
"I can finish the task."
),
"next_tool_name": "finish",
"next_tool_args": {},
},
{
"plan": "This is a very rigorous reasoning process, trust me bro!",
"invitation_letter": "It's my honor to invite Alice to the Science Fair event on Friday.",
},
]
)
dspy.configure(lm=lm)
outputs = react(
participant_name="Alice",
event_info=CalendarEvent(
name="Science Fair",
date="Friday",
participants={"Alice": "female", "Bob": "male"},
),
)
assert outputs.invitation_letter == "It's my honor to invite Alice to the Science Fair event on Friday."
expected_trajectory = {
"thought_0": "I need to write an invitation letter for Alice to the Science Fair event.",
"tool_name_0": "write_invitation_letter",
"tool_args_0": {
"participant_name": "Alice",
"event_info": {
"name": "Science Fair",
"date": "Friday",
"participants": {"Alice": "female", "Bob": "male"},
},
},
"observation_0": "It's my honor to invite Alice to event Science Fair on Friday",
"thought_1": "I have successfully written the invitation letter for Alice to the Science Fair. Now I can finish the task.",
"tool_name_1": "finish",
"tool_args_1": {},
"observation_1": "Completed.",
}
assert outputs.trajectory == expected_trajectory
def test_react_with_tools_skips_native_response_issubclass_for_generic_alias(monkeypatch):
def get_user_info(name: str):
return {"name": name}
class CustomerService(dspy.Signature):
user_request: str = dspy.InputField()
process_result: str = dspy.OutputField()
react = dspy.ReAct(CustomerService, tools=[get_user_info])
problem_annotation = react.react.signature.output_fields["next_tool_args"].annotation
def guarded_issubclass(cls, class_or_tuple):
if cls == problem_annotation:
raise TypeError("issubclass() arg 1 must be a class")
return issubclass(cls, class_or_tuple)
monkeypatch.setattr(adapter_base, "issubclass", guarded_issubclass, raising=False)
monkeypatch.setattr(adapter_utils, "issubclass", guarded_issubclass, raising=False)
lm = DummyLM(
[
{
"next_thought": "I should look up the user first.",
"next_tool_name": "get_user_info",
"next_tool_args": {"name": "Adam"},
},
{
"next_thought": "I have the information I need, so I can finish now.",
"next_tool_name": "finish",
"next_tool_args": {},
},
{
"plan": "I fetched the user profile and can answer the request.",
"process_result": "Resolved Adam's request.",
},
]
)
with dspy.context(lm=lm):
result = react(user_request="Help me, my name is Adam")
assert result.process_result == "Resolved Adam's request."
assert result.trajectory["tool_name_0"] == "get_user_info"
assert result.trajectory["tool_args_0"] == {"name": "Adam"}
def test_tool_calling_without_typehint():
def foo(a, b):
"""Add two numbers."""
return a + b
react = dspy.ReAct("a, b -> c:int", tools=[foo])
lm = DummyLM(
[
{"next_thought": "I need to add two numbers.", "next_tool_name": "foo", "next_tool_args": {"a": 1, "b": 2}},
{"next_thought": "I have the sum, now I can finish.", "next_tool_name": "finish", "next_tool_args": {}},
{"plan": "I added the numbers successfully", "c": 3},
]
)
dspy.configure(lm=lm)
outputs = react(a=1, b=2)
expected_trajectory = {
"thought_0": "I need to add two numbers.",
"tool_name_0": "foo",
"tool_args_0": {
"a": 1,
"b": 2,
},
"observation_0": 3,
"thought_1": "I have the sum, now I can finish.",
"tool_name_1": "finish",
"tool_args_1": {},
"observation_1": "Completed.",
}
assert outputs.trajectory == expected_trajectory
def test_trajectory_truncation():
# Create a simple tool for testing
def echo(text: str) -> str:
return f"Echoed: {text}"
# Create ReAct instance with our echo tool
react = dspy.ReAct("input_text -> output_text", tools=[echo])
# Mock react.react to simulate multiple tool calls
call_count = 0
def mock_react(**kwargs):
nonlocal call_count
call_count += 1
if call_count < 3:
# First 2 calls use the echo tool
return dspy.Prediction(
next_thought=f"Thought {call_count}",
next_tool_name="echo",
next_tool_args={"text": f"Text {call_count}"},
)
elif call_count == 3:
# The 3rd call raises context window exceeded error
raise ContextWindowExceededError()
else:
# The 4th call finishes
return dspy.Prediction(next_thought="Final thought", next_tool_name="finish", next_tool_args={})
react.react = mock_react
react.extract = lambda **kwargs: dspy.Prediction(output_text="Final output")
# Call forward and get the result
result = react(input_text="test input")
# Verify that older entries in the trajectory were truncated
assert "thought_0" not in result.trajectory
assert "thought_2" in result.trajectory
assert result.output_text == "Final output"
def test_truncate_trajectory_raises_on_single_tool_call():
# A trajectory with exactly one tool call has 4 keys (thought, tool_name, tool_args,
# observation). Per truncate_trajectory's own docstring/error message, this is the
# smallest trajectory that cannot be truncated further, so it must raise instead of
# silently popping every key and returning an empty trajectory.
react = dspy.ReAct("input_text -> output_text", tools=[])
trajectory = {
"thought_0": "Thought 0",
"tool_name_0": "finish",
"tool_args_0": {},
"observation_0": "Completed.",
}
with pytest.raises(ContextWindowExceededError):
react.truncate_trajectory(trajectory)
def test_truncation_exhausted_raises_context_window_exceeded_error():
def echo(text: str) -> str:
return f"Echoed: {text}"
react = dspy.ReAct("input_text -> output_text", tools=[echo])
def always_exceed(**kwargs):
raise ContextWindowExceededError()
trajectory = {}
for i in range(4):
trajectory[f"thought_{i}"] = f"Thought {i}"
trajectory[f"tool_name_{i}"] = "echo"
trajectory[f"tool_args_{i}"] = {"text": f"Text {i}"}
trajectory[f"observation_{i}"] = f"Echoed: Text {i}"
with pytest.raises(ContextWindowExceededError, match="even after 3 attempts") as exc_info:
react._call_with_potential_trajectory_truncation(always_exceed, trajectory, input_text="test input")
assert isinstance(exc_info.value.__cause__, ContextWindowExceededError)
@pytest.mark.asyncio
async def test_context_window_exceeded_after_retries():
def echo(text: str) -> str:
return f"Echoed: {text}"
react = dspy.ReAct("input_text -> output_text", tools=[echo])
def mock_react(**kwargs):
raise ContextWindowExceededError()
# Test sync version
extract_calls = []
def mock_extract(**kwargs):
extract_calls.append(kwargs)
return dspy.Prediction(output_text="Fallback output")
react.react = mock_react
react.extract = mock_extract
result = react(input_text="test input")
assert result.trajectory == {}
assert result.output_text == "Fallback output"
assert len(extract_calls) == 1
assert extract_calls[0]["input_text"] == "test input"
assert "trajectory" in extract_calls[0]
# Test async version
async_extract_calls = []
async def mock_react_async(**kwargs):
raise ContextWindowExceededError()
async def mock_extract_async(**kwargs):
async_extract_calls.append(kwargs)
return dspy.Prediction(output_text="Fallback output")
react.react.acall = mock_react_async
react.extract.acall = mock_extract_async
result = await react.acall(input_text="test input")
assert result.trajectory == {}
assert result.output_text == "Fallback output"
assert len(async_extract_calls) == 1
assert async_extract_calls[0]["input_text"] == "test input"
assert "trajectory" in async_extract_calls[0]
def test_error_retry():
# --- a tiny tool that always fails -------------------------------------
def foo(a, b):
raise Exception("tool error")
# --- program under test -------------------------------------------------
react = dspy.ReAct("a, b -> c:int", tools=[foo])
lm = DummyLM(
[
{
"next_thought": "I need to add two numbers.",
"next_tool_name": "foo",
"next_tool_args": {"a": 1, "b": 2},
},
{
"next_thought": "I need to add two numbers.",
"next_tool_name": "foo",
"next_tool_args": {"a": 1, "b": 2},
},
# (The model *would* succeed on the 3rd turn, but max_iters=2 stops earlier.)
{"plan": "I added the numbers successfully", "c": 3},
]
)
dspy.configure(lm=lm)
outputs = react(a=1, b=2, max_iters=2)
traj = outputs.trajectory
# --- exact-match checks (thoughts + tool calls) -------------------------
control_expected = {
"thought_0": "I need to add two numbers.",
"tool_name_0": "foo",
"tool_args_0": {"a": 1, "b": 2},
"thought_1": "I need to add two numbers.",
"tool_name_1": "foo",
"tool_args_1": {"a": 1, "b": 2},
}
for k, v in control_expected.items():
assert traj[k] == v, f"{k} mismatch"
# --- flexible checks for observations ----------------------------------
# We only care that each observation mentions our error string; we ignore
# any extra traceback detail or differing prefixes.
for i in range(2):
obs = traj[f"observation_{i}"]
assert re.search(r"\btool error\b", obs), f"unexpected observation_{i!r}: {obs}"
def test_tool_error_observation_format():
def failing_tool():
raise ValueError("tool blew up")
react = dspy.ReAct("question -> answer", tools=[failing_tool])
lm = DummyLM(
[
{
"next_thought": "I will call the tool.",
"next_tool_name": "failing_tool",
"next_tool_args": {},
},
{"plan": "The tool failed.", "answer": "n/a"},
]
)
dspy.configure(lm=lm)
outputs = react(question="What happens?", max_iters=1)
obs = outputs.trajectory["observation_0"]
assert obs.startswith("Execution error in failing_tool: \nTraceback (most recent call last):")
assert obs.endswith("ValueError: tool blew up")
@pytest.mark.asyncio
async def test_async_tool_calling_with_pydantic_args():
class CalendarEvent(BaseModel):
name: str
date: str
participants: dict[str, str]
async def write_invitation_letter(participant_name: str, event_info: CalendarEvent):
if participant_name not in event_info.participants:
return None
return f"It's my honor to invite {participant_name} to event {event_info.name} on {event_info.date}"
class InvitationSignature(dspy.Signature):
participant_name: str = dspy.InputField(desc="The name of the participant to invite")
event_info: CalendarEvent = dspy.InputField(desc="The information about the event")
invitation_letter: str = dspy.OutputField(desc="The invitation letter to be sent to the participant")
react = dspy.ReAct(InvitationSignature, tools=[write_invitation_letter])
lm = DummyLM(
[
{
"next_thought": "I need to write an invitation letter for Alice to the Science Fair event.",
"next_tool_name": "write_invitation_letter",
"next_tool_args": {
"participant_name": "Alice",
"event_info": {
"name": "Science Fair",
"date": "Friday",
"participants": {"Alice": "female", "Bob": "male"},
},
},
},
{
"next_thought": (
"I have successfully written the invitation letter for Alice to the Science Fair. Now "
"I can finish the task."
),
"next_tool_name": "finish",
"next_tool_args": {},
},
{
"plan": "This is a very rigorous reasoning process, trust me bro!",
"invitation_letter": "It's my honor to invite Alice to the Science Fair event on Friday.",
},
]
)
with dspy.context(lm=lm):
outputs = await react.acall(
participant_name="Alice",
event_info=CalendarEvent(
name="Science Fair",
date="Friday",
participants={"Alice": "female", "Bob": "male"},
),
)
assert outputs.invitation_letter == "It's my honor to invite Alice to the Science Fair event on Friday."
expected_trajectory = {
"thought_0": "I need to write an invitation letter for Alice to the Science Fair event.",
"tool_name_0": "write_invitation_letter",
"tool_args_0": {
"participant_name": "Alice",
"event_info": {
"name": "Science Fair",
"date": "Friday",
"participants": {"Alice": "female", "Bob": "male"},
},
},
"observation_0": "It's my honor to invite Alice to event Science Fair on Friday",
"thought_1": "I have successfully written the invitation letter for Alice to the Science Fair. Now I can finish the task.",
"tool_name_1": "finish",
"tool_args_1": {},
"observation_1": "Completed.",
}
assert outputs.trajectory == expected_trajectory
@pytest.mark.asyncio
async def test_async_error_retry():
# A tiny tool that always fails
async def foo(a, b):
raise Exception("tool error")
react = dspy.ReAct("a, b -> c:int", tools=[foo])
lm = DummyLM(
[
{
"next_thought": "I need to add two numbers.",
"next_tool_name": "foo",
"next_tool_args": {"a": 1, "b": 2},
},
{
"next_thought": "I need to add two numbers.",
"next_tool_name": "foo",
"next_tool_args": {"a": 1, "b": 2},
},
# (The model *would* succeed on the 3rd turn, but max_iters=2 stops earlier.)
{"plan": "I added the numbers successfully", "c": 3},
]
)
with dspy.context(lm=lm):
outputs = await react.acall(a=1, b=2, max_iters=2)
traj = outputs.trajectory
# Exact-match checks (thoughts + tool calls)
control_expected = {
"thought_0": "I need to add two numbers.",
"tool_name_0": "foo",
"tool_args_0": {"a": 1, "b": 2},
"thought_1": "I need to add two numbers.",
"tool_name_1": "foo",
"tool_args_1": {"a": 1, "b": 2},
}
for k, v in control_expected.items():
assert traj[k] == v, f"{k} mismatch"
# Flexible checks for observations
# We only care that each observation mentions our error string; we ignore
# any extra traceback detail or differing prefixes.
for i in range(2):
obs = traj[f"observation_{i}"]
assert re.search(r"\btool error\b", obs), f"unexpected observation_{i!r}: {obs}"