DSPy: The framework for programming—not prompting—language models
Python
import csv
import json
import signal
import tempfile
import threading
from unittest.mock import patch
import pytest
import dspy
from dspy.evaluate.evaluate import Evaluate, EvaluationResult
from dspy.evaluate.metrics import answer_exact_match
from dspy.predict import Predict
from dspy.utils.callback import ACTIVE_CALL_ID, BaseCallback
from dspy.utils.dummies import DummyLM
def new_example(question, answer):
"""Helper function to create a new example."""
return dspy.Example(
question=question,
answer=answer,
).with_inputs("question")
def test_evaluate_initialization():
devset = [new_example("What is 1+1?", "2")]
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
)
assert ev.devset == devset
assert ev.metric == answer_exact_match
assert ev.num_threads is None
assert not ev.display_progress
def test_evaluate_call():
dspy.configure(
lm=DummyLM(
{
"What is 1+1?": {"answer": "2"},
"What is 2+2?": {"answer": "4"},
}
)
)
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
assert program(question="What is 1+1?").answer == "2"
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
)
score = ev(program)
assert score.score == 100.0
def test_evaluate_single_thread_runs_in_main_thread():
"""Evaluate with num_threads=1 should run in the main thread."""
dspy.configure(
lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}})
)
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
execution_threads = []
original_metric = answer_exact_match
def tracking_metric(example, prediction, **kwargs):
execution_threads.append(threading.current_thread())
return original_metric(example, prediction, **kwargs)
program = Predict("question -> answer")
ev = Evaluate(
devset=devset,
metric=tracking_metric,
display_progress=False,
num_threads=1,
)
result = ev(program)
assert result.score == 100.0
assert all(t is threading.main_thread() for t in execution_threads)
@pytest.mark.extra
def test_construct_result_df():
import pandas as pd
devset = [
new_example("What is 1+1?", "2"),
new_example("What is 2+2?", "4"),
new_example("What is 3+3?", "-1"),
]
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
)
results = [
(devset[0], {"answer": "2"}, 100.0),
(devset[1], {"answer": "4"}, 100.0),
(devset[2], {"answer": "-1"}, 0.0),
]
result_df = ev._construct_result_table(results, answer_exact_match.__name__)
pd.testing.assert_frame_equal(
result_df,
pd.DataFrame(
{
"question": ["What is 1+1?", "What is 2+2?", "What is 3+3?"],
"example_answer": ["2", "4", "-1"],
"pred_answer": ["2", "4", "-1"],
"answer_exact_match": [100.0, 100.0, 0.0],
}
),
)
def test_multithread_evaluate_call():
dspy.configure(lm=DummyLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
assert program(question="What is 1+1?").answer == "2"
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
num_threads=2,
)
result = ev(program)
assert result.score == 100.0
def test_multi_thread_evaluate_call_cancelled(monkeypatch):
from dspy.utils import parallelizer
started = threading.Barrier(3, timeout=10) # two workers and the main thread
release = threading.Event()
worker_finished = threading.Event()
pools = []
running = []
original_pool = parallelizer.ThreadPoolExecutor
def make_pool(*args, **kwargs):
pool = original_pool(*args, **kwargs)
pools.append(pool)
return pool
class SlowLM(DummyLM):
def __call__(self, *args, **kwargs):
try:
started.wait()
if not release.wait(10):
raise TimeoutError("Test did not release the evaluation worker")
return super().__call__(*args, **kwargs)
finally:
worker_finished.set()
def interrupt_when_running(futures, **kwargs):
running.extend(futures)
started.wait()
# Deliver a real SIGINT with the evaluation handler installed, without
# a timer thread that could signal an unrelated test later.
signal.raise_signal(signal.SIGINT)
pytest.fail("SIGINT did not interrupt evaluation")
monkeypatch.setattr(parallelizer, "ThreadPoolExecutor", make_pool)
monkeypatch.setattr(parallelizer, "wait", interrupt_when_running)
dspy.configure(lm=SlowLM({"What is 1+1?": {"answer": "2"}, "What is 2+2?": {"answer": "4"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
ev = Evaluate(devset=devset, metric=answer_exact_match, display_progress=False, num_threads=2)
try:
with pytest.raises(KeyboardInterrupt):
ev(program)
# Cancellation must return while the LM calls are still blocked.
assert not worker_finished.is_set()
assert len(running) == 2 and all(future.running() for future in running)
finally:
started.abort()
release.set()
# Production intentionally uses shutdown(wait=False). The test owns
# these workers and must drain them before fixtures/patches are reset.
for pool in pools:
pool.shutdown(wait=True, cancel_futures=True)
assert all(future.done() for future in running)
def test_evaluate_call_wrong_answer():
dspy.configure(lm=DummyLM({"What is 1+1?": {"answer": "0"}, "What is 2+2?": {"answer": "0"}}))
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
)
result = ev(program)
assert result.score == 0.0
@pytest.mark.extra
@pytest.mark.parametrize(
"program_with_example",
[
(Predict("question -> answer"), new_example("What is 1+1?", "2")),
# Create programs that do not return dictionary-like objects because Evaluate()
# has failed for such cases in the past
(
lambda text: Predict("text: str -> entities: list[str]")(text=text).entities,
dspy.Example(text="United States", entities=["United States"]).with_inputs("text"),
),
(
lambda text: Predict("text: str -> entities: list[dict[str, str]]")(text=text).entities,
dspy.Example(text="United States", entities=[{"name": "United States", "type": "location"}]).with_inputs(
"text"
),
),
(
lambda text: Predict("text: str -> first_word: Tuple[str, int]")(text=text).words,
dspy.Example(text="United States", first_word=("United", 6)).with_inputs("text"),
),
],
)
@pytest.mark.parametrize("display_table", [True, False, 1])
@pytest.mark.parametrize("is_in_ipython_notebook_environment", [True, False])
def test_evaluate_display_table(program_with_example, display_table, is_in_ipython_notebook_environment, capfd):
program, example = program_with_example
example_input = next(iter(example.inputs().values()))
example_output = {key: value for key, value in example.toDict().items() if key not in example.inputs()}
dspy.configure(
lm=DummyLM(
{
example_input: example_output,
}
)
)
ev = Evaluate(
devset=[example],
metric=lambda example, pred, **kwargs: example == pred,
display_table=display_table,
)
assert ev.display_table == display_table
with patch(
"dspy.evaluate.evaluate.is_in_ipython_notebook_environment", return_value=is_in_ipython_notebook_environment
):
ev(program)
out, _ = capfd.readouterr()
if not is_in_ipython_notebook_environment and display_table:
# In console environments where IPython is not available, the table should be printed
# to the console
example_input = next(iter(example.inputs().values()))
assert example_input in out
def test_evaluate_callback():
class TestCallback(BaseCallback):
def __init__(self):
self.start_call_inputs = None
self.start_call_count = 0
self.end_call_outputs = None
self.end_call_count = 0
def on_evaluate_start(
self,
call_id: str,
instance,
inputs,
):
self.start_call_inputs = inputs
self.start_call_count += 1
def on_evaluate_end(
self,
call_id: str,
outputs,
exception=None,
):
self.end_call_outputs = outputs
self.end_call_count += 1
callback = TestCallback()
dspy.configure(
lm=DummyLM(
{
"What is 1+1?": {"answer": "2"},
"What is 2+2?": {"answer": "4"},
}
),
callbacks=[callback],
)
devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
program = Predict("question -> answer")
assert program(question="What is 1+1?").answer == "2"
ev = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
)
result = ev(program)
assert result.score == 100.0
assert callback.start_call_inputs["program"] == program
assert callback.start_call_count == 1
assert callback.end_call_outputs.score == 100.0
assert callback.end_call_count == 1
def test_parallel_program_callbacks_are_children_of_evaluate_callback():
class LineageCallback(BaseCallback):
def __init__(self):
self.evaluate_call_id = None
self.module_parent_call_ids = []
self.lock = threading.Lock()
def on_evaluate_start(self, call_id, instance, inputs):
self.evaluate_call_id = call_id
def on_module_start(self, call_id, instance, inputs):
with self.lock:
self.module_parent_call_ids.append(ACTIVE_CALL_ID.get())
callback = LineageCallback()
dspy.configure(
lm=DummyLM(
{
"What is 1+1?": {"answer": "2"},
"What is 2+2?": {"answer": "4"},
}
),
callbacks=[callback],
)
evaluator = Evaluate(
devset=[new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")],
metric=answer_exact_match,
num_threads=2,
display_progress=False,
)
evaluator(Predict("question -> answer"))
assert callback.evaluate_call_id is not None
assert callback.module_parent_call_ids == [callback.evaluate_call_id, callback.evaluate_call_id]
def test_evaluation_result_repr():
result = EvaluationResult(score=100.0, results=[(new_example("What is 1+1?", "2"), {"answer": "2"}, 100.0)])
assert repr(result) == "EvaluationResult(score=100.0, results=<list of 1 results>)"
def test_evaluate_save_as_json_with_history():
"""Test that save_as_json works with Examples containing dspy.History objects."""
# Setup
dspy.settings.configure(
lm=DummyLM(
{
"What is 1+1?": {"answer": "2"},
"What is 2+2?": {"answer": "4"},
}
)
)
# Create history objects
history1 = dspy.History(
messages=[
{"question": "Previous Q1", "answer": "Previous A1"},
]
)
history2 = dspy.History(
messages=[
{"question": "Previous Q2", "answer": "Previous A2"},
{"question": "Previous Q3", "answer": "Previous A3"},
]
)
# Create examples with history
devset = [
dspy.Example(question="What is 1+1?", answer="2", history=history1).with_inputs("question"),
dspy.Example(question="What is 2+2?", answer="4", history=history2).with_inputs("question"),
]
program = Predict("question -> answer")
# Create evaluator with save_as_json
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
temp_json = f.name
try:
evaluator = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
save_as_json=temp_json,
)
result = evaluator(program)
assert result.score == 100.0
# Verify JSON file was created and is valid
with open(temp_json) as f:
data = json.load(f)
assert len(data) == 2
# Verify history was properly serialized in first record
assert "history" in data[0]
assert isinstance(data[0]["history"], dict)
assert "messages" in data[0]["history"]
assert len(data[0]["history"]["messages"]) == 1
assert data[0]["history"]["messages"][0] == {"question": "Previous Q1", "answer": "Previous A1"}
# Verify history was properly serialized in second record
assert "history" in data[1]
assert isinstance(data[1]["history"], dict)
assert "messages" in data[1]["history"]
assert len(data[1]["history"]["messages"]) == 2
assert data[1]["history"]["messages"][0] == {"question": "Previous Q2", "answer": "Previous A2"}
assert data[1]["history"]["messages"][1] == {"question": "Previous Q3", "answer": "Previous A3"}
finally:
import os
if os.path.exists(temp_json):
os.unlink(temp_json)
def test_evaluate_save_as_csv_with_history():
"""Test that save_as_csv works with Examples containing dspy.History objects."""
# Setup
dspy.settings.configure(
lm=DummyLM(
{
"What is 1+1?": {"answer": "2"},
}
)
)
# Create history object
history = dspy.History(
messages=[
{"question": "Previous Q", "answer": "Previous A"},
]
)
# Create example with history
devset = [
dspy.Example(question="What is 1+1?", answer="2", history=history).with_inputs("question"),
]
program = Predict("question -> answer")
# Create evaluator with save_as_csv
with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f:
temp_csv = f.name
try:
evaluator = Evaluate(
devset=devset,
metric=answer_exact_match,
display_progress=False,
save_as_csv=temp_csv,
)
result = evaluator(program)
assert result.score == 100.0
# Verify CSV file was created
import csv
with open(temp_csv) as f:
reader = csv.DictReader(f)
rows = list(reader)
assert len(rows) == 1
assert "history" in rows[0]
# CSV will have string representation of the dict
assert "messages" in rows[0]["history"]
finally:
import os
if os.path.exists(temp_csv):
os.unlink(temp_csv)
@pytest.mark.parametrize("example_order", [(0, 1), (1, 0)])
@pytest.mark.parametrize("num_threads", [1, 2])
@pytest.mark.parametrize("failure_score", [0.0, 0.25])
def test_evaluate_save_as_csv_with_failed_prediction(tmp_path, example_order, num_threads, failure_score):
class Program(dspy.Module):
def forward(self, x):
if x == 1:
raise RuntimeError("controlled failure")
return dspy.Prediction(answer="pred-0")
examples = [dspy.Example(x=i, answer=f"gold-{i}").with_inputs("x") for i in example_order]
csv_path = tmp_path / "results.csv"
def graded(example, prediction):
return 1.0
evaluator = Evaluate(
devset=examples,
metric=graded,
num_threads=num_threads,
max_errors=10,
display_progress=False,
failure_score=failure_score,
save_as_csv=str(csv_path),
)
result = evaluator(Program())
assert result.score == round(100 * (1 + failure_score) / 2, 2)
assert [score for _, _, score in result.results] == [1.0 if i == 0 else failure_score for i in example_order]
with csv_path.open(newline="", encoding="utf-8") as csvfile:
reader = csv.DictReader(csvfile)
rows = list(reader)
assert len(rows) == 2
assert [int(row["x"]) for row in rows] == list(example_order)
expected_fieldnames = (
["x", "example_answer", "pred_answer", "graded", "answer"]
if example_order[0] == 0
else ["x", "answer", "graded", "example_answer", "pred_answer"]
)
assert reader.fieldnames == expected_fieldnames
successful_row = rows[example_order.index(0)]
failed_row = rows[example_order.index(1)]
assert successful_row["example_answer"] == "gold-0"
assert successful_row["pred_answer"] == "pred-0"
assert successful_row["answer"] == ""
assert successful_row["graded"] == "1.0"
assert failed_row["answer"] == "gold-1"
assert failed_row["example_answer"] == ""
assert failed_row["pred_answer"] == ""
assert failed_row["graded"] == str(failure_score)
def test_evaluate_save_as_csv_with_later_prediction_fields(tmp_path):
class Program(dspy.Module):
def forward(self, x):
if x == 0:
return dspy.Prediction(answer="pred-0")
return dspy.Prediction(answer="pred-1", explanation="later field")
examples = [dspy.Example(x=i, answer=f"gold-{i}").with_inputs("x") for i in range(2)]
csv_path = tmp_path / "results.csv"
def graded(example, prediction):
return 1.0
evaluator = Evaluate(
devset=examples,
metric=graded,
display_progress=False,
save_as_csv=str(csv_path),
)
result = evaluator(Program())
assert result.score == 100.0
with csv_path.open(newline="", encoding="utf-8") as csvfile:
reader = csv.DictReader(csvfile)
rows = list(reader)
assert reader.fieldnames == ["x", "example_answer", "pred_answer", "graded", "explanation"]
assert rows[0]["explanation"] == ""
assert rows[1]["explanation"] == "later field"
def test_evaluate_raises_on_empty_devset():
program = Predict("question -> answer")
ev = Evaluate(
devset=[],
metric=answer_exact_match,
display_progress=False,
)
with pytest.raises(ValueError, match="devset"):
ev(program)