DSPy: The framework for programming—not prompting—language models
Python
import pytest
import dspy
from dspy import Example
from dspy.predict import Predict
from dspy.teleprompt import BootstrapFewShot
from dspy.utils.dummies import DummyLM
# Define a simple metric function for testing
def simple_metric(example, prediction, trace=None):
# Simplified metric for testing: true if prediction matches expected output
return example.output == prediction.output
examples = [
Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!"),
]
trainset = [examples[0]]
valset = [examples[1]]
def test_bootstrap_initialization():
# Initialize BootstrapFewShot with a dummy metric and minimal setup
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
assert bootstrap.metric == simple_metric, "Metric not correctly initialized"
class SimpleModule(dspy.Module):
def __init__(self, signature):
super().__init__()
self.predictor = Predict(signature)
def forward(self, **kwargs):
return self.predictor(**kwargs)
class TwoPredictorModule(dspy.Module):
def __init__(self):
super().__init__()
self.p1 = Predict("input -> output")
self.p2 = Predict("input -> output")
def forward(self, **kwargs):
return self.p2(**self.p1(**kwargs))
def test_bootstrap_labeled_demos_sampled_independently_per_predictor():
# Each predictor must draw its labeled demos from the FULL validation pool.
# Regression: _train reassigned the shared `raw_demos` to the sampled subset
# inside the per-predictor loop, so later predictors sampled from the prior
# predictor's leftovers -- starving them when an earlier predictor consumed
# most of the pool via bootstrapped (augmented) demos.
student = TwoPredictorModule()
names = [name for name, _ in student.named_predictors()]
pool = [Example(input=f"q{i}", output=f"a{i}").with_inputs("input") for i in range(10)]
augmented = [Example(input=f"aug{i}", output=f"x{i}").with_inputs("input") for i in range(5)]
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=5, max_labeled_demos=8)
bootstrap.student = student
bootstrap.validation = pool
# First predictor has 5 bootstrapped demos; second has none.
bootstrap.name2traces = {names[0]: list(augmented), names[1]: []}
bootstrap._train()
p1, p2 = (p for _, p in student.named_predictors())
# p1: 5 augmented + 3 labeled = 8
assert len(p1.demos) == 8
# p2: 0 augmented, so it must still get the full labeled quota from the full pool.
assert len(p2.demos) == 8
def test_compile_with_predict_instances():
# Create Predict instances for student and teacher
# Note that dspy.Predict is not itself a module, so we can't use it directly here
student = SimpleModule("input -> output")
teacher = SimpleModule("input -> output")
lm = DummyLM(["Initial thoughts", "Finish[blue]"])
dspy.configure(lm=lm)
# Initialize BootstrapFewShot and compile the student
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
compiled_student = bootstrap.compile(student, teacher=teacher, trainset=trainset)
assert compiled_student is not None, "Failed to compile student"
assert hasattr(compiled_student, "_compiled") and compiled_student._compiled, "Student compilation flag not set"
def test_bootstrap_effectiveness():
# This test verifies if the bootstrapping process improves the student's predictions
student = SimpleModule("input -> output")
teacher = SimpleModule("input -> output")
lm = DummyLM([{"output": "blue"}, {"output": "Ring-ding-ding-ding-dingeringeding!"}], follow_examples=True)
dspy.configure(lm=lm, trace=[])
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
compiled_student = bootstrap.compile(student, teacher=teacher, trainset=trainset)
# Check that the compiled student has the correct demos
assert len(compiled_student.predictor.demos) == 1
assert compiled_student.predictor.demos[0].input == trainset[0].input
assert compiled_student.predictor.demos[0].output == trainset[0].output
# Test the compiled student's prediction.
# We are using a DummyLM with follow_examples=True, which means that
# even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"
# on the second output, if it seems an example that perfectly matches the
# prompt, it will use that instead. That is why we expect "blue" here.
prediction = compiled_student(input=trainset[0].input)
assert prediction.output == trainset[0].output
def test_error_handling_during_bootstrap():
"""
Test to verify error handling during the bootstrapping process
"""
class BuggyModule(dspy.Module):
def __init__(self, signature):
super().__init__()
self.predictor = Predict(signature)
def forward(self, **kwargs):
raise RuntimeError("Simulated error")
student = SimpleModule("input -> output")
teacher = BuggyModule("input -> output")
# Setup DummyLM to simulate an error scenario
lm = DummyLM(
[
{"output": "Initial thoughts"}, # Simulate initial teacher's prediction
]
)
dspy.configure(lm=lm)
bootstrap = BootstrapFewShot(
metric=simple_metric,
max_bootstrapped_demos=1,
max_labeled_demos=1,
max_errors=1,
)
with pytest.raises(RuntimeError, match="Simulated error"):
bootstrap.compile(student, teacher=teacher, trainset=trainset)
def test_validation_set_usage():
"""
Test to ensure the validation set is correctly used during bootstrapping
"""
student = SimpleModule("input -> output")
teacher = SimpleModule("input -> output")
lm = DummyLM(
[
{"output": "Initial thoughts"},
{"output": "Finish[blue]"}, # Expected output for both training and validation
]
)
dspy.configure(lm=lm)
bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
compiled_student = bootstrap.compile(student, teacher=teacher, trainset=trainset)
# Check that validation examples are part of student's demos after compilation
assert len(compiled_student.predictor.demos) >= len(valset), "Validation set not used in compiled student demos"
@pytest.mark.parametrize(
("metric_value", "metric_threshold", "expected_demos"),
[
(0.0, 0.0, 1),
(-0.1, 0.0, 0),
(0.0, 0.01, 0),
(0.0, -0.01, 1),
# Bool metrics are compared numerically too, so False clears a threshold of 0.
(True, 0, 1),
(False, 0, 1),
],
)
def test_metric_threshold_zero_is_applied(metric_value, metric_threshold, expected_demos):
# A threshold of 0.0 must be compared numerically rather than treated as unset.
def metric(example, prediction, trace=None):
return metric_value
dspy.configure(lm=DummyLM([{"output": "blue"}]))
bootstrap = BootstrapFewShot(
metric=metric,
metric_threshold=metric_threshold,
max_bootstrapped_demos=1,
max_labeled_demos=0,
)
compiled_student = bootstrap.compile(
SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset
)
assert len(compiled_student.predictor.demos) == expected_demos
@pytest.mark.parametrize(
("metric_value", "expected_demos"),
[
(0.0, 0),
(False, 0),
(1.0, 1),
(True, 1),
],
)
def test_metric_threshold_none_uses_metric_value_as_success(metric_value, expected_demos):
# With no threshold, the metric's own (truthy/falsy) value decides success.
def metric(example, prediction, trace=None):
return metric_value
dspy.configure(lm=DummyLM([{"output": "blue"}]))
bootstrap = BootstrapFewShot(
metric=metric,
metric_threshold=None,
max_bootstrapped_demos=1,
max_labeled_demos=0,
)
compiled_student = bootstrap.compile(
SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset
)
assert len(compiled_student.predictor.demos) == expected_demos