DSPy: The framework for programming—not prompting—language models
Python
"""Decision parameters exercised through Predict, independent of provider transport."""
import copy
import json
from typing import Annotated, Literal, get_args
import pytest
import dspy
from dspy.clients.engines.dummy_engine import AsyncDummyEngine
from dspy.experimental import Choice, Noul, Score
from dspy.lm15 import Message, Request, Response, Usage
Rating = Score["bad", "fair", "great"]
Label = Choice[(2, ""), ("other", "")]
class DecisionEngine:
supports_decision_requests = True
def __init__(self, owner):
self.owner = owner
def complete(self, request):
assert isinstance(request, Request)
state = request.messages[0].parts[0].value
questions = request.config.extensions["questions"]
self.owner.calls.append(copy.deepcopy((state, questions)))
response = Response(id=None, model=request.model, message=Message.assistant(""), finish_reason="stop", usage=Usage(),
provider_data={"typesafe": {}})
# Inject raw evidence after construction so malformed-evidence tests reach DSPy's decoder,
# including non-JSON keys and nonfinite values rejected by Response's generic JSON validation.
response.provider_data["typesafe"]["answers"] = self.owner.evidence(state, questions)
return response
class FakeClient(dspy.LM):
def __init__(self, probability=0.8, choice="2", answers=None):
engine = DecisionEngine(self)
super().__init__("typesafe:jev-test", engine=engine, async_engine=AsyncDummyEngine(engine), cache=False)
self.probability = probability
self.choice = choice
self.answers = answers
self.calls = []
def evidence(self, state, questions):
if self.answers is not None:
return copy.deepcopy(self.answers)
answers = {}
for name, question in questions.items():
if question["type"] == "noul":
answers[name] = {"noul": self.probability}
elif question["type"] == "score":
answers[name] = {"score": 1.7, "confidence": 0.61, "probabilities": {0: 0.1, 1: 0.3, 2: 0.6}}
else:
answers[name] = {
"choice": self.choice,
"confidence": 0.73,
"probabilities": {key: 0.8 if key == self.choice else 0.2 for key in question["criteria"]},
}
return answers
def signature(rich=False):
return dspy.Signature(
{
"text": (str, dspy.InputField()),
"flag": (Noul if rich else bool, dspy.OutputField(desc="Is it relevant?")),
"rating": (Rating, dspy.OutputField(desc="Rate usefulness.")),
"label": (Label if rich else Literal[2, "other"], dspy.OutputField(desc="Classify the document.")),
},
"Assess the document.",
)
def predict(rich=False, client=None):
module = dspy.Predict(signature(rich))
module.set_lm(client)
# Opt outputs into configurable evidence decoding without pinning numeric defaults.
module.fields = {"flag": {}, "rating": {}, "label": {}}
return module
@pytest.mark.parametrize("overrides", [{}, {"flag": {"threshold": 0.9}}])
def test_only_explicit_overrides_survive_inference_and_save_load(tmp_path, overrides):
module = dspy.Predict(signature(True))
assert module.fields == {}
module.fields = copy.deepcopy(overrides)
assert module.get_criteria("rating") == ["bad", "fair", "great"]
result = module(text="x", lm=FakeClient())
assert result.flag.value is (not bool(overrides))
assert result.rating.level == 2
assert result.label.value == 2
assert module.fields == overrides
path = tmp_path / "overrides.json"
module.save(path)
assert json.loads(path.read_text()).get("fields", {}) == overrides
restored = dspy.Predict(signature(True))
restored.load(path)
assert restored(text="x", lm=FakeClient()).toDict() == result.toDict()
assert restored.fields == overrides
def test_native_rich_equivalence_and_request_mapping():
client = FakeClient()
native, rich = predict(client=client), predict(True, client)
native.fields["flag"]["threshold"] = rich.fields["flag"]["threshold"] = 0.7
a, b = native(text="example"), rich(text="example")
assert a.flag is b.flag.value is True
assert a.rating.value == b.rating.value == pytest.approx(1.5)
assert a.label == b.label.value == 2
assert type(a.label) is type(b.label.value) is int
assert b.flag.confidence == pytest.approx(1 / 7)
assert b.flag.probability == 0.8
assert b.rating.confidence == 0.61
assert b.label.confidence == 0.73
assert client.calls[0] == client.calls[1]
state, questions = client.calls[0]
assert state == {
"instructions": "Assess the document.",
"input_fields": "1. `text` (str):",
"inputs": {"text": "example"},
}
assert questions == {
"flag": {"type": "noul", "instructions": "Is it relevant?"},
"rating": {"type": "score", "instructions": "Rate usefulness.", "criteria": ["bad", "fair", "great"]},
"label": {"type": "choice", "instructions": "Classify the document.", "criteria": {"2": None, "other": None}},
}
@pytest.mark.parametrize(
"p,value,confidence", [(0, False, 1), (0.6, False, 0.2), (0.75, True, 0), (0.9, True, 0.2), (1, True, 1 / 3)]
)
def test_threshold_boundary_and_confidence(p, value, confidence):
module = predict(True, FakeClient(p))
module.fields["flag"]["threshold"] = 0.75
result = module(text="x").flag
assert result.value is value
assert result.confidence == pytest.approx(confidence)
assert bool(result) is value
@pytest.mark.parametrize("threshold,p,value", [(0, 0, True), (1, 0.99, False), (1, 1, True)])
def test_threshold_endpoints(threshold, p, value):
module = predict(client=FakeClient(p))
module.fields["flag"]["threshold"] = threshold
assert module(text="x").flag is value
@pytest.mark.parametrize(
"options,selected,expected",
[
(Literal[1, 2], "2", 2),
(Literal[False, True], "False", False),
(Literal[True, 1, "other"], "1", 1),
(Literal[None, "other"], "None", None),
(Literal["a", "b"], "b", "b"),
],
)
@pytest.mark.parametrize("rich", [False, True])
def test_literal_membership_and_exact_type(options, selected, expected, rich):
kind = Choice[tuple((v, "") for v in get_args(options))] if rich else options
sig = signature().with_updated_fields("label", type_=kind)
result = dspy.Predict(sig)(text="x", lm=FakeClient(choice=selected)).label
value = result.value if rich else result
assert value == expected
assert type(value) is type(expected)
@pytest.mark.parametrize("cuts,level", [([0.5, 1.5], 2), ([0.5, 1.6], 1), ([0.5, 1.4], 2)])
def test_score_cuts_normalization_and_snapshot(cuts, level):
client = FakeClient()
module = predict(True, client)
before = module(text="x").rating
module.fields["rating"]["cuts"] = cuts
after = module(text="x").rating
assert before.value == after.value == pytest.approx(1.5)
assert before.level == 2 and after.level == level
assert before.confidence == after.confidence == 0.61
assert client.calls[0] == client.calls[1]
answers = client.evidence(*client.calls[0])
answers["rating"]["probabilities"] = {0: 0.099, 1: 0.297, 2: 0.594}
result = module(text="x", lm=FakeClient(answers=answers)).rating
assert result.value == pytest.approx(1.5)
assert sum(result.probabilities.values()) == pytest.approx(0.99)
@pytest.mark.parametrize("rich", [False, True])
def test_choice_weights_selection_evidence_and_request(rich):
client = FakeClient()
module = predict(rich, client)
before = module(text="x").label
module.fields["label"]["weights"] = {"2": 0.1}
after = module(text="x").label
assert (before.value if rich else before) == 2
assert (after.value if rich else after) == "other"
assert client.calls[0] == client.calls[1]
if rich:
assert after.probabilities == before.probabilities == {"2": 0.8, "other": 0.2}
assert after.confidence == before.confidence == 0.73
module.fields["label"]["weights"] = {"2": 10, "other": 100}
result = module(text="x").label
assert (result.value if rich else result) == "other"
@pytest.mark.parametrize("weights", [{}, {"2": 1}, {"2": 0.25}, {"2": 0}])
def test_choice_weights_defaults_ties_and_zero(weights):
module = predict(True, FakeClient())
module.fields["label"]["weights"] = weights
assert module(text="x").label.value == ("other" if weights.get("2") == 0 else 2)
def test_choice_weighted_tie_prefers_raw_winner():
module = predict(True, FakeClient(choice="other"))
module.fields["label"]["weights"] = {"other": 0.25}
assert module(text="x").label.value == "other"
client = FakeClient(
answers={
"flag": {"noul": 0.8},
"rating": {"probabilities": {0: 0.1, 1: 0.3, 2: 0.6}, "confidence": 0.6},
"label": {"probabilities": {"2": 1, "other": 0}, "confidence": 0.7},
}
)
module.fields["label"]["weights"] = {"2": 0}
with pytest.raises(ValueError, match="no positive probability mass"):
module(text="x", lm=client)
@pytest.mark.parametrize(
"field,config",
[
("flag", {"threshold": -0.1}),
("flag", {"threshold": float("nan")}),
("missing", {"threshold": 0.3}),
("flag", None),
("rating", {"cuts": [0.5, 1.5], "weights": [0, 3]}),
("flag", {"threshold": 0.5, "unknown": "typo"}),
*[
("label", {"weights": w})
for w in [
{"unknown": 1},
{2: 1},
{"2": -1},
{"2": float("nan")},
{"2": float("inf")},
{"2": True},
{"2": 0, "other": 0},
[1, 2],
]
],
*[
("rating", {"cuts": c})
for c in [
[0, 1.5],
[0.5, 2],
[1.5, 0.5],
[0.5, 0.5],
[0.5],
[True, 1.5],
[0.5, float("nan")],
]
],
],
)
@pytest.mark.parametrize("operation", ["call", "save", "load"])
def test_invalid_parameters_rejected(field, config, operation, tmp_path):
module = predict(True)
original = module.dump_state()
client = FakeClient()
if operation == "load":
state = copy.deepcopy(original)
state["fields"][field] = config
with pytest.raises(ValueError):
module.load_state(state)
assert module.dump_state() == original
else:
module.fields[field] = config
with pytest.raises(ValueError):
if operation == "call":
module(text="x", lm=client)
else:
module.save(tmp_path / "invalid.json")
assert not client.calls
@pytest.mark.parametrize(
"field,update",
[
("flag", {"noul": 1.1}),
("flag", {"noul": float("nan")}),
("rating", {"probabilities": {0: 0.1, 1: 0.9}}),
("rating", {"probabilities": {0: 0, 1: 0, 2: 0}}),
("rating", {"probabilities": {0: -0.1, 1: 0.3, 2: 0.8}}),
("rating", {"probabilities": {0.9: 0.1, 1.9: 0.3, 2.9: 0.6}}),
("rating", {"probabilities": {False: 0.1, True: 0.3, 2: 0.6}}),
("rating", {"probabilities": {"00": 0.1, "1": 0.3, "2": 0.6}}),
("rating", {"probabilities": {0: 0.1, "0": 0.1, 1: 0.3, 2: 0.6}}),
("label", {"probabilities": {"unknown": 1}}),
("label", {"confidence": 1.2}),
],
)
def test_malformed_evidence_is_rejected(field, update):
client = FakeClient()
module = predict(True, client)
module(text="x")
answers = client.evidence(*client.calls[0])
answers[field].update(update)
with pytest.raises(ValueError):
module(text="x", lm=FakeClient(answers=answers))
@pytest.mark.parametrize("rich", [False, True])
@pytest.mark.parametrize("operation", ["save", "load"])
@pytest.mark.parametrize(
"field,invalid",
[
("flag", ["yes", "no"]),
("flag", {"yes": "Relevant"}),
("label", ["Known", "Other"]),
("label", {"2": "Known"}),
("label", {"2": "Known", "other": None, "unknown": "Extra"}),
("rating", {"0": "bad", "1": "fair", "2": "great"}),
("rating", ["bad", "great"]),
],
)
def test_criteria_shape_validated_on_save_and_load(tmp_path, rich, operation, field, invalid):
module = predict(rich)
original = module.dump_state()
if operation == "save":
module.fields[field]["criteria"] = invalid
with pytest.raises(ValueError, match="criteria"):
module.save(tmp_path / "invalid.json")
else:
state = copy.deepcopy(original)
state["fields"][field]["criteria"] = invalid
with pytest.raises(ValueError, match="criteria"):
module.load_state(state)
assert module.dump_state() == original
@pytest.mark.parametrize("entry", [{"nested": {1: "integer key"}}, {"nested": float("nan")}, {"nested": {"set"}}, 42])
def test_instructions_validate_json(entry):
module = predict(True)
module.fields["flag"]["instructions"] = entry
with pytest.raises(ValueError, match="instructions"):
module.dump_state()
@pytest.mark.parametrize("setting", ["instructions", "criteria"])
def test_nested_mutations_are_revalidated(setting):
client = FakeClient()
module = predict(True, client)
examples = ["Relevant"]
module.fields["flag"][setting] = {"true": {"examples": examples}}
module(text="x")
examples.append("Another example")
module(text="x")
assert client.calls[-1][1]["flag"][setting]["true"]["examples"] == examples
for invalid in ({"not JSON"}, float("nan")):
examples.append(invalid)
with pytest.raises(ValueError, match=setting):
module(text="x")
examples.pop()
assert len(client.calls) == 2
def test_inflight_decoding_uses_configuration_snapshot():
class UpdatingClient(FakeClient):
def evidence(self, state, questions):
module.fields["flag"]["threshold"] = 0.9
module.fields["rating"]["cuts"][1] = 1.6
module.fields["label"]["weights"]["2"] = 0.1
return super().evidence(state, questions)
module = predict(True, UpdatingClient())
module.fields["rating"] = {"cuts": [0.5, 1.5]}
module.fields["label"] = {"weights": {"2": 1.0, "other": 1.0}}
first = module(text="x")
second = module(text="x")
assert first.flag.value is True and second.flag.value is False
assert first.rating.level == 2 and second.rating.level == 1
assert first.label.value == 2 and second.label.value == "other"
assert first.rating.probabilities == second.rating.probabilities
assert first.label.probabilities == second.label.probabilities
def test_json_state_preserves_instructions_criteria_and_numeric_settings(tmp_path):
module = predict(True)
module.fields = {
"flag": {
"threshold": 0.7,
"instructions": {"focus": ["impact"]},
"criteria": {"true": {"examples": ["Outage"]}, "false": None},
},
"rating": {
"cuts": [0.4, 1.6],
"instructions": ["Assess severity.", {"ignore": "tone"}],
"criteria": [{"what": "bad", "not_for": "cosmetic"}, "fair", {"examples": ["Resolved"]}],
},
"label": {
"weights": {"2": 0.25, "other": 1.5},
"instructions": "Choose category.",
"criteria": {"2": {"what": "Known"}, "other": None},
},
}
expected = copy.deepcopy(module.fields)
path = tmp_path / "parameters.json"
module.save(path)
assert json.loads(path.read_text())["fields"] == expected
restored = predict(True)
restored.load(path)
client = FakeClient()
assert module(text="x", lm=client).toDict() == restored(text="x", lm=client).toDict()
assert client.calls[0] == client.calls[1]
for name, config in expected.items():
question = client.calls[0][1][name]
assert question["instructions"] == config["instructions"]
assert question["criteria"] == config["criteria"]
assert not {"threshold", "cuts", "weights"} & question.keys()
restored.fields["flag"]["instructions"]["focus"].append("urgency")
restored.fields["rating"]["cuts"][0] = 0.2
assert module.fields == expected
@pytest.mark.parametrize(
"kind,expected",
[(Noul, None), (Noul[(True, "blocked")], {"true": "blocked"}),
(Rating, ["bad", "fair", "great"]), (Label, {"2": None, "other": None})],
)
def test_get_criteria_does_not_require_request_instructions(kind, expected):
sig = dspy.Signature({"flag": (kind, dspy.OutputField())})
module = dspy.Predict(sig)
assert module.get_criteria("flag") == expected
with pytest.raises(ValueError, match="requires an OutputField"):
module(lm=FakeClient())
def test_set_criteria_validates_replacement_and_preserves_other_settings():
module = predict(True)
module.fields["flag"].update(threshold=0.8, criteria={"wrong": "invalid"})
before = copy.deepcopy(module.fields)
with pytest.raises(ValueError, match="criteria"):
module.set_criteria("flag", {"also_wrong": "invalid"})
assert module.fields == before
replacement = {"true": {"examples": ["Outage"]}}
module.set_criteria("flag", replacement)
replacement["true"]["examples"].clear()
assert module.get_criteria("flag") == {"true": {"examples": ["Outage"]}}
assert module.fields["flag"]["threshold"] == 0.8
assert module.fields["rating"] == before["rating"]
assert module.fields["label"] == before["label"]
@pytest.mark.parametrize("rich", [False, True])
def test_criteria_accessors_and_whole_program_roundtrip(tmp_path, rich):
availability = Noul[(False, "Workaround available")]
sig = signature(rich).with_updated_fields("flag", type_=availability if rich else Annotated[bool, availability])
module = dspy.Predict(sig)
assert module.get_criteria("flag") == {"false": "Workaround available"}
criteria = {"true": {"examples": ["Outage"]}, "false": None}
module.set_criteria("flag", criteria)
criteria["true"]["examples"].clear()
retrieved = module.get_criteria("flag")
assert retrieved["true"]["examples"] == ["Outage"]
retrieved["true"]["examples"].clear()
assert module.get_criteria("flag")["true"]["examples"] == ["Outage"]
module.fields["flag"]["threshold"] = 0.8
module.save(tmp_path / "program", save_program=True)
restored = dspy.load(tmp_path / "program", allow_pickle=True)
result = restored(text="x", lm=FakeClient()).flag
assert (result.value if rich else result) is True
assert restored.fields == module.fields
module.set_criteria("flag", None)
assert module.get_criteria("flag") is None
def test_null_instructions_and_criteria_survive_reload(tmp_path):
module = predict(True)
module.fields["flag"].update(instructions=None, criteria=None)
module.save(tmp_path / "null.json")
restored = predict(True)
restored.load(tmp_path / "null.json")
client = FakeClient()
restored(text="x", lm=client)
assert client.calls[0][1]["flag"] == {"type": "noul", "instructions": None, "criteria": None}
def test_reset_copy_and_endpoint_state(tmp_path):
module = predict(True, dspy.LM("typesafe:jev-test", engine="lm15", api_key="test-credential", api_base="https://example.test"))
module.fields["flag"]["threshold"] = 0.9
module.demos = [{"text": "example", "flag": False}]
duplicate = module.reset_copy()
assert duplicate.lm is None and duplicate.demos == []
duplicate.fields["flag"]["threshold"] = 0.5
assert module.fields["flag"]["threshold"] == 0.9
path = tmp_path / "client.json"
module.save(path)
assert "test-credential" not in path.read_text()
restored = predict(True)
restored.load(path)
assert restored.lm.kwargs.get("api_base") != "https://example.test"
restored.load(path, allow_unsafe_lm_state=True)
assert restored.lm.kwargs["api_base"] == "https://example.test"
assert restored.lm.kwargs.get("api_key") is None
@pytest.mark.asyncio
async def test_client_resolution_batch_and_input_context():
module = predict(True)
client = FakeClient(0.1)
with dspy.context(lm=client):
assert (await module.acall(text="x")).flag.value is False
assert (await module.acall(text="x", lm=FakeClient(0.9))).flag.value is True
results = module.batch([dspy.Example(text=str(i)).with_inputs("text") for i in range(3)], num_threads=2)
assert len(results) == 3 and all(result.rating.value == pytest.approx(1.5) for result in results)
sig = dspy.Signature(
{"inputs": (str, dspy.InputField()), "flag": (bool, dspy.OutputField(desc="Is it actionable?"))},
"Shared task context.",
)
dspy.Predict(sig)(inputs="User content", lm=client)
state, questions = client.calls[-1]
assert state["inputs"] == {"inputs": "User content"}
assert state["instructions"] == "Shared task context."
assert "Shared task context." not in json.dumps(questions)
@pytest.mark.parametrize(
"base,override",
[
(Score["low", "high"], Rating),
(Rating, Score["great", "fair", "bad"]),
(Literal[True], Literal[1]),
(Literal["a", "b"], Literal["a", "c"]),
(bool, Literal[False, True]),
],
)
@pytest.mark.asyncio
async def test_output_override_preserves_answer_space(base, override):
sig = signature().with_updated_fields("flag", type_=base)
client = FakeClient()
module = dspy.Predict(sig)
module.set_lm(client)
module.fields.setdefault("flag", {})
changed = sig.with_updated_fields("flag", type_=override)
with pytest.raises(ValueError, match="preserve the answer space"):
module(text="x", signature=changed)
with pytest.raises(ValueError, match="preserve the answer space"):
await module.acall(text="x", signature=changed)
assert not client.calls
@pytest.mark.parametrize("kind", [str, int, float, Choice])
def test_unsupported_decision_outputs_fail_before_request(kind):
client = FakeClient()
module = dspy.Predict(signature().with_updated_fields("flag", type_=kind))
module.set_lm(client)
with pytest.raises(ValueError, match="Unsupported"):
module(text="x")
assert not client.calls
def test_mixed_program_discovery_demos_trace_and_persistence(tmp_path):
from dspy.utils.dummies import DummyLM
class Pipeline(dspy.Module):
def __init__(self):
sig = dspy.Signature(
{
"text": (str, dspy.InputField()),
"flag": (Annotated[bool, Noul], dspy.OutputField(desc="Is it relevant?")),
}
)
self.assess = dspy.Predict(sig)
self.explain = dspy.Predict("flag: bool -> explanation: str")
def forward(self, text):
return self.explain(flag=self.assess(text=text).flag)
program = Pipeline()
program.assess.fields["flag"] = {"threshold": 0.9}
assert program.named_predictors() == [("assess", program.assess), ("explain", program.explain)]
trained = dspy.LabeledFewShot(k=1).compile(
program, trainset=[dspy.Example(text="x", flag=False, explanation="Below threshold").with_inputs("text")]
)
assert len(trained.assess.demos) == len(trained.explain.demos) == 1
trained.save(tmp_path / "mixed.json")
restored = Pipeline()
restored.load(tmp_path / "mixed.json")
assert restored.assess.fields == {"flag": {"threshold": 0.9}}
restored.assess.lm = FakeClient()
with dspy.context(lm=DummyLM([{"explanation": "Below threshold"}]), trace=[]):
assert restored(text="x").explanation == "Below threshold"
assert [step[0] for step in dspy.settings.trace] == [restored.assess, restored.explain]
assert dspy.settings.trace[1][1] == {"flag": False}