DSPy: The framework for programming—not prompting—language models
Python
fix(gepa): keep trace-capture outputs aligned to the batch when the program raises (#10305)
Drew Breunig3f06959ebbb8
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.pyindex 1f8879867..183478978 100644--- a/dspy/teleprompt/gepa/gepa_utils.py+++ b/dspy/teleprompt/gepa/gepa_utils.py@@ -248,11 +248,14 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):format_failure_score=self.failure_score,callback_metadata=callback_metadata,)- return self._make_evaluation_batch(- outputs=[t["prediction"] for t in trajs],- raw_scores=[t.get("score") for t in trajs],- trajectories=trajs,- )+ # bootstrap_trace_data drops examples whose program raised, but gepa indexes results by+ # batch position, so keep outputs and scores batch-sized (dropped -> failure_score).+ outputs: list[Any] = [None] * len(batch)+ raw_scores: list[Any] = [None] * len(batch)+ for t in trajs:+ outputs[t["example_ind"]] = t["prediction"]+ raw_scores[t["example_ind"]] = t.get("score")+ return self._make_evaluation_batch(outputs=outputs, raw_scores=raw_scores, trajectories=trajs)else:evaluator = Evaluate(devset=batch,diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.pyindex 6cebd6c6b..5af96120f 100644--- a/tests/teleprompt/test_gepa.py+++ b/tests/teleprompt/test_gepa.py@@ -97,6 +97,50 @@ def test_gepa_adapter_disables_logging_on_minibatch_eval(monkeypatch, reflectionassert captured_kwargs["callback_metadata"] == expected_callback_metadata+def test_gepa_adapter_evaluate_keeps_outputs_aligned_when_program_crashes(monkeypatch):+ from dspy.teleprompt.gepa import gepa_utils++ class CrashingModule(dspy.Module):+ def forward(self, **kwargs):+ if kwargs["input"] == "crash":+ raise RuntimeError("boom")+ return dspy.Prediction(output=kwargs["input"])++ def always_one_metric(example, prediction, trace=None, pred_name=None, pred_trace=None):+ return 1.0++ adapter = gepa_utils.DspyAdapter(+ student_module=SimpleModule("input -> output"),+ metric_fn=always_one_metric,+ feedback_map={},+ failure_score=-0.5,+ )+ monkeypatch.setattr(+ gepa_utils.DspyAdapter,+ "build_program",+ lambda self, candidate: CrashingModule(),+ )++ batch = [+ Example(input="first", output="first").with_inputs("input"),+ Example(input="crash", output="crash").with_inputs("input"),+ Example(input="third", output="third").with_inputs("input"),+ ]++ with dspy.context(max_errors=100):+ result = adapter.evaluate(batch=batch, candidate={}, capture_traces=True)++ # gepa's valset evaluation indexes outputs and scores by batch position, so a+ # crashed example must keep its slot rather than shrinking the lists.+ assert len(result.outputs) == 3+ assert len(result.scores) == 3+ assert result.outputs[0].output == "first"+ assert result.outputs[1] is None+ assert result.outputs[2].output == "third"+ assert result.scores == [1.0, -0.5, 1.0]+ assert [t["example_ind"] for t in result.trajectories] == [0, 2]++def test_gepa_adapter_forwards_sparse_objective_scores():from dspy.teleprompt.gepa import gepa_utils