Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(gepa): keep trace-capture outputs aligned to the batch when the program raises (#10305)

Drew Breunig3f06959ebbb8
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.py
index 1f8879867..183478978 100644
--- a/dspy/teleprompt/gepa/gepa_utils.py
+++ b/dspy/teleprompt/gepa/gepa_utils.py
@@ -248,11 +248,14 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
format_failure_score=self.failure_score,
callback_metadata=callback_metadata,
)
- return self._make_evaluation_batch(
- outputs=[t["prediction"] for t in trajs],
- raw_scores=[t.get("score") for t in trajs],
- trajectories=trajs,
- )
+ # bootstrap_trace_data drops examples whose program raised, but gepa indexes results by
+ # batch position, so keep outputs and scores batch-sized (dropped -> failure_score).
+ outputs: list[Any] = [None] * len(batch)
+ raw_scores: list[Any] = [None] * len(batch)
+ for t in trajs:
+ outputs[t["example_ind"]] = t["prediction"]
+ raw_scores[t["example_ind"]] = t.get("score")
+ return self._make_evaluation_batch(outputs=outputs, raw_scores=raw_scores, trajectories=trajs)
else:
evaluator = Evaluate(
devset=batch,
diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.py
index 6cebd6c6b..5af96120f 100644
--- a/tests/teleprompt/test_gepa.py
+++ b/tests/teleprompt/test_gepa.py
@@ -97,6 +97,50 @@ def test_gepa_adapter_disables_logging_on_minibatch_eval(monkeypatch, reflection
assert captured_kwargs["callback_metadata"] == expected_callback_metadata
+def test_gepa_adapter_evaluate_keeps_outputs_aligned_when_program_crashes(monkeypatch):
+ from dspy.teleprompt.gepa import gepa_utils
+
+ class CrashingModule(dspy.Module):
+ def forward(self, **kwargs):
+ if kwargs["input"] == "crash":
+ raise RuntimeError("boom")
+ return dspy.Prediction(output=kwargs["input"])
+
+ def always_one_metric(example, prediction, trace=None, pred_name=None, pred_trace=None):
+ return 1.0
+
+ adapter = gepa_utils.DspyAdapter(
+ student_module=SimpleModule("input -> output"),
+ metric_fn=always_one_metric,
+ feedback_map={},
+ failure_score=-0.5,
+ )
+ monkeypatch.setattr(
+ gepa_utils.DspyAdapter,
+ "build_program",
+ lambda self, candidate: CrashingModule(),
+ )
+
+ batch = [
+ Example(input="first", output="first").with_inputs("input"),
+ Example(input="crash", output="crash").with_inputs("input"),
+ Example(input="third", output="third").with_inputs("input"),
+ ]
+
+ with dspy.context(max_errors=100):
+ result = adapter.evaluate(batch=batch, candidate={}, capture_traces=True)
+
+ # gepa's valset evaluation indexes outputs and scores by batch position, so a
+ # crashed example must keep its slot rather than shrinking the lists.
+ assert len(result.outputs) == 3
+ assert len(result.scores) == 3
+ assert result.outputs[0].output == "first"
+ assert result.outputs[1] is None
+ assert result.outputs[2].output == "third"
+ assert result.scores == [1.0, -0.5, 1.0]
+ assert [t["example_ind"] for t in result.trajectories] == [0, 2]
+
+
def test_gepa_adapter_forwards_sparse_objective_scores():
from dspy.teleprompt.gepa import gepa_utils