Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(gepa): support objective-aware optimization (#10259)

Isaac Miller638e155cf725
diff --git a/docs/docs/api/optimizers/GEPA/overview.md b/docs/docs/api/optimizers/GEPA/overview.md
index b9f3cd66a..97bef5b82 100644
--- a/docs/docs/api/optimizers/GEPA/overview.md
+++ b/docs/docs/api/optimizers/GEPA/overview.md
@@ -67,6 +67,29 @@ highest_score_achieved_per_task = new_prog.detailed_results.highest_score_achiev
best_outputs = new_prog.detailed_results.best_outputs_valset
```
+### Objective-Aware Frontier Tracking
+
+Metrics can return named `objective_scores` alongside the scalar score and feedback:
+
+```python
+def metric(gold, pred, trace=None, pred_name=None, pred_trace=None):
+ quality = measure_quality(gold, pred)
+ privacy = measure_privacy(gold, pred)
+ return dspy.Prediction(
+ score=(quality + privacy) / 2,
+ objective_scores={"quality": quality, "privacy": privacy},
+ feedback="...",
+ )
+
+gepa = dspy.GEPA(metric=metric, gepa_kwargs={"frontier_type": "objective"}, track_stats=True)
+```
+
+`frontier_type` can be `"instance"` (the default), `"objective"`, `"hybrid"` (both instance and objective),
+or `"cartesian"` (each validation-instance/objective pair). The scalar score still gates acceptance and determines
+the final candidate; objectives only affect parent and merge selection. Each objective is maximized and averaged
+over examples that report it. Predictor-level objective scores are ignored. `objective_pareto_front` contains the
+independent maximum for each objective, not nondominated objective vectors.
+
## How Does GEPA Work?
### 1. **Reflective Prompt Mutation**
diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.py
index 48be13b29..60f2fec26 100644
--- a/dspy/teleprompt/gepa/gepa.py
+++ b/dspy/teleprompt/gepa/gepa.py
@@ -74,6 +74,9 @@ class DspyGEPAResult:
- val_subscores: per-candidate scores keyed by validation instance id
- per_val_instance_best_candidates: for each val instance id, a set of candidate indices achieving the best score
- discovery_eval_counts: Budget (number of metric calls / rollouts) consumed up to the discovery of each candidate
+ - val_aggregate_subscores: per-candidate objective scores aggregated over the validation set
+ - per_objective_best_candidates: candidate indices achieving the best score for each objective
+ - objective_pareto_front: highest score achieved for each objective
- total_metric_calls: total number of metric calls made across the run
- num_full_val_evals: number of full validation evaluations performed
@@ -101,6 +104,11 @@ class DspyGEPAResult:
log_dir: str | None = None
seed: int | None = None
+ # Multi-objective data
+ val_aggregate_subscores: list[dict[str, float]] | None = None
+ per_objective_best_candidates: dict[str, set[int]] | None = None
+ objective_pareto_front: dict[str, float] | None = None
+
@property
def best_idx(self) -> int:
scores = self.val_aggregate_scores
@@ -140,6 +148,13 @@ class DspyGEPAResult:
per_val_instance_best_candidates={
val_id: list(s) for val_id, s in self.per_val_instance_best_candidates.items()
},
+ val_aggregate_subscores=self.val_aggregate_subscores,
+ per_objective_best_candidates=(
+ {objective: list(s) for objective, s in self.per_objective_best_candidates.items()}
+ if self.per_objective_best_candidates is not None
+ else None
+ ),
+ objective_pareto_front=self.objective_pareto_front,
discovery_eval_counts=self.discovery_eval_counts,
total_metric_calls=self.total_metric_calls,
num_full_val_evals=self.num_full_val_evals,
@@ -158,6 +173,9 @@ class DspyGEPAResult:
val_subscores=gepa_result.val_subscores,
per_val_instance_best_candidates=gepa_result.per_val_instance_best_candidates,
discovery_eval_counts=gepa_result.discovery_eval_counts,
+ val_aggregate_subscores=gepa_result.val_aggregate_subscores,
+ per_objective_best_candidates=gepa_result.per_objective_best_candidates,
+ objective_pareto_front=gepa_result.objective_pareto_front,
total_metric_calls=gepa_result.total_metric_calls,
num_full_val_evals=gepa_result.num_full_val_evals,
log_dir=gepa_result.run_dir,
diff --git a/dspy/teleprompt/gepa/gepa_flex_utils.py b/dspy/teleprompt/gepa/gepa_flex_utils.py
index fe55e6db1..202559cb2 100644
--- a/dspy/teleprompt/gepa/gepa_flex_utils.py
+++ b/dspy/teleprompt/gepa/gepa_flex_utils.py
@@ -234,6 +234,7 @@ def evaluate_with_trace(
)
outputs: list[Any] = [None] * len(batch)
scores: list[float] = [failure_score] * len(batch)
+ objective_scores: list[dict[str, float]] = [{} for _ in batch]
for t in trajs:
pred = t["prediction"]
outputs[t["example_ind"]] = pred
@@ -250,6 +251,17 @@ def evaluate_with_trace(
)
result = failure_score
t["score"] = result # make_reflective_dataset reads this for the (trace-aware) feedback
- score = result["score"] if hasattr(result, "score") else result
+ if isinstance(result, Prediction):
+ score = result.score
+ objectives = dict(result.get("objective_scores") or {})
+ else:
+ score = result
+ objectives = {}
scores[t["example_ind"]] = failure_score if score is None else score
- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=trajs if capture_traces else None)
+ objective_scores[t["example_ind"]] = objectives
+ return EvaluationBatch(
+ outputs=outputs,
+ scores=scores,
+ trajectories=trajs if capture_traces else None,
+ objective_scores=objective_scores if any(objective_scores) else None,
+ )
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.py
index 16d067e01..1f8879867 100644
--- a/dspy/teleprompt/gepa/gepa_utils.py
+++ b/dspy/teleprompt/gepa/gepa_utils.py
@@ -57,6 +57,13 @@ Each example contains the predictor inputs, generated outputs, and feedback from
class ScoreWithFeedback(Prediction):
score: float
feedback: str
+ objective_scores: dict[str, float] | None
+
+
+def _extract_score_and_objective_scores(score: Any) -> tuple[float | None, dict[str, float]]:
+ if isinstance(score, Prediction):
+ return score.score, dict(score.get("objective_scores") or {})
+ return score, {}
class PredictorFeedbackFn(Protocol):
@@ -201,9 +208,9 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
# (e.g., an LM provider or rate-limit error) is not the candidate's fault and must
# propagate.
logger.warning("Candidate failed to build (%s); scoring the batch as failures.", e)
- return EvaluationBatch(
+ return self._make_evaluation_batch(
outputs=[None] * len(batch),
- scores=[self.failure_score] * len(batch),
+ raw_scores=[None] * len(batch),
trajectories=[] if capture_traces else None,
)
callback_metadata = (
@@ -241,19 +248,11 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
format_failure_score=self.failure_score,
callback_metadata=callback_metadata,
)
- scores = []
- outputs = []
- for t in trajs:
- outputs.append(t["prediction"])
- if hasattr(t["prediction"], "__class__") and t.get("score") is None:
- scores.append(self.failure_score)
- else:
- score = t["score"]
- if hasattr(score, "score"):
- score = score["score"]
- scores.append(score)
-
- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=trajs)
+ return self._make_evaluation_batch(
+ outputs=[t["prediction"] for t in trajs],
+ raw_scores=[t.get("score") for t in trajs],
+ trajectories=trajs,
+ )
else:
evaluator = Evaluate(
devset=batch,
@@ -266,10 +265,22 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
callback_metadata=callback_metadata,
)
res = evaluator(program)
- outputs = [r[1] for r in res.results]
- scores = [r[2] for r in res.results]
- scores = [s["score"] if hasattr(s, "score") else s for s in scores]
- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=None)
+ return self._make_evaluation_batch(
+ outputs=[result[1] for result in res.results],
+ raw_scores=[result[2] for result in res.results],
+ trajectories=None,
+ )
+
+ def _make_evaluation_batch(self, outputs, raw_scores, trajectories) -> EvaluationBatch:
+ scores_and_objectives = [_extract_score_and_objective_scores(score) for score in raw_scores]
+ scores = [self.failure_score if score is None else score for score, _ in scores_and_objectives]
+ objective_scores = [objectives for _, objectives in scores_and_objectives]
+ return EvaluationBatch(
+ outputs=outputs,
+ scores=scores,
+ trajectories=trajectories,
+ objective_scores=objective_scores if any(objective_scores) else None,
+ )
def batch_evaluate(self, items, *, capture_traces=True):
total_threads = self.num_threads or dspy.settings.num_threads
diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.py
index 2999aa18d..6cebd6c6b 100644
--- a/tests/teleprompt/test_gepa.py
+++ b/tests/teleprompt/test_gepa.py
@@ -40,6 +40,15 @@ def simple_metric(example, prediction, trace=None, pred_name=None, pred_trace=No
return dspy.Prediction(score=example.output == prediction.output, feedback="Wrong answer.")
+def multi_objective_metric(example, prediction, trace=None, pred_name=None, pred_trace=None):
+ correct = float(example.output == prediction.output)
+ return dspy.Prediction(
+ score=(correct + 1.0) / 2,
+ objective_scores={"correctness": correct, "incorrectness": 1.0 - correct},
+ feedback="Return the expected answer.",
+ )
+
+
def bad_metric(example, prediction):
return 0.0
@@ -88,6 +97,48 @@ def test_gepa_adapter_disables_logging_on_minibatch_eval(monkeypatch, reflection
assert captured_kwargs["callback_metadata"] == expected_callback_metadata
+def test_gepa_adapter_forwards_sparse_objective_scores():
+ from dspy.teleprompt.gepa import gepa_utils
+
+ adapter = gepa_utils.DspyAdapter(SimpleModule("input -> output"), multi_objective_metric, {})
+ evaluation = adapter._make_evaluation_batch(
+ outputs=[None, None],
+ raw_scores=[
+ dspy.Prediction(score=0.75, objective_scores={"quality": 1.0, "cost": 0.5}),
+ dspy.Prediction(score=0.25),
+ ],
+ trajectories=None,
+ )
+
+ assert evaluation.scores == [0.75, 0.25]
+ assert evaluation.objective_scores == [{"quality": 1.0, "cost": 0.5}, {}]
+
+
+def test_gepa_flex_evaluation_forwards_metric_objective_scores(monkeypatch):
+ from dspy.teleprompt.gepa import gepa_flex_utils
+
+ example = Example(input="question", output="answer")
+ prediction = dspy.Prediction(output="answer")
+ monkeypatch.setattr(
+ gepa_flex_utils.bootstrap_trace_module,
+ "bootstrap_trace_data",
+ lambda **kwargs: [{"example_ind": 0, "example": example, "prediction": prediction, "trace": []}],
+ )
+
+ evaluation = gepa_flex_utils.evaluate_with_trace(
+ program=SimpleModule("input -> output"),
+ batch=[example],
+ metric_fn=multi_objective_metric,
+ num_threads=1,
+ failure_score=0.0,
+ callback_metadata={},
+ capture_traces=True,
+ )
+
+ assert evaluation.scores == [1.0]
+ assert evaluation.objective_scores == [{"correctness": 1.0, "incorrectness": 0.0}]
+
+
@pytest.fixture
def mock_mlflow():
mock_mlflow = mock.MagicMock()
@@ -602,6 +653,49 @@ def test_track_stats_result_structure():
assert isinstance(best_list, list)
+def test_multi_objective_frontier_results():
+ from gepa.strategies.candidate_selector import select_program_candidate_from_pareto_front
+
+ student = SimpleModule("input -> output")
+
+ with open("tests/teleprompt/gepa_dummy_lm.json") as f:
+ data = json.load(f)
+
+ dspy.configure(lm=DictDummyLM(data["lm"]))
+ optimizer = dspy.GEPA(
+ metric=multi_objective_metric,
+ reflection_lm=DictDummyLM(data["reflection_lm"]),
+ max_metric_calls=5,
+ track_stats=True,
+ gepa_kwargs={"frontier_type": "objective"},
+ )
+ trainset = [
+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),
+ ]
+
+ result = optimizer.compile(student, trainset=trainset, valset=trainset).detailed_results
+
+ assert result.val_aggregate_subscores is not None
+ assert result.per_objective_best_candidates is not None
+ assert result.objective_pareto_front is not None
+ assert result.per_objective_best_candidates == {"correctness": {1}, "incorrectness": {0}}
+ assert result.objective_pareto_front == {"correctness": 0.5, "incorrectness": 1.0}
+ serialized = result.to_dict()
+ assert serialized["val_aggregate_subscores"] == result.val_aggregate_subscores
+ assert serialized["per_objective_best_candidates"] == {"correctness": [1], "incorrectness": [0]}
+ assert serialized["objective_pareto_front"] == result.objective_pareto_front
+ selected_parents = {
+ select_program_candidate_from_pareto_front(
+ result.per_objective_best_candidates,
+ result.val_aggregate_scores,
+ random.Random(seed),
+ )
+ for seed in range(10)
+ }
+ assert selected_parents == {0, 1}
+
+
def test_track_best_outputs_result_structure():
"""
Verify best_outputs_valset has the correct type from GEPA 0.1.1: