DSPy: The framework for programming—not prompting—language models
Python
feat(gepa): support objective-aware optimization (#10259)
Isaac Miller638e155cf725
diff --git a/docs/docs/api/optimizers/GEPA/overview.md b/docs/docs/api/optimizers/GEPA/overview.mdindex b9f3cd66a..97bef5b82 100644--- a/docs/docs/api/optimizers/GEPA/overview.md+++ b/docs/docs/api/optimizers/GEPA/overview.md@@ -67,6 +67,29 @@ highest_score_achieved_per_task = new_prog.detailed_results.highest_score_achievbest_outputs = new_prog.detailed_results.best_outputs_valset```+### Objective-Aware Frontier Tracking++Metrics can return named `objective_scores` alongside the scalar score and feedback:++```python+def metric(gold, pred, trace=None, pred_name=None, pred_trace=None):+ quality = measure_quality(gold, pred)+ privacy = measure_privacy(gold, pred)+ return dspy.Prediction(+ score=(quality + privacy) / 2,+ objective_scores={"quality": quality, "privacy": privacy},+ feedback="...",+ )++gepa = dspy.GEPA(metric=metric, gepa_kwargs={"frontier_type": "objective"}, track_stats=True)+```++`frontier_type` can be `"instance"` (the default), `"objective"`, `"hybrid"` (both instance and objective),+or `"cartesian"` (each validation-instance/objective pair). The scalar score still gates acceptance and determines+the final candidate; objectives only affect parent and merge selection. Each objective is maximized and averaged+over examples that report it. Predictor-level objective scores are ignored. `objective_pareto_front` contains the+independent maximum for each objective, not nondominated objective vectors.+## How Does GEPA Work?### 1. **Reflective Prompt Mutation**diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.pyindex 48be13b29..60f2fec26 100644--- a/dspy/teleprompt/gepa/gepa.py+++ b/dspy/teleprompt/gepa/gepa.py@@ -74,6 +74,9 @@ class DspyGEPAResult:- val_subscores: per-candidate scores keyed by validation instance id- per_val_instance_best_candidates: for each val instance id, a set of candidate indices achieving the best score- discovery_eval_counts: Budget (number of metric calls / rollouts) consumed up to the discovery of each candidate+ - val_aggregate_subscores: per-candidate objective scores aggregated over the validation set+ - per_objective_best_candidates: candidate indices achieving the best score for each objective+ - objective_pareto_front: highest score achieved for each objective- total_metric_calls: total number of metric calls made across the run- num_full_val_evals: number of full validation evaluations performed@@ -101,6 +104,11 @@ class DspyGEPAResult:log_dir: str | None = Noneseed: int | None = None+ # Multi-objective data+ val_aggregate_subscores: list[dict[str, float]] | None = None+ per_objective_best_candidates: dict[str, set[int]] | None = None+ objective_pareto_front: dict[str, float] | None = None+@propertydef best_idx(self) -> int:scores = self.val_aggregate_scores@@ -140,6 +148,13 @@ class DspyGEPAResult:per_val_instance_best_candidates={val_id: list(s) for val_id, s in self.per_val_instance_best_candidates.items()},+ val_aggregate_subscores=self.val_aggregate_subscores,+ per_objective_best_candidates=(+ {objective: list(s) for objective, s in self.per_objective_best_candidates.items()}+ if self.per_objective_best_candidates is not None+ else None+ ),+ objective_pareto_front=self.objective_pareto_front,discovery_eval_counts=self.discovery_eval_counts,total_metric_calls=self.total_metric_calls,num_full_val_evals=self.num_full_val_evals,@@ -158,6 +173,9 @@ class DspyGEPAResult:val_subscores=gepa_result.val_subscores,per_val_instance_best_candidates=gepa_result.per_val_instance_best_candidates,discovery_eval_counts=gepa_result.discovery_eval_counts,+ val_aggregate_subscores=gepa_result.val_aggregate_subscores,+ per_objective_best_candidates=gepa_result.per_objective_best_candidates,+ objective_pareto_front=gepa_result.objective_pareto_front,total_metric_calls=gepa_result.total_metric_calls,num_full_val_evals=gepa_result.num_full_val_evals,log_dir=gepa_result.run_dir,diff --git a/dspy/teleprompt/gepa/gepa_flex_utils.py b/dspy/teleprompt/gepa/gepa_flex_utils.pyindex fe55e6db1..202559cb2 100644--- a/dspy/teleprompt/gepa/gepa_flex_utils.py+++ b/dspy/teleprompt/gepa/gepa_flex_utils.py@@ -234,6 +234,7 @@ def evaluate_with_trace()outputs: list[Any] = [None] * len(batch)scores: list[float] = [failure_score] * len(batch)+ objective_scores: list[dict[str, float]] = [{} for _ in batch]for t in trajs:pred = t["prediction"]outputs[t["example_ind"]] = pred@@ -250,6 +251,17 @@ def evaluate_with_trace()result = failure_scoret["score"] = result # make_reflective_dataset reads this for the (trace-aware) feedback- score = result["score"] if hasattr(result, "score") else result+ if isinstance(result, Prediction):+ score = result.score+ objectives = dict(result.get("objective_scores") or {})+ else:+ score = result+ objectives = {}scores[t["example_ind"]] = failure_score if score is None else score- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=trajs if capture_traces else None)+ objective_scores[t["example_ind"]] = objectives+ return EvaluationBatch(+ outputs=outputs,+ scores=scores,+ trajectories=trajs if capture_traces else None,+ objective_scores=objective_scores if any(objective_scores) else None,+ )diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.pyindex 16d067e01..1f8879867 100644--- a/dspy/teleprompt/gepa/gepa_utils.py+++ b/dspy/teleprompt/gepa/gepa_utils.py@@ -57,6 +57,13 @@ Each example contains the predictor inputs, generated outputs, and feedback fromclass ScoreWithFeedback(Prediction):score: floatfeedback: str+ objective_scores: dict[str, float] | None+++def _extract_score_and_objective_scores(score: Any) -> tuple[float | None, dict[str, float]]:+ if isinstance(score, Prediction):+ return score.score, dict(score.get("objective_scores") or {})+ return score, {}class PredictorFeedbackFn(Protocol):@@ -201,9 +208,9 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):# (e.g., an LM provider or rate-limit error) is not the candidate's fault and must# propagate.logger.warning("Candidate failed to build (%s); scoring the batch as failures.", e)- return EvaluationBatch(+ return self._make_evaluation_batch(outputs=[None] * len(batch),- scores=[self.failure_score] * len(batch),+ raw_scores=[None] * len(batch),trajectories=[] if capture_traces else None,)callback_metadata = (@@ -241,19 +248,11 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):format_failure_score=self.failure_score,callback_metadata=callback_metadata,)- scores = []- outputs = []- for t in trajs:- outputs.append(t["prediction"])- if hasattr(t["prediction"], "__class__") and t.get("score") is None:- scores.append(self.failure_score)- else:- score = t["score"]- if hasattr(score, "score"):- score = score["score"]- scores.append(score)-- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=trajs)+ return self._make_evaluation_batch(+ outputs=[t["prediction"] for t in trajs],+ raw_scores=[t.get("score") for t in trajs],+ trajectories=trajs,+ )else:evaluator = Evaluate(devset=batch,@@ -266,10 +265,22 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):callback_metadata=callback_metadata,)res = evaluator(program)- outputs = [r[1] for r in res.results]- scores = [r[2] for r in res.results]- scores = [s["score"] if hasattr(s, "score") else s for s in scores]- return EvaluationBatch(outputs=outputs, scores=scores, trajectories=None)+ return self._make_evaluation_batch(+ outputs=[result[1] for result in res.results],+ raw_scores=[result[2] for result in res.results],+ trajectories=None,+ )++ def _make_evaluation_batch(self, outputs, raw_scores, trajectories) -> EvaluationBatch:+ scores_and_objectives = [_extract_score_and_objective_scores(score) for score in raw_scores]+ scores = [self.failure_score if score is None else score for score, _ in scores_and_objectives]+ objective_scores = [objectives for _, objectives in scores_and_objectives]+ return EvaluationBatch(+ outputs=outputs,+ scores=scores,+ trajectories=trajectories,+ objective_scores=objective_scores if any(objective_scores) else None,+ )def batch_evaluate(self, items, *, capture_traces=True):total_threads = self.num_threads or dspy.settings.num_threadsdiff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.pyindex 2999aa18d..6cebd6c6b 100644--- a/tests/teleprompt/test_gepa.py+++ b/tests/teleprompt/test_gepa.py@@ -40,6 +40,15 @@ def simple_metric(example, prediction, trace=None, pred_name=None, pred_trace=Noreturn dspy.Prediction(score=example.output == prediction.output, feedback="Wrong answer.")+def multi_objective_metric(example, prediction, trace=None, pred_name=None, pred_trace=None):+ correct = float(example.output == prediction.output)+ return dspy.Prediction(+ score=(correct + 1.0) / 2,+ objective_scores={"correctness": correct, "incorrectness": 1.0 - correct},+ feedback="Return the expected answer.",+ )++def bad_metric(example, prediction):return 0.0@@ -88,6 +97,48 @@ def test_gepa_adapter_disables_logging_on_minibatch_eval(monkeypatch, reflectionassert captured_kwargs["callback_metadata"] == expected_callback_metadata+def test_gepa_adapter_forwards_sparse_objective_scores():+ from dspy.teleprompt.gepa import gepa_utils++ adapter = gepa_utils.DspyAdapter(SimpleModule("input -> output"), multi_objective_metric, {})+ evaluation = adapter._make_evaluation_batch(+ outputs=[None, None],+ raw_scores=[+ dspy.Prediction(score=0.75, objective_scores={"quality": 1.0, "cost": 0.5}),+ dspy.Prediction(score=0.25),+ ],+ trajectories=None,+ )++ assert evaluation.scores == [0.75, 0.25]+ assert evaluation.objective_scores == [{"quality": 1.0, "cost": 0.5}, {}]+++def test_gepa_flex_evaluation_forwards_metric_objective_scores(monkeypatch):+ from dspy.teleprompt.gepa import gepa_flex_utils++ example = Example(input="question", output="answer")+ prediction = dspy.Prediction(output="answer")+ monkeypatch.setattr(+ gepa_flex_utils.bootstrap_trace_module,+ "bootstrap_trace_data",+ lambda **kwargs: [{"example_ind": 0, "example": example, "prediction": prediction, "trace": []}],+ )++ evaluation = gepa_flex_utils.evaluate_with_trace(+ program=SimpleModule("input -> output"),+ batch=[example],+ metric_fn=multi_objective_metric,+ num_threads=1,+ failure_score=0.0,+ callback_metadata={},+ capture_traces=True,+ )++ assert evaluation.scores == [1.0]+ assert evaluation.objective_scores == [{"correctness": 1.0, "incorrectness": 0.0}]++@pytest.fixturedef mock_mlflow():mock_mlflow = mock.MagicMock()@@ -602,6 +653,49 @@ def test_track_stats_result_structure():assert isinstance(best_list, list)+def test_multi_objective_frontier_results():+ from gepa.strategies.candidate_selector import select_program_candidate_from_pareto_front++ student = SimpleModule("input -> output")++ with open("tests/teleprompt/gepa_dummy_lm.json") as f:+ data = json.load(f)++ dspy.configure(lm=DictDummyLM(data["lm"]))+ optimizer = dspy.GEPA(+ metric=multi_objective_metric,+ reflection_lm=DictDummyLM(data["reflection_lm"]),+ max_metric_calls=5,+ track_stats=True,+ gepa_kwargs={"frontier_type": "objective"},+ )+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),+ ]++ result = optimizer.compile(student, trainset=trainset, valset=trainset).detailed_results++ assert result.val_aggregate_subscores is not None+ assert result.per_objective_best_candidates is not None+ assert result.objective_pareto_front is not None+ assert result.per_objective_best_candidates == {"correctness": {1}, "incorrectness": {0}}+ assert result.objective_pareto_front == {"correctness": 0.5, "incorrectness": 1.0}+ serialized = result.to_dict()+ assert serialized["val_aggregate_subscores"] == result.val_aggregate_subscores+ assert serialized["per_objective_best_candidates"] == {"correctness": [1], "incorrectness": [0]}+ assert serialized["objective_pareto_front"] == result.objective_pareto_front+ selected_parents = {+ select_program_candidate_from_pareto_front(+ result.per_objective_best_candidates,+ result.val_aggregate_scores,+ random.Random(seed),+ )+ for seed in range(10)+ }+ assert selected_parents == {0, 1}++def test_track_best_outputs_result_structure():"""Verify best_outputs_valset has the correct type from GEPA 0.1.1: