DSPy: The framework for programming—not prompting—language models
Python
Update DspyGEPAResult and documentation for gepa 0.1.1 changes (#9673)
Ben McHone9e8a323d998c
diff --git a/docs/docs/api/optimizers/GEPA/GEPA_Advanced.md b/docs/docs/api/optimizers/GEPA/GEPA_Advanced.mdindex 624e580ad..39ba410bc 100644--- a/docs/docs/api/optimizers/GEPA/GEPA_Advanced.md+++ b/docs/docs/api/optimizers/GEPA/GEPA_Advanced.md@@ -13,12 +13,12 @@ By default, GEPA uses the built-in instruction proposer from the [GEPA library](````I provided an assistant with the following instructions to perform a task for me:```-<curr_instructions>+<curr_param>```The following are examples of different task inputs provided to the assistant along with the assistant's response for each of them, and some feedback on how the assistant's response could be better:```-<inputs_outputs_feedback>+<side_info>```Your task is to write a new instruction for the assistant.@@ -32,8 +32,9 @@ Provide the new instructions within ``` blocks.This template is automatically filled with:-- `<curr_instructions>`: The current instruction being optimized-- `<inputs_outputs_feedback>`: Structured markdown containing predictor inputs, generated outputs, and evaluation feedback+- `<curr_param>`: The current instruction being optimized+- `<side_info>`: Structured markdown containing predictor inputs, generated outputs, and evaluation feedback+Example of default behavior:diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.pyindex 2fdc0494d..41fd6de58 100644--- a/dspy/teleprompt/gepa/gepa.py+++ b/dspy/teleprompt/gepa/gepa.py@@ -62,11 +62,11 @@ class DspyGEPAResult:Additional data related to the GEPA run.Fields:- - candidates: list of proposed candidates (component_name -> component_text)+ - candidates: list of proposed candidates (compiled DSPy modules)- parents: lineage info; for each candidate i, parents[i] is a list of parent indices or None- val_aggregate_scores: per-candidate aggregate score on the validation set (higher is better)- - val_subscores: per-candidate per-instance scores on the validation set (len == num_val_instances)- - per_val_instance_best_candidates: for each val instance t, a set of candidate indices achieving the best score on t+ - val_subscores: per-candidate scores keyed by validation instance id+ - per_val_instance_best_candidates: for each val instance id, a set of candidate indices achieving the best score- discovery_eval_counts: Budget (number of metric calls / rollouts) consumed up to the discovery of each candidate- total_metric_calls: total number of metric calls made across the run@@ -75,19 +75,19 @@ class DspyGEPAResult:- seed: RNG seed for reproducibility (if known)- best_idx: candidate index with the highest val_aggregate_scores- - best_candidate: the program text mapping for best_idx+ - best_candidate: the compiled DSPy module for best_idx"""# Data about the proposed candidatescandidates: list[Module]parents: list[list[int | None]]val_aggregate_scores: list[float]- val_subscores: list[list[float]]- per_val_instance_best_candidates: list[set[int]]+ val_subscores: list[dict[Any, float]]+ per_val_instance_best_candidates: dict[Any, set[int]]discovery_eval_counts: list[int]# Optional data- best_outputs_valset: list[list[tuple[int, list[Prediction]]]] | None = None+ best_outputs_valset: dict[Any, list[tuple[int, Prediction]]] | None = None# Optimization metadatatotal_metric_calls: int | None = None@@ -101,18 +101,21 @@ class DspyGEPAResult:return max(range(len(scores)), key=lambda i: scores[i])@property- def best_candidate(self) -> dict[str, str]:+ def best_candidate(self) -> Module:return self.candidates[self.best_idx]@property- def highest_score_achieved_per_val_task(self) -> list[float]:- return [- self.val_subscores[list(self.per_val_instance_best_candidates[val_idx])[0]][val_idx]- for val_idx in range(len(self.val_subscores[0]))- ]+ def highest_score_achieved_per_val_task(self) -> dict[Any, float]:+ return {+ val_id: self.val_subscores[list(self.per_val_instance_best_candidates[val_id])[0]][val_id]+ for val_id in self.per_val_instance_best_candidates+ }def to_dict(self) -> dict[str, Any]:- cands = [{k: v for k, v in cand.items()} for cand in self.candidates]+ cands = [+ {name: pred.signature.instructions for name, pred in cand.named_predictors()}+ for cand in self.candidates+ ]return dict(candidates=cands,@@ -120,7 +123,9 @@ class DspyGEPAResult:val_aggregate_scores=self.val_aggregate_scores,best_outputs_valset=self.best_outputs_valset,val_subscores=self.val_subscores,- per_val_instance_best_candidates=[list(s) for s in self.per_val_instance_best_candidates],+ per_val_instance_best_candidates={+ val_id: list(s) for val_id, s in self.per_val_instance_best_candidates.items()+ },discovery_eval_counts=self.discovery_eval_counts,total_metric_calls=self.total_metric_calls,num_full_val_evals=self.num_full_val_evals,@@ -428,6 +433,13 @@ class GEPA(Teleprompter):self.component_selector = component_selectorself.gepa_kwargs = gepa_kwargs or {}+ if "reflection_prompt_template" in self.gepa_kwargs:+ raise ValueError(+ "reflection_prompt_template cannot be passed via gepa_kwargs when using dspy.GEPA. "+ "DspyAdapter implements its own propose_new_texts, so reflection_prompt_template is unused. "+ "To customize reflection behavior, pass a custom ProposalFn via the instruction_proposer parameter instead."+ )+def auto_budget(self, num_preds, num_candidates, valset_size: int, minibatch_size: int = 35, full_eval_steps: int = 5) -> int:diff --git a/pyproject.toml b/pyproject.tomlindex 289d5306a..b708f99ff 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -37,7 +37,7 @@ dependencies = ["cloudpickle>=3.1.2","numpy>=1.26.0","xxhash>=3.5.0",- "gepa[dspy]==0.0.27",+ "gepa[dspy]==0.1.1","typeguard==4.4.3",]diff --git a/tests/teleprompt/test_gepa.py b/tests/teleprompt/test_gepa.pyindex c5b292ba5..4d88a78ea 100644--- a/tests/teleprompt/test_gepa.py+++ b/tests/teleprompt/test_gepa.py@@ -214,6 +214,18 @@ def test_metric_requires_feedback_signature():dspy.GEPA(metric=bad_metric, reflection_lm=reflection_lm, max_metric_calls=1)+def test_reflection_prompt_template_in_gepa_kwargs_raises():+ """reflection_prompt_template via gepa_kwargs is unsupported: DspyAdapter owns propose_new_texts."""+ reflection_lm = DictDummyLM([])+ with pytest.raises(ValueError, match="reflection_prompt_template"):+ dspy.GEPA(+ metric=simple_metric,+ reflection_lm=reflection_lm,+ max_metric_calls=1,+ gepa_kwargs={"reflection_prompt_template": "Instructions: <curr_param>\n\nExamples: <side_info>"},+ )++def any_metric(gold: dspy.Example,pred: dspy.Prediction,@@ -521,3 +533,104 @@ def test_alternating_half_component_selector():# Odd iteration should select second half: ["generator"]assert "generator" in selection["selected"], f"Odd iteration {selection['iteration']} should include generator"assert "classifier" not in selection["selected"], f"Odd iteration {selection['iteration']} should not include classifier"+++def test_track_stats_result_structure():+ """+ Verify DspyGEPAResult fields have the correct types from GEPA 0.1.1:+ - val_subscores is list[dict[DataId, float]] (not list[list[float]])+ - per_val_instance_best_candidates is dict[DataId, set[int]] (not list[set[int]])+ - best_candidate returns a Module+ - highest_score_achieved_per_val_task works without errors+ - to_dict() serializes per_val_instance_best_candidates as a dict+ """+ from dspy.teleprompt.gepa.gepa import DspyGEPAResult++ student = SimpleModule("input -> output")++ with open("tests/teleprompt/gepa_dummy_lm.json") as f:+ data = json.load(f)++ dspy.configure(lm=DictDummyLM(data["lm"]))+ optimizer = dspy.GEPA(+ metric=simple_metric,+ reflection_lm=DictDummyLM(data["reflection_lm"]),+ max_metric_calls=5,+ track_stats=True,+ )+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),+ ]+ prog = optimizer.compile(student, trainset=trainset, valset=trainset)++ dr = prog.detailed_results+ assert isinstance(dr, DspyGEPAResult)++ # val_subscores: list[dict[DataId, float]]+ assert isinstance(dr.val_subscores, list)+ for subscores in dr.val_subscores:+ assert isinstance(subscores, dict), f"Expected dict, got {type(subscores)}"+ for val, score in subscores.items():+ assert isinstance(score, (int, float))++ # per_val_instance_best_candidates: dict[DataId, set[int]]+ assert isinstance(dr.per_val_instance_best_candidates, dict), (+ f"Expected dict, got {type(dr.per_val_instance_best_candidates)}"+ )+ for val_id, best_set in dr.per_val_instance_best_candidates.items():+ assert isinstance(best_set, set)++ # best_candidate returns a Module+ assert isinstance(dr.best_candidate, dspy.Module), (+ f"Expected Module, got {type(dr.best_candidate)}"+ )++ # highest_score_achieved_per_val_task works and returns one float per val instance+ scores = dr.highest_score_achieved_per_val_task+ assert isinstance(scores, dict)+ assert len(scores) == len(dr.per_val_instance_best_candidates)+ for val_id, s in scores.items():+ assert val_id in dr.per_val_instance_best_candidates+ assert isinstance(s, (int, float))++ # to_dict() serializes per_val_instance_best_candidates as a dict (not a list)+ d = dr.to_dict()+ assert isinstance(d["per_val_instance_best_candidates"], dict), (+ f"Expected dict in to_dict output, got {type(d['per_val_instance_best_candidates'])}"+ )+ for val_id, best_list in d["per_val_instance_best_candidates"].items():+ assert isinstance(best_list, list)+++def test_track_best_outputs_result_structure():+ """+ Verify best_outputs_valset has the correct type from GEPA 0.1.1:+ dict[DataId, list[tuple[int, Prediction]]] (not list[list[tuple[...]]])+ """+ student = SimpleModule("input -> output")++ with open("tests/teleprompt/gepa_dummy_lm.json") as f:+ data = json.load(f)++ dspy.configure(lm=DictDummyLM(data["lm"]))+ optimizer = dspy.GEPA(+ metric=simple_metric,+ reflection_lm=DictDummyLM(data["reflection_lm"]),+ max_metric_calls=5,+ track_stats=True,+ track_best_outputs=True,+ )+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),+ ]+ prog = optimizer.compile(student, trainset=trainset, valset=trainset)++ best_outputs = prog.detailed_results.best_outputs_valset+ assert best_outputs is not None+ assert isinstance(best_outputs, dict), f"Expected dict, got {type(best_outputs)}"+ for val_id, entries in best_outputs.items():+ assert isinstance(entries, list)+ for cand_idx, output in entries:+ assert isinstance(cand_idx, int)diff --git a/uv.lock b/uv.lockindex a46bcbe91..7c311ce04 100644--- a/uv.lock+++ b/uv.lock@@ -823,7 +823,7 @@ requires-dist = [{ name = "datamodel-code-generator", marker = "extra == 'dev'", specifier = ">=0.26.3" },{ name = "datasets", marker = "extra == 'test-extras'", specifier = ">=2.14.6" },{ name = "diskcache", specifier = ">=5.6.0" },- { name = "gepa", extras = ["dspy"], specifier = "==0.0.27" },+ { name = "gepa", extras = ["dspy"], specifier = "==0.1.1" },{ name = "json-repair", specifier = ">=0.54.2" },{ name = "langchain-core", marker = "extra == 'langchain'" },{ name = "langchain-core", marker = "extra == 'test-extras'" },@@ -1037,11 +1037,11 @@ wheels = [[[package]]name = "gepa"-version = "0.0.27"+version = "0.1.1"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/4e/99/6840f84498f2dcbfd27e8a15eeb4e637e84e9dbbd6331977b71f6ffad9c7/gepa-0.0.27.tar.gz", hash = "sha256:02ecb19e4aa6a1f5bb2994cd54b2057f25cd5e8cd662fee514303b2a8ba0a738", size = 155106, upload-time = "2026-01-28T00:33:51.72Z" }+sdist = { url = "https://files.pythonhosted.org/packages/e4/62/10f5a8f24c075e3b64f952be73ba8e15f0055584bbcdf9ce48d754a36679/gepa-0.1.1.tar.gz", hash = "sha256:643fda01c23de4c9f01306e01305dd69facc29bcb34ad59e4cd07e6621d34aa1", size = 272251, upload-time = "2026-03-16T10:17:53.131Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/a3/bd/f9e83519099a9fd5d090520ab0c51a6278e473b914a9b32d1e812662dd3b/gepa-0.0.27-py3-none-any.whl", hash = "sha256:592beb084fd638d525edae84ca75ad8e9e9c3758e5498b933c17153addf5dd2d", size = 146454, upload-time = "2026-01-28T00:33:50.262Z" },+ { url = "https://files.pythonhosted.org/packages/dc/b7/8c72dedbb950d88a6f64588fcbc590d2a21e2b9f19b36aa6c5016c54ec75/gepa-0.1.1-py3-none-any.whl", hash = "sha256:71ead7c591eafcc727b83509cdc4182f20264800a6ddf8520d61419daeb47466", size = 244246, upload-time = "2026-03-16T10:17:51.922Z" },][[package]]