Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix: only suggest reducing valset in GEPA when it is large (#9320)

Blake Leddenedd0a5b26e7c
diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.py
index cb752150a..5995b67bb 100644
--- a/dspy/teleprompt/gepa/gepa.py
+++ b/dspy/teleprompt/gepa/gepa.py
@@ -506,9 +506,15 @@ class GEPA(Teleprompter):
"No valset provided; Using trainset as valset. This is useful as an inference-time scaling strategy where you want GEPA to find the best solutions for the provided tasks in the trainset, as it makes GEPA overfit prompts to the provided trainset. In order to ensure generalization and perform well on unseen tasks, please provide separate trainset and valset. Provide the smallest valset that is just large enough to match the downstream task distribution, while keeping trainset as large as possible."
)
valset = valset or trainset
- logger.info(
- f"Using {len(valset)} examples for tracking Pareto scores. You can consider using a smaller sample of the valset to allow GEPA to explore more diverse solutions within the same budget. GEPA requires you to provide the smallest valset that is just large enough to match your downstream task distribution, while providing as large trainset as possible."
- )
+ # 35 matches the default minibatch_size in auto_budget(); when the valset is
+ # at or below this size, suggesting further reduction is unhelpful since GEPA
+ # would already evaluate the full valset per step.
+ if len(valset) > 35:
+ logger.info(
+ f"Using {len(valset)} examples for tracking Pareto scores. You can consider using a smaller sample of the valset to allow GEPA to explore more diverse solutions within the same budget. GEPA requires you to provide the smallest valset that is just large enough to match your downstream task distribution, while providing as large trainset as possible."
+ )
+ else:
+ logger.info(f"Using {len(valset)} examples for tracking Pareto scores.")
rng = random.Random(self.seed)