DSPy: The framework for programming—not prompting—language models
Python
fix(teleprompt): validate restrict up front in BootstrapFewShotWithRandomSearch (#10033)
Andrew Chen965a39594df3
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex c6447e833..c27d59e9b 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -58,6 +58,14 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):self.trainset = trainsetself.valset = valset or trainset # TODO: FIXME: Note this choice.+ if restrict is not None:+ restrict = set(restrict)+ if not restrict.intersection(range(-3, self.num_candidate_sets)):+ raise ValueError(+ f"`restrict` {restrict!r} does not match any candidate seed in "+ f"range(-3, {self.num_candidate_sets}); no candidate programs would be evaluated."+ )+effective_max_errors = self.max_errors if self.max_errors is not None else dspy.settings.max_errorsscores = []diff --git a/tests/teleprompt/test_random_search.py b/tests/teleprompt/test_random_search.pyindex 3041be0d3..d0560c3ad 100644--- a/tests/teleprompt/test_random_search.py+++ b/tests/teleprompt/test_random_search.py@@ -1,3 +1,5 @@+import pytest+import dspyfrom dspy import Examplefrom dspy.predict import Predict@@ -37,3 +39,38 @@ def test_basic_workflow():Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),]optimizer.compile(student, teacher=teacher, trainset=trainset)+++def test_restrict_matching_no_candidate_seed_raises_clear_error():+ """restrict that matches no candidate seed should raise ValueError, not UnboundLocalError."""+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")++ lm = DummyLM(["Initial thoughts", "Finish[blue]"])+ dspy.configure(lm=lm)++ optimizer = BootstrapFewShotWithRandomSearch(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ ]++ with pytest.raises(ValueError, match="restrict"):+ optimizer.compile(student, teacher=teacher, trainset=trainset, restrict=[999])+++def test_restrict_as_single_use_iterator_still_matches_a_valid_seed():+ """A single-use iterable `restrict` must not be exhausted by the upfront validation check."""+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")++ lm = DummyLM(["Initial thoughts", "Finish[blue]"])+ dspy.configure(lm=lm)++ optimizer = BootstrapFewShotWithRandomSearch(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ ]++ # -3 (zero-shot) is a valid seed; a plain iterator is single-use.+ result = optimizer.compile(student, teacher=teacher, trainset=trainset, restrict=iter([-3]))+ assert result is not None