Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(teleprompt): validate restrict up front in BootstrapFewShotWithRandomSearch (#10033)

Andrew Chen965a39594df3
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index c6447e833..c27d59e9b 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -58,6 +58,14 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
self.trainset = trainset
self.valset = valset or trainset # TODO: FIXME: Note this choice.
+ if restrict is not None:
+ restrict = set(restrict)
+ if not restrict.intersection(range(-3, self.num_candidate_sets)):
+ raise ValueError(
+ f"`restrict` {restrict!r} does not match any candidate seed in "
+ f"range(-3, {self.num_candidate_sets}); no candidate programs would be evaluated."
+ )
+
effective_max_errors = self.max_errors if self.max_errors is not None else dspy.settings.max_errors
scores = []
diff --git a/tests/teleprompt/test_random_search.py b/tests/teleprompt/test_random_search.py
index 3041be0d3..d0560c3ad 100644
--- a/tests/teleprompt/test_random_search.py
+++ b/tests/teleprompt/test_random_search.py
@@ -1,3 +1,5 @@
+import pytest
+
import dspy
from dspy import Example
from dspy.predict import Predict
@@ -37,3 +39,38 @@ def test_basic_workflow():
Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),
]
optimizer.compile(student, teacher=teacher, trainset=trainset)
+
+
+def test_restrict_matching_no_candidate_seed_raises_clear_error():
+ """restrict that matches no candidate seed should raise ValueError, not UnboundLocalError."""
+ student = SimpleModule("input -> output")
+ teacher = SimpleModule("input -> output")
+
+ lm = DummyLM(["Initial thoughts", "Finish[blue]"])
+ dspy.configure(lm=lm)
+
+ optimizer = BootstrapFewShotWithRandomSearch(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
+ trainset = [
+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
+ ]
+
+ with pytest.raises(ValueError, match="restrict"):
+ optimizer.compile(student, teacher=teacher, trainset=trainset, restrict=[999])
+
+
+def test_restrict_as_single_use_iterator_still_matches_a_valid_seed():
+ """A single-use iterable `restrict` must not be exhausted by the upfront validation check."""
+ student = SimpleModule("input -> output")
+ teacher = SimpleModule("input -> output")
+
+ lm = DummyLM(["Initial thoughts", "Finish[blue]"])
+ dspy.configure(lm=lm)
+
+ optimizer = BootstrapFewShotWithRandomSearch(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)
+ trainset = [
+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
+ ]
+
+ # -3 (zero-shot) is a valid seed; a plain iterator is single-use.
+ result = optimizer.compile(student, teacher=teacher, trainset=trainset, restrict=iter([-3]))
+ assert result is not None