Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Sample labeled demos independently per predictor in BootstrapFewShot (#9948)

Shardul Deshpande1bc87de157f5
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 9905d5c88..58f45b412 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -266,7 +266,7 @@ class BootstrapFewShot(Teleprompter):
sample_size = min(self.max_labeled_demos - len(augmented_demos), len(raw_demos))
sample_size = max(0, sample_size)
- raw_demos = rng.sample(raw_demos, sample_size)
- predictor.demos = augmented_demos + raw_demos
+ sampled_demos = rng.sample(raw_demos, sample_size)
+ predictor.demos = augmented_demos + sampled_demos
return self.student
diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.py
index 75193eb24..d24cde1c3 100644
--- a/tests/teleprompt/test_bootstrap.py
+++ b/tests/teleprompt/test_bootstrap.py
@@ -36,6 +36,42 @@ class SimpleModule(dspy.Module):
return self.predictor(**kwargs)
+class TwoPredictorModule(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.p1 = Predict("input -> output")
+ self.p2 = Predict("input -> output")
+
+ def forward(self, **kwargs):
+ return self.p2(**self.p1(**kwargs))
+
+
+def test_bootstrap_labeled_demos_sampled_independently_per_predictor():
+ # Each predictor must draw its labeled demos from the FULL validation pool.
+ # Regression: _train reassigned the shared `raw_demos` to the sampled subset
+ # inside the per-predictor loop, so later predictors sampled from the prior
+ # predictor's leftovers -- starving them when an earlier predictor consumed
+ # most of the pool via bootstrapped (augmented) demos.
+ student = TwoPredictorModule()
+ names = [name for name, _ in student.named_predictors()]
+ pool = [Example(input=f"q{i}", output=f"a{i}").with_inputs("input") for i in range(10)]
+ augmented = [Example(input=f"aug{i}", output=f"x{i}").with_inputs("input") for i in range(5)]
+
+ bootstrap = BootstrapFewShot(metric=simple_metric, max_bootstrapped_demos=5, max_labeled_demos=8)
+ bootstrap.student = student
+ bootstrap.validation = pool
+ # First predictor has 5 bootstrapped demos; second has none.
+ bootstrap.name2traces = {names[0]: list(augmented), names[1]: []}
+
+ bootstrap._train()
+
+ p1, p2 = (p for _, p in student.named_predictors())
+ # p1: 5 augmented + 3 labeled = 8
+ assert len(p1.demos) == 8
+ # p2: 0 augmented, so it must still get the full labeled quota from the full pool.
+ assert len(p2.demos) == 8
+
+
def test_compile_with_predict_instances():
# Create Predict instances for student and teacher
# Note that dspy.Predict is not itself a module, so we can't use it directly here