Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(BootstrapFewShot): honor metric_threshold=0.0 (#10604)

Tom Rond4c090015fc0
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 58f45b412..dd6c10266 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -55,7 +55,10 @@ class BootstrapFewShot(Teleprompter):
outputting the result of that comparison.
metric_threshold (float, optional): If the metric yields a numerical value, then check it
against this threshold when deciding whether or not to accept a bootstrap example.
- Defaults to None.
+ An example is accepted when ``metric_value >= metric_threshold``. Any value other
+ than None is applied, including ``0``; note that a boolean metric returning False
+ passes a threshold of ``0`` because ``False >= 0``. When None, the metric's own
+ truthiness decides. Defaults to None.
teacher_settings (dict, optional): Settings for the `teacher` model.
Defaults to None.
max_bootstrapped_demos (int): Maximum number of bootstrapped demonstrations to include.
@@ -204,7 +207,7 @@ class BootstrapFewShot(Teleprompter):
if self.metric:
metric_val = self.metric(example, prediction, trace)
- if self.metric_threshold:
+ if self.metric_threshold is not None:
success = metric_val >= self.metric_threshold
else:
success = metric_val
diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.py
index d24cde1c3..713d549cc 100644
--- a/tests/teleprompt/test_bootstrap.py
+++ b/tests/teleprompt/test_bootstrap.py
@@ -168,3 +168,62 @@ def test_validation_set_usage():
# Check that validation examples are part of student's demos after compilation
assert len(compiled_student.predictor.demos) >= len(valset), "Validation set not used in compiled student demos"
+
+
+ ("metric_value", "metric_threshold", "expected_demos"),
+ [
+ (0.0, 0.0, 1),
+ (-0.1, 0.0, 0),
+ (0.0, 0.01, 0),
+ (0.0, -0.01, 1),
+ # Bool metrics are compared numerically too, so False clears a threshold of 0.
+ (True, 0, 1),
+ (False, 0, 1),
+ ],
+)
+def test_metric_threshold_zero_is_applied(metric_value, metric_threshold, expected_demos):
+ # A threshold of 0.0 must be compared numerically rather than treated as unset.
+ def metric(example, prediction, trace=None):
+ return metric_value
+
+ dspy.configure(lm=DummyLM([{"output": "blue"}]))
+ bootstrap = BootstrapFewShot(
+ metric=metric,
+ metric_threshold=metric_threshold,
+ max_bootstrapped_demos=1,
+ max_labeled_demos=0,
+ )
+ compiled_student = bootstrap.compile(
+ SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset
+ )
+
+ assert len(compiled_student.predictor.demos) == expected_demos
+
+
+ ("metric_value", "expected_demos"),
+ [
+ (0.0, 0),
+ (False, 0),
+ (1.0, 1),
+ (True, 1),
+ ],
+)
+def test_metric_threshold_none_uses_metric_value_as_success(metric_value, expected_demos):
+ # With no threshold, the metric's own (truthy/falsy) value decides success.
+ def metric(example, prediction, trace=None):
+ return metric_value
+
+ dspy.configure(lm=DummyLM([{"output": "blue"}]))
+ bootstrap = BootstrapFewShot(
+ metric=metric,
+ metric_threshold=None,
+ max_bootstrapped_demos=1,
+ max_labeled_demos=0,
+ )
+ compiled_student = bootstrap.compile(
+ SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset
+ )
+
+ assert len(compiled_student.predictor.demos) == expected_demos