DSPy: The framework for programming—not prompting—language models
Python
fix(BootstrapFewShot): honor metric_threshold=0.0 (#10604)
Tom Rond4c090015fc0
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 58f45b412..dd6c10266 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -55,7 +55,10 @@ class BootstrapFewShot(Teleprompter):outputting the result of that comparison.metric_threshold (float, optional): If the metric yields a numerical value, then check itagainst this threshold when deciding whether or not to accept a bootstrap example.- Defaults to None.+ An example is accepted when ``metric_value >= metric_threshold``. Any value other+ than None is applied, including ``0``; note that a boolean metric returning False+ passes a threshold of ``0`` because ``False >= 0``. When None, the metric's own+ truthiness decides. Defaults to None.teacher_settings (dict, optional): Settings for the `teacher` model.Defaults to None.max_bootstrapped_demos (int): Maximum number of bootstrapped demonstrations to include.@@ -204,7 +207,7 @@ class BootstrapFewShot(Teleprompter):if self.metric:metric_val = self.metric(example, prediction, trace)- if self.metric_threshold:+ if self.metric_threshold is not None:success = metric_val >= self.metric_thresholdelse:success = metric_valdiff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.pyindex d24cde1c3..713d549cc 100644--- a/tests/teleprompt/test_bootstrap.py+++ b/tests/teleprompt/test_bootstrap.py@@ -168,3 +168,62 @@ def test_validation_set_usage():# Check that validation examples are part of student's demos after compilationassert len(compiled_student.predictor.demos) >= len(valset), "Validation set not used in compiled student demos"+++ ("metric_value", "metric_threshold", "expected_demos"),+ [+ (0.0, 0.0, 1),+ (-0.1, 0.0, 0),+ (0.0, 0.01, 0),+ (0.0, -0.01, 1),+ # Bool metrics are compared numerically too, so False clears a threshold of 0.+ (True, 0, 1),+ (False, 0, 1),+ ],+)+def test_metric_threshold_zero_is_applied(metric_value, metric_threshold, expected_demos):+ # A threshold of 0.0 must be compared numerically rather than treated as unset.+ def metric(example, prediction, trace=None):+ return metric_value++ dspy.configure(lm=DummyLM([{"output": "blue"}]))+ bootstrap = BootstrapFewShot(+ metric=metric,+ metric_threshold=metric_threshold,+ max_bootstrapped_demos=1,+ max_labeled_demos=0,+ )+ compiled_student = bootstrap.compile(+ SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset+ )++ assert len(compiled_student.predictor.demos) == expected_demos+++ ("metric_value", "expected_demos"),+ [+ (0.0, 0),+ (False, 0),+ (1.0, 1),+ (True, 1),+ ],+)+def test_metric_threshold_none_uses_metric_value_as_success(metric_value, expected_demos):+ # With no threshold, the metric's own (truthy/falsy) value decides success.+ def metric(example, prediction, trace=None):+ return metric_value++ dspy.configure(lm=DummyLM([{"output": "blue"}]))+ bootstrap = BootstrapFewShot(+ metric=metric,+ metric_threshold=None,+ max_bootstrapped_demos=1,+ max_labeled_demos=0,+ )+ compiled_student = bootstrap.compile(+ SimpleModule("input -> output"), teacher=SimpleModule("input -> output"), trainset=trainset+ )++ assert len(compiled_student.predictor.demos) == expected_demos