DSPy: The framework for programming—not prompting—language models
Python
optimize w/ suggest failures in mind
manishshettymefc71760037d
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 4962c2733..c828fb7b5 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -181,6 +181,7 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):error_msg, result = None, Nonewith dspy.settings.lock:dspy.settings.backtrack_to = None+ dspy.settings.suggest_failure_count = 0# Predictor -> List[feedback_msg]dspy.settings.predictor_feedbacks = {}@@ -214,6 +215,9 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):e.state[-1],)+ # increment failure count for optimization+ dspy.settings.suggest_failure_count += 1+if dsp.settings.trace:if suggest_target_module:for i in range(len(dsp.settings.trace) - 1, -1, -1):diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 996b5dca7..b33ea2694 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -2,6 +2,8 @@ import dspimport tqdmimport randomimport threading++import dspyfrom dspy.predict.retry import Retryfrom dspy.primitives import Example@@ -42,6 +44,9 @@ class BootstrapFewShot(Teleprompter):self.max_errors= max_errorsself.error_count = 0self.error_lock = threading.Lock()++ self.suggest_failures = 0+ self.assert_failures = 0def compile(self, student, *, teacher=None, trainset, valset=None):self.trainset = trainset@@ -53,6 +58,10 @@ class BootstrapFewShot(Teleprompter):self.student = self._train()self.student._compiled = True++ # number of suggestion/assertion the student failed when it was learning+ self.student._suggest_failures = self.suggest_failures+ self.student._assert_failures = self.assert_failuresreturn self.student@@ -102,12 +111,14 @@ class BootstrapFewShot(Teleprompter):breakif example_idx not in bootstrapped:- success = self._bootstrap_one_example(example, round_idx)+ success, suggest_failures = self._bootstrap_one_example(example, round_idx)if success:bootstrapped[example_idx] = True-+ self.suggest_failures += suggest_failures+print(f'Bootstrapped {len(bootstrapped)} full traces after {example_idx+1} examples in round {round_idx}.')+ print(f"# of suggestion failures during bootstrapping: {self.suggest_failures}")# Unbootstrapped training examples@@ -124,6 +135,7 @@ class BootstrapFewShot(Teleprompter):name2traces = self.name2tracesteacher = self.teacher #.deepcopy()predictor_cache = {}+ suggest_failures = 0try:with dsp.settings.context(trace=[], **self.teacher_settings):@@ -141,8 +153,9 @@ class BootstrapFewShot(Teleprompter):for name, predictor in teacher.named_predictors():predictor.demos = predictor_cache[name]-+success = (self.metric is None) or self.metric(example, prediction, trace)+ suggest_failures = dspy.settings.suggest_failure_count# print(success, example, prediction)except Exception as e:success = False@@ -176,7 +189,7 @@ class BootstrapFewShot(Teleprompter):name2traces[predictor_name].append(demo)- return success+ return success, suggest_failuresdef _train(self):rng = random.Random(0)diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex f027f83fe..4c7f91624 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -99,6 +99,9 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):all_subscores.append(subscores)+ if hasattr(program2, '_suggest_failures'):+ score = score - program2._suggest_failures * 0.2+print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])if len(scores) == 0 or score > max(scores):