Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

optimize w/ suggest failures in mind

manishshettymefc71760037d
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index 4962c2733..c828fb7b5 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -181,6 +181,7 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
error_msg, result = None, None
with dspy.settings.lock:
dspy.settings.backtrack_to = None
+ dspy.settings.suggest_failure_count = 0
# Predictor -> List[feedback_msg]
dspy.settings.predictor_feedbacks = {}
@@ -214,6 +215,9 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
e.state[-1],
)
+ # increment failure count for optimization
+ dspy.settings.suggest_failure_count += 1
+
if dsp.settings.trace:
if suggest_target_module:
for i in range(len(dsp.settings.trace) - 1, -1, -1):
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 996b5dca7..b33ea2694 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -2,6 +2,8 @@ import dsp
import tqdm
import random
import threading
+
+import dspy
from dspy.predict.retry import Retry
from dspy.primitives import Example
@@ -42,6 +44,9 @@ class BootstrapFewShot(Teleprompter):
self.max_errors= max_errors
self.error_count = 0
self.error_lock = threading.Lock()
+
+ self.suggest_failures = 0
+ self.assert_failures = 0
def compile(self, student, *, teacher=None, trainset, valset=None):
self.trainset = trainset
@@ -53,6 +58,10 @@ class BootstrapFewShot(Teleprompter):
self.student = self._train()
self.student._compiled = True
+
+ # number of suggestion/assertion the student failed when it was learning
+ self.student._suggest_failures = self.suggest_failures
+ self.student._assert_failures = self.assert_failures
return self.student
@@ -102,12 +111,14 @@ class BootstrapFewShot(Teleprompter):
break
if example_idx not in bootstrapped:
- success = self._bootstrap_one_example(example, round_idx)
+ success, suggest_failures = self._bootstrap_one_example(example, round_idx)
if success:
bootstrapped[example_idx] = True
-
+ self.suggest_failures += suggest_failures
+
print(f'Bootstrapped {len(bootstrapped)} full traces after {example_idx+1} examples in round {round_idx}.')
+ print(f"# of suggestion failures during bootstrapping: {self.suggest_failures}")
# Unbootstrapped training examples
@@ -124,6 +135,7 @@ class BootstrapFewShot(Teleprompter):
name2traces = self.name2traces
teacher = self.teacher #.deepcopy()
predictor_cache = {}
+ suggest_failures = 0
try:
with dsp.settings.context(trace=[], **self.teacher_settings):
@@ -141,8 +153,9 @@ class BootstrapFewShot(Teleprompter):
for name, predictor in teacher.named_predictors():
predictor.demos = predictor_cache[name]
-
+
success = (self.metric is None) or self.metric(example, prediction, trace)
+ suggest_failures = dspy.settings.suggest_failure_count
# print(success, example, prediction)
except Exception as e:
success = False
@@ -176,7 +189,7 @@ class BootstrapFewShot(Teleprompter):
name2traces[predictor_name].append(demo)
- return success
+ return success, suggest_failures
def _train(self):
rng = random.Random(0)
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index f027f83fe..4c7f91624 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -99,6 +99,9 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
all_subscores.append(subscores)
+ if hasattr(program2, '_suggest_failures'):
+ score = score - program2._suggest_failures * 0.2
+
print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])
if len(scores) == 0 or score > max(scores):