Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Redesign BootstrapFewShotWithRandomSearch

Omar Khattabf01784f9f475
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py
index 651bacfb4..688e0ea30 100644
--- a/dspy/evaluate/evaluate.py
+++ b/dspy/evaluate/evaluate.py
@@ -19,13 +19,15 @@ we print the number of failures, the first N examples that failed, and the first
class Evaluate:
- def __init__(self, *, devset, metric=None, num_threads=1, display_progress=False, display_table=False, display=True):
+ def __init__(self, *, devset, metric=None, num_threads=1, display_progress=False,
+ display_table=False, display=True, max_errors=5):
self.devset = devset
self.metric = metric
self.num_threads = num_threads
self.display_progress = display_progress
self.display_table = display_table
self.display = display
+ self.max_errors = max_errors
def _execute_single_thread(self, wrapped_program, devset, display_progress):
ncorrect = 0
@@ -66,7 +68,9 @@ class Evaluate:
pbar.set_description(f"Average Metric: {ncorrect} / {ntotal} ({round(100 * ncorrect / ntotal, 1)})")
pbar.update()
- def __call__(self, program, metric=None, devset=None, num_threads=None, display_progress=None, display_table=None, display=None):
+ def __call__(self, program, metric=None, devset=None, num_threads=None,
+ display_progress=None, display_table=None, display=None,
+ return_all_scores=False):
metric = metric if metric is not None else self.metric
devset = devset if devset is not None else self.devset
num_threads = num_threads if num_threads is not None else self.num_threads
@@ -147,6 +151,8 @@ class Evaluate:
"""
ipython_display(HTML(message))
+ if return_all_scores:
+ return round(100 * ncorrect / ntotal, 2), [score for *_, score in predicted_devset]
return round(100 * ncorrect / ntotal, 2)
@@ -196,3 +202,4 @@ def configure_dataframe_display(df, metric_name):
})
# FIXME: TODO: The merge_dicts stuff above is way too quick and dirty.
+# TODO: the display_table can't handle False but can handle 0! Not sure how it works with True exactly, probably fails too.
\ No newline at end of file
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 96735261d..ba9d7f3a9 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -81,7 +81,7 @@ class Predict(Parameter):
x, C = dsp.generate(signature, **config)(x, stage=self.stage)
else:
with dsp.settings.context(lm=self.lm, query_only=True):
- print(f"using lm = {self.lm} !")
+ # print(f"using lm = {self.lm} !")
x, C = dsp.generate(signature, **config)(x, stage=self.stage)
completions = []
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index fa584d339..9f42ff9a7 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -25,6 +25,9 @@ from dspy.evaluate.evaluate import Evaluate
# won't hurt our "best effort" guarantees.)
+# TODO: Add baselines=[...]
+
+
class BootstrapFewShot(Teleprompter):
def __init__(self, metric=None, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1):
self.metric = metric
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index d77dccb50..be04411d6 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -2,6 +2,8 @@ import dsp
import tqdm
import random
+from dspy.teleprompt.teleprompt import Teleprompter
+
from .bootstrap import BootstrapFewShot
from .vanilla import LabeledFewShot
@@ -9,12 +11,8 @@ from dspy.evaluate.evaluate import Evaluate
# TODO: Don't forget dealing with the raw demos.
-# TODO: Deal with the (pretty common) case of having a metric for filtering and a metric for eval
-
-# TODO: There's an extremely strong case (> 90%) to switch this to NOT inherit from BootstrapFewShot.
-# Instead, it should wrap it: during compilation just loop, create a copy to compile, shuffle the full/sampled
-# trainset and compile with that. This will also make it easier to use raw demos.
-# Once all versions exist, define the validation set and evaluate.
+# TODO: Deal with the (pretty common) case of having a metric for filtering and a separate metric for eval.
+# The metric itself may tell though by the presence of trace.
# TODO: This function should take a max_budget and max_teacher_budget. That's in the number of program calls.
# In this case, max_student_budget is max_budget - max_teacher_budget.
@@ -28,11 +26,8 @@ from dspy.evaluate.evaluate import Evaluate
# In general, though, the early filtering is just saying: either there are some really bad ones, or some really really
# good ones, or most things are pretty close. In all of these cases, dropping the bottom third is not going to hurt.
-# Also, this function should be efficient in the following way:
-# seed -3, seed -2, and seed -1 so to speak should just be "zero shot", "labeled shots", and "bootstrap" without any tweaks.
-
-class BootstrapFewShotWithRandomSearch(BootstrapFewShot):
+class BootstrapFewShotWithRandomSearch(Teleprompter):
def __init__(self, metric, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, num_candidate_programs=16, num_threads=6):
self.metric = metric
self.teacher_settings = teacher_settings
@@ -43,64 +38,85 @@ class BootstrapFewShotWithRandomSearch(BootstrapFewShot):
self.min_num_samples = 1
self.max_num_samples = max_bootstrapped_demos
self.num_candidate_sets = num_candidate_programs
- self.max_num_traces = 1 + int(max_bootstrapped_demos / 2.0 * self.num_candidate_sets)
+ # self.max_num_traces = 1 + int(max_bootstrapped_demos / 2.0 * self.num_candidate_sets)
# Semi-hacky way to get the parent class's _boostrap function to stop early.
- self.max_bootstrapped_demos = self.max_num_traces
+ # self.max_bootstrapped_demos = self.max_num_traces
self.max_labeled_demos = max_labeled_demos
print("Going to sample between", self.min_num_samples, "and", self.max_num_samples, "traces per predictor.")
- print("Going to sample", self.max_num_traces, "traces in total.")
+ # print("Going to sample", self.max_num_traces, "traces in total.")
print("Will attempt to train", self.num_candidate_sets, "candidate sets.")
- # self.num_candidate_sets = 1
+ def compile(self, student, *, teacher=None, trainset, valset=None):
+ self.trainset = trainset
+ self.valset = valset or trainset # TODO: FIXME: Note this choice.
+
+ scores = []
+ all_subscores = []
+ score_data = []
+
+ for seed in range(-3, self.num_candidate_sets):
+ trainset2 = list(self.trainset)
+
+ if seed == -3:
+ # zero-shot
+ program2 = student.reset_copy()
+
+ elif seed == -2:
+ # labels only
+ teleprompter = LabeledFewShot(k=self.max_labeled_demos)
+ program2 = teleprompter.compile(student, trainset=trainset2)
+
+ elif seed == -1:
+ # unshuffled few-shot
+ program = BootstrapFewShot(metric=self.metric, max_bootstrapped_demos=self.max_num_samples,
+ max_labeled_demos=self.max_labeled_demos,
+ teacher_settings=self.teacher_settings, max_rounds=self.max_rounds)
+ program2 = program.compile(student, teacher=teacher, trainset=trainset2)
+
+ else:
+ assert seed >= 0, seed
+
+ random.Random(seed).shuffle(trainset2)
+ size = random.Random(seed).randint(self.min_num_samples, self.max_num_samples)
- # import time
- # time.sleep(10)
+ teleprompter = BootstrapFewShot(metric=self.metric, max_bootstrapped_demos=size,
+ max_labeled_demos=self.max_labeled_demos,
+ teacher_settings=self.teacher_settings,
+ max_rounds=self.max_rounds)
- def _random_search_instance(self, idx):
- print("Random search instance", idx)
+ program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)
- rng = random.Random(idx)
- program = self.student.deepcopy()
+ evaluate = Evaluate(devset=self.valset, metric=self.metric, num_threads=self.num_threads,
+ display_table=False, display_progress=True)
- for predictor in program.predictors():
- sample_size = rng.randint(self.min_num_samples, self.max_num_samples)
- print(f"[{idx}] \t Sampling {sample_size} traces from {len(predictor.traces)} traces.")
+ score, subscores = evaluate(program2, return_all_scores=True)
- augmented_demos = rng.sample(predictor.traces, min(sample_size, len(predictor.traces)))
+ all_subscores.append(subscores)
- # TODO: FIXME: Figuring out the raw demos here is a bit tricky. We can't just use the unused nor the unaugmented (validation) ones.
- augmented_uuids = set([x.dspy_uuid for x in augmented_demos])
- raw_demos_uuids = set([x.dspy_uuid for x in predictor.traces if x.dspy_uuid not in augmented_uuids])
+ print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])
+ scores.append(score)
- raw_demos = [x for x in self.trainset if x.dspy_uuid in raw_demos_uuids]
- raw_demos = rng.sample(raw_demos, min(self.max_labeled_demos - len(augmented_demos), len(raw_demos)))
+ print(f"Scores so far: {scores}")
- print(f'Got {len(augmented_demos)} augmented demos and {len(raw_demos)} raw demos.')
- predictor.demos = augmented_demos + raw_demos
-
- evaluate = Evaluate(devset=self.validation, metric=self.metric, num_threads=self.num_threads, display_table=False, display_progress=True)
- score = evaluate(program)
+ if score >= max(scores):
+ print('New best score:', score, 'for seed', seed)
+ best_program = program2
- print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program.predictors()])
- # dsp.settings.lm.inspect_history(n=1)
+ print('Best score:', max(scores))
- return (score, program)
+ score_data.append((score, subscores))
- def _train(self):
- for name, predictor in self.student.named_predictors():
- predictor.traces = self.name2traces[name]
- pass
+ if len(score_data) > 2: # We check if there are at least 3 scores to consider
+ for k in [1, 2, 3, 5, 8, 9999]:
+ top_3_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]
- self.candidate_sets = []
+ # Transpose the subscores to get max per entry and then calculate their average
+ transposed_subscores = zip(*[subscores for _, subscores in top_3_scores if subscores])
+ avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])
- for candidate_set_idx in range(self.num_candidate_sets):
- score, program = self._random_search_instance(candidate_set_idx)
- self.candidate_sets.append((score, program))
-
- best_score, best_program = max(self.candidate_sets, key=lambda x: x[0])
- print('Best score:', best_score)
+ print(f'Average of max per entry across top {k} scores: {avg_of_max_per_entry}')
return best_program