DSPy: The framework for programming—not prompting—language models
Python
Redesign BootstrapFewShotWithRandomSearch
Omar Khattabf01784f9f475
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 651bacfb4..688e0ea30 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -19,13 +19,15 @@ we print the number of failures, the first N examples that failed, and the firstclass Evaluate:- def __init__(self, *, devset, metric=None, num_threads=1, display_progress=False, display_table=False, display=True):+ def __init__(self, *, devset, metric=None, num_threads=1, display_progress=False,+ display_table=False, display=True, max_errors=5):self.devset = devsetself.metric = metricself.num_threads = num_threadsself.display_progress = display_progressself.display_table = display_tableself.display = display+ self.max_errors = max_errorsdef _execute_single_thread(self, wrapped_program, devset, display_progress):ncorrect = 0@@ -66,7 +68,9 @@ class Evaluate:pbar.set_description(f"Average Metric: {ncorrect} / {ntotal} ({round(100 * ncorrect / ntotal, 1)})")pbar.update()- def __call__(self, program, metric=None, devset=None, num_threads=None, display_progress=None, display_table=None, display=None):+ def __call__(self, program, metric=None, devset=None, num_threads=None,+ display_progress=None, display_table=None, display=None,+ return_all_scores=False):metric = metric if metric is not None else self.metricdevset = devset if devset is not None else self.devsetnum_threads = num_threads if num_threads is not None else self.num_threads@@ -147,6 +151,8 @@ class Evaluate:"""ipython_display(HTML(message))+ if return_all_scores:+ return round(100 * ncorrect / ntotal, 2), [score for *_, score in predicted_devset]return round(100 * ncorrect / ntotal, 2)@@ -196,3 +202,4 @@ def configure_dataframe_display(df, metric_name):})# FIXME: TODO: The merge_dicts stuff above is way too quick and dirty.+# TODO: the display_table can't handle False but can handle 0! Not sure how it works with True exactly, probably fails too.\ No newline at end of filediff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 96735261d..ba9d7f3a9 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -81,7 +81,7 @@ class Predict(Parameter):x, C = dsp.generate(signature, **config)(x, stage=self.stage)else:with dsp.settings.context(lm=self.lm, query_only=True):- print(f"using lm = {self.lm} !")+ # print(f"using lm = {self.lm} !")x, C = dsp.generate(signature, **config)(x, stage=self.stage)completions = []diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex fa584d339..9f42ff9a7 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -25,6 +25,9 @@ from dspy.evaluate.evaluate import Evaluate# won't hurt our "best effort" guarantees.)+# TODO: Add baselines=[...]++class BootstrapFewShot(Teleprompter):def __init__(self, metric=None, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1):self.metric = metricdiff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex d77dccb50..be04411d6 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -2,6 +2,8 @@ import dspimport tqdmimport random+from dspy.teleprompt.teleprompt import Teleprompter+from .bootstrap import BootstrapFewShotfrom .vanilla import LabeledFewShot@@ -9,12 +11,8 @@ from dspy.evaluate.evaluate import Evaluate# TODO: Don't forget dealing with the raw demos.-# TODO: Deal with the (pretty common) case of having a metric for filtering and a metric for eval--# TODO: There's an extremely strong case (> 90%) to switch this to NOT inherit from BootstrapFewShot.-# Instead, it should wrap it: during compilation just loop, create a copy to compile, shuffle the full/sampled-# trainset and compile with that. This will also make it easier to use raw demos.-# Once all versions exist, define the validation set and evaluate.+# TODO: Deal with the (pretty common) case of having a metric for filtering and a separate metric for eval.+# The metric itself may tell though by the presence of trace.# TODO: This function should take a max_budget and max_teacher_budget. That's in the number of program calls.# In this case, max_student_budget is max_budget - max_teacher_budget.@@ -28,11 +26,8 @@ from dspy.evaluate.evaluate import Evaluate# In general, though, the early filtering is just saying: either there are some really bad ones, or some really really# good ones, or most things are pretty close. In all of these cases, dropping the bottom third is not going to hurt.-# Also, this function should be efficient in the following way:-# seed -3, seed -2, and seed -1 so to speak should just be "zero shot", "labeled shots", and "bootstrap" without any tweaks.--class BootstrapFewShotWithRandomSearch(BootstrapFewShot):+class BootstrapFewShotWithRandomSearch(Teleprompter):def __init__(self, metric, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, num_candidate_programs=16, num_threads=6):self.metric = metricself.teacher_settings = teacher_settings@@ -43,64 +38,85 @@ class BootstrapFewShotWithRandomSearch(BootstrapFewShot):self.min_num_samples = 1self.max_num_samples = max_bootstrapped_demosself.num_candidate_sets = num_candidate_programs- self.max_num_traces = 1 + int(max_bootstrapped_demos / 2.0 * self.num_candidate_sets)+ # self.max_num_traces = 1 + int(max_bootstrapped_demos / 2.0 * self.num_candidate_sets)# Semi-hacky way to get the parent class's _boostrap function to stop early.- self.max_bootstrapped_demos = self.max_num_traces+ # self.max_bootstrapped_demos = self.max_num_tracesself.max_labeled_demos = max_labeled_demosprint("Going to sample between", self.min_num_samples, "and", self.max_num_samples, "traces per predictor.")- print("Going to sample", self.max_num_traces, "traces in total.")+ # print("Going to sample", self.max_num_traces, "traces in total.")print("Will attempt to train", self.num_candidate_sets, "candidate sets.")- # self.num_candidate_sets = 1+ def compile(self, student, *, teacher=None, trainset, valset=None):+ self.trainset = trainset+ self.valset = valset or trainset # TODO: FIXME: Note this choice.++ scores = []+ all_subscores = []+ score_data = []++ for seed in range(-3, self.num_candidate_sets):+ trainset2 = list(self.trainset)++ if seed == -3:+ # zero-shot+ program2 = student.reset_copy()++ elif seed == -2:+ # labels only+ teleprompter = LabeledFewShot(k=self.max_labeled_demos)+ program2 = teleprompter.compile(student, trainset=trainset2)++ elif seed == -1:+ # unshuffled few-shot+ program = BootstrapFewShot(metric=self.metric, max_bootstrapped_demos=self.max_num_samples,+ max_labeled_demos=self.max_labeled_demos,+ teacher_settings=self.teacher_settings, max_rounds=self.max_rounds)+ program2 = program.compile(student, teacher=teacher, trainset=trainset2)++ else:+ assert seed >= 0, seed++ random.Random(seed).shuffle(trainset2)+ size = random.Random(seed).randint(self.min_num_samples, self.max_num_samples)- # import time- # time.sleep(10)+ teleprompter = BootstrapFewShot(metric=self.metric, max_bootstrapped_demos=size,+ max_labeled_demos=self.max_labeled_demos,+ teacher_settings=self.teacher_settings,+ max_rounds=self.max_rounds)- def _random_search_instance(self, idx):- print("Random search instance", idx)+ program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)- rng = random.Random(idx)- program = self.student.deepcopy()+ evaluate = Evaluate(devset=self.valset, metric=self.metric, num_threads=self.num_threads,+ display_table=False, display_progress=True)- for predictor in program.predictors():- sample_size = rng.randint(self.min_num_samples, self.max_num_samples)- print(f"[{idx}] \t Sampling {sample_size} traces from {len(predictor.traces)} traces.")+ score, subscores = evaluate(program2, return_all_scores=True)- augmented_demos = rng.sample(predictor.traces, min(sample_size, len(predictor.traces)))+ all_subscores.append(subscores)- # TODO: FIXME: Figuring out the raw demos here is a bit tricky. We can't just use the unused nor the unaugmented (validation) ones.- augmented_uuids = set([x.dspy_uuid for x in augmented_demos])- raw_demos_uuids = set([x.dspy_uuid for x in predictor.traces if x.dspy_uuid not in augmented_uuids])+ print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])+ scores.append(score)- raw_demos = [x for x in self.trainset if x.dspy_uuid in raw_demos_uuids]- raw_demos = rng.sample(raw_demos, min(self.max_labeled_demos - len(augmented_demos), len(raw_demos)))+ print(f"Scores so far: {scores}")- print(f'Got {len(augmented_demos)} augmented demos and {len(raw_demos)} raw demos.')- predictor.demos = augmented_demos + raw_demos-- evaluate = Evaluate(devset=self.validation, metric=self.metric, num_threads=self.num_threads, display_table=False, display_progress=True)- score = evaluate(program)+ if score >= max(scores):+ print('New best score:', score, 'for seed', seed)+ best_program = program2- print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program.predictors()])- # dsp.settings.lm.inspect_history(n=1)+ print('Best score:', max(scores))- return (score, program)+ score_data.append((score, subscores))- def _train(self):- for name, predictor in self.student.named_predictors():- predictor.traces = self.name2traces[name]- pass+ if len(score_data) > 2: # We check if there are at least 3 scores to consider+ for k in [1, 2, 3, 5, 8, 9999]:+ top_3_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]- self.candidate_sets = []+ # Transpose the subscores to get max per entry and then calculate their average+ transposed_subscores = zip(*[subscores for _, subscores in top_3_scores if subscores])+ avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])- for candidate_set_idx in range(self.num_candidate_sets):- score, program = self._random_search_instance(candidate_set_idx)- self.candidate_sets.append((score, program))-- best_score, best_program = max(self.candidate_sets, key=lambda x: x[0])- print('Best score:', best_score)+ print(f'Average of max per entry across top {k} scores: {avg_of_max_per_entry}')return best_program