DSPy: The framework for programming—not prompting—language models
Python
typo fixes and add tests for BoostrapWithRandomSearch
chenmoneygithub9427411e3793
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex 23e3e03dd..d7a29102b 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -48,16 +48,10 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):self.max_num_samples = max_bootstrapped_demosself.max_errors = max_errorsself.num_candidate_sets = num_candidate_programs- # self.max_num_traces = 1 + int(max_bootstrapped_demos / 2.0 * self.num_candidate_sets)-- # Semi-hacky way to get the parent class's _bootstrap function to stop early.- # self.max_bootstrapped_demos = self.max_num_tracesself.max_labeled_demos = max_labeled_demos- print(- "Going to sample between", self.min_num_samples, "and", self.max_num_samples, "traces per predictor.",- )- print("Will attempt to bootstrap", self.num_candidate_sets, "candidate sets.")+ print(f"Going to sample between {self.min_num_samples} and {self.max_num_samples} traces per predictor.")+ print(f"Will attempt to bootstrap {self.num_candidate_sets} candidate sets.")def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None, labeled_sample=True):self.trainset = trainset@@ -71,20 +65,20 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):if (restrict is not None) and (seed not in restrict):continue- trainset2 = list(self.trainset)+ trainset_copy = list(self.trainset)if seed == -3:# zero-shot- program2 = student.reset_copy()+ program = student.reset_copy()elif seed == -2:# labels onlyteleprompter = LabeledFewShot(k=self.max_labeled_demos)- program2 = teleprompter.compile(student, trainset=trainset2, sample=labeled_sample)+ program = teleprompter.compile(student, trainset=trainset_copy, sample=labeled_sample)elif seed == -1:# unshuffled few-shot- program = BootstrapFewShot(+ optimizer = BootstrapFewShot(metric=self.metric,metric_threshold=self.metric_threshold,max_bootstrapped_demos=self.max_num_samples,@@ -92,15 +86,15 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):teacher_settings=self.teacher_settings,max_rounds=self.max_rounds,)- program2 = program.compile(student, teacher=teacher, trainset=trainset2)+ program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy)else:assert seed >= 0, seed- random.Random(seed).shuffle(trainset2)+ random.Random(seed).shuffle(trainset_copy)size = random.Random(seed).randint(self.min_num_samples, self.max_num_samples)- teleprompter = BootstrapFewShot(+ optimizer = BootstrapFewShot(metric=self.metric,metric_threshold=self.metric_threshold,max_bootstrapped_demos=size,@@ -109,7 +103,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):max_rounds=self.max_rounds,)- program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)+ program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy)evaluate = Evaluate(devset=self.valset,@@ -120,37 +114,36 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):display_progress=True,)- score, subscores = evaluate(program2, return_all_scores=True)+ score, subscores = evaluate(program, return_all_scores=True)all_subscores.append(subscores)############ Assertion-aware Optimization ############- if hasattr(program2, "_suggest_failures"):- score = score - program2._suggest_failures * 0.2- if hasattr(program2, "_assert_failures"):- score = 0 if program2._assert_failures > 0 else score+ if hasattr(program, "_suggest_failures"):+ score = score - program._suggest_failures * 0.2+ if hasattr(program, "_assert_failures"):+ score = 0 if program._assert_failures > 0 else score######################################################- print("Score:", score, "for set:", [len(predictor.demos) for predictor in program2.predictors()])+ print("Score:", score, "for set:", [len(predictor.demos) for predictor in program.predictors()])if len(scores) == 0 or score > max(scores):- print("New best sscore:", score, "for seed", seed)- best_program = program2+ print("New best score:", score, "for seed", seed)+ best_program = programscores.append(score)- print(f"Scores so far: {scores}")-- print("Best score:", max(scores))+ print(f"Scores so far: {scores}.")+ print(f"Best score: {max(scores)}")- score_data.append((score, subscores, seed, program2))+ score_data.append((score, subscores, seed, program))if len(score_data) > 2: # We check if there are at least 3 scores to considerfor k in [1, 2, 3, 5, 8, 9999]:- top_3_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]+ top_k_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]# Transpose the subscores to get max per entry and then calculate their average- transposed_subscores = zip(*[subscores for _, subscores, *_ in top_3_scores if subscores])- avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])+ transposed_subscores = zip(*[subscores for _, subscores, *_ in top_k_scores if subscores])+ avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_k_scores[0][1])print(f"Average of max per entry across top {k} scores: {avg_of_max_per_entry}")diff --git a/tests/teleprompt/test_random_search.py b/tests/teleprompt/test_random_search.pynew file mode 100644index 000000000..e76178c80--- /dev/null+++ b/tests/teleprompt/test_random_search.py@@ -0,0 +1,37 @@+import dspy+from dspy.predict import Predict+from dspy.utils.dummies import DummyLM+from dspy import Example+from dspy.teleprompt import BootstrapFewShotWithRandomSearch++class SimpleModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ self.predictor = Predict(signature)++ def forward(self, **kwargs):+ return self.predictor(**kwargs)++def simple_metric(example, prediction, trace=None):+ return example.output == prediction.output++def test_basic_workflow():+ """Test to ensure the basic compile flow runs without errors."""+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")++ lm = DummyLM(+ [+ "Initial thoughts",+ "Finish[blue]", # Expected output for both training and validation+ ]+ )+ dspy.settings.configure(lm=lm)++ optimizer = BootstrapFewShotWithRandomSearch(metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1)+ trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),+ ]+ optimizer.compile(student, teacher=teacher, trainset=trainset)+