DSPy: The framework for programming—not prompting—language models
Python
updates to mipro optimizer (show score trajectory in print statements, rename some vars, add vars to compile
klopsahlongcd3bf705d1ca
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex 5947451d0..b1914b4de 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -30,8 +30,8 @@ To start off, you can use a preconfigured auto-run setting, which sets hyperpara``` pythonfrom dspy.teleprompt import MIPROv2-teleprompter = MIPROv2(metric=metric, auto_run_mode = "light")-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset, max_bootstrapped_demos=3, max_labeled_demos=5)+teleprompter = MIPROv2(metric=metric, auto = "light")+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset, self.max_bootstrapped_demos=3, self.max_labeled_demos=5)eval_score = evaluate(compiled_prompt_opt, devset=testset, display_progress=True, num_threads=1)```@@ -42,7 +42,7 @@ from dspy.teleprompt import MIPROv2teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5)+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, self.max_bootstrapped_demos=3, self.max_labeled_demos=5)eval_score = evaluate(compiled_prompt_opt, devset=valset[:EVAL_NUM], **kwargs)```@@ -88,16 +88,16 @@ class MIPROv2(Teleprompter):def _set_hyperparams_from_run_mode(self, program, num_trials, minibatch, zeroshot_opt, valset):- if self.auto_run_mode is None:+ if self.auto is None:return num_trials, valset, minibatchnum_vars = (len(program.predictors()) if zeroshot_opt else len(program.predictors()) * 2) # (multiplying by two to account for few-shot examples + instruction variables)- num_trials = auto_run_settings[self.auto_run_mode]["num_trials"]+ num_trials = auto_run_settings[self.auto]["num_trials"]valset = create_minibatch(- valset, batch_size=auto_run_settings[self.auto_run_mode]["val_size"]+ valset, batch_size=auto_run_settings[self.auto]["val_size"])minibatch = len(valset) > MIN_MINIBATCH_SIZEself.num_candidates = int(@@ -109,32 +109,37 @@ class MIPROv2(Teleprompter):def __init__(self,metric,- prompt_model=dspy.settings.lm,- task_model=dspy.settings.lm,+ prompt_model=None,+ task_model=None,teacher_settings={},- auto_run_mode=None,+ max_bootstrapped_demos=4,+ max_labeled_demos=16,+ auto=None,num_candidates=10,num_threads=6,max_errors=10,+ seed=9,init_temperature=0.5,verbose=False,track_stats=True,log_dir=None,metric_threshold=None,):- # Check for valid auto_run_mode+ # Check for valid autoallowed_modes = {None, "light", "medium", "heavy"}- if auto_run_mode not in allowed_modes:+ if auto not in allowed_modes:raise ValueError(- f"Invalid value for auto_run_mode: {auto_run_mode}. Must be one of {allowed_modes}."+ f"Invalid value for auto: {auto}. Must be one of {allowed_modes}.")- self.auto_run_mode = auto_run_mode+ self.auto = autoself.num_candidates = num_candidatesself.metric = metricself.init_temperature = init_temperature- self.task_model = task_model- self.prompt_model = prompt_model+ self.task_model = task_model if task_model else dspy.settings.lm+ self.prompt_model = prompt_model if prompt_model else dspy.settings.lm+ self.max_bootstrapped_demos = max_bootstrapped_demos+ self.max_labeled_demos = max_labeled_demosself.verbose = verboseself.track_stats = track_statsself.log_dir = log_dir@@ -152,9 +157,9 @@ class MIPROv2(Teleprompter):trainset,valset=None,num_trials=30,- max_bootstrapped_demos=4,- max_labeled_demos=16,- seed=9,+ max_bootstrapped_demos=None,+ max_labeled_demos=None,+ seed=None,minibatch=True,minibatch_size=25,minibatch_full_eval_steps=10,@@ -173,7 +178,12 @@ class MIPROv2(Teleprompter):ENDC = "\033[0m" # Resets the color to defaultrandom.seed(seed)-+ np.random.seed(seed)++ if max_bootstrapped_demos or max_labeled_demos:+ self.max_bootstrapped_demos = max_bootstrapped_demos+ self.max_labeled_demos = max_labeled_demos+# Validate inputsif not trainset:raise ValueError("Trainset cannot be empty.")@@ -198,14 +208,14 @@ class MIPROv2(Teleprompter):)# Set hyperparams- zeroshot_opt = (max_bootstrapped_demos == 0) and (max_labeled_demos == 0)+ zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(student, num_trials, minibatch, zeroshot_opt, valset)- if self.auto_run_mode:+ if self.auto:print(- f"RUNNING WITH THE FOLLOWING {self.auto_run_mode.upper()} AUTO RUN SETTINGS:\nnum_trials: {num_trials}\nminibatch: {minibatch}\nnum_candidates: {self.num_candidates}\nvalset size: {len(valset)}P"+ f"\nRUNNING WITH THE FOLLOWING {self.auto.upper()} AUTO RUN SETTINGS:\nnum_trials: {num_trials}\nminibatch: {minibatch}\nnum_candidates: {self.num_candidates}\nvalset size: {len(valset)}\n")if minibatch and minibatch_size > len(valset):@@ -289,7 +299,6 @@ class MIPROv2(Teleprompter):if run:# Setup random seeds- random.seed(seed)np.random.seed(seed)# Set up program and evaluation function@@ -304,21 +313,21 @@ class MIPROv2(Teleprompter):)# Determine the number of fewshot examples to use to generate demos for prompt- if max_bootstrapped_demos == 0 and max_labeled_demos == 0:- max_bootstrapped_demos_for_candidate_gen = (+ if self.max_bootstrapped_demos == 0 and self.max_labeled_demos == 0:+ self.max_bootstrapped_demos_for_candidate_gen = (BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT)- max_labeled_demos_for_candidate_gen = (+ self.max_labeled_demos_for_candidate_gen = (LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT)else:- max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos- max_labeled_demos_for_candidate_gen = max_labeled_demos+ self.max_bootstrapped_demos_for_candidate_gen = self.max_bootstrapped_demos+ self.max_labeled_demos_for_candidate_gen = self.max_labeled_demos# Generate N few shot example sets (these will inform instruction creation, and be used as few-shot examples in our prompt)print("Beginning MIPROv2 optimization process...")print("\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")- if max_bootstrapped_demos > 0:+ if self.max_bootstrapped_demos > 0:print("These will be used for as few-shot examples candidates for our program and for creating instructions.\n")@@ -330,8 +339,8 @@ class MIPROv2(Teleprompter):student=program,num_candidate_sets=self.num_candidates,trainset=trainset,- max_labeled_demos=max_labeled_demos_for_candidate_gen,- max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,+ max_labeled_demos=self.max_labeled_demos_for_candidate_gen,+ max_bootstrapped_demos=self.max_bootstrapped_demos_for_candidate_gen,metric=self.metric,max_errors=self.max_errors,teacher_settings=self.teacher_settings,@@ -355,7 +364,6 @@ class MIPROv2(Teleprompter):program=program,trainset=trainset,prompt_model=self.prompt_model,- # program_code_string=self.program_code_string,view_data_batch_size=view_data_batch_size,program_aware=program_aware_proposer,use_dataset_summary=data_aware_proposer,@@ -384,7 +392,7 @@ class MIPROv2(Teleprompter):print("\n")# If we're doing zero-shot, reset demo_candidates to none now that we've used them for instruction proposal- if max_bootstrapped_demos == 0 and max_labeled_demos == 0:+ if self.max_bootstrapped_demos == 0 and self.max_labeled_demos == 0:demo_candidates = None# Initialize variables to track during the optimization process@@ -405,6 +413,8 @@ class MIPROv2(Teleprompter):)print(f"Default program score: {default_score}\n")+ scores = []+ full_eval_scores = [default_score]best_score = default_scorebest_program = program.deepcopy()@@ -417,7 +427,7 @@ class MIPROv2(Teleprompter):valset,):def objective(trial):- nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls # Allow access to the outer variables+ nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls, scores, full_eval_scores # Allow access to the outer variables# Kick off trialif minibatch:@@ -507,6 +517,8 @@ class MIPROv2(Teleprompter):evaluate,)+ scores.append(score)+# Print out a full trace of the program in useif self.verbose:print("Full trace of prompts in use on an example...")@@ -550,18 +562,28 @@ class MIPROv2(Teleprompter):if minibatch:print(- f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n"+ f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n")+ print(+ f"Minibatch Score Trajectory: {', '.join([f'{s}' for s in scores])}"+ )+ trajectory = ", ".join([f"{s}" for s in full_eval_scores])+ print(f"Full Eval Trajectory: {trajectory}")+ print(f'{"="*len(f"== Minibatch Trial {trial.number+1} / {num_trials} ==")}\n\n')else:print(f"Score: {score} with parameters {chosen_params}.")if best_score_updated:print(- f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n"- )- else:- print(- f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n"+ f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.")+ print()+ print(+ f"Score Trajectory: {', '.join([f'{s}' for s in scores])}"+ )+ print(+ f"Best score so far: {best_score} on trial {best_scoring_trial}."+ )+ print(f'{"="*len(f"===== Trial {trial.number+1} / {num_trials} =====")}\n\n')# If we're doing minibatching, check to see if it's time to do a full evalif minibatch and (@@ -599,6 +621,7 @@ class MIPROv2(Teleprompter):highest_mean_program,evaluate,)+ full_eval_scores.append(full_val_score)# Log relevant informationfully_evaled_param_combos[combo_key] = {@@ -631,6 +654,10 @@ class MIPROv2(Teleprompter):else:print(f"Full eval score: {full_val_score}")print(f"Best full eval score so far: {best_score}")+ print()+ trajectory = ", ".join([f"{s}" for s in full_eval_scores])+ print(f"Full Eval Trajectory: {trajectory}")+print("=======================\n\n")return score