DSPy: The framework for programming—not prompting—language models
Python
adding auto run mode for miprov2
klopsahlongbe581df924dc
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.pyindex d1d6192bb..740197f6f 100644--- a/dspy/propose/grounded_proposer.py+++ b/dspy/propose/grounded_proposer.py@@ -270,20 +270,27 @@ class GroundedProposer(Proposer):self.verbose = verboseself.prompt_model = get_prompt_model(prompt_model)++ self.program_code_string = Noneif self.program_aware:try:self.program_code_string = get_dspy_source_code(program)if self.verbose: print("SOURCE CODE:",self.program_code_string)except Exception as e:print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")- self.program_code_string = Noneself.program_aware = False- else:- self.program_code_string = None- self.data_summary = create_dataset_summary(- trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,- )- if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")++ self.data_summary = None+ if self.use_dataset_summary:+ try:+ self.data_summary = create_dataset_summary(+ trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,+ )+ if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")+ except Exception as e:+ print(f"Error getting data summary: {e}.\n\nRunning without data aware proposer.")+ self.use_dataset_summary = False+ print(f"")def propose_instructions_for_program(self,diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex ab758f7e6..6e238e383 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -19,6 +19,7 @@ from dspy.teleprompt.utils import (print_full_program,save_candidate_program,set_signature,+ create_minibatch,)"""@@ -63,13 +64,45 @@ Note that this teleprompter takes in the following parameters:BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0+MIN_MINIBATCH_SIZE = 50++auto_run_settings = {+ "light": {"num_trials": 7, "val_size": 100},+ "medium": {"num_trials": 25, "val_size": 300},+ "heavy": {"num_trials": 50, "val_size": 1000},+}++class MIPROv2(Teleprompter):++ def _set_hyperparams_from_run_mode(+ self, program, num_trials, minibatch, zeroshot_opt, valset+ ):+ if self.auto_run_mode is None:+ return num_trials, valset, minibatch++ num_vars = (+ len(program.predictors()) if zeroshot_opt else len(program.predictors()) * 2+ ) # (multiplying by two to account for few-shot examples + instruction variables)++ num_trials = auto_run_settings[self.auto_run_mode]["num_trials"]+ valset = create_minibatch(+ valset, batch_size=auto_run_settings[self.auto_run_mode]["val_size"]+ )+ minibatch = len(valset) > MIN_MINIBATCH_SIZE+ self.num_candidates = int(+ np.round(np.min([num_trials * num_vars, (3 / 2 * num_trials) / num_vars]))+ )++ return num_trials, valset, minibatch+def __init__(self,metric,prompt_model=dspy.settings.lm,task_model=dspy.settings.lm,teacher_settings={},+ auto_run_mode=None,num_candidates=10,num_threads=6,max_errors=10,@@ -79,6 +112,14 @@ class MIPROv2(Teleprompter):log_dir=None,metric_threshold=None,):+ # Check for valid auto_run_mode+ allowed_modes = {None, "light", "medium", "heavy"}+ if auto_run_mode not in allowed_modes:+ raise ValueError(+ f"Invalid value for auto_run_mode: {auto_run_mode}. Must be one of {allowed_modes}."+ )+ self.auto_run_mode = auto_run_mode+self.num_candidates = num_candidatesself.metric = metricself.init_temperature = init_temperature@@ -129,27 +170,48 @@ class MIPROv2(Teleprompter):if not valset:if len(trainset) < 2:- raise ValueError("Trainset must have at least 2 examples if no valset specified, or at least 1 example with external validation set.")-- valset_size = min(500, max(1, int(len(trainset) * 0.80))) # 80% of trainset, capped at 500+ raise ValueError(+ "Trainset must have at least 2 examples if no valset specified, or at least 1 example with external validation set."+ )++ valset_size = min(+ 500, max(1, int(len(trainset) * 0.80))+ ) # 80% of trainset, capped at 500cutoff = len(trainset) - valset_sizevalset = trainset[cutoff:]trainset = trainset[:cutoff]else:if len(valset) < 1:- raise ValueError("Validation set must have at least 1 example if specified.")-+ raise ValueError(+ "Validation set must have at least 1 example if specified."+ )++ # Set hyperparams+ zeroshot_opt = (max_bootstrapped_demos == 0) and (max_labeled_demos == 0)+ num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(+ student, num_trials, minibatch, zeroshot_opt, valset+ )++ if self.auto_run_mode:+ print(+ f"RUNNING WITH THE FOLLOWING {self.auto_run_mode.upper()} AUTO RUN SETTINGS:\nnum_trials: {num_trials}\nminibatch: {minibatch}\nnum_candidates: {self.num_candidates}\nvalset size: {len(valset)}P"+ )+if minibatch and minibatch_size > len(valset):- raise ValueError(f"Minibatch size cannot exceed the size of the valset. Note that your validation set contains {len(valset)} examples. Your train set contains {len(trainset)} examples.")-- if minibatch and num_trials < minibatch_full_eval_steps:- raise ValueError(f"Number of trials (num_trials={num_trials}) must be greater than or equal to the number of minibatch full eval steps (minibatch_full_eval_steps={minibatch_full_eval_steps}).")+ raise ValueError(+ f"Minibatch size cannot exceed the size of the valset. Note that your validation set contains {len(valset)} examples. Your train set contains {len(trainset)} examples."+ )- estimated_prompt_model_calls = 10 + self.num_candidates * len(+ estimated_prompt_model_calls = (+ 10+ + self.num_candidates+ * len(student.predictors(),- ) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)-+ )+ + (0 if not program_aware_proposer else len(student.predictors()) + 1)+ ) # num data summary calls + N * P + (P + 1)+prompt_model_line = ""if not program_aware_proposer:prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""@@ -159,14 +221,18 @@ class MIPROv2(Teleprompter):estimated_task_model_calls_wo_module_calls = 0task_model_line = ""if not minibatch:- estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P+ estimated_task_model_calls_wo_module_calls = (+ len(trainset) * num_trials+ ) # M * T * Ptask_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(valset)}{ENDC}{YELLOW} examples in val set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""else:- estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (len(trainset) * (num_trials // minibatch_full_eval_steps)) # B * T * P+ estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (+ len(trainset) * (num_trials // minibatch_full_eval_steps)+ ) # B * T * Ptask_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(valset)}{ENDC}{YELLOW} examples in val set * {BLUE}{BOLD}{num_trials // minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""-- user_message = textwrap.dedent(f"""\+ user_message = textwrap.dedent(+ f"""\{YELLOW}{BOLD}Projected Language Model (LM) Calls{ENDC}Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:@@ -184,17 +250,21 @@ class MIPROv2(Teleprompter):and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:{YELLOW}- Reducing the number of trials (`num_trials`), the size of the valset, or the number of LM calls in your program.{ENDC}- {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n""")+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n"""+ )- user_confirmation_message = textwrap.dedent(f"""\+ user_confirmation_message = textwrap.dedent(+ f"""\To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.{YELLOW}Awaiting your input...{ENDC}- """)+ """+ )- if requires_permission_to_run: print(user_message)+ if requires_permission_to_run:+ print(user_message)sys.stdout.flush() # Flush the output buffer to force the message to print@@ -225,17 +295,23 @@ class MIPROv2(Teleprompter):# Determine the number of fewshot examples to use to generate demos for promptif max_bootstrapped_demos == 0 and max_labeled_demos == 0:- max_bootstrapped_demos_for_candidate_gen = BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT- max_labeled_demos_for_candidate_gen = LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT+ max_bootstrapped_demos_for_candidate_gen = (+ BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT+ )+ max_labeled_demos_for_candidate_gen = (+ LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT+ )else:max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demosmax_labeled_demos_for_candidate_gen = max_labeled_demos-+# Generate N few shot example sets (these will inform instruction creation, and be used as few-shot examples in our prompt)print("Beginning MIPROv2 optimization process...")print("\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")if max_bootstrapped_demos > 0:- print("These will be used for as few-shot examples candidates for our program and for creating instructions.\n")+ print(+ "These will be used for as few-shot examples candidates for our program and for creating instructions.\n"+ )else:print("These will be used for informing instruction proposal.\n")print(f"Bootstrapping N={self.num_candidates} sets of demonstrations...")@@ -256,12 +332,14 @@ class MIPROv2(Teleprompter):print(f"Error generating fewshot examples: {e}")print("Running without fewshot examples.")demo_candidates = None-+# Generate N candidate instructions- # Setup our proposer+ # Setup our proposerprint("\n==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==")- print("In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.")+ print(+ "In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions."+ )proposer = GroundedProposer(program=program,@@ -276,9 +354,9 @@ class MIPROv2(Teleprompter):set_tip_randomly=tip_aware_proposer,use_instruct_history=False,set_history_randomly=False,- verbose = self.verbose,+ verbose=self.verbose,)-+print("\nProposing instructions...\n")instruction_candidates = proposer.propose_instructions_for_program(trainset=trainset,@@ -303,18 +381,23 @@ class MIPROv2(Teleprompter):best_scoring_trial = 0trial_logs = {}total_eval_calls = 0- param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scores- fully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of+ param_score_dict = defaultdict(+ list+ ) # Dictionaries of paramater combinations we've tried, and their associated scores+ fully_evaled_param_combos = (+ {}+ ) # List of the parameter combinations we've done full evals of# Evaluate the default programprint("Evaluating the default program...\n")- default_score = eval_candidate_program(len(valset), valset, program, evaluate)+ default_score = eval_candidate_program(+ len(valset), valset, program, evaluate+ )print(f"Default program score: {default_score}\n")best_score = default_scorebest_program = program.deepcopy()-# Define our trial objectivedef create_objective(baseline_program,@@ -325,13 +408,13 @@ class MIPROv2(Teleprompter):):def objective(trial):nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls # Allow access to the outer variables-+# Kick off trialif minibatch:print(f"== Minibatch Trial {trial.number+1} / {num_trials} ==")else:print(f"===== Trial {trial.number+1} / {num_trials} =====")- trial_logs[trial.number+1] = {}+ trial_logs[trial.number + 1] = {}# Create a new candidate programcandidate_program = baseline_program.deepcopy()@@ -351,19 +434,26 @@ class MIPROv2(Teleprompter):range(len(p_instruction_candidates)),)# chosen_params.append(instruction_idx)- chosen_params.append(f"Predictor {i+1}: Instruction {instruction_idx}")+ chosen_params.append(+ f"Predictor {i+1}: Instruction {instruction_idx}"+ )if demo_candidates:demos_idx = trial.suggest_categorical(- f"{i}_predictor_demos", range(len(p_demo_candidates)),+ f"{i}_predictor_demos",+ range(len(p_demo_candidates)),+ )+ chosen_params.append(+ f"Predictor {i+1}: Few-Shot Set {demos_idx}")- chosen_params.append(f"Predictor {i+1}: Few-Shot Set {demos_idx}")# Log the selected instruction / demo candidate- trial_logs[trial.number+1][+ trial_logs[trial.number + 1][f"{i}_predictor_instruction"] = instruction_idxif demo_candidates:- trial_logs[trial.number+1][f"{i}_predictor_demos"] = demos_idx+ trial_logs[trial.number + 1][+ f"{i}_predictor_demos"+ ] = demos_idxdspy.logger.debug(f"instruction_idx {instruction_idx}")if demo_candidates:@@ -381,21 +471,29 @@ class MIPROv2(Teleprompter):p_new.demos = p_demo_candidates[demos_idx]# Log assembled program- if self.verbose: print("Evaluating the following candidate program...\n")- if self.verbose: print_full_program(candidate_program)+ if self.verbose:+ print("Evaluating the following candidate program...\n")+ if self.verbose:+ print_full_program(candidate_program)# Save the candidate program- trial_logs[trial.number+1]["program_path"] = save_candidate_program(- candidate_program, self.log_dir, trial.number+1,+ trial_logs[trial.number + 1]["program_path"] = (+ save_candidate_program(+ candidate_program,+ self.log_dir,+ trial.number + 1,+ ))- trial_logs[trial.number+1]["num_eval_calls"] = 0+ trial_logs[trial.number + 1]["num_eval_calls"] = 0# Evaluate the candidate program with relevant batch sizebatch_size = minibatch_size if minibatch else len(valset)score = eval_candidate_program(- batch_size, valset, candidate_program,+ batch_size,+ valset,+ candidate_program,evaluate,)@@ -403,7 +501,10 @@ class MIPROv2(Teleprompter):if self.verbose:print("Full trace of prompts in use on an example...")get_task_model_history_for_full_example(- candidate_program, self.task_model, valset, evaluate,+ candidate_program,+ self.task_model,+ valset,+ evaluate,)# Log relevant information@@ -411,72 +512,117 @@ class MIPROv2(Teleprompter):param_score_dict[categorical_key].append((score, candidate_program),)- trial_logs[trial.number+1]["num_eval_calls"] = batch_size- trial_logs[trial.number+1]["full_eval"] = batch_size >= len(valset)- trial_logs[trial.number+1]["score"] = score- trial_logs[trial.number+1]["pruned"] = False- total_eval_calls += trial_logs[trial.number+1]["num_eval_calls"]- trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number+1]["program"] = candidate_program.deepcopy()+ trial_logs[trial.number + 1]["num_eval_calls"] = batch_size+ trial_logs[trial.number + 1]["full_eval"] = batch_size >= len(+ valset+ )+ trial_logs[trial.number + 1]["score"] = score+ trial_logs[trial.number + 1]["pruned"] = False+ total_eval_calls += trial_logs[trial.number + 1]["num_eval_calls"]+ trial_logs[trial.number + 1][+ "total_eval_calls_so_far"+ ] = total_eval_calls+ trial_logs[trial.number + 1][+ "program"+ ] = candidate_program.deepcopy()# If this score was from a full evaluation, update the best program if the new score is betterbest_score_updated = False- if score > best_score and trial_logs[trial.number+1]["full_eval"] and not minibatch:+ if (+ score > best_score+ and trial_logs[trial.number + 1]["full_eval"]+ and not minibatch+ ):best_score = score- best_scoring_trial = trial.number+1+ best_scoring_trial = trial.number + 1best_program = candidate_program.deepcopy()best_score_updated = True-+if minibatch:- print(f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n")+ print(+ f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n"+ )else:print(f"Score: {score} with parameters {chosen_params}.")if best_score_updated:- print(f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n")+ print(+ f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n"+ )else:- print(f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n")-+ print(+ f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n"+ )# If we're doing minibatching, check to see if it's time to do a full eval- if minibatch and (((trial.number+1) % minibatch_full_eval_steps == 0) or (trial.number+1 == num_trials)):- print(f"===== Full Eval {len(fully_evaled_param_combos)+1} =====")-+ if minibatch and (+ ((trial.number + 1) % minibatch_full_eval_steps == 0)+ or (trial.number + 1 == num_trials)+ ):+ print(+ f"===== Full Eval {len(fully_evaled_param_combos)+1} ====="+ )+# Save old information as the minibatch version- trial_logs[trial.number+1]["mb_score"] = score- trial_logs[trial.number+1]["mb_program_path"] = trial_logs[trial.number+1]["program_path"]+ trial_logs[trial.number + 1]["mb_score"] = score+ trial_logs[trial.number + 1]["mb_program_path"] = trial_logs[+ trial.number + 1+ ]["program_path"]# Identify our best program (based on mean of scores so far, and do a full eval on it)- highest_mean_program, mean, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)-- if trial.number+1 // minibatch_full_eval_steps > 0:- print(f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials...")+ highest_mean_program, mean, combo_key = (+ get_program_with_highest_avg_score(+ param_score_dict, fully_evaled_param_combos+ )+ )++ if trial.number + 1 // minibatch_full_eval_steps > 0:+ print(+ f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials..."+ )else:- print(f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials...")+ print(+ f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials..."+ )full_val_score = eval_candidate_program(- len(valset), valset, highest_mean_program, evaluate,+ len(valset),+ valset,+ highest_mean_program,+ evaluate,)# Log relevant information- fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_val_score}+ fully_evaled_param_combos[combo_key] = {+ "program": highest_mean_program,+ "score": full_val_score,+ }total_eval_calls += len(valset)- trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial.number+1]["full_eval"] = True- trial_logs[trial.number+1]["program_path"] = save_candidate_program(- program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number+1, note="full_eval",+ trial_logs[trial.number + 1][+ "total_eval_calls_so_far"+ ] = total_eval_calls+ trial_logs[trial.number + 1]["full_eval"] = True+ trial_logs[trial.number + 1]["program_path"] = (+ save_candidate_program(+ program=highest_mean_program,+ log_dir=self.log_dir,+ trial_num=trial.number + 1,+ note="full_eval",+ ))- trial_logs[trial.number+1]["score"] = full_val_score-+ trial_logs[trial.number + 1]["score"] = full_val_score+if full_val_score > best_score:- print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_val_score}")+ print(+ f"{GREEN}Best full eval score so far!{ENDC} Score: {full_val_score}"+ )best_score = full_val_score- best_scoring_trial = trial.number+1+ best_scoring_trial = trial.number + 1best_program = highest_mean_program.deepcopy()best_score_updated = Trueelse:print(f"Full eval score: {full_val_score}")print(f"Best full eval score so far: {best_score}")print("=======================\n\n")-+return scorereturn objective@@ -484,9 +630,15 @@ class MIPROv2(Teleprompter):# Run the trialoptuna.logging.set_verbosity(optuna.logging.WARNING)print("==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==")- print("In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n")+ print(+ "In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n"+ )objective_function = create_objective(- program, instruction_candidates, demo_candidates, evaluate, valset,+ program,+ instruction_candidates,+ demo_candidates,+ evaluate,+ valset,)sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)study = optuna.create_study(direction="maximize", sampler=sampler)