DSPy: The framework for programming—not prompting—language models
Python
MIPROv2 Updates: New defaults, fixed Trial # Logging (#8029)
klopsahlong6922a8b014bf
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex 772e0c5b0..0f92ece37 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -52,8 +52,8 @@ class MIPROv2(Teleprompter):task_model: Optional[Any] = None,teacher_settings: Dict = {},max_bootstrapped_demos: int = 4,- max_labeled_demos: int = 16,- auto: Optional[Literal["light", "medium", "heavy"]] = None,+ max_labeled_demos: int = 4,+ auto: Optional[Literal["light", "medium", "heavy"]] = "medium",num_candidates: int = 10,num_threads: int = 6,max_errors: int = 10,@@ -101,8 +101,8 @@ class MIPROv2(Teleprompter):max_labeled_demos: Optional[int] = None,seed: Optional[int] = None,minibatch: bool = True,- minibatch_size: int = 25,- minibatch_full_eval_steps: int = 10,+ minibatch_size: int = 35,+ minibatch_full_eval_steps: int = 5,program_aware_proposer: bool = True,data_aware_proposer: bool = True,view_data_batch_size: int = 10,@@ -464,7 +464,8 @@ class MIPROv2(Teleprompter):)# Compute the adjusted total trials that we will run (including full evals)- adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1) if minibatch else num_trials+ run_additional_full_eval_at_end = 1 if num_trials % minibatch_full_eval_steps != 0 else 0+ adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1 + run_additional_full_eval_at_end) if minibatch else num_trialslogger.info(f"== Trial {1} / {adjusted_num_trials} - Full Evaluation of Default Program ==")default_score, _ = eval_candidate_program(@@ -473,11 +474,11 @@ class MIPROv2(Teleprompter):logger.info(f"Default program score: {default_score}\n")trial_logs = {}- trial_logs[-1] = {}- trial_logs[-1]["full_eval_program_path"] = save_candidate_program(program, self.log_dir, -1)- trial_logs[-1]["full_eval_score"] = default_score- trial_logs[-1]["total_eval_calls_so_far"] = len(valset)- trial_logs[-1]["full_eval_program"] = program.deepcopy()+ trial_logs[1] = {}+ trial_logs[1]["full_eval_program_path"] = save_candidate_program(program, self.log_dir, -1)+ trial_logs[1]["full_eval_score"] = default_score+ trial_logs[1]["total_eval_calls_so_far"] = len(valset)+ trial_logs[1]["full_eval_program"] = program.deepcopy()# Initialize optimization variablesbest_score = default_score@@ -567,7 +568,7 @@ class MIPROv2(Teleprompter):)# If minibatch, perform full evaluation at intervals (and at the very end)- if minibatch and ((trial_num % minibatch_full_eval_steps == 0) or (trial_num == (adjusted_num_trials - 1))):+ if minibatch and ((trial_num % (minibatch_full_eval_steps+1) == 0) or (trial_num == (adjusted_num_trials-1))):best_score, best_program, total_eval_calls = self._perform_full_evaluation(trial_num,adjusted_num_trials,@@ -601,7 +602,7 @@ class MIPROv2(Teleprompter):value=default_score,)study.add_trial(trial)- study.optimize(objective, n_trials=num_trials - 1)+ study.optimize(objective, n_trials=num_trials)# Attach logs to best programif best_program is not None and self.track_stats:@@ -767,15 +768,16 @@ class MIPROv2(Teleprompter):"score": full_eval_score,}total_eval_calls += len(valset)- trial_logs[trial_num]["total_eval_calls_so_far"] = total_eval_calls- trial_logs[trial_num]["full_eval_program_path"] = save_candidate_program(+ trial_logs[trial_num + 1] = {}+ trial_logs[trial_num + 1]["total_eval_calls_so_far"] = total_eval_calls+ trial_logs[trial_num + 1]["full_eval_program_path"] = save_candidate_program(program=highest_mean_program,log_dir=self.log_dir,- trial_num=trial_num,+ trial_num=trial_num + 1,note="full_eval",)- trial_logs[trial_num]["full_eval_program"] = highest_mean_program- trial_logs[trial_num]["full_eval_score"] = full_eval_score+ trial_logs[trial_num + 1]["full_eval_program"] = highest_mean_program+ trial_logs[trial_num + 1]["full_eval_score"] = full_eval_score# Update best score and program if necessaryif full_eval_score > best_score: