Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

MIPROv2 Updates: New defaults, fixed Trial # Logging (#8029)

klopsahlong6922a8b014bf
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index 772e0c5b0..0f92ece37 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -52,8 +52,8 @@ class MIPROv2(Teleprompter):
task_model: Optional[Any] = None,
teacher_settings: Dict = {},
max_bootstrapped_demos: int = 4,
- max_labeled_demos: int = 16,
- auto: Optional[Literal["light", "medium", "heavy"]] = None,
+ max_labeled_demos: int = 4,
+ auto: Optional[Literal["light", "medium", "heavy"]] = "medium",
num_candidates: int = 10,
num_threads: int = 6,
max_errors: int = 10,
@@ -101,8 +101,8 @@ class MIPROv2(Teleprompter):
max_labeled_demos: Optional[int] = None,
seed: Optional[int] = None,
minibatch: bool = True,
- minibatch_size: int = 25,
- minibatch_full_eval_steps: int = 10,
+ minibatch_size: int = 35,
+ minibatch_full_eval_steps: int = 5,
program_aware_proposer: bool = True,
data_aware_proposer: bool = True,
view_data_batch_size: int = 10,
@@ -464,7 +464,8 @@ class MIPROv2(Teleprompter):
)
# Compute the adjusted total trials that we will run (including full evals)
- adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1) if minibatch else num_trials
+ run_additional_full_eval_at_end = 1 if num_trials % minibatch_full_eval_steps != 0 else 0
+ adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1 + run_additional_full_eval_at_end) if minibatch else num_trials
logger.info(f"== Trial {1} / {adjusted_num_trials} - Full Evaluation of Default Program ==")
default_score, _ = eval_candidate_program(
@@ -473,11 +474,11 @@ class MIPROv2(Teleprompter):
logger.info(f"Default program score: {default_score}\n")
trial_logs = {}
- trial_logs[-1] = {}
- trial_logs[-1]["full_eval_program_path"] = save_candidate_program(program, self.log_dir, -1)
- trial_logs[-1]["full_eval_score"] = default_score
- trial_logs[-1]["total_eval_calls_so_far"] = len(valset)
- trial_logs[-1]["full_eval_program"] = program.deepcopy()
+ trial_logs[1] = {}
+ trial_logs[1]["full_eval_program_path"] = save_candidate_program(program, self.log_dir, -1)
+ trial_logs[1]["full_eval_score"] = default_score
+ trial_logs[1]["total_eval_calls_so_far"] = len(valset)
+ trial_logs[1]["full_eval_program"] = program.deepcopy()
# Initialize optimization variables
best_score = default_score
@@ -567,7 +568,7 @@ class MIPROv2(Teleprompter):
)
# If minibatch, perform full evaluation at intervals (and at the very end)
- if minibatch and ((trial_num % minibatch_full_eval_steps == 0) or (trial_num == (adjusted_num_trials - 1))):
+ if minibatch and ((trial_num % (minibatch_full_eval_steps+1) == 0) or (trial_num == (adjusted_num_trials-1))):
best_score, best_program, total_eval_calls = self._perform_full_evaluation(
trial_num,
adjusted_num_trials,
@@ -601,7 +602,7 @@ class MIPROv2(Teleprompter):
value=default_score,
)
study.add_trial(trial)
- study.optimize(objective, n_trials=num_trials - 1)
+ study.optimize(objective, n_trials=num_trials)
# Attach logs to best program
if best_program is not None and self.track_stats:
@@ -767,15 +768,16 @@ class MIPROv2(Teleprompter):
"score": full_eval_score,
}
total_eval_calls += len(valset)
- trial_logs[trial_num]["total_eval_calls_so_far"] = total_eval_calls
- trial_logs[trial_num]["full_eval_program_path"] = save_candidate_program(
+ trial_logs[trial_num + 1] = {}
+ trial_logs[trial_num + 1]["total_eval_calls_so_far"] = total_eval_calls
+ trial_logs[trial_num + 1]["full_eval_program_path"] = save_candidate_program(
program=highest_mean_program,
log_dir=self.log_dir,
- trial_num=trial_num,
+ trial_num=trial_num + 1,
note="full_eval",
)
- trial_logs[trial_num]["full_eval_program"] = highest_mean_program
- trial_logs[trial_num]["full_eval_score"] = full_eval_score
+ trial_logs[trial_num + 1]["full_eval_program"] = highest_mean_program
+ trial_logs[trial_num + 1]["full_eval_score"] = full_eval_score
# Update best score and program if necessary
if full_eval_score > best_score: