DSPy: The framework for programming—not prompting—language models
Python
MIPROv2 Ensure that if the default program is best that is returned
Michael Ryan21fbb10ba7fc
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex fa4293d00..25fb413db 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -308,6 +308,15 @@ class MIPROv2(Teleprompter):param_score_dict = defaultdict(list) # Dictionaries of paramater combinations we've tried, and their associated scoresfully_evaled_param_combos = {} # List of the parameter combinations we've done full evals of+ # Evaluate the default program+ if self.verbose: print("Evaluating the default program...\n")+ default_score = eval_candidate_program(len(valset), valset, program, evaluate)+ if self.verbose: print(f"Default program score: {default_score}\n")++ best_score = default_score+ best_program = program.deepcopy()++# Define our trial objectivedef create_objective(baseline_program,@@ -385,8 +394,8 @@ class MIPROv2(Teleprompter):trial_logs[trial.number+1]["num_eval_calls"] = 0# Evaluate the candidate program with relevant batch size- batch_size = minibatch_size if minibatch else len(trainset)evalset = trainset if minibatch else valset+ batch_size = minibatch_size if minibatch else len(evalset)score = eval_candidate_program(batch_size, evalset, candidate_program,@@ -446,28 +455,28 @@ class MIPROv2(Teleprompter):print(f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials...")else:print(f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials...")- full_train_score = eval_candidate_program(- len(trainset), trainset, highest_mean_program, evaluate,+ full_val_score = eval_candidate_program(+ len(valset), valset, highest_mean_program, evaluate,)# Log relevant information- fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}+ fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_val_score}total_eval_calls += len(trainset)trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_callstrial_logs[trial.number+1]["full_eval"] = Truetrial_logs[trial.number+1]["program_path"] = save_candidate_program(program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number+1, note="full_eval",)- trial_logs[trial.number+1]["score"] = full_train_score+ trial_logs[trial.number+1]["score"] = full_val_score- if full_train_score > best_score:- print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_train_score}")- best_score = full_train_score+ if full_val_score > best_score:+ print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_val_score}")+ best_score = full_val_scorebest_scoring_trial = trial.number+1best_program = highest_mean_program.deepcopy()best_score_updated = Trueelse:- print(f"Full eval score: {full_train_score}")+ print(f"Full eval score: {full_val_score}")print(f"Best full eval score so far: {best_score}")print("=======================\n\n")diff --git a/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_5 b/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_5new file mode 100644index 000000000..f3d37d9ba--- /dev/null+++ b/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_5@@ -0,0 +1 @@+ "generate_query": {