DSPy: The framework for programming—not prompting—language models
Python
Fix error handling and add tests for eval_candidate_program (#7936)
TomuHirata16023c8dda84
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex c92fcba06..673d59ca1 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -467,7 +467,7 @@ class MIPROv2(Teleprompter):adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1) if minibatch else num_trialslogger.info(f"== Trial {1} / {adjusted_num_trials} - Full Evaluation of Default Program ==")- default_score, baseline_results = eval_candidate_program(+ default_score, _ = eval_candidate_program(len(valset), valset, program, evaluate, self.rng, return_all_scores=True)logger.info(f"Default program score: {default_score}\n")@@ -649,6 +649,8 @@ class MIPROv2(Teleprompter):trajectory = "[" + full_eval_scores + "]"logger.info(f"Full eval scores so far: {trajectory}")logger.info(f"Best full score so far: {best_score}")+ import mlflow+ mlflow.log_metric("best score", best_score, step=trial.number)logger.info(f"{'=' * len(f'== Trial {trial.number + 1} / {adjusted_num_trials} - Minibatch Evaluation ==')}\n\n")@@ -679,6 +681,8 @@ class MIPROv2(Teleprompter):full_eval_scores = ", ".join([f"{s['score']}" for s in score_data if s["full_eval"]])logger.info(f"Scores so far: {'[' + full_eval_scores + ']'}")logger.info(f"Best score so far: {best_score}")+ import mlflow+ mlflow.log_metric("best score", best_score, step=trial.number)logger.info(f"{'=' * len(f'===== Trial {trial.number + 1} / {num_trials} =====')}\n\n")def _select_and_insert_instructions_and_demos(diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pyindex 46cf5d6d4..9414c7bd7 100644--- a/dspy/teleprompt/utils.py+++ b/dspy/teleprompt/utils.py@@ -23,6 +23,7 @@ This file consists of helper functions for our variety of optimizers.### OPTIMIZER TRAINING UTILS ###+logger = logging.getLogger(__name__)def create_minibatch(trainset, batch_size=50, rng=None):"""Create a minibatch from the trainset."""@@ -56,8 +57,10 @@ def eval_candidate_program(batch_size, trainset, candidate_program, evaluate, rndevset=create_minibatch(trainset, batch_size, rng),return_all_scores=return_all_scores)- except Exception as e:- print(f"Exception occurred: {e}")+ except Exception:+ logger.error("An exception occurred during evaluation", exc_info=True)+ if return_all_scores:+ return 0.0, [0.0] * len(trainset)return 0.0 # TODO: Handle this better, as -ve scores are possibledef eval_candidate_program_with_pruning(diff --git a/tests/teleprompt/test_utils.py b/tests/teleprompt/test_utils.pynew file mode 100644index 000000000..f067e488d--- /dev/null+++ b/tests/teleprompt/test_utils.py@@ -0,0 +1,53 @@+from unittest.mock import Mock+import pytest++import dspy+from dspy.teleprompt.utils import eval_candidate_program++class DummyModule(dspy.Module):+ def __init__(self):+ super().__init__()++ def forward(self, **kwargs):+ pass+++def test_eval_candidate_program_full_trainset():+ trainset = [1, 2, 3, 4, 5]+ candidate_program = DummyModule()+ evaluate = Mock(return_value=0)+ batch_size = 10++ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate)++ evaluate.assert_called_once()+ _, called_kwargs = evaluate.call_args+ assert len(called_kwargs['devset']) == len(trainset)+ assert result == 0++def test_eval_candidate_program_minibatch():+ trainset = [1, 2, 3, 4, 5]+ candidate_program = DummyModule()+ evaluate = Mock(return_value=0)+ batch_size = 3++ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate)++ evaluate.assert_called_once()+ _, called_kwargs = evaluate.call_args+ assert len(called_kwargs['devset']) == batch_size+ assert result == 0+[email protected]("return_all_scores", [True, False])+def test_eval_candidate_program_failure(return_all_scores):+ trainset = [1, 2, 3, 4, 5]+ candidate_program = DummyModule()+ evaluate = Mock(side_effect=ValueError("Error"))+ batch_size = 3++ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate, return_all_scores=return_all_scores)++ if return_all_scores:+ assert result == (0.0, [0.0]*len(trainset))+ else:+ assert result == 0.0