Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix error handling and add tests for eval_candidate_program (#7936)

TomuHirata16023c8dda84
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index c92fcba06..673d59ca1 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -467,7 +467,7 @@ class MIPROv2(Teleprompter):
adjusted_num_trials = (num_trials + num_trials // minibatch_full_eval_steps + 1) if minibatch else num_trials
logger.info(f"== Trial {1} / {adjusted_num_trials} - Full Evaluation of Default Program ==")
- default_score, baseline_results = eval_candidate_program(
+ default_score, _ = eval_candidate_program(
len(valset), valset, program, evaluate, self.rng, return_all_scores=True
)
logger.info(f"Default program score: {default_score}\n")
@@ -649,6 +649,8 @@ class MIPROv2(Teleprompter):
trajectory = "[" + full_eval_scores + "]"
logger.info(f"Full eval scores so far: {trajectory}")
logger.info(f"Best full score so far: {best_score}")
+ import mlflow
+ mlflow.log_metric("best score", best_score, step=trial.number)
logger.info(
f"{'=' * len(f'== Trial {trial.number + 1} / {adjusted_num_trials} - Minibatch Evaluation ==')}\n\n"
)
@@ -679,6 +681,8 @@ class MIPROv2(Teleprompter):
full_eval_scores = ", ".join([f"{s['score']}" for s in score_data if s["full_eval"]])
logger.info(f"Scores so far: {'[' + full_eval_scores + ']'}")
logger.info(f"Best score so far: {best_score}")
+ import mlflow
+ mlflow.log_metric("best score", best_score, step=trial.number)
logger.info(f"{'=' * len(f'===== Trial {trial.number + 1} / {num_trials} =====')}\n\n")
def _select_and_insert_instructions_and_demos(
diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py
index 46cf5d6d4..9414c7bd7 100644
--- a/dspy/teleprompt/utils.py
+++ b/dspy/teleprompt/utils.py
@@ -23,6 +23,7 @@ This file consists of helper functions for our variety of optimizers.
### OPTIMIZER TRAINING UTILS ###
+logger = logging.getLogger(__name__)
def create_minibatch(trainset, batch_size=50, rng=None):
"""Create a minibatch from the trainset."""
@@ -56,8 +57,10 @@ def eval_candidate_program(batch_size, trainset, candidate_program, evaluate, rn
devset=create_minibatch(trainset, batch_size, rng),
return_all_scores=return_all_scores
)
- except Exception as e:
- print(f"Exception occurred: {e}")
+ except Exception:
+ logger.error("An exception occurred during evaluation", exc_info=True)
+ if return_all_scores:
+ return 0.0, [0.0] * len(trainset)
return 0.0 # TODO: Handle this better, as -ve scores are possible
def eval_candidate_program_with_pruning(
diff --git a/tests/teleprompt/test_utils.py b/tests/teleprompt/test_utils.py
new file mode 100644
index 000000000..f067e488d
--- /dev/null
+++ b/tests/teleprompt/test_utils.py
@@ -0,0 +1,53 @@
+from unittest.mock import Mock
+import pytest
+
+import dspy
+from dspy.teleprompt.utils import eval_candidate_program
+
+class DummyModule(dspy.Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, **kwargs):
+ pass
+
+
+def test_eval_candidate_program_full_trainset():
+ trainset = [1, 2, 3, 4, 5]
+ candidate_program = DummyModule()
+ evaluate = Mock(return_value=0)
+ batch_size = 10
+
+ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate)
+
+ evaluate.assert_called_once()
+ _, called_kwargs = evaluate.call_args
+ assert len(called_kwargs['devset']) == len(trainset)
+ assert result == 0
+
+def test_eval_candidate_program_minibatch():
+ trainset = [1, 2, 3, 4, 5]
+ candidate_program = DummyModule()
+ evaluate = Mock(return_value=0)
+ batch_size = 3
+
+ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate)
+
+ evaluate.assert_called_once()
+ _, called_kwargs = evaluate.call_args
+ assert len(called_kwargs['devset']) == batch_size
+ assert result == 0
+
[email protected]("return_all_scores", [True, False])
+def test_eval_candidate_program_failure(return_all_scores):
+ trainset = [1, 2, 3, 4, 5]
+ candidate_program = DummyModule()
+ evaluate = Mock(side_effect=ValueError("Error"))
+ batch_size = 3
+
+ result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate, return_all_scores=return_all_scores)
+
+ if return_all_scores:
+ assert result == (0.0, [0.0]*len(trainset))
+ else:
+ assert result == 0.0