DSPy: The framework for programming—not prompting—language models
Python
propagate failure counts during evaluation
manishshettym1e229057ad15
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex ba832c560..4eccff47c 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -96,6 +96,13 @@ class Evaluate:try:prediction = program(**example.inputs())score = metric(example, prediction) # FIXME: TODO: What's the right order? Maybe force name-based kwargs!++ # increment assert and suggest failures to program's attributes+ if hasattr(program, '_assert_failures'):+ program._assert_failures += dsp.settings.assert_failures+ if hasattr(program, '_suggest_failures'):+ program._suggest_failures += dsp.settings.suggest_failures+return example_idx, example, prediction, scoreexcept Exception as e:with self.error_lock: