DSPy: The framework for programming—not prompting—language models
Python
Add ability return outputs from evaluate
jphillipsa92a90a108d6
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 4eccff47c..ed14ae388 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -21,7 +21,7 @@ we print the number of failures, the first N examples that failed, and the firstclass Evaluate:def __init__(self, *, devset, metric=None, num_threads=1, display_progress=False,- display_table=False, display=True, max_errors=5):+ display_table=False, display=True, max_errors=5, return_outputs=False):self.devset = devsetself.metric = metricself.num_threads = num_threads@@ -31,6 +31,7 @@ class Evaluate:self.max_errors = max_errorsself.error_count = 0self.error_lock = threading.Lock()+ self.return_outputs = return_outputsdef _execute_single_thread(self, wrapped_program, devset, display_progress):ncorrect = 0@@ -73,7 +74,7 @@ class Evaluate:def __call__(self, program, metric=None, devset=None, num_threads=None,display_progress=None, display_table=None, display=None,- return_all_scores=False):+ return_all_scores=False, return_outputs=False):metric = metric if metric is not None else self.metricdevset = devset if devset is not None else self.devsetnum_threads = num_threads if num_threads is not None else self.num_threads@@ -83,7 +84,9 @@ class Evaluate:display = self.display if display is None else displaydisplay_progress = display_progress and displaydisplay_table = display_table if display else False-+ return_outputs = return_outputs if return_outputs is not False else self.return_outputs+ results = []+def wrapped_program(example_idx, example):# NOTE: TODO: Won't work if threads create threads!creating_new_thread = threading.get_ident() not in dsp.settings.stack_by_thread@@ -122,6 +125,8 @@ class Evaluate:reordered_devset, ncorrect, ntotal = self._execute_single_thread(wrapped_program, devset, display_progress)else:reordered_devset, ncorrect, ntotal = self._execute_multi_thread(wrapped_program, devset, num_threads, display_progress)+ if return_outputs: # Handle the return_outputs logic+ results = [(example, prediction, score) for _, example, prediction, score in reordered_devset]if display:print(f"Average Metric: {ncorrect} / {ntotal} ({round(100 * ncorrect / ntotal, 1)}%)")@@ -166,8 +171,12 @@ class Evaluate:"""ipython_display(HTML(message))- if return_all_scores:- return round(100 * ncorrect / ntotal, 2), [score for *_, score in predicted_devset]+ if return_all_scores and return_outputs:+ return round(100 * ncorrect / ntotal, 2), results+ elif return_all_scores:+ return round(100 * ncorrect / ntotal, 2), [score for *_, score in reordered_devset]+ elif return_outputs:+ return round(100 * ncorrect / ntotal, 2), resultsreturn round(100 * ncorrect / ntotal, 2)