DSPy: The framework for programming—not prompting—language models
Python
Fix additional evaluation display failures for non-dict objects (#1820)
Corey Zumar62526fcd5fb8
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 6b74645a8..04a135537 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -1,10 +1,9 @@import logging-import tqdmimport typesfrom typing import Anyimport pandas as pd-+import tqdmimport dspyfrom dspy.utils.parallelizer import ParallelExecutor@@ -14,7 +13,7 @@ try:from IPython.display import display as displayexcept ImportError:-+def display(obj: Any):"""Display the specified Python object in the console.@@ -41,6 +40,7 @@ logger = logging.getLogger(__name__)logger = logging.getLogger(__name__)+class Evaluate:def __init__(self,@@ -66,7 +66,6 @@ class Evaluate:self.return_outputs = return_outputsself.provide_traceback = provide_traceback-def __call__(self,program,@@ -131,11 +130,9 @@ class Evaluate:results = [(example, prediction, score) for _, example, prediction, score in predicted_devset]def prediction_is_dictlike(prediction):- try:- dict(prediction)- return True- except Exception:- return False+ # Downstream logic for displaying dictionary-like predictions depends solely on the predictions+ # having a method called `items()` for iterating through key/value pairs+ return hasattr(prediction, "items") and callable(getattr(prediction, "items"))data = [(diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.pyindex 7615c70b3..048d4c08b 100644--- a/tests/evaluate/test_evaluate.py+++ b/tests/evaluate/test_evaluate.py@@ -8,7 +8,6 @@ import dspimport dspyfrom dspy.evaluate.evaluate import Evaluatefrom dspy.evaluate.metrics import answer_exact_match-from dspy.functional import TypedPredictorfrom dspy.predict import Predictfrom dspy.utils.dummies import DummyLM@@ -125,14 +124,22 @@ def test_evaluate_call_bad():"program_with_example",[(Predict("question -> answer"), new_example("What is 1+1?", "2")),+ # Create programs that do not return dictionary-like objects because Evaluate()+ # has failed for such cases in the past(- # Create a program that extracts entities from text and returns them as a list,- # rather than returning a Predictor() wrapper. This is done intentionally to test- # the case where the program does not output a dictionary-like object because- # Evaluate() has failed for this case in the past- lambda text: TypedPredictor("text: str -> entities: List[str]")(text=text).entities,+ lambda text: Predict("text: str -> entities: List[str]")(text=text).entities,dspy.Example(text="United States", entities=["United States"]).with_inputs("text"),),+ (+ lambda text: Predict("text: str -> entities: List[Dict[str, str]]")(text=text).entities,+ dspy.Example(text="United States", entities=[{"name": "United States", "type": "location"}]).with_inputs(+ "text"+ ),+ ),+ (+ lambda text: Predict("text: str -> first_word: Tuple[str, int]")(text=text).words,+ dspy.Example(text="United States", first_word=("United", 6)).with_inputs("text"),+ ),],)@pytest.mark.parametrize("display_table", [True, False, 1])