DSPy: The framework for programming—not prompting—language models
Python
fix(dspy): SemanticF1 and CompleteAndGrounded now return dspy.Prediction (#9302)
Copilot3dde431bd020
diff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.pyindex bca596df7..a5390811f 100644--- a/dspy/evaluate/auto_evaluation.py+++ b/dspy/evaluate/auto_evaluation.py@@ -1,5 +1,6 @@from dspy.predict.chain_of_thought import ChainOfThoughtfrom dspy.primitives import Module+from dspy.primitives.prediction import Predictionfrom dspy.signatures import InputField, OutputField, Signature@@ -50,7 +51,7 @@ class SemanticF1(Module):scores = self.module(question=example.question, ground_truth=example.response, system_response=pred.response)score = f1_score(scores.precision, scores.recall)- return score if trace is None else score >= self.threshold+ return Prediction(score=score if trace is None else score >= self.threshold)@@ -99,4 +100,4 @@ class CompleteAndGrounded(Module):groundedness = self.groundedness_module(question=example.question, retrieved_context=pred.context, system_response=pred.response)score = f1_score(groundedness.groundedness, completeness.completeness)- return score if trace is None else score >= self.threshold+ return Prediction(score=score if trace is None else score >= self.threshold)diff --git a/tests/evaluate/test_auto_evaluation.py b/tests/evaluate/test_auto_evaluation.pynew file mode 100644index 000000000..4643f6cd6--- /dev/null+++ b/tests/evaluate/test_auto_evaluation.py@@ -0,0 +1,191 @@+import dspy+from dspy.evaluate.auto_evaluation import CompleteAndGrounded, SemanticF1+from dspy.primitives.prediction import Prediction+from dspy.utils.dummies import DummyLM+++def test_semantic_f1_returns_prediction_without_trace():+ # Configure with a dummy LM that returns precision and recall values+ # ChainOfThought adds a "reasoning" field to the output+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Comparing the responses",+ "precision": 1.0,+ "recall": 1.0,+ }+ ]+ )+ )++ # Create example and prediction+ example = dspy.Example(question="What is 1+1?", response="2")+ pred = dspy.Prediction(response="2")++ # Test SemanticF1+ metric = SemanticF1()+ result = metric(example, pred)++ assert isinstance(result, Prediction)+ assert hasattr(result, "score")+ assert isinstance(result.score, (int, float, bool))+++def test_semantic_f1_returns_prediction_with_trace():+ # Configure with a dummy LM+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Comparing the responses",+ "precision": 1.0,+ "recall": 1.0,+ }+ ]+ )+ )++ # Create example and prediction+ example = dspy.Example(question="What is 1+1?", response="2")+ pred = dspy.Prediction(response="2")++ # Test SemanticF1 with trace+ metric = SemanticF1(threshold=0.5)+ result = metric(example, pred, trace=True)++ assert isinstance(result, Prediction)+ assert hasattr(result, "score")+ assert isinstance(result.score, bool)+++def test_semantic_f1_score_value():+ # Configure with a dummy LM that returns specific precision and recall+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Comparing the responses",+ "precision": 0.8,+ "recall": 0.6,+ }+ ]+ )+ )++ # Create example and prediction+ example = dspy.Example(question="test", response="answer")+ pred = dspy.Prediction(response="response")++ # Test SemanticF1+ metric = SemanticF1()+ result = metric(example, pred)++ expected_f1 = 2 * (0.8 * 0.6) / (0.8 + 0.6)+ assert isinstance(result, Prediction)+ assert abs(result.score - expected_f1) < 0.001+++def test_complete_and_grounded_returns_prediction_without_trace():+ # Configure with a dummy LM+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Analyzing completeness",+ "ground_truth_key_ideas": "the answer is 2",+ "system_response_key_ideas": "the answer is 2",+ "discussion": "both match",+ "completeness": 1.0,+ },+ {+ "reasoning": "Analyzing groundedness",+ "system_response_claims": "1+1=2",+ "discussion": "supported by context",+ "groundedness": 1.0,+ },+ ]+ )+ )++ # Create example and prediction with context+ example = dspy.Example(question="What is 1+1?", response="2")+ pred = dspy.Prediction(response="2", context="context")++ # Test CompleteAndGrounded+ metric = CompleteAndGrounded()+ result = metric(example, pred)++ assert isinstance(result, Prediction)+ assert hasattr(result, "score")+ assert isinstance(result.score, (int, float, bool))+++def test_complete_and_grounded_returns_prediction_with_trace():+ # Configure with a dummy LM+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Analyzing completeness",+ "ground_truth_key_ideas": "the answer is 2",+ "system_response_key_ideas": "the answer is 2",+ "discussion": "both match",+ "completeness": 0.9,+ },+ {+ "reasoning": "Analyzing groundedness",+ "system_response_claims": "1+1=2",+ "discussion": "supported by context",+ "groundedness": 0.8,+ },+ ]+ )+ )++ # Create example and prediction with context+ example = dspy.Example(question="What is 1+1?", response="2")+ pred = dspy.Prediction(response="2", context="context")++ # Test CompleteAndGrounded with trace+ metric = CompleteAndGrounded(threshold=0.7)+ result = metric(example, pred, trace=True)++ assert isinstance(result, Prediction)+ assert hasattr(result, "score")+ assert isinstance(result.score, bool)+++def test_semantic_f1_prediction_can_be_compared():+ # Configure with a dummy LM+ dspy.configure(+ lm=DummyLM(+ [+ {+ "reasoning": "Comparing first response",+ "precision": 0.8,+ "recall": 0.6,+ },+ {+ "reasoning": "Comparing second response",+ "precision": 0.9,+ "recall": 0.7,+ },+ ]+ )+ )++ metric = SemanticF1()++ # Create two predictions with different scores+ example1 = dspy.Example(question="test1", response="answer1")+ pred1 = dspy.Prediction(response="response1")+ result1 = metric(example1, pred1)++ example2 = dspy.Example(question="test2", response="answer2")+ pred2 = dspy.Prediction(response="response2")+ result2 = metric(example2, pred2)++ assert isinstance(result1, Prediction)+ assert isinstance(result2, Prediction)+ assert result2.score > result1.score