DSPy: The framework for programming—not prompting—language models
Python
Move dsp/metrics into evaluate/metrics (#8402)
TomuHirata58914cb4cd5c
diff --git a/dspy/dsp/utils/__init__.py b/dspy/dsp/utils/__init__.pyindex 5384fa47f..1258a8e7a 100644--- a/dspy/dsp/utils/__init__.py+++ b/dspy/dsp/utils/__init__.py@@ -1,4 +1,3 @@from dspy.dsp.utils.dpr import *-from dspy.dsp.utils.metrics import *from dspy.dsp.utils.settings import *from dspy.dsp.utils.utils import *diff --git a/dspy/dsp/utils/metrics.py b/dspy/dsp/utils/metrics.pydeleted file mode 100644index 9d3cc255d..000000000--- a/dspy/dsp/utils/metrics.py+++ /dev/null@@ -1,113 +0,0 @@-import re-import string-import unicodedata-from collections import Counter--from dspy.dsp.utils.utils import print_message---def EM(prediction, answers_list): # noqa: N802- assert isinstance(answers_list, list)-- return max(em_score(prediction, ans) for ans in answers_list)---def F1(prediction, answers_list): # noqa: N802- assert isinstance(answers_list, list)-- return max(f1_score(prediction, ans) for ans in answers_list)---def HotPotF1(prediction, answers_list): # noqa: N802- assert isinstance(answers_list, list)-- return max(hotpot_f1_score(prediction, ans) for ans in answers_list)---def normalize_text(s):- s = unicodedata.normalize("NFD", s)-- def remove_articles(text):- return re.sub(r"\b(a|an|the)\b", " ", text)-- def white_space_fix(text):- return " ".join(text.split())-- def remove_punc(text):- exclude = set(string.punctuation)- return "".join(ch for ch in text if ch not in exclude)-- def lower(text):- return text.lower()-- return white_space_fix(remove_articles(remove_punc(lower(s))))---def em_score(prediction, ground_truth):- return normalize_text(prediction) == normalize_text(ground_truth)---# See: https://github.com/hotpotqa/hotpot/blob/master/hotpot_evaluate_v1.py-# See: https://rajpurkar.github.io/SQuAD-explorer/ under Evaluation Script-# See: QReCC's---def f1_score(prediction, ground_truth):- prediction_tokens = normalize_text(prediction).split()- ground_truth_tokens = normalize_text(ground_truth).split()-- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)- num_same = sum(common.values())-- if len(prediction_tokens) == len(ground_truth_tokens) == 0:- # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.- print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")-- if num_same == 0:- return 0-- precision = 1.0 * num_same / len(prediction_tokens)- recall = 1.0 * num_same / len(ground_truth_tokens)- f1 = (2 * precision * recall) / (precision + recall)-- return f1---def hotpot_f1_score(prediction, ground_truth):- normalized_prediction = normalize_text(prediction)- normalized_ground_truth = normalize_text(ground_truth)-- if normalized_prediction in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:- return 0- if normalized_ground_truth in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:- return 0-- prediction_tokens = normalized_prediction.split()- ground_truth_tokens = normalized_ground_truth.split()- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)- num_same = sum(common.values())- if num_same == 0:- return 0- precision = 1.0 * num_same / len(prediction_tokens)- recall = 1.0 * num_same / len(ground_truth_tokens)- f1 = (2 * precision * recall) / (precision + recall)- return f1---def precision_score(prediction, ground_truth):- prediction_tokens = normalize_text(prediction).split()- ground_truth_tokens = normalize_text(ground_truth).split()-- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)- num_same = sum(common.values())-- if len(prediction_tokens) == len(ground_truth_tokens) == 0:- # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.- print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")-- if num_same == 0:- return 0-- precision = 1.0 * num_same / len(prediction_tokens)-- return precisiondiff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.pyindex 24c0a2fef..d9d9668ee 100644--- a/dspy/evaluate/__init__.py+++ b/dspy/evaluate/__init__.py@@ -1,7 +1,6 @@-from dspy.dsp.utils import EM, normalize_textfrom dspy.evaluate.auto_evaluation import CompleteAndGrounded, SemanticF1from dspy.evaluate.evaluate import Evaluate-from dspy.evaluate.metrics import answer_exact_match, answer_passage_match+from dspy.evaluate.metrics import EM, answer_exact_match, answer_passage_match, normalize_text__all__ = ["EM",diff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.pyindex 001a9043e..bca596df7 100644--- a/dspy/evaluate/auto_evaluation.py+++ b/dspy/evaluate/auto_evaluation.py@@ -1,33 +1,35 @@-import dspy+from dspy.predict.chain_of_thought import ChainOfThought+from dspy.primitives import Module+from dspy.signatures import InputField, OutputField, Signature-class SemanticRecallPrecision(dspy.Signature):+class SemanticRecallPrecision(Signature):"""Compare a system's response to the ground truth to compute its recall and precision.If asked to reason, enumerate key ideas in each response, and whether they are present in the other response."""- question: str = dspy.InputField()- ground_truth: str = dspy.InputField()- system_response: str = dspy.InputField()- recall: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")- precision: float = dspy.OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")+ question: str = InputField()+ ground_truth: str = InputField()+ system_response: str = InputField()+ recall: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")+ precision: float = OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")-class DecompositionalSemanticRecallPrecision(dspy.Signature):+class DecompositionalSemanticRecallPrecision(Signature):"""Compare a system's response to the ground truth to compute recall and precision of key ideas.You will first enumerate key ideas in each response, discuss their overlap, and then report recall and precision."""- question: str = dspy.InputField()- ground_truth: str = dspy.InputField()- system_response: str = dspy.InputField()- ground_truth_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the ground truth")- system_response_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the system response")- discussion: str = dspy.OutputField(desc="discussion of the overlap between ground truth and system response")- recall: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")- precision: float = dspy.OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")+ question: str = InputField()+ ground_truth: str = InputField()+ system_response: str = InputField()+ ground_truth_key_ideas: str = OutputField(desc="enumeration of key ideas in the ground truth")+ system_response_key_ideas: str = OutputField(desc="enumeration of key ideas in the system response")+ discussion: str = OutputField(desc="discussion of the overlap between ground truth and system response")+ recall: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")+ precision: float = OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")def f1_score(precision, recall):@@ -35,14 +37,14 @@ def f1_score(precision, recall):return 0.0 if precision + recall == 0 else 2 * (precision * recall) / (precision + recall)-class SemanticF1(dspy.Module):+class SemanticF1(Module):def __init__(self, threshold=0.66, decompositional=False):self.threshold = thresholdif decompositional:- self.module = dspy.ChainOfThought(DecompositionalSemanticRecallPrecision)+ self.module = ChainOfThought(DecompositionalSemanticRecallPrecision)else:- self.module = dspy.ChainOfThought(SemanticRecallPrecision)+ self.module = ChainOfThought(SemanticRecallPrecision)def forward(self, example, pred, trace=None):scores = self.module(question=example.question, ground_truth=example.response, system_response=pred.response)@@ -55,42 +57,42 @@ class SemanticF1(dspy.Module):###########-class AnswerCompleteness(dspy.Signature):+class AnswerCompleteness(Signature):"""Estimate the completeness of a system's responses, against the ground truth.You will first enumerate key ideas in each response, discuss their overlap, and then report completeness."""- question: str = dspy.InputField()- ground_truth: str = dspy.InputField()- system_response: str = dspy.InputField()- ground_truth_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the ground truth")- system_response_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the system response")- discussion: str = dspy.OutputField(desc="discussion of the overlap between ground truth and system response")- completeness: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")+ question: str = InputField()+ ground_truth: str = InputField()+ system_response: str = InputField()+ ground_truth_key_ideas: str = OutputField(desc="enumeration of key ideas in the ground truth")+ system_response_key_ideas: str = OutputField(desc="enumeration of key ideas in the system response")+ discussion: str = OutputField(desc="discussion of the overlap between ground truth and system response")+ completeness: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")-class AnswerGroundedness(dspy.Signature):+class AnswerGroundedness(Signature):"""Estimate the groundedness of a system's responses, against real retrieved documents written by people.You will first enumerate whatever non-trivial or check-worthy claims are made in the system response, and thendiscuss the extent to which some or all of them can be deduced from the retrieved context and basic commonsense."""- question: str = dspy.InputField()- retrieved_context: str = dspy.InputField()- system_response: str = dspy.InputField()- system_response_claims: str = dspy.OutputField(desc="enumeration of non-trivial or check-worthy claims in the system response")- discussion: str = dspy.OutputField(desc="discussion of how supported the claims are by the retrieved context")- groundedness: float = dspy.OutputField(desc="fraction (out of 1.0) of system response supported by the retrieved context")+ question: str = InputField()+ retrieved_context: str = InputField()+ system_response: str = InputField()+ system_response_claims: str = OutputField(desc="enumeration of non-trivial or check-worthy claims in the system response")+ discussion: str = OutputField(desc="discussion of how supported the claims are by the retrieved context")+ groundedness: float = OutputField(desc="fraction (out of 1.0) of system response supported by the retrieved context")-class CompleteAndGrounded(dspy.Module):+class CompleteAndGrounded(Module):def __init__(self, threshold=0.66):self.threshold = threshold- self.completeness_module = dspy.ChainOfThought(AnswerCompleteness)- self.groundedness_module = dspy.ChainOfThought(AnswerGroundedness)+ self.completeness_module = ChainOfThought(AnswerCompleteness)+ self.groundedness_module = ChainOfThought(AnswerGroundedness)def forward(self, example, pred, trace=None):completeness = self.completeness_module(question=example.question, ground_truth=example.response, system_response=pred.response)diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.pyindex 8f0b50f3c..ea86c0c04 100644--- a/dspy/evaluate/evaluate.py+++ b/dspy/evaluate/evaluate.py@@ -204,7 +204,7 @@ class Evaluate:return round(100 * ncorrect / ntotal, 2)def _construct_result_table(- self, results: list[Tuple[dspy.Example, dspy.Example, Any]], metric_name: str+ self, results: list[Tuple["dspy.Example", "dspy.Example", Any]], metric_name: str) -> "pd.DataFrame":"""Construct a pandas DataFrame from the specified result list.diff --git a/dspy/evaluate/metrics.py b/dspy/evaluate/metrics.pyindex dc9a3db41..65ae37fcf 100644--- a/dspy/evaluate/metrics.py+++ b/dspy/evaluate/metrics.py@@ -1,9 +1,123 @@# TODO: This should move internally. Same for passage_match. dspy.metrics.answer_exact_match, dspy.metrics.answer_passage_match+import re+import string+import unicodedata+from collections import Counter++from dspy.dsp.utils.utils import print_message+++def EM(prediction, answers_list): # noqa: N802+ assert isinstance(answers_list, list)++ return max(em_score(prediction, ans) for ans in answers_list)+++def F1(prediction, answers_list): # noqa: N802+ assert isinstance(answers_list, list)++ return max(f1_score(prediction, ans) for ans in answers_list)+++def HotPotF1(prediction, answers_list): # noqa: N802+ assert isinstance(answers_list, list)++ return max(hotpot_f1_score(prediction, ans) for ans in answers_list)+++def normalize_text(s):+ s = unicodedata.normalize("NFD", s)++ def remove_articles(text):+ return re.sub(r"\b(a|an|the)\b", " ", text)++ def white_space_fix(text):+ return " ".join(text.split())++ def remove_punc(text):+ exclude = set(string.punctuation)+ return "".join(ch for ch in text if ch not in exclude)++ def lower(text):+ return text.lower()++ return white_space_fix(remove_articles(remove_punc(lower(s))))+++def em_score(prediction, ground_truth):+ return normalize_text(prediction) == normalize_text(ground_truth)+++# See: https://github.com/hotpotqa/hotpot/blob/master/hotpot_evaluate_v1.py+# See: https://rajpurkar.github.io/SQuAD-explorer/ under Evaluation Script+# See: QReCC's+++def f1_score(prediction, ground_truth):+ prediction_tokens = normalize_text(prediction).split()+ ground_truth_tokens = normalize_text(ground_truth).split()++ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())++ if len(prediction_tokens) == len(ground_truth_tokens) == 0:+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.+ print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")++ if num_same == 0:+ return 0++ precision = 1.0 * num_same / len(prediction_tokens)+ recall = 1.0 * num_same / len(ground_truth_tokens)+ f1 = (2 * precision * recall) / (precision + recall)++ return f1+++def hotpot_f1_score(prediction, ground_truth):+ normalized_prediction = normalize_text(prediction)+ normalized_ground_truth = normalize_text(ground_truth)++ if normalized_prediction in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:+ return 0+ if normalized_ground_truth in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:+ return 0++ prediction_tokens = normalized_prediction.split()+ ground_truth_tokens = normalized_ground_truth.split()+ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())+ if num_same == 0:+ return 0+ precision = 1.0 * num_same / len(prediction_tokens)+ recall = 1.0 * num_same / len(ground_truth_tokens)+ f1 = (2 * precision * recall) / (precision + recall)+ return f1+++def precision_score(prediction, ground_truth):+ prediction_tokens = normalize_text(prediction).split()+ ground_truth_tokens = normalize_text(ground_truth).split()++ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())++ if len(prediction_tokens) == len(ground_truth_tokens) == 0:+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.+ print_message("\n#> Precision Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")++ if num_same == 0:+ return 0++ precision = 1.0 * num_same / len(prediction_tokens)++ return precision+def _passage_match(passages: list[str], answers: list[str]) -> bool:"""Returns True if any of the passages contains the answer."""- from dspy.dsp.utils import DPR_normalize, has_answer, normalize_text+ from dspy.dsp.utils import DPR_normalize, has_answerdef passage_has_answers(passage: str, answers: list[str]) -> bool:"""Returns True if the passage contains the answer."""@@ -17,7 +131,6 @@ def _passage_match(passages: list[str], answers: list[str]) -> bool:def _answer_match(prediction, answers, frac=1.0):"""Returns True if the prediction matches any of the answers."""- from dspy.dsp.utils import EM, F1if frac >= 1.0:return EM(prediction, answers)diff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.pyindex 9d56d3810..8f0e33448 100644--- a/dspy/predict/aggregation.py+++ b/dspy/predict/aggregation.py@@ -1,4 +1,4 @@-from dspy.dsp.utils import normalize_text+from dspy.evaluate import normalize_textfrom dspy.primitives.prediction import Completions, Predictiondiff --git a/tests/predict/test_aggregation.py b/tests/predict/test_aggregation.pyindex 1a020ab39..a92f44a0b 100644--- a/tests/predict/test_aggregation.py+++ b/tests/predict/test_aggregation.py@@ -1,4 +1,4 @@-from dspy.dsp.utils import normalize_text+from dspy.evaluate import normalize_textfrom dspy.predict.aggregation import majorityfrom dspy.primitives.prediction import Completions, Prediction