Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Move dsp/metrics into evaluate/metrics (#8402)

TomuHirata58914cb4cd5c
diff --git a/dspy/dsp/utils/__init__.py b/dspy/dsp/utils/__init__.py
index 5384fa47f..1258a8e7a 100644
--- a/dspy/dsp/utils/__init__.py
+++ b/dspy/dsp/utils/__init__.py
@@ -1,4 +1,3 @@
from dspy.dsp.utils.dpr import *
-from dspy.dsp.utils.metrics import *
from dspy.dsp.utils.settings import *
from dspy.dsp.utils.utils import *
diff --git a/dspy/dsp/utils/metrics.py b/dspy/dsp/utils/metrics.py
deleted file mode 100644
index 9d3cc255d..000000000
--- a/dspy/dsp/utils/metrics.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import re
-import string
-import unicodedata
-from collections import Counter
-
-from dspy.dsp.utils.utils import print_message
-
-
-def EM(prediction, answers_list): # noqa: N802
- assert isinstance(answers_list, list)
-
- return max(em_score(prediction, ans) for ans in answers_list)
-
-
-def F1(prediction, answers_list): # noqa: N802
- assert isinstance(answers_list, list)
-
- return max(f1_score(prediction, ans) for ans in answers_list)
-
-
-def HotPotF1(prediction, answers_list): # noqa: N802
- assert isinstance(answers_list, list)
-
- return max(hotpot_f1_score(prediction, ans) for ans in answers_list)
-
-
-def normalize_text(s):
- s = unicodedata.normalize("NFD", s)
-
- def remove_articles(text):
- return re.sub(r"\b(a|an|the)\b", " ", text)
-
- def white_space_fix(text):
- return " ".join(text.split())
-
- def remove_punc(text):
- exclude = set(string.punctuation)
- return "".join(ch for ch in text if ch not in exclude)
-
- def lower(text):
- return text.lower()
-
- return white_space_fix(remove_articles(remove_punc(lower(s))))
-
-
-def em_score(prediction, ground_truth):
- return normalize_text(prediction) == normalize_text(ground_truth)
-
-
-# See: https://github.com/hotpotqa/hotpot/blob/master/hotpot_evaluate_v1.py
-# See: https://rajpurkar.github.io/SQuAD-explorer/ under Evaluation Script
-# See: QReCC's
-
-
-def f1_score(prediction, ground_truth):
- prediction_tokens = normalize_text(prediction).split()
- ground_truth_tokens = normalize_text(ground_truth).split()
-
- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
- num_same = sum(common.values())
-
- if len(prediction_tokens) == len(ground_truth_tokens) == 0:
- # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.
- print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")
-
- if num_same == 0:
- return 0
-
- precision = 1.0 * num_same / len(prediction_tokens)
- recall = 1.0 * num_same / len(ground_truth_tokens)
- f1 = (2 * precision * recall) / (precision + recall)
-
- return f1
-
-
-def hotpot_f1_score(prediction, ground_truth):
- normalized_prediction = normalize_text(prediction)
- normalized_ground_truth = normalize_text(ground_truth)
-
- if normalized_prediction in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:
- return 0
- if normalized_ground_truth in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:
- return 0
-
- prediction_tokens = normalized_prediction.split()
- ground_truth_tokens = normalized_ground_truth.split()
- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
- num_same = sum(common.values())
- if num_same == 0:
- return 0
- precision = 1.0 * num_same / len(prediction_tokens)
- recall = 1.0 * num_same / len(ground_truth_tokens)
- f1 = (2 * precision * recall) / (precision + recall)
- return f1
-
-
-def precision_score(prediction, ground_truth):
- prediction_tokens = normalize_text(prediction).split()
- ground_truth_tokens = normalize_text(ground_truth).split()
-
- common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
- num_same = sum(common.values())
-
- if len(prediction_tokens) == len(ground_truth_tokens) == 0:
- # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.
- print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")
-
- if num_same == 0:
- return 0
-
- precision = 1.0 * num_same / len(prediction_tokens)
-
- return precision
diff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.py
index 24c0a2fef..d9d9668ee 100644
--- a/dspy/evaluate/__init__.py
+++ b/dspy/evaluate/__init__.py
@@ -1,7 +1,6 @@
-from dspy.dsp.utils import EM, normalize_text
from dspy.evaluate.auto_evaluation import CompleteAndGrounded, SemanticF1
from dspy.evaluate.evaluate import Evaluate
-from dspy.evaluate.metrics import answer_exact_match, answer_passage_match
+from dspy.evaluate.metrics import EM, answer_exact_match, answer_passage_match, normalize_text
__all__ = [
"EM",
diff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.py
index 001a9043e..bca596df7 100644
--- a/dspy/evaluate/auto_evaluation.py
+++ b/dspy/evaluate/auto_evaluation.py
@@ -1,33 +1,35 @@
-import dspy
+from dspy.predict.chain_of_thought import ChainOfThought
+from dspy.primitives import Module
+from dspy.signatures import InputField, OutputField, Signature
-class SemanticRecallPrecision(dspy.Signature):
+class SemanticRecallPrecision(Signature):
"""
Compare a system's response to the ground truth to compute its recall and precision.
If asked to reason, enumerate key ideas in each response, and whether they are present in the other response.
"""
- question: str = dspy.InputField()
- ground_truth: str = dspy.InputField()
- system_response: str = dspy.InputField()
- recall: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
- precision: float = dspy.OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")
+ question: str = InputField()
+ ground_truth: str = InputField()
+ system_response: str = InputField()
+ recall: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
+ precision: float = OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")
-class DecompositionalSemanticRecallPrecision(dspy.Signature):
+class DecompositionalSemanticRecallPrecision(Signature):
"""
Compare a system's response to the ground truth to compute recall and precision of key ideas.
You will first enumerate key ideas in each response, discuss their overlap, and then report recall and precision.
"""
- question: str = dspy.InputField()
- ground_truth: str = dspy.InputField()
- system_response: str = dspy.InputField()
- ground_truth_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the ground truth")
- system_response_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the system response")
- discussion: str = dspy.OutputField(desc="discussion of the overlap between ground truth and system response")
- recall: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
- precision: float = dspy.OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")
+ question: str = InputField()
+ ground_truth: str = InputField()
+ system_response: str = InputField()
+ ground_truth_key_ideas: str = OutputField(desc="enumeration of key ideas in the ground truth")
+ system_response_key_ideas: str = OutputField(desc="enumeration of key ideas in the system response")
+ discussion: str = OutputField(desc="discussion of the overlap between ground truth and system response")
+ recall: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
+ precision: float = OutputField(desc="fraction (out of 1.0) of system response covered by the ground truth")
def f1_score(precision, recall):
@@ -35,14 +37,14 @@ def f1_score(precision, recall):
return 0.0 if precision + recall == 0 else 2 * (precision * recall) / (precision + recall)
-class SemanticF1(dspy.Module):
+class SemanticF1(Module):
def __init__(self, threshold=0.66, decompositional=False):
self.threshold = threshold
if decompositional:
- self.module = dspy.ChainOfThought(DecompositionalSemanticRecallPrecision)
+ self.module = ChainOfThought(DecompositionalSemanticRecallPrecision)
else:
- self.module = dspy.ChainOfThought(SemanticRecallPrecision)
+ self.module = ChainOfThought(SemanticRecallPrecision)
def forward(self, example, pred, trace=None):
scores = self.module(question=example.question, ground_truth=example.response, system_response=pred.response)
@@ -55,42 +57,42 @@ class SemanticF1(dspy.Module):
###########
-class AnswerCompleteness(dspy.Signature):
+class AnswerCompleteness(Signature):
"""
Estimate the completeness of a system's responses, against the ground truth.
You will first enumerate key ideas in each response, discuss their overlap, and then report completeness.
"""
- question: str = dspy.InputField()
- ground_truth: str = dspy.InputField()
- system_response: str = dspy.InputField()
- ground_truth_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the ground truth")
- system_response_key_ideas: str = dspy.OutputField(desc="enumeration of key ideas in the system response")
- discussion: str = dspy.OutputField(desc="discussion of the overlap between ground truth and system response")
- completeness: float = dspy.OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
+ question: str = InputField()
+ ground_truth: str = InputField()
+ system_response: str = InputField()
+ ground_truth_key_ideas: str = OutputField(desc="enumeration of key ideas in the ground truth")
+ system_response_key_ideas: str = OutputField(desc="enumeration of key ideas in the system response")
+ discussion: str = OutputField(desc="discussion of the overlap between ground truth and system response")
+ completeness: float = OutputField(desc="fraction (out of 1.0) of ground truth covered by the system response")
-class AnswerGroundedness(dspy.Signature):
+class AnswerGroundedness(Signature):
"""
Estimate the groundedness of a system's responses, against real retrieved documents written by people.
You will first enumerate whatever non-trivial or check-worthy claims are made in the system response, and then
discuss the extent to which some or all of them can be deduced from the retrieved context and basic commonsense.
"""
- question: str = dspy.InputField()
- retrieved_context: str = dspy.InputField()
- system_response: str = dspy.InputField()
- system_response_claims: str = dspy.OutputField(desc="enumeration of non-trivial or check-worthy claims in the system response")
- discussion: str = dspy.OutputField(desc="discussion of how supported the claims are by the retrieved context")
- groundedness: float = dspy.OutputField(desc="fraction (out of 1.0) of system response supported by the retrieved context")
+ question: str = InputField()
+ retrieved_context: str = InputField()
+ system_response: str = InputField()
+ system_response_claims: str = OutputField(desc="enumeration of non-trivial or check-worthy claims in the system response")
+ discussion: str = OutputField(desc="discussion of how supported the claims are by the retrieved context")
+ groundedness: float = OutputField(desc="fraction (out of 1.0) of system response supported by the retrieved context")
-class CompleteAndGrounded(dspy.Module):
+class CompleteAndGrounded(Module):
def __init__(self, threshold=0.66):
self.threshold = threshold
- self.completeness_module = dspy.ChainOfThought(AnswerCompleteness)
- self.groundedness_module = dspy.ChainOfThought(AnswerGroundedness)
+ self.completeness_module = ChainOfThought(AnswerCompleteness)
+ self.groundedness_module = ChainOfThought(AnswerGroundedness)
def forward(self, example, pred, trace=None):
completeness = self.completeness_module(question=example.question, ground_truth=example.response, system_response=pred.response)
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py
index 8f0b50f3c..ea86c0c04 100644
--- a/dspy/evaluate/evaluate.py
+++ b/dspy/evaluate/evaluate.py
@@ -204,7 +204,7 @@ class Evaluate:
return round(100 * ncorrect / ntotal, 2)
def _construct_result_table(
- self, results: list[Tuple[dspy.Example, dspy.Example, Any]], metric_name: str
+ self, results: list[Tuple["dspy.Example", "dspy.Example", Any]], metric_name: str
) -> "pd.DataFrame":
"""
Construct a pandas DataFrame from the specified result list.
diff --git a/dspy/evaluate/metrics.py b/dspy/evaluate/metrics.py
index dc9a3db41..65ae37fcf 100644
--- a/dspy/evaluate/metrics.py
+++ b/dspy/evaluate/metrics.py
@@ -1,9 +1,123 @@
# TODO: This should move internally. Same for passage_match. dspy.metrics.answer_exact_match, dspy.metrics.answer_passage_match
+import re
+import string
+import unicodedata
+from collections import Counter
+
+from dspy.dsp.utils.utils import print_message
+
+
+def EM(prediction, answers_list): # noqa: N802
+ assert isinstance(answers_list, list)
+
+ return max(em_score(prediction, ans) for ans in answers_list)
+
+
+def F1(prediction, answers_list): # noqa: N802
+ assert isinstance(answers_list, list)
+
+ return max(f1_score(prediction, ans) for ans in answers_list)
+
+
+def HotPotF1(prediction, answers_list): # noqa: N802
+ assert isinstance(answers_list, list)
+
+ return max(hotpot_f1_score(prediction, ans) for ans in answers_list)
+
+
+def normalize_text(s):
+ s = unicodedata.normalize("NFD", s)
+
+ def remove_articles(text):
+ return re.sub(r"\b(a|an|the)\b", " ", text)
+
+ def white_space_fix(text):
+ return " ".join(text.split())
+
+ def remove_punc(text):
+ exclude = set(string.punctuation)
+ return "".join(ch for ch in text if ch not in exclude)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def em_score(prediction, ground_truth):
+ return normalize_text(prediction) == normalize_text(ground_truth)
+
+
+# See: https://github.com/hotpotqa/hotpot/blob/master/hotpot_evaluate_v1.py
+# See: https://rajpurkar.github.io/SQuAD-explorer/ under Evaluation Script
+# See: QReCC's
+
+
+def f1_score(prediction, ground_truth):
+ prediction_tokens = normalize_text(prediction).split()
+ ground_truth_tokens = normalize_text(ground_truth).split()
+
+ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+ num_same = sum(common.values())
+
+ if len(prediction_tokens) == len(ground_truth_tokens) == 0:
+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.
+ print_message("\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")
+
+ if num_same == 0:
+ return 0
+
+ precision = 1.0 * num_same / len(prediction_tokens)
+ recall = 1.0 * num_same / len(ground_truth_tokens)
+ f1 = (2 * precision * recall) / (precision + recall)
+
+ return f1
+
+
+def hotpot_f1_score(prediction, ground_truth):
+ normalized_prediction = normalize_text(prediction)
+ normalized_ground_truth = normalize_text(ground_truth)
+
+ if normalized_prediction in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:
+ return 0
+ if normalized_ground_truth in ["yes", "no", "noanswer"] and normalized_prediction != normalized_ground_truth:
+ return 0
+
+ prediction_tokens = normalized_prediction.split()
+ ground_truth_tokens = normalized_ground_truth.split()
+ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+ num_same = sum(common.values())
+ if num_same == 0:
+ return 0
+ precision = 1.0 * num_same / len(prediction_tokens)
+ recall = 1.0 * num_same / len(ground_truth_tokens)
+ f1 = (2 * precision * recall) / (precision + recall)
+ return f1
+
+
+def precision_score(prediction, ground_truth):
+ prediction_tokens = normalize_text(prediction).split()
+ ground_truth_tokens = normalize_text(ground_truth).split()
+
+ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+ num_same = sum(common.values())
+
+ if len(prediction_tokens) == len(ground_truth_tokens) == 0:
+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.
+ print_message("\n#> Precision Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")
+
+ if num_same == 0:
+ return 0
+
+ precision = 1.0 * num_same / len(prediction_tokens)
+
+ return precision
+
def _passage_match(passages: list[str], answers: list[str]) -> bool:
"""Returns True if any of the passages contains the answer."""
- from dspy.dsp.utils import DPR_normalize, has_answer, normalize_text
+ from dspy.dsp.utils import DPR_normalize, has_answer
def passage_has_answers(passage: str, answers: list[str]) -> bool:
"""Returns True if the passage contains the answer."""
@@ -17,7 +131,6 @@ def _passage_match(passages: list[str], answers: list[str]) -> bool:
def _answer_match(prediction, answers, frac=1.0):
"""Returns True if the prediction matches any of the answers."""
- from dspy.dsp.utils import EM, F1
if frac >= 1.0:
return EM(prediction, answers)
diff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.py
index 9d56d3810..8f0e33448 100644
--- a/dspy/predict/aggregation.py
+++ b/dspy/predict/aggregation.py
@@ -1,4 +1,4 @@
-from dspy.dsp.utils import normalize_text
+from dspy.evaluate import normalize_text
from dspy.primitives.prediction import Completions, Prediction
diff --git a/tests/predict/test_aggregation.py b/tests/predict/test_aggregation.py
index 1a020ab39..a92f44a0b 100644
--- a/tests/predict/test_aggregation.py
+++ b/tests/predict/test_aggregation.py
@@ -1,4 +1,4 @@
-from dspy.dsp.utils import normalize_text
+from dspy.evaluate import normalize_text
from dspy.predict.aggregation import majority
from dspy.primitives.prediction import Completions, Prediction