DSPy: The framework for programming—not prompting—language models
Python
added AnswerCorrectness and AnswerFaithfulness auto-eval modules
Arnav Singhvi4e34fa753a33
diff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.pyindex 1af0d8f5c..0b71ccfac 100644--- a/dspy/evaluate/__init__.py+++ b/dspy/evaluate/__init__.py@@ -1,2 +1,3 @@from .evaluate import Evaluatefrom .metrics import *+from .auto_evaluation import *\ No newline at end of filediff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.pynew file mode 100644index 000000000..f55832ab7--- /dev/null+++ b/dspy/evaluate/auto_evaluation.py@@ -0,0 +1,34 @@+import dspy++class AnswerCorrectnessSignature(dspy.Signature):+ """Determines if predicted answer matches the gold answer."""++ question = dspy.InputField()+ gold_answer = dspy.InputField(desc="correct answer for question")+ predicted_answer = dspy.InputField(desc="predicted answer for question")+ is_correct = dspy.OutputField(desc='True or False')++class AnswerCorrectness(dspy.Module):+ def __init__(self):+ super().__init__()+ self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)++ def forward(self, question, gold_answer, predicted_answer):+ return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)+++class AnswerFaithfulnessSignature(dspy.Signature):+ """Checks if answer for question is based on rationale."""++ context = dspy.InputField(desc="relevant facts for producing answer")+ question = dspy.InputField()+ answer = dspy.InputField(desc="often between 1 and 5 words")+ faithful = dspy.OutputField(desc='True or False')++class AnswerFaithfulness(dspy.Module):+ def __init__(self):+ super().__init__()+ self.evaluate_faithfulness = dspy.ChainOfThought(AnswerFaithfulnessSignature)++ def forward(self, context, question, answer):+ return self.evaluate_faithfulness(context=context, question=question, answer=answer)\ No newline at end of filediff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.pyindex 7bb1f802d..5a92964ae 100644--- a/dspy/signatures/signature.py+++ b/dspy/signatures/signature.py@@ -33,6 +33,10 @@ class SignatureMeta(type):return new_class+ @property+ def kwargs(cls):+ return cls.signature.fields+def __call__(cls, *args, **kwargs):if len(args) == 1 and isinstance(args[0], str):instance = super(SignatureMeta, cls).__call__(*args, **kwargs)@@ -42,7 +46,9 @@ class SignatureMeta(type):def __getattr__(cls, attr):# Redirect attribute access to the template object when accessed on the class directly- return getattr(cls._template, attr)+ if attr not in cls.__dict__:+ return getattr(cls._template, attr)+ return super().__getattr__(attr)class Signature(metaclass=SignatureMeta):def __init__(self, signature: str = "", instructions: str = ""):@@ -51,6 +57,11 @@ class Signature(metaclass=SignatureMeta):self.fields = {}self.parse_structure()+ def __getattr__(self, attr):+ if attr not in self.__dict__:+ return getattr(self.__class__, attr)+ return super().__getattr__(attr)+@propertydef kwargs(self):return {k: v for k, v in self.fields.items()}