Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added AnswerCorrectness and AnswerFaithfulness auto-eval modules

Arnav Singhvi4e34fa753a33
diff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.py
index 1af0d8f5c..0b71ccfac 100644
--- a/dspy/evaluate/__init__.py
+++ b/dspy/evaluate/__init__.py
@@ -1,2 +1,3 @@
from .evaluate import Evaluate
from .metrics import *
+from .auto_evaluation import *
\ No newline at end of file
diff --git a/dspy/evaluate/auto_evaluation.py b/dspy/evaluate/auto_evaluation.py
new file mode 100644
index 000000000..f55832ab7
--- /dev/null
+++ b/dspy/evaluate/auto_evaluation.py
@@ -0,0 +1,34 @@
+import dspy
+
+class AnswerCorrectnessSignature(dspy.Signature):
+ """Determines if predicted answer matches the gold answer."""
+
+ question = dspy.InputField()
+ gold_answer = dspy.InputField(desc="correct answer for question")
+ predicted_answer = dspy.InputField(desc="predicted answer for question")
+ is_correct = dspy.OutputField(desc='True or False')
+
+class AnswerCorrectness(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)
+
+ def forward(self, question, gold_answer, predicted_answer):
+ return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)
+
+
+class AnswerFaithfulnessSignature(dspy.Signature):
+ """Checks if answer for question is based on rationale."""
+
+ context = dspy.InputField(desc="relevant facts for producing answer")
+ question = dspy.InputField()
+ answer = dspy.InputField(desc="often between 1 and 5 words")
+ faithful = dspy.OutputField(desc='True or False')
+
+class AnswerFaithfulness(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.evaluate_faithfulness = dspy.ChainOfThought(AnswerFaithfulnessSignature)
+
+ def forward(self, context, question, answer):
+ return self.evaluate_faithfulness(context=context, question=question, answer=answer)
\ No newline at end of file
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index 7bb1f802d..5a92964ae 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -33,6 +33,10 @@ class SignatureMeta(type):
return new_class
+ @property
+ def kwargs(cls):
+ return cls.signature.fields
+
def __call__(cls, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], str):
instance = super(SignatureMeta, cls).__call__(*args, **kwargs)
@@ -42,7 +46,9 @@ class SignatureMeta(type):
def __getattr__(cls, attr):
# Redirect attribute access to the template object when accessed on the class directly
- return getattr(cls._template, attr)
+ if attr not in cls.__dict__:
+ return getattr(cls._template, attr)
+ return super().__getattr__(attr)
class Signature(metaclass=SignatureMeta):
def __init__(self, signature: str = "", instructions: str = ""):
@@ -51,6 +57,11 @@ class Signature(metaclass=SignatureMeta):
self.fields = {}
self.parse_structure()
+ def __getattr__(self, attr):
+ if attr not in self.__dict__:
+ return getattr(self.__class__, attr)
+ return super().__getattr__(attr)
+
@property
def kwargs(self):
return {k: v for k, v in self.fields.items()}