Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Resetting signature after current run; fix LM assert

Shangyint56b6e8dab1a5
diff --git a/assert_example.py b/assert_example.py
index bfe6a6092..f01efd4e3 100644
--- a/assert_example.py
+++ b/assert_example.py
@@ -73,31 +73,31 @@ def validate_query_distinction_local(previous_queries, query):
return True
-# class GenerateSearchQuery(dspy.Signature):
-# query = dspy.InputField(desc="current query")
-# previous_queries = dspy.InputField(desc="previous queries")
-# duplication = dspy.OutputField(
-# desc="Whether the current query is similar to previous queries, only return True or False"
-# )
+class EvaluateSearchQueries(dspy.Signature):
+ query = dspy.InputField(desc="current query")
+ previous_queries = dspy.InputField(desc="previous queries")
+ duplication = dspy.OutputField(
+ desc="Whether the current query is similar to previous queries, only return True or False"
+ )
-# class QueryDistinction(dspy.Module):
+class QueryDistinction(dspy.Module):
-# def __init__(self):
-# super().__init__()
-# self.generate_query = dspy.Predict(GenerateSearchQuery)
+ def __init__(self):
+ super().__init__()
+ self.evaluate_query = dspy.Predict(EvaluateSearchQueries)
-# def forward(self, query, previous_queries):
-# duplication = self.generate_query(
-# query=query, previous_queries=previous_queries
-# ).duplication
-# return duplication
+ def forward(self, query, previous_queries):
+ duplication = self.evaluate_query(
+ query=query, previous_queries=previous_queries
+ ).duplication
+ return duplication
-# def validate_query_distinction_LM(previous_queries, query):
-# query_distinction = QueryDistinction()
-# duplication = query_distinction(query=query, previous_queries=previous_queries)
-# return True if duplication == "False" else False
+def validate_query_distinction_LM(previous_queries, query):
+ query_distinction = QueryDistinction()
+ duplication = query_distinction(query=query, previous_queries=previous_queries)
+ return True if duplication == "False" else False
# declaration of dspy program
@@ -120,13 +120,13 @@ class SimplifiedBaleen(dspy.Module):
for hop in range(self.max_hops):
query = self.generate_query[hop](context=context, question=question).query
- # turbo.inspect_history(n=1)
- print("query is ", query)
+ turbo.inspect_history(n=1)
+ # print("query is ", query)
dspy.Assert(
validate_query_distinction_local,
previous_queries,
query,
- msg=f"Queries should be different with previous queries {previous_queries}",
+ msg=f"Queries should be really different with previous queries {('; ').join(previous_queries)}",
)
previous_queries.append(query)
passages = self.retrieve(query).passages
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index ee95a8651..41f2456e0 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -22,9 +22,9 @@ class Assert:
self.assert_fun = assert_fun
self.args = args
- if 'msg' in kwargs:
- self.msg = kwargs['msg']
- del kwargs['msg']
+ if "msg" in kwargs:
+ self.msg = kwargs["msg"]
+ del kwargs["msg"]
else:
self.msg = ""
@@ -150,21 +150,21 @@ def assert_latest_transform(backtrack=2):
def assert_latest_feedback_transform(backtrack=2):
- """"Decorator that simply re-runs the function but updates the signature of
+ """ "Decorator that simply re-runs the function but updates the signature of
the latest predictor in the trace if assertion fails, up to `backtrack` times.
-
+
The updated signature passes a new input to the predictor, which is the feedback
from the failed assertion. This feedback will be used by the predictor to self-repair.
"""
-
+
def wrapper(func):
def inner(*args, **kwargs):
backtrack_to = None
error_msg = None
-
+ extended_backtrack_to = None
+
def _extend_predictor_signature(predictor, **kwargs):
- """Update the signature of a predictor instance with a new field.
- """
+ """Update the signature of a predictor instance with a new field."""
old_signature = predictor.extended_signature
*keys, last_key = old_signature.kwargs.keys()
@@ -175,12 +175,20 @@ def assert_latest_feedback_transform(backtrack=2):
new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)
predictor.extended_signature = new_signature
+ def _remove_predictor_signature(predictor, *args):
+ """Remove the signature of a predictor."""
+ old_signature_kwargs = predictor.extended_signature.kwargs
+ for key in args:
+ old_signature_kwargs.pop(key, None)
+ new_signature = dsp.Template(
+ predictor.extended_signature.instructions, **old_signature_kwargs
+ )
+ predictor.extended_signature = new_signature
def _wrap_forward_with_set_fields(predictor, default_args):
- """Wrap the forward method of a predictor instance to enforce default arguments.
- """
+ """Wrap the forward method of a predictor instance to enforce default arguments."""
original_forward = predictor.forward
-
+
def new_forward(**kwargs):
for arg, value in default_args.items():
kwargs.setdefault(arg, value)
@@ -189,17 +197,22 @@ def assert_latest_feedback_transform(backtrack=2):
predictor.forward = new_forward
-
for i in range(backtrack):
if i > 0 and backtrack_to is not None:
print(f"rewinding to {id(backtrack_to)}")
- # udpate the signature of the `backtrack_to` predictor with a feedback field
- feedback = dspy.InputField(prefix="Note:", desc="common mistakes to avoid")
+ # udpate the signature of the `backtrack_to` predictor with a feedback field
+ feedback = dspy.InputField(
+ prefix="Note:", desc="common mistakes to avoid"
+ )
+ extended_backtrack_to = backtrack_to
+ original_forward = extended_backtrack_to.forward
_extend_predictor_signature(backtrack_to, feedback=feedback)
-
+
# set the feedback field as a default kwarg to the predictor's forward function
- _wrap_forward_with_set_fields(backtrack_to, default_args={'feedback': error_msg})
+ _wrap_forward_with_set_fields(
+ backtrack_to, default_args={"feedback": error_msg}
+ )
try:
return func(*args, **kwargs)
@@ -213,7 +226,11 @@ def assert_latest_feedback_transform(backtrack=2):
print(
"UNREACHABLE: No trace available, this should not happen. Is this run time?"
)
+ finally:
+ if extended_backtrack_to:
+ _remove_predictor_signature(extended_backtrack_to, "feedback")
+ extended_backtrack_to.forward = original_forward
return inner
- return wrapper
\ No newline at end of file
+ return wrapper