Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

backtracking w/ past attempts passed as traces

manishshettym5d6f7cd0da81
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.py
index b33ed91fe..e96c8348a 100644
--- a/assert_hotpotqa.py
+++ b/assert_hotpotqa.py
@@ -114,7 +114,7 @@ class SimplifiedBaleen(dspy.Module):
dspy.Suggest(
validate_query_distinction_local(prev_queries, query),
- "Query should not be the following: "
+ "Query should be distinct from: "
+ "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),
)
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index 52ae84a95..0407349fe 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -37,14 +37,12 @@ def _build_error_msg(feedback_msgs):
def _build_trace_passages(failure_traces):
"""Build a list of trace passages from a list of failure traces."""
+ trace_template = """Failed Instruction: {} | Output: {}"""
+ trace_passages = []
+ for id, (ip, op, msg) in failure_traces.items():
+ trace_passages.append(trace_template.format(msg, op))
- # print("Traces:")
- # for id, (ip, op, msg) in failure_traces.items():
- # print(id, op, msg)
- # print()
-
- # TODO complete this
- return failure_traces
+ return trace_passages
def _extend_predictor_signature(predictor, **kwargs):
@@ -116,11 +114,19 @@ class DSPySuggestionError(AssertionError):
class Constraint:
+ # a registry of all constraints created
+ _registry = {}
+
def __init__(self, result: bool, msg: str = "", target_module=None):
- self.id = str(uuid.uuid4())
- self.result = result
- self.msg = msg
- self.target_module = target_module
+ key = (result, msg, target_module)
+ if key in Constraint._registry:
+ self = Constraint._registry[key]
+ else:
+ self.id = str(uuid.uuid4())
+ self.result = result
+ self.msg = msg
+ self.target_module = target_module
+ Constraint._registry[key] = self
self.__call__()
@@ -232,46 +238,50 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
backtrack_to = None
error_msg = None
result = None
- extended_predictors_to_original_forward = {}
+
+ predictors_to_original_forward = {}
# predictor_feedback: Predictor -> List[feedback_msg]
predictor_feedbacks = {}
- # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]
+ # failure_traces: Predictor -> Dict[assertion, (ip, op, msg)]
failure_traces = {}
for i in range(max_backtracks + 1):
if i > 0 and backtrack_to is not None:
# create a new traces field
- traces = dspy.InputField(
- prefix="Trace:",
- desc="Traces from your past attempts",
+ traces_field = dspy.InputField(
+ prefix="Counter Examples:",
+ desc="Traces of some incorrect outputs that violate instructions",
format=dsp.passages2text,
)
# create a new feedback field
- feedback = dspy.InputField(
+ feedback_field = dspy.InputField(
prefix="Instruction:",
desc="Some instructions you must satisfy",
format=str,
)
# save the original forward function to revert back to
- extended_predictors_to_original_forward[
- backtrack_to
- ] = backtrack_to.forward
+ predictors_to_original_forward[backtrack_to] = backtrack_to.forward
- # extend signature with feedback field
- _extend_predictor_signature(backtrack_to, feedback=feedback)
+ # extend signature with new fields
+ _extend_predictor_signature(
+ backtrack_to,
+ _assert_feedback=feedback_field,
+ _assert_traces=traces_field,
+ )
- # set feedback field for predictor's forward function
+ # set the new fields
feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])
-
- # set traces field for predictor's forward function
trace_passages = _build_trace_passages(failure_traces[backtrack_to])
- # TODO: set this in the predictor's forward function
_wrap_forward_with_set_fields(
- backtrack_to, default_args={"feedback": feedback_msg}
+ backtrack_to,
+ default_args={
+ "_assert_feedback": feedback_msg,
+ "_assert_traces": trace_passages,
+ },
)
# if last backtrack: ignore suggestion errors
@@ -284,9 +294,7 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
result = func(*args, **kwargs)
break
except DSPySuggestionError as e:
- suggest_id, error_msg = e.id, e.msg
- error_ip = e.state[-1][1]
- error_op = e.state[-1][2].__dict__["_store"]
+ suggest_id, error_msg, error_state = e.id, e.msg, e.state[-1]
if dsp.settings.trace:
if target_module:
@@ -309,6 +317,11 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
predictor_feedbacks[backtrack_to].append(error_msg)
# save latest failure trace for predictor per suggestion
+ error_ip = error_state[1]
+ error_op = error_state[2].__dict__["_store"]
+ error_op.pop("_assert_feedback", None)
+ error_op.pop("_assert_traces", None)
+
failure_traces.setdefault(backtrack_to, {})[suggest_id] = (
error_ip,
error_op,
@@ -321,11 +334,8 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
)
# revert any extended predictors to their originals
- if extended_predictors_to_original_forward:
- for (
- predictor,
- original_forward,
- ) in extended_predictors_to_original_forward.items():
+ if predictors_to_original_forward:
+ for predictor, original_forward in predictors_to_original_forward.items():
_revert_predictor_signature(predictor, "feedback")
predictor.forward = original_forward
@@ -334,6 +344,9 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):
return wrapper
+#################### Assertion Module Transformer ####################
+
+
def handle_assert_forward(assertion_handler, **handler_args):
def forward(self, *args, **kwargs):
args_to_vals = inspect.getcallargs(self._forward, *args, **kwargs)