DSPy: The framework for programming—not prompting—language models
Python
backtracking w/ past attempts passed as traces
manishshettym5d6f7cd0da81
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.pyindex b33ed91fe..e96c8348a 100644--- a/assert_hotpotqa.py+++ b/assert_hotpotqa.py@@ -114,7 +114,7 @@ class SimplifiedBaleen(dspy.Module):dspy.Suggest(validate_query_distinction_local(prev_queries, query),- "Query should not be the following: "+ "Query should be distinct from: "+ "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 52ae84a95..0407349fe 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -37,14 +37,12 @@ def _build_error_msg(feedback_msgs):def _build_trace_passages(failure_traces):"""Build a list of trace passages from a list of failure traces."""+ trace_template = """Failed Instruction: {} | Output: {}"""+ trace_passages = []+ for id, (ip, op, msg) in failure_traces.items():+ trace_passages.append(trace_template.format(msg, op))- # print("Traces:")- # for id, (ip, op, msg) in failure_traces.items():- # print(id, op, msg)- # print()-- # TODO complete this- return failure_traces+ return trace_passagesdef _extend_predictor_signature(predictor, **kwargs):@@ -116,11 +114,19 @@ class DSPySuggestionError(AssertionError):class Constraint:+ # a registry of all constraints created+ _registry = {}+def __init__(self, result: bool, msg: str = "", target_module=None):- self.id = str(uuid.uuid4())- self.result = result- self.msg = msg- self.target_module = target_module+ key = (result, msg, target_module)+ if key in Constraint._registry:+ self = Constraint._registry[key]+ else:+ self.id = str(uuid.uuid4())+ self.result = result+ self.msg = msg+ self.target_module = target_module+ Constraint._registry[key] = selfself.__call__()@@ -232,46 +238,50 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):backtrack_to = Noneerror_msg = Noneresult = None- extended_predictors_to_original_forward = {}++ predictors_to_original_forward = {}# predictor_feedback: Predictor -> List[feedback_msg]predictor_feedbacks = {}- # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]+ # failure_traces: Predictor -> Dict[assertion, (ip, op, msg)]failure_traces = {}for i in range(max_backtracks + 1):if i > 0 and backtrack_to is not None:# create a new traces field- traces = dspy.InputField(- prefix="Trace:",- desc="Traces from your past attempts",+ traces_field = dspy.InputField(+ prefix="Counter Examples:",+ desc="Traces of some incorrect outputs that violate instructions",format=dsp.passages2text,)# create a new feedback field- feedback = dspy.InputField(+ feedback_field = dspy.InputField(prefix="Instruction:",desc="Some instructions you must satisfy",format=str,)# save the original forward function to revert back to- extended_predictors_to_original_forward[- backtrack_to- ] = backtrack_to.forward+ predictors_to_original_forward[backtrack_to] = backtrack_to.forward- # extend signature with feedback field- _extend_predictor_signature(backtrack_to, feedback=feedback)+ # extend signature with new fields+ _extend_predictor_signature(+ backtrack_to,+ _assert_feedback=feedback_field,+ _assert_traces=traces_field,+ )- # set feedback field for predictor's forward function+ # set the new fieldsfeedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])-- # set traces field for predictor's forward functiontrace_passages = _build_trace_passages(failure_traces[backtrack_to])- # TODO: set this in the predictor's forward function_wrap_forward_with_set_fields(- backtrack_to, default_args={"feedback": feedback_msg}+ backtrack_to,+ default_args={+ "_assert_feedback": feedback_msg,+ "_assert_traces": trace_passages,+ },)# if last backtrack: ignore suggestion errors@@ -284,9 +294,7 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):result = func(*args, **kwargs)breakexcept DSPySuggestionError as e:- suggest_id, error_msg = e.id, e.msg- error_ip = e.state[-1][1]- error_op = e.state[-1][2].__dict__["_store"]+ suggest_id, error_msg, error_state = e.id, e.msg, e.state[-1]if dsp.settings.trace:if target_module:@@ -309,6 +317,11 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):predictor_feedbacks[backtrack_to].append(error_msg)# save latest failure trace for predictor per suggestion+ error_ip = error_state[1]+ error_op = error_state[2].__dict__["_store"]+ error_op.pop("_assert_feedback", None)+ error_op.pop("_assert_traces", None)+failure_traces.setdefault(backtrack_to, {})[suggest_id] = (error_ip,error_op,@@ -321,11 +334,8 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):)# revert any extended predictors to their originals- if extended_predictors_to_original_forward:- for (- predictor,- original_forward,- ) in extended_predictors_to_original_forward.items():+ if predictors_to_original_forward:+ for predictor, original_forward in predictors_to_original_forward.items():_revert_predictor_signature(predictor, "feedback")predictor.forward = original_forward@@ -334,6 +344,9 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):return wrapper+#################### Assertion Module Transformer ####################++def handle_assert_forward(assertion_handler, **handler_args):def forward(self, *args, **kwargs):args_to_vals = inspect.getcallargs(self._forward, *args, **kwargs)