DSPy: The framework for programming—not prompting—language models
Python
[wip] tracking failure traces and using it for backtrack
manishshettym76715411d7bf
diff --git a/.gitignore b/.gitignoreindex 3abff4193..f048c4639 100644--- a/.gitignore+++ b/.gitignore@@ -40,3 +40,4 @@ finetuning_ckpts/# cache/* copy*.idea+assertion.logdiff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex ee4b6e2b1..52ae84a95 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -4,6 +4,10 @@ import dspimport dspyimport logging+import uuid+++#################### Assertion Helpers ####################def setup_logger():@@ -26,11 +30,74 @@ def setup_logger():logger = setup_logger()+def _build_error_msg(feedback_msgs):+ """Build an error message from a list of feedback messages."""+ return "\n".join([msg for msg in feedback_msgs])+++def _build_trace_passages(failure_traces):+ """Build a list of trace passages from a list of failure traces."""++ # print("Traces:")+ # for id, (ip, op, msg) in failure_traces.items():+ # print(id, op, msg)+ # print()++ # TODO complete this+ return failure_traces+++def _extend_predictor_signature(predictor, **kwargs):+ """Update the signature of a predictor instance with specified fields."""+ old_signature = predictor.extended_signature+ old_keys = list(old_signature.kwargs.keys())++ # include other input fields after question+ position = old_keys.index("question") + 1+ for key in reversed(kwargs):+ old_keys.insert(position, key)++ extended_kwargs = {+ key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys+ }++ new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)+ predictor.extended_signature = new_signature+++def _revert_predictor_signature(predictor, *args):+ """Revert the signature of a predictor by removing specified fields."""+ old_signature_kwargs = predictor.extended_signature.kwargs+ for key in args:+ old_signature_kwargs.pop(key, None)+ new_signature = dsp.Template(+ predictor.extended_signature.instructions, **old_signature_kwargs+ )+ predictor.extended_signature = new_signature+++def _wrap_forward_with_set_fields(predictor, default_args):+ """Wrap the forward method of a predictor instance to enforce default arguments."""+ original_forward = predictor.forward++ def new_forward(**kwargs):+ for arg, value in default_args.items():+ kwargs.setdefault(arg, value)++ return original_forward(**kwargs)++ predictor.forward = new_forward+++#################### Assertion Exceptions ####################++class DSPyAssertionError(AssertionError):"""Custom exception raised when a DSPy `Assert` fails."""- def __init__(self, msg: str, state: Any = None) -> None:+ def __init__(self, id: str, msg: str, state: Any = None) -> None:super().__init__(msg)+ self.id = idself.msg = msgself.state = state@@ -38,15 +105,19 @@ class DSPyAssertionError(AssertionError):class DSPySuggestionError(AssertionError):"""Custom exception raised when a DSPy `Suggest` fails."""- def __init__(self, msg: str, state: Any = None) -> None:+ def __init__(self, id: str, msg: str, state: Any = None) -> None:super().__init__(msg)+ self.id = idself.msg = msgself.state = state-class Constraint:+#################### Assertion Primitives ####################++class Constraint:def __init__(self, result: bool, msg: str = "", target_module=None):+ self.id = str(uuid.uuid4())self.result = resultself.msg = msgself.target_module = target_module@@ -66,7 +137,9 @@ class Assert(Constraint):return Trueelse:logger.error(f"AssertionError: {self.msg}")- raise DSPyAssertionError(msg=self.msg, state=dsp.settings.trace)+ raise DSPyAssertionError(+ id=self.id, msg=self.msg, state=dsp.settings.trace+ )else:raise ValueError("Assertion function should always return [bool]")@@ -83,11 +156,16 @@ class Suggest(Constraint):return Trueelse:logger.error(f"SuggestionFailed: {self.msg}")- raise DSPySuggestionError(msg=self.msg, state=dsp.settings.trace)+ raise DSPySuggestionError(+ id=self.id, msg=self.msg, state=dsp.settings.trace+ )else:raise ValueError("Suggestion function should always return [bool]")+#################### Assertion Handlers ####################++def noop_handler(func):"""Handler to bypass assertions and suggestions.@@ -111,7 +189,7 @@ def bypass_suggest_handler(func):def wrapper(*args, **kwargs):with dspy.settings.context(bypass_suggest=True, bypass_assert=False):return func(*args, **kwargs)-+return wrapper@@ -125,7 +203,7 @@ def bypass_assert_handler(func):def wrapper(*args, **kwargs):with dspy.settings.context(bypass_assert=True):return func(*args, **kwargs)-+return wrapper@@ -137,7 +215,7 @@ def assert_no_except_handler(func):return func(*args, **kwargs)except DSPyAssertionError as e:return None-+return wrapper@@ -150,62 +228,26 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):"""def wrapper(*args, **kwargs):- target_module = handler_args.get('target_module', None)+ target_module = handler_args.get("target_module", None)backtrack_to = Noneerror_msg = Noneresult = Noneextended_predictors_to_original_forward = {}# predictor_feedback: Predictor -> List[feedback_msg]- # this will contain all assertion failure msgs for each predictor- # including msgs for the same assertion failurepredictor_feedbacks = {}- def _extend_predictor_signature(predictor, **kwargs):- """Update the signature of a predictor instance with specified fields."""- old_signature = predictor.extended_signature- old_keys = list(old_signature.kwargs.keys())-- # include other input fields after question- position = old_keys.index("question") + 1- for key in reversed(kwargs):- old_keys.insert(position, key)-- extended_kwargs = {- key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys- }-- new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)- predictor.extended_signature = new_signature-- def _build_error_msg(feedback_msgs):- """Build an error message from a list of feedback messages."""- return "\n".join([msg for msg in feedback_msgs])-- def _revert_predictor_signature(predictor, *args):- """Revert the signature of a predictor by removing specified fields."""- old_signature_kwargs = predictor.extended_signature.kwargs- for key in args:- old_signature_kwargs.pop(key, None)- new_signature = dsp.Template(- predictor.extended_signature.instructions, **old_signature_kwargs- )- predictor.extended_signature = new_signature-- def _wrap_forward_with_set_fields(predictor, default_args):- """Wrap the forward method of a predictor instance to enforce default arguments."""- original_forward = predictor.forward-- def new_forward(**kwargs):- for arg, value in default_args.items():- kwargs.setdefault(arg, value)-- return original_forward(**kwargs)-- predictor.forward = new_forward+ # failure_traces: Predictor -> Dict[assertion, (input, op, msg)]+ failure_traces = {}for i in range(max_backtracks + 1):if i > 0 and backtrack_to is not None:- # TODO clean up this function+ # create a new traces field+ traces = dspy.InputField(+ prefix="Trace:",+ desc="Traces from your past attempts",+ format=dsp.passages2text,+ )+# create a new feedback fieldfeedback = dspy.InputField(prefix="Instruction:",@@ -221,8 +263,13 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):# extend signature with feedback field_extend_predictor_signature(backtrack_to, feedback=feedback)- # set feedback field as a default kwarg to the predictor's forward function+ # set feedback field for predictor's forward functionfeedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])++ # set traces field for predictor's forward function+ trace_passages = _build_trace_passages(failure_traces[backtrack_to])+ # TODO: set this in the predictor's forward function+_wrap_forward_with_set_fields(backtrack_to, default_args={"feedback": feedback_msg})@@ -236,9 +283,10 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):try:result = func(*args, **kwargs)break- # TODO Change this to suggest instead of assertexcept DSPySuggestionError as e:- error_msg = e.msg+ suggest_id, error_msg = e.id, e.msg+ error_ip = e.state[-1][1]+ error_op = e.state[-1][2].__dict__["_store"]if dsp.settings.trace:if target_module:@@ -250,10 +298,23 @@ def suggest_backtrack_handler(func, max_backtracks=10, **handler_args):breakelse:backtrack_to = dsp.settings.trace[-1][0]- if error_msg not in predictor_feedbacks.setdefault(backtrack_to, []):- predictor_feedbacks[backtrack_to].append(error_msg)+if backtrack_to is None:logger.error("Specified module not found in trace")++ # save unique feedback message for predictor+ if error_msg not in predictor_feedbacks.setdefault(+ backtrack_to, []+ ):+ predictor_feedbacks[backtrack_to].append(error_msg)++ # save latest failure trace for predictor per suggestion+ failure_traces.setdefault(backtrack_to, {})[suggest_id] = (+ error_ip,+ error_op,+ error_msg,+ )+else:logger.error(f"UNREACHABLE: No trace available, this should not happen. Is this run time?"@@ -290,7 +351,9 @@ def handle_assert_forward(assertion_handler, **handler_args):default_assertion_handler = suggest_backtrack_handler-def assert_transform_module(module, assertion_handler=default_assertion_handler, **handler_args):+def assert_transform_module(+ module, assertion_handler=default_assertion_handler, **handler_args+):"""Transform a module to handle assertions."""@@ -305,6 +368,8 @@ def assert_transform_module(module, assertion_handler=default_assertion_handler,pass # TODO warning: might be overwriting a previous _forward methodmodule._forward = module.forward- module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(module)+ module.forward = handle_assert_forward(assertion_handler, **handler_args).__get__(+ module+ )return module