DSPy: The framework for programming—not prompting—language models
Python
make assertion handler cleaner
Shangyint0f0e2e6b1140
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 311f991f3..c33f5658c 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -35,18 +35,30 @@ class DSPyAssertionError(AssertionError):self.state = state-class Assert:- """DSPy Assertion"""+class DSPySuggestionError(AssertionError):+ """Custom exception raised when a DSPy `Suggest` fails."""- def __init__(self, assert_fun_res: bool, msg: str = ""):- self.assert_fun_res = assert_fun_res+ def __init__(self, msg: str, state: Any = None) -> None:+ super().__init__(msg)+ self.msg = msg+ self.state = state++class Constraint:++ def __init__(self, result: bool, msg: str = ""):+ self.result = resultself.msg = msg+self.__call__()++class Assert(Constraint):+ """DSPy Assertion"""+def __call__(self) -> bool:- if isinstance(self.assert_fun_res, bool):- if self.assert_fun_res:+ if isinstance(self.result, bool):+ if self.result:return Trueelif dspy.settings.bypass_assert:logger.error(f"AssertionError: {self.msg}")@@ -58,136 +70,155 @@ class Assert:raise ValueError("Assertion function should always return [bool]")-def assert_backtrack_policy(max_backtracks=2):- """Decorator that defines the backtracking policy for assertions.+class Suggest(Constraint):+ """DSPy Suggestion"""- current policy: re-run the latest predictor up to `max_backtracks` times,- with updated signature if assertion fails. updated signature adds a new- input field to the signature, which is the feedback.- """+ def __call__(self) -> Any:+ if isinstance(self.result, bool):+ if self.result:+ return True+ elif dspy.settings.bypass_assert:+ logger.error(f"SuggestionFailed: {self.msg}")+ return True+ else:+ logger.error(f"SuggestionFailed: {self.msg}")+ raise DSPyAssertionError(msg=self.msg, state=dsp.settings.trace)- def wrapper(func):- def inner(*args, **kwargs):- backtrack_to = None- error_msg = None- result = None- extended_predictors_to_original_forward = {}- # predictor_feedback: Predictor -> List[feedback_msg]- # this will contain all assertion failure msgs for each predictor- # including msgs for the same assertion failure- predictor_feedbacks = {}-- def _extend_predictor_signature(predictor, **kwargs):- """Update the signature of a predictor instance with specified fields."""- old_signature = predictor.extended_signature- old_keys = list(old_signature.kwargs.keys())-- # include other input fields after question- position = old_keys.index("question") + 1- for key in reversed(kwargs):- old_keys.insert(position, key)-- extended_kwargs = {- key: kwargs.get(key, old_signature.kwargs.get(key))- for key in old_keys- }-- new_signature = dsp.Template(- old_signature.instructions, **extended_kwargs- )- predictor.extended_signature = new_signature-- def _build_error_msg(feedback_msgs):- """Build an error message from a list of feedback messages."""- return "\n".join([msg for msg in feedback_msgs])-- def _revert_predictor_signature(predictor, *args):- """Revert the signature of a predictor by removing specified fields."""- old_signature_kwargs = predictor.extended_signature.kwargs- for key in args:- old_signature_kwargs.pop(key, None)- new_signature = dsp.Template(- predictor.extended_signature.instructions, **old_signature_kwargs- )- predictor.extended_signature = new_signature- def _wrap_forward_with_set_fields(predictor, default_args):- """Wrap the forward method of a predictor instance to enforce default arguments."""- original_forward = predictor.forward+def noop_handler(func):+ """ "Handler to bypass assertions."""- def new_forward(**kwargs):- for arg, value in default_args.items():- kwargs.setdefault(arg, value)+ def wrapper(*args, **kwargs):+ with dspy.settings.context(bypass_assert=True):+ return func(*args, **kwargs)- return original_forward(**kwargs)+ return wrapper- predictor.forward = new_forward- for i in range(max_backtracks + 1):- if i > 0 and backtrack_to is not None:- # create a new feedback field- feedback = dspy.InputField(- prefix="Instruction:",- desc="Some instructions you must satisfy",- format=str,- )+def assert_backtrack_handler(func, max_backtracks=2):+ """Handler for backtracking assertions.- # save the original forward function to revert back to- extended_predictors_to_original_forward[- backtrack_to- ] = backtrack_to.forward+ Re-run the latest predictor up to `max_backtracks` times,+ with updated signature if assertion fails. updated signature adds a new+ input field to the signature, which is the feedback.+ """- # extend signature with feedback field- _extend_predictor_signature(backtrack_to, feedback=feedback)+ def wrapper(*args, **kwargs):+ backtrack_to = None+ error_msg = None+ result = None+ extended_predictors_to_original_forward = {}+ # predictor_feedback: Predictor -> List[feedback_msg]+ # this will contain all assertion failure msgs for each predictor+ # including msgs for the same assertion failure+ predictor_feedbacks = {}++ def _extend_predictor_signature(predictor, **kwargs):+ """Update the signature of a predictor instance with specified fields."""+ old_signature = predictor.extended_signature+ old_keys = list(old_signature.kwargs.keys())++ # include other input fields after question+ position = old_keys.index("question") + 1+ for key in reversed(kwargs):+ old_keys.insert(position, key)++ extended_kwargs = {+ key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys+ }++ new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)+ predictor.extended_signature = new_signature++ def _build_error_msg(feedback_msgs):+ """Build an error message from a list of feedback messages."""+ return "\n".join([msg for msg in feedback_msgs])++ def _revert_predictor_signature(predictor, *args):+ """Revert the signature of a predictor by removing specified fields."""+ old_signature_kwargs = predictor.extended_signature.kwargs+ for key in args:+ old_signature_kwargs.pop(key, None)+ new_signature = dsp.Template(+ predictor.extended_signature.instructions, **old_signature_kwargs+ )+ predictor.extended_signature = new_signature++ def _wrap_forward_with_set_fields(predictor, default_args):+ """Wrap the forward method of a predictor instance to enforce default arguments."""+ original_forward = predictor.forward++ def new_forward(**kwargs):+ for arg, value in default_args.items():+ kwargs.setdefault(arg, value)++ return original_forward(**kwargs)++ predictor.forward = new_forward++ for i in range(max_backtracks + 1):+ if i > 0 and backtrack_to is not None:+ # create a new feedback field+ feedback = dspy.InputField(+ prefix="Instruction:",+ desc="Some instructions you must satisfy",+ format=str,+ )- # set feedback field as a default kwarg to the predictor's forward function- feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])- _wrap_forward_with_set_fields(- backtrack_to, default_args={"feedback": feedback_msg}- )+ # save the original forward function to revert back to+ extended_predictors_to_original_forward[+ backtrack_to+ ] = backtrack_to.forward- # if user set bypass_assert: ignore assertion errors- if dsp.settings.bypass_assert:- result = func(*args, **kwargs)- break+ # extend signature with feedback field+ _extend_predictor_signature(backtrack_to, feedback=feedback)++ # set feedback field as a default kwarg to the predictor's forward function+ feedback_msg = _build_error_msg(predictor_feedbacks[backtrack_to])+ _wrap_forward_with_set_fields(+ backtrack_to, default_args={"feedback": feedback_msg}+ )- # if last backtrack: ignore assertion errors- elif i == max_backtracks and dspy.settings.bypass_assert is False:- dspy.settings.configure(bypass_assert=True)+ # if user set bypass_assert: ignore assertion errors+ if dsp.settings.bypass_assert:+ result = func(*args, **kwargs)+ break++ # if last backtrack: ignore assertion errors+ elif i == max_backtracks and dspy.settings.bypass_assert is False:+ dspy.settings.configure(bypass_assert=True)+ result = func(*args, **kwargs)+ dspy.settings.configure(bypass_assert=False)+ break++ else:+ try:result = func(*args, **kwargs)- dspy.settings.configure(bypass_assert=False)break-- else:- try:- result = func(*args, **kwargs)- break- except DSPyAssertionError as e:- error_msg = e.msg-- if dsp.settings.trace:- backtrack_to = dsp.settings.trace[-1][0]- if error_msg not in predictor_feedbacks.setdefault(- backtrack_to, []- ):- predictor_feedbacks[backtrack_to].append(error_msg)- else:- logger.error(- f"UNREACHABLE: No trace available, this should not happen. Is this run time?"- )-- # revert any extended predictors to their originals- if extended_predictors_to_original_forward:- for (- predictor,- original_forward,- ) in extended_predictors_to_original_forward.items():- _revert_predictor_signature(predictor, "feedback")- predictor.forward = original_forward-- return result-- return inner+ except DSPyAssertionError as e:+ error_msg = e.msg++ if dsp.settings.trace:+ backtrack_to = dsp.settings.trace[-1][0]+ if error_msg not in predictor_feedbacks.setdefault(+ backtrack_to, []+ ):+ predictor_feedbacks[backtrack_to].append(error_msg)+ else:+ logger.error(+ f"UNREACHABLE: No trace available, this should not happen. Is this run time?"+ )++ # revert any extended predictors to their originals+ if extended_predictors_to_original_forward:+ for (+ predictor,+ original_forward,+ ) in extended_predictors_to_original_forward.items():+ _revert_predictor_signature(predictor, "feedback")+ predictor.forward = original_forward++ return resultreturn wrapper@@ -200,13 +231,13 @@ def handle_assert_forward(assertion_handler):if "bypass_assert" in args_to_vals:dspy.settings.configure(bypass_assert=args_to_vals["bypass_assert"])- wrapped_forward = assertion_handler()(self._forward)+ wrapped_forward = assertion_handler(self._forward)return wrapped_forward(*args, **kwargs)return forward-default_assertion_handler = assert_backtrack_policy+default_assertion_handler = assert_backtrack_handlerdef assert_transform_module(module, assertion_handler=default_assertion_handler):