DSPy: The framework for programming—not prompting—language models
Python
moved bypass_assert to dspy.settings + DSPyAssertionError not raised when bypass=True
manishshettym5a94ffaf540f
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex b7f001e45..b471fefeb 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -2,6 +2,7 @@ from contextlib import contextmanagerfrom dsp.utils.utils import dotdictimport threading+class Settings(object):"""DSP configuration settings."""@@ -17,7 +18,9 @@ class Settings(object):cls._instance.main_tid = threading.get_ident()cls._instance.main_stack = []cls._instance.stack_by_thread = {}- cls._instance.stack_by_thread[threading.get_ident()] = cls._instance.main_stack+ cls._instance.stack_by_thread[+ threading.get_ident()+ ] = cls._instance.main_stack# TODO: remove first-class support for re-ranker and potentially combine with RM to form a pipeline of sorts# eg: RetrieveThenRerankPipeline(RetrievalModel, Reranker)@@ -33,6 +36,7 @@ class Settings(object):skip_logprobs=False,trace=None,release=0,+ bypass_assert=False,)cls._instance.__append(config)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 43ccabddd..2ca3452f2 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -4,8 +4,7 @@ import dspyclass DSPyAssertionError(AssertionError):- """Custom exception raised when a DSPy `Assert` fails.- """+ """Custom exception raised when a DSPy `Assert` fails."""def __init__(self, msg: str, state: Any = None) -> None:super().__init__(msg)@@ -26,6 +25,9 @@ class Assert:if isinstance(self.assert_fun_res, bool):if self.assert_fun_res:return True+ elif dspy.settings.bypass_assert:+ # TODO: log the assertion failure+ return Trueelse:raise DSPyAssertionError(msg=self.msg, state=dsp.settings.trace)else:@@ -34,9 +36,9 @@ class Assert:def assert_backtrack_policy(max_backtracks=2):"""Decorator that defines the backtracking policy for assertions.-+current policy: re-run the latest predictor up to `max_backtracks` times,- with updated signature if assertion fails. updated signature adds a new+ with updated signature if assertion fails. updated signature adds a newinput field to the signature, which is the feedback."""@@ -56,19 +58,24 @@ def assert_backtrack_policy(max_backtracks=2):old_signature = predictor.extended_signatureold_keys = list(old_signature.kwargs.keys())- #include other input fields after question- position = old_keys.index('question') + 1+ # include other input fields after question+ position = old_keys.index("question") + 1for key in reversed(kwargs):old_keys.insert(position, key)- extended_kwargs = {key: kwargs.get(key, old_signature.kwargs.get(key)) for key in old_keys}+ extended_kwargs = {+ key: kwargs.get(key, old_signature.kwargs.get(key))+ for key in old_keys+ }- new_signature = dsp.Template(old_signature.instructions, **extended_kwargs)+ new_signature = dsp.Template(+ old_signature.instructions, **extended_kwargs+ )predictor.extended_signature = new_signaturedef _build_error_msg(feedback_msgs):"""Build an error message from a list of feedback messages."""- return '\n'.join([ msg for msg in feedback_msgs])+ return "\n".join([msg for msg in feedback_msgs])def _revert_predictor_signature(predictor, *args):"""Revert the signature of a predictor by removing specified fields."""@@ -91,13 +98,14 @@ def assert_backtrack_policy(max_backtracks=2):return original_forward(**kwargs)predictor.forward = new_forward-- bypass_assert = False+for i in range(max_backtracks + 1):if i > 0 and backtrack_to is not None:# create a new feedback fieldfeedback = dspy.InputField(- prefix="Instruction:", desc="Some instructions you must satisfy", format=str+ prefix="Instruction:",+ desc="Some instructions you must satisfy",+ format=str,)# save the original forward function to revert back to@@ -114,25 +122,31 @@ def assert_backtrack_policy(max_backtracks=2):backtrack_to, default_args={"feedback": feedback_msg})- if i == max_backtracks: # Check if it's the last backtrack attempt- bypass_assert = True- result = func(*args, **kwargs, bypass_assert=bypass_assert)+ # if last attempt and run ignoring assertion errors+ if i == max_backtracks and dspy.settings.bypass_assert is False:+ dspy.settings.configure(bypass_assert=True)+ result = func(*args, **kwargs)+ dspy.settings.configure(bypass_assert=False)break- try:- result = func(*args, **kwargs, bypass_assert=bypass_assert)- break- except DSPyAssertionError as e:- print(f"AssertionError: {e.msg}")- error_msg = e.msg-- if dsp.settings.trace:- backtrack_to = dsp.settings.trace[-1][0]- if error_msg not in predictor_feedbacks.setdefault(backtrack_to, []):- predictor_feedbacks[backtrack_to].append(error_msg)- else:- print(- "UNREACHABLE: No trace available, this should not happen. Is this run time?"- )++ else:+ try:+ result = func(*args, **kwargs)+ break # if no error, break out of backtrack loop+ except DSPyAssertionError as e:+ print(f"AssertionError: {e.msg}")+ error_msg = e.msg++ if dsp.settings.trace:+ backtrack_to = dsp.settings.trace[-1][0]+ if error_msg not in predictor_feedbacks.setdefault(+ backtrack_to, []+ ):+ predictor_feedbacks[backtrack_to].append(error_msg)+ else:+ print(+ "UNREACHABLE: No trace available, this should not happen. Is this run time?"+ )# revert any extended predictors to their originalsif extended_predictors_to_original_forward: