DSPy: The framework for programming—not prompting—language models
Python
change behavior for suggest and assert
Shangyinta3f7fe83c8e0
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.pyindex 0d9848bc9..b33ed91fe 100644--- a/assert_hotpotqa.py+++ b/assert_hotpotqa.py@@ -107,12 +107,12 @@ class SimplifiedBaleen(dspy.Module):for hop in range(self.max_hops):query = self.generate_query[hop](context=context, question=question).query- dspy.Assert(+ dspy.Suggest(len(query) <= 100,"Query should be short and less than 100 characters",)- dspy.Assert(+ dspy.Suggest(validate_query_distinction_local(prev_queries, query),"Query should not be the following: "+ "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex b471fefeb..f4d207b2f 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -37,6 +37,7 @@ class Settings(object):trace=None,release=0,bypass_assert=False,+ bypass_suggest=False,)cls._instance.__append(config)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex c66f50202..b07af0b46 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -77,7 +77,7 @@ class Suggest(Constraint):if isinstance(self.result, bool):if self.result:return True- elif dspy.settings.bypass_assert:+ elif dspy.settings.bypass_suggest:logger.error(f"SuggestionFailed: {self.msg}")return Trueelse:@@ -88,15 +88,32 @@ class Suggest(Constraint):def noop_handler(func):- """ "Handler to bypass assertions."""+ """Handler to bypass assertions and suggestions.++ Now both assertions and suggestions will become noops.+ """def wrapper(*args, **kwargs):- with dspy.settings.context(bypass_assert=True):+ with dspy.settings.context(bypass_assert=True, bypass_suggest=True):return func(*args, **kwargs)return wrapper+def bypass_suggest_handler(func):+ """Handler to bypass suggest only.++ If a suggestion fails, it will be logged but not raised.+ And If an assertion fails, it will be raised.+ """++ def wrapper(*args, **kwargs):+ with dspy.settings.context(bypass_suggest=True, bypass_assert=False):+ return func(*args, **kwargs)++ return wrapper++def assert_backtrack_handler(func, max_backtracks=2):"""Handler for backtracking assertions.@@ -160,6 +177,7 @@ def assert_backtrack_handler(func, max_backtracks=2):for i in range(max_backtracks + 1):if i > 0 and backtrack_to is not None:+ # TODO clean up this function# create a new feedback fieldfeedback = dspy.InputField(prefix="Instruction:",@@ -181,23 +199,17 @@ def assert_backtrack_handler(func, max_backtracks=2):backtrack_to, default_args={"feedback": feedback_msg})- # if user set bypass_assert: ignore assertion errors- if dsp.settings.bypass_assert:- result = func(*args, **kwargs)- break-- # if last backtrack: ignore assertion errors- elif i == max_backtracks and dspy.settings.bypass_assert is False:- dspy.settings.configure(bypass_assert=True)- result = func(*args, **kwargs)- dspy.settings.configure(bypass_assert=False)+ # if last backtrack: ignore suggestion errors+ if i == max_backtracks:+ result = bypass_suggest_handler(func)(*args, **kwargs)breakelse:try:result = func(*args, **kwargs)break- except DSPyAssertionError as e:+ # TODO Change this to suggest instead of assert+ except DSPySuggestionError as e:error_msg = e.msgif dsp.settings.trace: