Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

change behavior for suggest and assert

Shangyinta3f7fe83c8e0
diff --git a/assert_hotpotqa.py b/assert_hotpotqa.py
index 0d9848bc9..b33ed91fe 100644
--- a/assert_hotpotqa.py
+++ b/assert_hotpotqa.py
@@ -107,12 +107,12 @@ class SimplifiedBaleen(dspy.Module):
for hop in range(self.max_hops):
query = self.generate_query[hop](context=context, question=question).query
- dspy.Assert(
+ dspy.Suggest(
len(query) <= 100,
"Query should be short and less than 100 characters",
)
- dspy.Assert(
+ dspy.Suggest(
validate_query_distinction_local(prev_queries, query),
"Query should not be the following: "
+ "; ".join(f"{i+1}) {q}" for i, q in enumerate(prev_queries)),
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py
index b471fefeb..f4d207b2f 100644
--- a/dsp/utils/settings.py
+++ b/dsp/utils/settings.py
@@ -37,6 +37,7 @@ class Settings(object):
trace=None,
release=0,
bypass_assert=False,
+ bypass_suggest=False,
)
cls._instance.__append(config)
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index c66f50202..b07af0b46 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -77,7 +77,7 @@ class Suggest(Constraint):
if isinstance(self.result, bool):
if self.result:
return True
- elif dspy.settings.bypass_assert:
+ elif dspy.settings.bypass_suggest:
logger.error(f"SuggestionFailed: {self.msg}")
return True
else:
@@ -88,15 +88,32 @@ class Suggest(Constraint):
def noop_handler(func):
- """ "Handler to bypass assertions."""
+ """Handler to bypass assertions and suggestions.
+
+ Now both assertions and suggestions will become noops.
+ """
def wrapper(*args, **kwargs):
- with dspy.settings.context(bypass_assert=True):
+ with dspy.settings.context(bypass_assert=True, bypass_suggest=True):
return func(*args, **kwargs)
return wrapper
+def bypass_suggest_handler(func):
+ """Handler to bypass suggest only.
+
+ If a suggestion fails, it will be logged but not raised.
+ And If an assertion fails, it will be raised.
+ """
+
+ def wrapper(*args, **kwargs):
+ with dspy.settings.context(bypass_suggest=True, bypass_assert=False):
+ return func(*args, **kwargs)
+
+ return wrapper
+
+
def assert_backtrack_handler(func, max_backtracks=2):
"""Handler for backtracking assertions.
@@ -160,6 +177,7 @@ def assert_backtrack_handler(func, max_backtracks=2):
for i in range(max_backtracks + 1):
if i > 0 and backtrack_to is not None:
+ # TODO clean up this function
# create a new feedback field
feedback = dspy.InputField(
prefix="Instruction:",
@@ -181,23 +199,17 @@ def assert_backtrack_handler(func, max_backtracks=2):
backtrack_to, default_args={"feedback": feedback_msg}
)
- # if user set bypass_assert: ignore assertion errors
- if dsp.settings.bypass_assert:
- result = func(*args, **kwargs)
- break
-
- # if last backtrack: ignore assertion errors
- elif i == max_backtracks and dspy.settings.bypass_assert is False:
- dspy.settings.configure(bypass_assert=True)
- result = func(*args, **kwargs)
- dspy.settings.configure(bypass_assert=False)
+ # if last backtrack: ignore suggestion errors
+ if i == max_backtracks:
+ result = bypass_suggest_handler(func)(*args, **kwargs)
break
else:
try:
result = func(*args, **kwargs)
break
- except DSPyAssertionError as e:
+ # TODO Change this to suggest instead of assert
+ except DSPySuggestionError as e:
error_msg = e.msg
if dsp.settings.trace: