Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

refactor backtrack_handler to support assert

Shangyint342f0269d708
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index c46df4f88..7fa89d839 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -185,7 +185,7 @@ def assert_no_except_handler(func):
return wrapper
-def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
+def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
"""Handler for backtracking suggestion.
Re-run the latest predictor up to `max_backtracks` times,
@@ -202,6 +202,7 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
# Predictor -> List[feedback_msg]
dspy.settings.predictor_feedbacks = {}
+ current_error = None
for i in range(max_backtracks + 1):
if i > 0 and dspy.settings.backtrack_to is not None:
# generate values for new fields
@@ -216,18 +217,21 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):
# if last backtrack: ignore suggestion errors
if i == max_backtracks:
+ if isinstance(current_error, DSPyAssertionError):
+ raise current_error
result = (
bypass_suggest_handler(func)(*args, **kwargs)
if bypass_suggest
else None
)
break
-
else:
try:
result = func(*args, **kwargs)
break
- except DSPySuggestionError as e:
+ except (DSPySuggestionError, DSPyAssertionError) as e:
+ if not current_error:
+ current_error = e
suggest_id, error_msg, suggest_target_module, error_state = (
e.id,
e.msg,
@@ -302,7 +306,7 @@ def handle_assert_forward(assertion_handler, **handler_args):
return forward
-default_assertion_handler = suggest_backtrack_handler
+default_assertion_handler = backtrack_handler
def assert_transform_module(
diff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynb
index 236837b0c..f3ae4326a 100644
--- a/examples/longformqa/longformqa_assertions.ipynb
+++ b/examples/longformqa/longformqa_assertions.ipynb
@@ -495,7 +495,7 @@
"source": [
"Let's now evaluate our `LongFormQAWithAssertions` program over the devset.\n",
"\n",
- "Note that this requires wrapping the module with the `Retry` module which handles the backtracking logic. This wrapped module is then passed to the `assert_transform_module` function to prepare and execute the backtracking logic. This is passed alongside the `suggest_backtrack_handler` which configures the backtracking logic to account for the feedback messages passed in to the `dspy.Suggest` statements within the program."
+ "Note that this requires wrapping the module with the `Retry` module which handles the backtracking logic. This wrapped module is then passed to the `assert_transform_module` function to prepare and execute the backtracking logic. This is passed alongside the `backtrack_handler` which configures the backtracking logic to account for the feedback messages passed in to the `dspy.Suggest` statements within the program."
]
},
{
@@ -504,8 +504,8 @@
"metadata": {},
"outputs": [],
"source": [
- "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler\n",
- "longformqa_with_assertions = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
+ "from dspy.primitives.assertions import assert_transform_module, backtrack_handler\n",
+ "longformqa_with_assertions = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler) \n",
"evaluate(longformqa_with_assertions)"
]
},
@@ -580,7 +580,7 @@
"source": [
"longformqa = LongFormQA()\n",
"teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",
- "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), suggest_backtrack_handler), teacher = longformqa, trainset=trainset, valset=devset[:100])\n",
+ "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = longformqa, trainset=trainset, valset=devset[:100])\n",
"evaluate(cited_longformqa)"
]
}
diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
index 36c3daa20..77fb52f37 100644
--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb
@@ -13,7 +13,7 @@
"from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n",
"from dspy.evaluate.evaluate import Evaluate\n",
"\n",
- "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"
+ "from dspy.primitives.assertions import assert_transform_module, backtrack_handler"
]
},
{
@@ -228,7 +228,7 @@
"outputs": [],
"source": [
"# No Compilation + Yes Assertion\n",
- "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",
+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), backtrack_handler) \n",
"evaluate(baleen_with_assertions)"
]
},
@@ -266,7 +266,7 @@
"compiled_baleen = teleprompter.compile(\n",
" student=assert_transform_module(\n",
" SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",
- " suggest_backtrack_handler,\n",
+ " backtrack_handler,\n",
" ),\n",
" teacher=baleen,\n",
" trainset=trainset,\n",