DSPy: The framework for programming—not prompting—language models
Python
refactor backtrack_handler to support assert
Shangyint342f0269d708
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex c46df4f88..7fa89d839 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -185,7 +185,7 @@ def assert_no_except_handler(func):return wrapper-def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):+def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):"""Handler for backtracking suggestion.Re-run the latest predictor up to `max_backtracks` times,@@ -202,6 +202,7 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):# Predictor -> List[feedback_msg]dspy.settings.predictor_feedbacks = {}+ current_error = Nonefor i in range(max_backtracks + 1):if i > 0 and dspy.settings.backtrack_to is not None:# generate values for new fields@@ -216,18 +217,21 @@ def suggest_backtrack_handler(func, bypass_suggest=True, max_backtracks=2):# if last backtrack: ignore suggestion errorsif i == max_backtracks:+ if isinstance(current_error, DSPyAssertionError):+ raise current_errorresult = (bypass_suggest_handler(func)(*args, **kwargs)if bypass_suggestelse None)break-else:try:result = func(*args, **kwargs)break- except DSPySuggestionError as e:+ except (DSPySuggestionError, DSPyAssertionError) as e:+ if not current_error:+ current_error = esuggest_id, error_msg, suggest_target_module, error_state = (e.id,e.msg,@@ -302,7 +306,7 @@ def handle_assert_forward(assertion_handler, **handler_args):return forward-default_assertion_handler = suggest_backtrack_handler+default_assertion_handler = backtrack_handlerdef assert_transform_module(diff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynbindex 236837b0c..f3ae4326a 100644--- a/examples/longformqa/longformqa_assertions.ipynb+++ b/examples/longformqa/longformqa_assertions.ipynb@@ -495,7 +495,7 @@"source": ["Let's now evaluate our `LongFormQAWithAssertions` program over the devset.\n","\n",- "Note that this requires wrapping the module with the `Retry` module which handles the backtracking logic. This wrapped module is then passed to the `assert_transform_module` function to prepare and execute the backtracking logic. This is passed alongside the `suggest_backtrack_handler` which configures the backtracking logic to account for the feedback messages passed in to the `dspy.Suggest` statements within the program."+ "Note that this requires wrapping the module with the `Retry` module which handles the backtracking logic. This wrapped module is then passed to the `assert_transform_module` function to prepare and execute the backtracking logic. This is passed alongside the `backtrack_handler` which configures the backtracking logic to account for the feedback messages passed in to the `dspy.Suggest` statements within the program."]},{@@ -504,8 +504,8 @@"metadata": {},"outputs": [],"source": [- "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler\n",- "longformqa_with_assertions = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",+ "from dspy.primitives.assertions import assert_transform_module, backtrack_handler\n",+ "longformqa_with_assertions = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler) \n","evaluate(longformqa_with_assertions)"]},@@ -580,7 +580,7 @@"source": ["longformqa = LongFormQA()\n","teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",- "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), suggest_backtrack_handler), teacher = longformqa, trainset=trainset, valset=devset[:100])\n",+ "cited_longformqa = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = longformqa, trainset=trainset, valset=devset[:100])\n","evaluate(cited_longformqa)"]}diff --git a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb b/examples/qa/hotpot/hotpotqa_with_assertions.ipynbindex 36c3daa20..77fb52f37 100644--- a/examples/qa/hotpot/hotpotqa_with_assertions.ipynb+++ b/examples/qa/hotpot/hotpotqa_with_assertions.ipynb@@ -13,7 +13,7 @@"from dspy.teleprompt import BootstrapFewShot, BootstrapFewShotWithRandomSearch\n","from dspy.evaluate.evaluate import Evaluate\n","\n",- "from dspy.primitives.assertions import assert_transform_module, suggest_backtrack_handler"+ "from dspy.primitives.assertions import assert_transform_module, backtrack_handler"]},{@@ -228,7 +228,7 @@"outputs": [],"source": ["# No Compilation + Yes Assertion\n",- "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), suggest_backtrack_handler) \n",+ "baleen_with_assertions = assert_transform_module(SimplifiedBaleenAssertions().map_named_predictors(Retry), backtrack_handler) \n","evaluate(baleen_with_assertions)"]},@@ -266,7 +266,7 @@"compiled_baleen = teleprompter.compile(\n"," student=assert_transform_module(\n"," SimplifiedBaleenAssertions().map_named_predictors(Retry),\n",- " suggest_backtrack_handler,\n",+ " backtrack_handler,\n"," ),\n"," teacher=baleen,\n"," trainset=trainset,\n",