DSPy: The framework for programming—not prompting—language models
Python
fix(dspy): preserve CoT parse failures in bootstrap traces (#10683)
Isaac Miller4706e3cd6b5e
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.pyindex 274e80635..1d6c3ad97 100644--- a/dspy/adapters/base.py+++ b/dspy/adapters/base.py@@ -167,7 +167,19 @@ class Adapter:tool_calls = output.get("tool_calls")if text and not (tool_calls and tool_call_output_field_name):- value = self.parse(processed_signature, text)+ try:+ value = self.parse(processed_signature, text)+ except AdapterParseError as e:+ # Expose the predictor's signature and field names to callers, while+ # preserving the parser's diagnostic message and raw LM response.+ e.signature = original_signature+ if e.parsed_result is not None:+ e.parsed_result = e.parsed_result.copy()+ for name, field in processed_signature.output_fields.items():+ if original_name := field.json_schema_extra.get("__dspy_reasoning_field"):+ if name in e.parsed_result:+ e.parsed_result[original_name] = e.parsed_result.pop(name)+ raiseelif tool_calls and tool_call_output_field_name:try:value = self.parse(processed_signature, text) if text and processed_signature.output_fields else {}diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.pyindex 84837f49b..1048e6d43 100644--- a/dspy/teleprompt/bootstrap_trace.py+++ b/dspy/teleprompt/bootstrap_trace.py@@ -54,7 +54,7 @@ def bootstrap_trace_data(def wrapped_metric(example, prediction, trace=None):prediction, _ = predictionif isinstance(prediction, FailedPrediction):- return prediction.format_reward or format_failure_score+ return prediction.format_reward if prediction.format_reward is not None else format_failure_scorereturn metric(example, prediction, trace) if metric else True# Use `object.__getattribute__` to bypass the custom hook `Module.__getattribute__` so that we avoid@@ -88,7 +88,7 @@ def bootstrap_trace_data(failed_pred = FailedPrediction(completion_text=completion_str,format_reward=format_failure_score- + (failure_score - format_failure_score) * (present / expected),+ + (failure_score - format_failure_score) * (len(present) / len(expected)),)else:failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pyindex 44d18f6aa..ee30dd098 100644--- a/tests/predict/test_chain_of_thought.py+++ b/tests/predict/test_chain_of_thought.py@@ -6,6 +6,7 @@ from litellm.utils import Choices, Message, ModelResponseimport dspyfrom dspy import ChainOfThoughtfrom dspy.utils import DummyLM+from dspy.utils.exceptions import AdapterParseErrordef test_initialization_with_string_signature():@@ -173,3 +174,29 @@ def test_explicit_rationale_is_not_renamed(custom):result = program(question="Question")assert result.reasoning == "Working"assert isinstance(result.reasoning, str)++[email protected]("async_mode", [False, True])[email protected]("collision", [False, True])[email protected]("adapter", [+ dspy.ChatAdapter(use_json_adapter_fallback=False),+ dspy.JSONAdapter(),+ dspy.XMLAdapter(use_json_adapter_fallback=False),+])+async def test_parse_failure_preserves_predictor_signature(adapter, collision, async_mode):+ program = ChainOfThought("question -> plan, answer" if collision else "question -> answer")+ response = {"plan_2": "Working", "plan": "Task plan"} if collision else {"plan": "Working"}+ lm = DummyLM([response], adapter=adapter)++ with dspy.context(lm=lm, adapter=adapter), pytest.raises(AdapterParseError) as exc_info:+ if async_mode:+ await program.acall(question="Question")+ else:+ program(question="Question")++ error = exc_info.value+ assert error.signature is program.predict.signature+ assert error.parsed_result == ({"reasoning": "Working", "plan": "Task plan"} if collision else {"reasoning": "Working"})+ assert error.lm_response == lm.history[-1]["outputs"][0]+ assert error.adapter_name == type(adapter).__name__diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.pyindex d325de13e..977d4678c 100644--- a/tests/teleprompt/test_bootstrap_trace.py+++ b/tests/teleprompt/test_bootstrap_trace.py@@ -1,11 +1,13 @@from typing import Any, ClassVarfrom unittest import mock+import pytestfrom litellm import Choices, Message, ModelResponseimport dspyfrom dspy.primitives.example import Examplefrom dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data+from dspy.utils import DummyLMdef test_bootstrap_trace_data():@@ -98,6 +100,43 @@ def test_bootstrap_trace_data():assert len(trace_entry) == 3, "Trace entry should have 3 elements"[email protected]("response, format_failure_score, expected_score", [+ ({}, -2, -2),+ ({"answer": "4"}, -2, -0.5),+ ({"answer": "4"}, -1, 0),+])[email protected]("adapter", [+ dspy.ChatAdapter(use_json_adapter_fallback=False),+ dspy.JSONAdapter(),+ dspy.XMLAdapter(use_json_adapter_fallback=False),+])+def test_chain_of_thought_parse_failure_is_captured(adapter, response, format_failure_score, expected_score):+ program = dspy.ChainOfThought("question -> answer")+ example = Example(question="2+2?", answer="4").with_inputs("question")+ lm = DummyLM([response], adapter=adapter)++ with dspy.context(lm=lm, adapter=adapter):+ data = bootstrap_trace_data(+ program=program,+ dataset=[example],+ metric=lambda e, p, t=None: 1.0,+ num_threads=1,+ raise_on_error=False,+ capture_failed_parses=True,+ failure_score=1,+ format_failure_score=format_failure_score,+ )++ assert len(data) == 1+ result = data[0]+ prediction = result["prediction"]+ assert isinstance(prediction, FailedPrediction)+ assert prediction.completion_text == lm.history[-1]["outputs"][0]+ assert prediction.format_reward == expected_score+ assert result["score"] == expected_score+ assert result["trace"] == [(program.predict, {"question": "2+2?"}, prediction)]++def test_bootstrap_trace_data_passes_callback_metadata(monkeypatch):from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module