Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(dspy): preserve CoT parse failures in bootstrap traces (#10683)

Isaac Miller4706e3cd6b5e
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py
index 274e80635..1d6c3ad97 100644
--- a/dspy/adapters/base.py
+++ b/dspy/adapters/base.py
@@ -167,7 +167,19 @@ class Adapter:
tool_calls = output.get("tool_calls")
if text and not (tool_calls and tool_call_output_field_name):
- value = self.parse(processed_signature, text)
+ try:
+ value = self.parse(processed_signature, text)
+ except AdapterParseError as e:
+ # Expose the predictor's signature and field names to callers, while
+ # preserving the parser's diagnostic message and raw LM response.
+ e.signature = original_signature
+ if e.parsed_result is not None:
+ e.parsed_result = e.parsed_result.copy()
+ for name, field in processed_signature.output_fields.items():
+ if original_name := field.json_schema_extra.get("__dspy_reasoning_field"):
+ if name in e.parsed_result:
+ e.parsed_result[original_name] = e.parsed_result.pop(name)
+ raise
elif tool_calls and tool_call_output_field_name:
try:
value = self.parse(processed_signature, text) if text and processed_signature.output_fields else {}
diff --git a/dspy/teleprompt/bootstrap_trace.py b/dspy/teleprompt/bootstrap_trace.py
index 84837f49b..1048e6d43 100644
--- a/dspy/teleprompt/bootstrap_trace.py
+++ b/dspy/teleprompt/bootstrap_trace.py
@@ -54,7 +54,7 @@ def bootstrap_trace_data(
def wrapped_metric(example, prediction, trace=None):
prediction, _ = prediction
if isinstance(prediction, FailedPrediction):
- return prediction.format_reward or format_failure_score
+ return prediction.format_reward if prediction.format_reward is not None else format_failure_score
return metric(example, prediction, trace) if metric else True
# Use `object.__getattribute__` to bypass the custom hook `Module.__getattribute__` so that we avoid
@@ -88,7 +88,7 @@ def bootstrap_trace_data(
failed_pred = FailedPrediction(
completion_text=completion_str,
format_reward=format_failure_score
- + (failure_score - format_failure_score) * (present / expected),
+ + (failure_score - format_failure_score) * (len(present) / len(expected)),
)
else:
failed_pred = FailedPrediction(completion_text=completion_str, format_reward=format_failure_score)
diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.py
index 44d18f6aa..ee30dd098 100644
--- a/tests/predict/test_chain_of_thought.py
+++ b/tests/predict/test_chain_of_thought.py
@@ -6,6 +6,7 @@ from litellm.utils import Choices, Message, ModelResponse
import dspy
from dspy import ChainOfThought
from dspy.utils import DummyLM
+from dspy.utils.exceptions import AdapterParseError
def test_initialization_with_string_signature():
@@ -173,3 +174,29 @@ def test_explicit_rationale_is_not_renamed(custom):
result = program(question="Question")
assert result.reasoning == "Working"
assert isinstance(result.reasoning, str)
+
+
[email protected]("async_mode", [False, True])
[email protected]("collision", [False, True])
[email protected]("adapter", [
+ dspy.ChatAdapter(use_json_adapter_fallback=False),
+ dspy.JSONAdapter(),
+ dspy.XMLAdapter(use_json_adapter_fallback=False),
+])
+async def test_parse_failure_preserves_predictor_signature(adapter, collision, async_mode):
+ program = ChainOfThought("question -> plan, answer" if collision else "question -> answer")
+ response = {"plan_2": "Working", "plan": "Task plan"} if collision else {"plan": "Working"}
+ lm = DummyLM([response], adapter=adapter)
+
+ with dspy.context(lm=lm, adapter=adapter), pytest.raises(AdapterParseError) as exc_info:
+ if async_mode:
+ await program.acall(question="Question")
+ else:
+ program(question="Question")
+
+ error = exc_info.value
+ assert error.signature is program.predict.signature
+ assert error.parsed_result == ({"reasoning": "Working", "plan": "Task plan"} if collision else {"reasoning": "Working"})
+ assert error.lm_response == lm.history[-1]["outputs"][0]
+ assert error.adapter_name == type(adapter).__name__
diff --git a/tests/teleprompt/test_bootstrap_trace.py b/tests/teleprompt/test_bootstrap_trace.py
index d325de13e..977d4678c 100644
--- a/tests/teleprompt/test_bootstrap_trace.py
+++ b/tests/teleprompt/test_bootstrap_trace.py
@@ -1,11 +1,13 @@
from typing import Any, ClassVar
from unittest import mock
+import pytest
from litellm import Choices, Message, ModelResponse
import dspy
from dspy.primitives.example import Example
from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data
+from dspy.utils import DummyLM
def test_bootstrap_trace_data():
@@ -98,6 +100,43 @@ def test_bootstrap_trace_data():
assert len(trace_entry) == 3, "Trace entry should have 3 elements"
[email protected]("response, format_failure_score, expected_score", [
+ ({}, -2, -2),
+ ({"answer": "4"}, -2, -0.5),
+ ({"answer": "4"}, -1, 0),
+])
[email protected]("adapter", [
+ dspy.ChatAdapter(use_json_adapter_fallback=False),
+ dspy.JSONAdapter(),
+ dspy.XMLAdapter(use_json_adapter_fallback=False),
+])
+def test_chain_of_thought_parse_failure_is_captured(adapter, response, format_failure_score, expected_score):
+ program = dspy.ChainOfThought("question -> answer")
+ example = Example(question="2+2?", answer="4").with_inputs("question")
+ lm = DummyLM([response], adapter=adapter)
+
+ with dspy.context(lm=lm, adapter=adapter):
+ data = bootstrap_trace_data(
+ program=program,
+ dataset=[example],
+ metric=lambda e, p, t=None: 1.0,
+ num_threads=1,
+ raise_on_error=False,
+ capture_failed_parses=True,
+ failure_score=1,
+ format_failure_score=format_failure_score,
+ )
+
+ assert len(data) == 1
+ result = data[0]
+ prediction = result["prediction"]
+ assert isinstance(prediction, FailedPrediction)
+ assert prediction.completion_text == lm.history[-1]["outputs"][0]
+ assert prediction.format_reward == expected_score
+ assert result["score"] == expected_score
+ assert result["trace"] == [(program.predict, {"question": "2+2?"}, prediction)]
+
+
def test_bootstrap_trace_data_passes_callback_metadata(monkeypatch):
from dspy.teleprompt import bootstrap_trace as bootstrap_trace_module