DSPy: The framework for programming—not prompting—language models
Python
Add AdapterParseError to dspy (#8212)
Chen Qianc2b3007b6f39
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pyindex b11e5c1ca..5d5f2400f 100644--- a/dspy/adapters/chat_adapter.py+++ b/dspy/adapters/chat_adapter.py@@ -16,6 +16,7 @@ from dspy.adapters.utils import (from dspy.clients.lm import LMfrom dspy.signatures.signature import Signaturefrom dspy.utils.callback import BaseCallback+from dspy.utils.exceptions import AdapterParseErrorfield_header_pattern = re.compile(r"\[\[ ## (\w+) ## \]\]")@@ -168,11 +169,19 @@ class ChatAdapter(Adapter):try:fields[k] = parse_value(v, signature.output_fields[k].annotation)except Exception as e:- raise ValueError(- f"Error parsing field {k}: {e}.\n\n\t\tOn attempting to parse the value\n```\n{v}\n```"+ raise AdapterParseError(+ adapter_name="ChatAdapter",+ signature=signature,+ lm_response=completion,+ message=f"Failed to parse field {k} with value {v} from the LM response. Error message: {e}",)if fields.keys() != signature.output_fields.keys():- raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")+ raise AdapterParseError(+ adapter_name="ChatAdapter",+ signature=signature,+ lm_response=completion,+ parsed_result=fields,+ )return fieldsdiff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.pyindex 14c5230eb..b5941a7cf 100644--- a/dspy/adapters/json_adapter.py+++ b/dspy/adapters/json_adapter.py@@ -18,6 +18,7 @@ from dspy.adapters.utils import ()from dspy.clients.lm import LMfrom dspy.signatures.signature import Signature, SignatureMeta+from dspy.utils.exceptions import AdapterParseErrorlogger = logging.getLogger(__name__)@@ -66,7 +67,11 @@ class JSONAdapter(ChatAdapter):try:lm_kwargs["response_format"] = {"type": "json_object"}return super().__call__(lm, lm_kwargs, signature, demos, inputs)+ except AdapterParseError as e:+ # On AdapterParseError, we raise the original error.+ raise eexcept Exception as e:+ # On any other error, we raise a RuntimeError with the original error message.raise RuntimeError("Both structured output format and JSON mode failed. Please choose a model that supports "f"`response_format` argument. Original error: {e}"@@ -124,7 +129,12 @@ class JSONAdapter(ChatAdapter):fields = json_repair.loads(completion)if not isinstance(fields, dict):- raise ValueError(f"Expected a JSON object but parsed a {type(fields)}")+ raise AdapterParseError(+ adapter_name="JSONAdapter",+ signature=signature,+ lm_response=completion,+ message="LM response cannot be serialized to a JSON object.",+ )fields = {k: v for k, v in fields.items() if k in signature.output_fields}@@ -134,7 +144,12 @@ class JSONAdapter(ChatAdapter):fields[k] = parse_value(v, signature.output_fields[k].annotation)if fields.keys() != signature.output_fields.keys():- raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")+ raise AdapterParseError(+ adapter_name="JSONAdapter",+ signature=signature,+ lm_response=completion,+ parsed_result=fields,+ )return fieldsdiff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.pyindex 84178e491..fd8f00f70 100644--- a/dspy/utils/__init__.py+++ b/dspy/utils/__init__.py@@ -1,6 +1,7 @@from dspy.utils.callback import BaseCallback, with_callbacksfrom dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rmfrom dspy.streaming.messages import StatusMessageProvider, StatusMessage+from dspy.utils import exceptionsimport osimport requests@@ -20,6 +21,7 @@ def download(url):__all__ = ["download",+ "exceptions","BaseCallback","with_callbacks","DummyLM",diff --git a/dspy/utils/exceptions.py b/dspy/utils/exceptions.pynew file mode 100644index 000000000..70b2f443d--- /dev/null+++ b/dspy/utils/exceptions.py@@ -0,0 +1,33 @@+from typing import Optional++from dspy.signatures.signature import Signature+++class AdapterParseError(Exception):+ """Exception raised when adapter cannot parse the LM response."""++ def __init__(+ self,+ adapter_name: str,+ signature: Signature,+ lm_response: str,+ message: Optional[str] = None,+ parsed_result: Optional[str] = None,+ ):+ self.adapter_name = adapter_name+ self.signature = signature+ self.lm_response = lm_response+ self.parsed_result = parsed_result++ message = f"{message}\n\n" if message else ""+ message = (+ f"{message}"+ f"Adapter {adapter_name} failed to parse the LM response. \n\n"+ f"LM Response: {lm_response} \n\n"+ f"Expected to find output fields in the LM response: [{', '.join(signature.output_fields.keys())}] \n\n"+ )++ if parsed_result is not None:+ message += f"Actual output fields parsed from the LM response: [{', '.join(parsed_result.keys())}] \n\n"++ super().__init__(message)diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.pyindex 64f6285b5..8396126ef 100644--- a/tests/adapters/test_json_adapter.py+++ b/tests/adapters/test_json_adapter.py@@ -3,6 +3,7 @@ from unittest import mockimport pydanticimport pytestfrom pydantic import create_model+from litellm.utils import ModelResponse, Message, Choicesimport dspy@@ -195,9 +196,28 @@ def test_json_adapter_on_pydantic_model():def test_json_adapter_parse_raise_error_on_mismatch_fields():- signature = dspy.make_signature("input1->output1")+ signature = dspy.make_signature("question->answer")adapter = dspy.JSONAdapter()- invalid_completion = '{"output": "Test output"}'- with pytest.raises(ValueError) as error:- adapter.parse(signature, invalid_completion)- assert str(error.value) == "Expected dict_keys(['output1']) but got dict_keys([])"++ with mock.patch("litellm.completion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[+ Choices(message=Message(content="{'answer1': 'Paris'}")),+ ],+ model="openai/gpt4o",+ )+ lm = dspy.LM(model="openai/gpt-4o-mini")+ with pytest.raises(dspy.utils.exceptions.AdapterParseError) as e:+ adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})++ assert e.value.adapter_name == "JSONAdapter"+ assert e.value.signature == signature+ assert e.value.lm_response == "{'answer1': 'Paris'}"+ assert e.value.parsed_result == {}++ assert str(e.value) == (+ "Adapter JSONAdapter failed to parse the LM response. \n\n"+ "LM Response: {'answer1': 'Paris'} \n\n"+ "Expected to find output fields in the LM response: [answer] \n\n"+ "Actual output fields parsed from the LM response: [] \n\n"+ )diff --git a/tests/utils/test_exceptions.py b/tests/utils/test_exceptions.pynew file mode 100644index 000000000..b3b63ea41--- /dev/null+++ b/tests/utils/test_exceptions.py@@ -0,0 +1,63 @@+import pytest+from dspy.utils.exceptions import AdapterParseError+from dspy.signatures.signature import Signature+import dspy+++def test_adapter_parse_error_basic():+ adapter_name = "ChatAdapter"+ signature = dspy.make_signature("question->answer1, answer2")+ lm_response = "[[ ## answer1 ## ]]\nanswer1"++ error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response)++ assert error.adapter_name == adapter_name+ assert error.signature == signature+ assert error.lm_response == lm_response++ error_message = str(error)+ assert error_message == (+ "Adapter ChatAdapter failed to parse the LM response. \n\n"+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"+ )+++def test_adapter_parse_error_with_message():+ adapter_name = "ChatAdapter"+ signature = dspy.make_signature("question->answer1, answer2")+ lm_response = "[[ ## answer1 ## ]]\nanswer1"+ message = "Critical error, please fix!"++ error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response, message=message)++ assert error.adapter_name == adapter_name+ assert error.signature == signature+ assert error.lm_response == lm_response++ error_message = str(error)+ assert error_message == (+ "Critical error, please fix!\n\n"+ "Adapter ChatAdapter failed to parse the LM response. \n\n"+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"+ )+++def test_adapter_parse_error_with_parsed_result():+ adapter_name = "ChatAdapter"+ signature = dspy.make_signature("question->answer1, answer2")+ lm_response = "[[ ## answer1 ## ]]\nanswer1"+ parsed_result = {"answer1": "value1"}++ error = AdapterParseError(+ adapter_name=adapter_name, signature=signature, lm_response=lm_response, parsed_result=parsed_result+ )++ error_message = str(error)+ assert error_message == (+ "Adapter ChatAdapter failed to parse the LM response. \n\n"+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"+ "Actual output fields parsed from the LM response: [answer1] \n\n"+ )