Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add AdapterParseError to dspy (#8212)

Chen Qianc2b3007b6f39
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index b11e5c1ca..5d5f2400f 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -16,6 +16,7 @@ from dspy.adapters.utils import (
from dspy.clients.lm import LM
from dspy.signatures.signature import Signature
from dspy.utils.callback import BaseCallback
+from dspy.utils.exceptions import AdapterParseError
field_header_pattern = re.compile(r"\[\[ ## (\w+) ## \]\]")
@@ -168,11 +169,19 @@ class ChatAdapter(Adapter):
try:
fields[k] = parse_value(v, signature.output_fields[k].annotation)
except Exception as e:
- raise ValueError(
- f"Error parsing field {k}: {e}.\n\n\t\tOn attempting to parse the value\n```\n{v}\n```"
+ raise AdapterParseError(
+ adapter_name="ChatAdapter",
+ signature=signature,
+ lm_response=completion,
+ message=f"Failed to parse field {k} with value {v} from the LM response. Error message: {e}",
)
if fields.keys() != signature.output_fields.keys():
- raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+ raise AdapterParseError(
+ adapter_name="ChatAdapter",
+ signature=signature,
+ lm_response=completion,
+ parsed_result=fields,
+ )
return fields
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 14c5230eb..b5941a7cf 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -18,6 +18,7 @@ from dspy.adapters.utils import (
)
from dspy.clients.lm import LM
from dspy.signatures.signature import Signature, SignatureMeta
+from dspy.utils.exceptions import AdapterParseError
logger = logging.getLogger(__name__)
@@ -66,7 +67,11 @@ class JSONAdapter(ChatAdapter):
try:
lm_kwargs["response_format"] = {"type": "json_object"}
return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ except AdapterParseError as e:
+ # On AdapterParseError, we raise the original error.
+ raise e
except Exception as e:
+ # On any other error, we raise a RuntimeError with the original error message.
raise RuntimeError(
"Both structured output format and JSON mode failed. Please choose a model that supports "
f"`response_format` argument. Original error: {e}"
@@ -124,7 +129,12 @@ class JSONAdapter(ChatAdapter):
fields = json_repair.loads(completion)
if not isinstance(fields, dict):
- raise ValueError(f"Expected a JSON object but parsed a {type(fields)}")
+ raise AdapterParseError(
+ adapter_name="JSONAdapter",
+ signature=signature,
+ lm_response=completion,
+ message="LM response cannot be serialized to a JSON object.",
+ )
fields = {k: v for k, v in fields.items() if k in signature.output_fields}
@@ -134,7 +144,12 @@ class JSONAdapter(ChatAdapter):
fields[k] = parse_value(v, signature.output_fields[k].annotation)
if fields.keys() != signature.output_fields.keys():
- raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+ raise AdapterParseError(
+ adapter_name="JSONAdapter",
+ signature=signature,
+ lm_response=completion,
+ parsed_result=fields,
+ )
return fields
diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.py
index 84178e491..fd8f00f70 100644
--- a/dspy/utils/__init__.py
+++ b/dspy/utils/__init__.py
@@ -1,6 +1,7 @@
from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm
from dspy.streaming.messages import StatusMessageProvider, StatusMessage
+from dspy.utils import exceptions
import os
import requests
@@ -20,6 +21,7 @@ def download(url):
__all__ = [
"download",
+ "exceptions",
"BaseCallback",
"with_callbacks",
"DummyLM",
diff --git a/dspy/utils/exceptions.py b/dspy/utils/exceptions.py
new file mode 100644
index 000000000..70b2f443d
--- /dev/null
+++ b/dspy/utils/exceptions.py
@@ -0,0 +1,33 @@
+from typing import Optional
+
+from dspy.signatures.signature import Signature
+
+
+class AdapterParseError(Exception):
+ """Exception raised when adapter cannot parse the LM response."""
+
+ def __init__(
+ self,
+ adapter_name: str,
+ signature: Signature,
+ lm_response: str,
+ message: Optional[str] = None,
+ parsed_result: Optional[str] = None,
+ ):
+ self.adapter_name = adapter_name
+ self.signature = signature
+ self.lm_response = lm_response
+ self.parsed_result = parsed_result
+
+ message = f"{message}\n\n" if message else ""
+ message = (
+ f"{message}"
+ f"Adapter {adapter_name} failed to parse the LM response. \n\n"
+ f"LM Response: {lm_response} \n\n"
+ f"Expected to find output fields in the LM response: [{', '.join(signature.output_fields.keys())}] \n\n"
+ )
+
+ if parsed_result is not None:
+ message += f"Actual output fields parsed from the LM response: [{', '.join(parsed_result.keys())}] \n\n"
+
+ super().__init__(message)
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index 64f6285b5..8396126ef 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -3,6 +3,7 @@ from unittest import mock
import pydantic
import pytest
from pydantic import create_model
+from litellm.utils import ModelResponse, Message, Choices
import dspy
@@ -195,9 +196,28 @@ def test_json_adapter_on_pydantic_model():
def test_json_adapter_parse_raise_error_on_mismatch_fields():
- signature = dspy.make_signature("input1->output1")
+ signature = dspy.make_signature("question->answer")
adapter = dspy.JSONAdapter()
- invalid_completion = '{"output": "Test output"}'
- with pytest.raises(ValueError) as error:
- adapter.parse(signature, invalid_completion)
- assert str(error.value) == "Expected dict_keys(['output1']) but got dict_keys([])"
+
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[
+ Choices(message=Message(content="{'answer1': 'Paris'}")),
+ ],
+ model="openai/gpt4o",
+ )
+ lm = dspy.LM(model="openai/gpt-4o-mini")
+ with pytest.raises(dspy.utils.exceptions.AdapterParseError) as e:
+ adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+
+ assert e.value.adapter_name == "JSONAdapter"
+ assert e.value.signature == signature
+ assert e.value.lm_response == "{'answer1': 'Paris'}"
+ assert e.value.parsed_result == {}
+
+ assert str(e.value) == (
+ "Adapter JSONAdapter failed to parse the LM response. \n\n"
+ "LM Response: {'answer1': 'Paris'} \n\n"
+ "Expected to find output fields in the LM response: [answer] \n\n"
+ "Actual output fields parsed from the LM response: [] \n\n"
+ )
diff --git a/tests/utils/test_exceptions.py b/tests/utils/test_exceptions.py
new file mode 100644
index 000000000..b3b63ea41
--- /dev/null
+++ b/tests/utils/test_exceptions.py
@@ -0,0 +1,63 @@
+import pytest
+from dspy.utils.exceptions import AdapterParseError
+from dspy.signatures.signature import Signature
+import dspy
+
+
+def test_adapter_parse_error_basic():
+ adapter_name = "ChatAdapter"
+ signature = dspy.make_signature("question->answer1, answer2")
+ lm_response = "[[ ## answer1 ## ]]\nanswer1"
+
+ error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response)
+
+ assert error.adapter_name == adapter_name
+ assert error.signature == signature
+ assert error.lm_response == lm_response
+
+ error_message = str(error)
+ assert error_message == (
+ "Adapter ChatAdapter failed to parse the LM response. \n\n"
+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+ )
+
+
+def test_adapter_parse_error_with_message():
+ adapter_name = "ChatAdapter"
+ signature = dspy.make_signature("question->answer1, answer2")
+ lm_response = "[[ ## answer1 ## ]]\nanswer1"
+ message = "Critical error, please fix!"
+
+ error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response, message=message)
+
+ assert error.adapter_name == adapter_name
+ assert error.signature == signature
+ assert error.lm_response == lm_response
+
+ error_message = str(error)
+ assert error_message == (
+ "Critical error, please fix!\n\n"
+ "Adapter ChatAdapter failed to parse the LM response. \n\n"
+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+ )
+
+
+def test_adapter_parse_error_with_parsed_result():
+ adapter_name = "ChatAdapter"
+ signature = dspy.make_signature("question->answer1, answer2")
+ lm_response = "[[ ## answer1 ## ]]\nanswer1"
+ parsed_result = {"answer1": "value1"}
+
+ error = AdapterParseError(
+ adapter_name=adapter_name, signature=signature, lm_response=lm_response, parsed_result=parsed_result
+ )
+
+ error_message = str(error)
+ assert error_message == (
+ "Adapter ChatAdapter failed to parse the LM response. \n\n"
+ "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+ "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+ "Actual output fields parsed from the LM response: [answer1] \n\n"
+ )