Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Support async path for ChatAdapter and JSONAdapter (#8419)

Chen Qian1df67689ee5f
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index 97bb173c2..7e7859978 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -50,6 +50,26 @@ class ChatAdapter(Adapter):
raise e
return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs)
+ async def acall(
+ self,
+ lm: LM,
+ lm_kwargs: dict[str, Any],
+ signature: Type[Signature],
+ demos: list[dict[str, Any]],
+ inputs: dict[str, Any],
+ ) -> list[dict[str, Any]]:
+ try:
+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)
+ except Exception as e:
+ # fallback to JSONAdapter
+ from dspy.adapters.json_adapter import JSONAdapter
+
+ if isinstance(e, ContextWindowExceededError) or isinstance(self, JSONAdapter):
+ # On context window exceeded error or already using JSONAdapter, we don't want to retry with a different
+ # adapter.
+ raise e
+ return await JSONAdapter().acall(lm, lm_kwargs, signature, demos, inputs)
+
def format_field_description(self, signature: Type[Signature]) -> str:
return (
f"Your input fields are:\n{get_field_description_string(signature.input_fields)}\n"
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index e43561b0e..5e599d29c 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -22,6 +22,11 @@ from dspy.utils.exceptions import AdapterParseError
logger = logging.getLogger(__name__)
+ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE = (
+ "Both structured output format and JSON mode failed. Please choose a model that supports "
+ "`response_format` argument. Original error: {}"
+)
+
def _has_open_ended_mapping(signature: SignatureMeta) -> bool:
"""
@@ -37,6 +42,18 @@ def _has_open_ended_mapping(signature: SignatureMeta) -> bool:
class JSONAdapter(ChatAdapter):
+ def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, call_fn):
+ """Common call logic to be used for both sync and async calls."""
+ provider = lm.model.split("/", 1)[0] or "openai"
+ params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)
+
+ if not params or "response_format" not in params:
+ return call_fn(lm, lm_kwargs, signature, demos, inputs)
+
+ if _has_open_ended_mapping(signature):
+ lm_kwargs["response_format"] = {"type": "json_object"}
+ return call_fn(lm, lm_kwargs, signature, demos, inputs)
+
def __call__(
self,
lm: LM,
@@ -45,37 +62,49 @@ class JSONAdapter(ChatAdapter):
demos: list[dict[str, Any]],
inputs: dict[str, Any],
) -> list[dict[str, Any]]:
- provider = lm.model.split("/", 1)[0] or "openai"
- params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)
+ result = self._json_adapter_call_common(lm, lm_kwargs, signature, demos, inputs, super().__call__)
+ if result:
+ return result
- # If response_format is not supported, use basic call
- if not params or "response_format" not in params:
+ try:
+ structured_output_model = _get_structured_outputs_response_format(signature)
+ lm_kwargs["response_format"] = structured_output_model
return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ except Exception:
+ logger.warning("Failed to use structured output format, falling back to JSON mode.")
+ try:
+ lm_kwargs["response_format"] = {"type": "json_object"}
+ return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ except AdapterParseError as e:
+ raise e
+ except Exception as e:
+ raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from e
- # Check early for open-ended mapping types before trying structured outputs.
- if _has_open_ended_mapping(signature):
- lm_kwargs["response_format"] = {"type": "json_object"}
- return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ async def acall(
+ self,
+ lm: LM,
+ lm_kwargs: dict[str, Any],
+ signature: Type[Signature],
+ demos: list[dict[str, Any]],
+ inputs: dict[str, Any],
+ ) -> list[dict[str, Any]]:
+ result = self._json_adapter_call_common(lm, lm_kwargs, signature, demos, inputs, super().acall)
+ if result:
+ return await result
- # Try structured output first, fall back to basic JSON if it fails.
try:
structured_output_model = _get_structured_outputs_response_format(signature)
lm_kwargs["response_format"] = structured_output_model
- return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)
except Exception:
logger.warning("Failed to use structured output format, falling back to JSON mode.")
try:
lm_kwargs["response_format"] = {"type": "json_object"}
- return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)
except AdapterParseError as e:
- # On AdapterParseError, we raise the original error.
raise e
except Exception as e:
- # On any other error, we raise a RuntimeError with the original error message.
- raise RuntimeError(
- "Both structured output format and JSON mode failed. Please choose a model that supports "
- f"`response_format` argument. Original error: {e}"
- ) from e
+ raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from e
def _call_preprocess(
self,
diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py
index 969a8f177..e599da585 100644
--- a/tests/adapters/test_chat_adapter.py
+++ b/tests/adapters/test_chat_adapter.py
@@ -3,6 +3,7 @@ from unittest import mock
import pydantic
import pytest
+from litellm.utils import Choices, Message, ModelResponse
import dspy
@@ -376,3 +377,48 @@ def test_chat_adapter_formats_conversation_history():
assert messages[2]["content"] == "[[ ## answer ## ]]\nParis\n\n[[ ## completed ## ]]\n"
assert messages[3]["content"] == "[[ ## question ## ]]\nWhat is the capital of Germany?"
assert messages[4]["content"] == "[[ ## answer ## ]]\nBerlin\n\n[[ ## completed ## ]]\n"
+
+
+def test_chat_adapter_fallback_to_json_adapter_on_exception():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+
+ with mock.patch("litellm.completion") as mock_completion:
+ # Mock returning a response compatible with JSONAdapter but not ChatAdapter
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],
+ model="openai/gpt-4o-mini",
+ )
+
+ lm = dspy.LM("openai/gpt-4o-mini", cache=False)
+
+ with mock.patch("dspy.adapters.json_adapter.JSONAdapter.__call__") as mock_json_adapter_call:
+ adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ mock_json_adapter_call.assert_called_once()
+
+ # The parse should succeed
+ result = adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
+
+
+async def test_chat_adapter_fallback_to_json_adapter_on_exception_async():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+
+ with mock.patch("litellm.acompletion") as mock_completion:
+ # Mock returning a response compatible with JSONAdapter but not ChatAdapter
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],
+ model="openai/gpt-4o-mini",
+ )
+
+ lm = dspy.LM("openai/gpt-4o-mini", cache=False)
+
+ with mock.patch("dspy.adapters.json_adapter.JSONAdapter.acall") as mock_json_adapter_acall:
+ await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ mock_json_adapter_acall.assert_called_once()
+
+ # The parse should succeed
+ result = await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index 7db4a3202..f688af43f 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -5,6 +5,7 @@ import pytest
from litellm.utils import Choices, Message, ModelResponse
import dspy
+from dspy.adapters.json_adapter import ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE
def test_json_adapter_passes_structured_output_when_supported_by_model():
@@ -22,7 +23,7 @@ def test_json_adapter_passes_structured_output_when_supported_by_model():
program = dspy.Predict(TestSignature)
# Configure DSPy to use an OpenAI LM that supports structured outputs
- dspy.configure(lm=dspy.LM(model="openai/gpt4o"), adapter=dspy.JSONAdapter())
+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o"), adapter=dspy.JSONAdapter())
with mock.patch("litellm.completion") as mock_completion:
program(input1="Test input")
@@ -43,14 +44,28 @@ def test_json_adapter_passes_structured_output_when_supported_by_model():
assert issubclass(response_format, pydantic.BaseModel)
assert response_format.model_fields.keys() == {"output1", "output2", "output3", "output4_unannotated"}
+
+def test_json_adapter_not_using_structured_outputs_when_not_supported_by_model():
+ class TestSignature(dspy.Signature):
+ input1: str = dspy.InputField()
+ output1: str = dspy.OutputField()
+ output2: bool = dspy.OutputField()
+
+ program = dspy.Predict(TestSignature)
+
# Configure DSPy to use a model from a fake provider that doesn't support structured outputs
- dspy.configure(lm=dspy.LM(model="fakeprovider/fakemodel"), adapter=dspy.JSONAdapter())
+ dspy.configure(lm=dspy.LM(model="fakeprovider/fakemodel", cache=False), adapter=dspy.JSONAdapter())
with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[Choices(message=Message(content=("{'output1': 'Test output', 'output2': True}")))],
+ model="openai/gpt-4o",
+ )
+
program(input1="Test input")
mock_completion.assert_called_once()
_, call_kwargs = mock_completion.call_args
- assert response_format not in call_kwargs
+ assert "response_format" not in call_kwargs
def test_json_adapter_falls_back_when_structured_outputs_fails():
@@ -442,3 +457,186 @@ def test_json_adapter_formats_conversation_history():
assert messages[2]["content"] == '{\n "answer": "Paris"\n}'
assert messages[3]["content"] == "[[ ## question ## ]]\nWhat is the capital of Germany?"
assert messages[4]["content"] == '{\n "answer": "Berlin"\n}'
+
+
+async def test_json_adapter_on_pydantic_model_async():
+ from litellm.utils import Choices, Message, ModelResponse
+
+ class User(pydantic.BaseModel):
+ id: int
+ name: str
+ email: str
+
+ class Answer(pydantic.BaseModel):
+ analysis: str
+ result: str
+
+ class TestSignature(dspy.Signature):
+ user: User = dspy.InputField(desc="The user who asks the question")
+ question: str = dspy.InputField(desc="Question the user asks")
+ answer: Answer = dspy.OutputField(desc="Answer to this question")
+
+ program = dspy.Predict(TestSignature)
+
+ with mock.patch("litellm.acompletion") as mock_completion:
+ mock_completion.return_value = ModelResponse(
+ choices=[
+ Choices(
+ message=Message(
+ content="{'answer': {'analysis': 'Paris is the captial of France', 'result': 'Paris'}}"
+ )
+ )
+ ],
+ model="openai/gpt4o",
+ )
+
+ with dspy.context(lm=dspy.LM(model="openai/gpt4o", cache=False), adapter=dspy.JSONAdapter()):
+ result = await program.acall(
+ user={"id": 5, "name": "name_test", "email": "email_test"}, question="What is the capital of France?"
+ )
+
+ # Check that litellm.acompletion was called exactly once
+ mock_completion.assert_called_once()
+
+ _, call_kwargs = mock_completion.call_args
+ # Assert that there are exactly 2 messages (system + user)
+ assert len(call_kwargs["messages"]) == 2
+
+ assert call_kwargs["messages"][0]["role"] == "system"
+ content = call_kwargs["messages"][0]["content"]
+ assert content is not None
+
+ # Assert that system prompt includes correct input field descriptions
+ expected_input_fields = (
+ "1. `user` (User): The user who asks the question\n2. `question` (str): Question the user asks\n"
+ )
+ assert expected_input_fields in content
+
+ # Assert that system prompt includes correct output field description
+ expected_output_fields = "1. `answer` (Answer): Answer to this question\n"
+ assert expected_output_fields in content
+
+ # Assert that system prompt includes input formatting structure
+ expected_input_structure = "[[ ## user ## ]]\n{user}\n\n[[ ## question ## ]]\n{question}\n\n"
+ assert expected_input_structure in content
+
+ # Assert that system prompt includes output formatting structure
+ expected_output_structure = (
+ "Outputs will be a JSON object with the following fields.\n\n{\n "
+ '"answer": "{answer} # note: the value you produce must adhere to the JSON schema: '
+ '{\\"type\\": \\"object\\", \\"properties\\": {\\"analysis\\": {\\"type\\": \\"string\\", \\"title\\": '
+ '\\"Analysis\\"}, \\"result\\": {\\"type\\": \\"string\\", \\"title\\": \\"Result\\"}}, \\"required\\": '
+ '[\\"analysis\\", \\"result\\"], \\"title\\": \\"Answer\\"}"\n}'
+ )
+ assert expected_output_structure in content
+
+ assert call_kwargs["messages"][1]["role"] == "user"
+ user_message_content = call_kwargs["messages"][1]["content"]
+ assert user_message_content is not None
+
+ # Assert that the user input data is formatted correctly
+ expected_input_data = (
+ '[[ ## user ## ]]\n{"id": 5, "name": "name_test", "email": "email_test"}\n\n[[ ## question ## ]]\n'
+ "What is the capital of France?\n\n"
+ )
+ assert expected_input_data in user_message_content
+
+ # Assert that the adapter output has expected fields and values
+ assert result.answer.analysis == "Paris is the captial of France"
+ assert result.answer.result == "Paris"
+
+
+def test_json_adapter_fallback_to_json_mode_on_structured_output_failure():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField(desc="String output field")
+
+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
+ program = dspy.Predict(TestSignature)
+
+ with mock.patch("litellm.completion") as mock_completion:
+ # First call raises error to simulate structured output failure, second call returns a valid response
+ mock_completion.side_effect = [
+ RuntimeError("Structured output failed!"),
+ ModelResponse(choices=[Choices(message=Message(content="{'answer': 'Test output'}"))]),
+ ]
+
+ result = program(question="Dummy question!")
+ # The parse should succeed on the second call
+ assert mock_completion.call_count == 2
+ assert result.answer == "Test output"
+
+ # The first call should have tried structured output
+ _, first_call_kwargs = mock_completion.call_args_list[0]
+ assert issubclass(first_call_kwargs.get("response_format"), pydantic.BaseModel)
+
+ # The second call should have used JSON mode
+ _, second_call_kwargs = mock_completion.call_args_list[1]
+ assert second_call_kwargs.get("response_format") == {"type": "json_object"}
+
+
+async def test_json_adapter_fallback_to_json_mode_on_structured_output_failure_async():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField(desc="String output field")
+
+ program = dspy.Predict(TestSignature)
+
+ with mock.patch("litellm.acompletion") as mock_acompletion:
+ # First call raises error to simulate structured output failure, second call returns a valid response
+ mock_acompletion.side_effect = [
+ RuntimeError("Structured output failed!"),
+ ModelResponse(choices=[Choices(message=Message(content="{'answer': 'Test output'}"))]),
+ ]
+
+ with dspy.context(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+ result = await program.acall(question="Dummy question!")
+ # The parse should succeed on the second call
+ assert mock_acompletion.call_count == 2
+ assert result.answer == "Test output"
+
+ # The first call should have tried structured output
+ _, first_call_kwargs = mock_acompletion.call_args_list[0]
+ assert issubclass(first_call_kwargs.get("response_format"), pydantic.BaseModel)
+
+ # The second call should have used JSON mode
+ _, second_call_kwargs = mock_acompletion.call_args_list[1]
+ assert second_call_kwargs.get("response_format") == {"type": "json_object"}
+
+
+def test_error_message_on_json_adapter_failure():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField(desc="String output field")
+
+ program = dspy.Predict(TestSignature)
+
+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
+
+ with mock.patch("litellm.completion") as mock_completion:
+ mock_completion.side_effect = RuntimeError("Failed!")
+
+ with pytest.raises(RuntimeError) as error:
+ program(question="Dummy question!")
+
+ assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)
+
+
+async def test_error_message_on_json_adapter_failure_async():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField(desc="String output field")
+
+ program = dspy.Predict(TestSignature)
+
+ with mock.patch("litellm.acompletion") as mock_acompletion:
+ mock_acompletion.side_effect = RuntimeError("Failed!")
+
+ with dspy.context(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+ with pytest.raises(RuntimeError) as error:
+ await program.acall(question="Dummy question!")
+
+ assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)