DSPy: The framework for programming—not prompting—language models
Python
Support async path for ChatAdapter and JSONAdapter (#8419)
Chen Qian1df67689ee5f
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pyindex 97bb173c2..7e7859978 100644--- a/dspy/adapters/chat_adapter.py+++ b/dspy/adapters/chat_adapter.py@@ -50,6 +50,26 @@ class ChatAdapter(Adapter):raise ereturn JSONAdapter()(lm, lm_kwargs, signature, demos, inputs)+ async def acall(+ self,+ lm: LM,+ lm_kwargs: dict[str, Any],+ signature: Type[Signature],+ demos: list[dict[str, Any]],+ inputs: dict[str, Any],+ ) -> list[dict[str, Any]]:+ try:+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)+ except Exception as e:+ # fallback to JSONAdapter+ from dspy.adapters.json_adapter import JSONAdapter++ if isinstance(e, ContextWindowExceededError) or isinstance(self, JSONAdapter):+ # On context window exceeded error or already using JSONAdapter, we don't want to retry with a different+ # adapter.+ raise e+ return await JSONAdapter().acall(lm, lm_kwargs, signature, demos, inputs)+def format_field_description(self, signature: Type[Signature]) -> str:return (f"Your input fields are:\n{get_field_description_string(signature.input_fields)}\n"diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.pyindex e43561b0e..5e599d29c 100644--- a/dspy/adapters/json_adapter.py+++ b/dspy/adapters/json_adapter.py@@ -22,6 +22,11 @@ from dspy.utils.exceptions import AdapterParseErrorlogger = logging.getLogger(__name__)+ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE = (+ "Both structured output format and JSON mode failed. Please choose a model that supports "+ "`response_format` argument. Original error: {}"+)+def _has_open_ended_mapping(signature: SignatureMeta) -> bool:"""@@ -37,6 +42,18 @@ def _has_open_ended_mapping(signature: SignatureMeta) -> bool:class JSONAdapter(ChatAdapter):+ def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, call_fn):+ """Common call logic to be used for both sync and async calls."""+ provider = lm.model.split("/", 1)[0] or "openai"+ params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)++ if not params or "response_format" not in params:+ return call_fn(lm, lm_kwargs, signature, demos, inputs)++ if _has_open_ended_mapping(signature):+ lm_kwargs["response_format"] = {"type": "json_object"}+ return call_fn(lm, lm_kwargs, signature, demos, inputs)+def __call__(self,lm: LM,@@ -45,37 +62,49 @@ class JSONAdapter(ChatAdapter):demos: list[dict[str, Any]],inputs: dict[str, Any],) -> list[dict[str, Any]]:- provider = lm.model.split("/", 1)[0] or "openai"- params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)+ result = self._json_adapter_call_common(lm, lm_kwargs, signature, demos, inputs, super().__call__)+ if result:+ return result- # If response_format is not supported, use basic call- if not params or "response_format" not in params:+ try:+ structured_output_model = _get_structured_outputs_response_format(signature)+ lm_kwargs["response_format"] = structured_output_modelreturn super().__call__(lm, lm_kwargs, signature, demos, inputs)+ except Exception:+ logger.warning("Failed to use structured output format, falling back to JSON mode.")+ try:+ lm_kwargs["response_format"] = {"type": "json_object"}+ return super().__call__(lm, lm_kwargs, signature, demos, inputs)+ except AdapterParseError as e:+ raise e+ except Exception as e:+ raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from e- # Check early for open-ended mapping types before trying structured outputs.- if _has_open_ended_mapping(signature):- lm_kwargs["response_format"] = {"type": "json_object"}- return super().__call__(lm, lm_kwargs, signature, demos, inputs)+ async def acall(+ self,+ lm: LM,+ lm_kwargs: dict[str, Any],+ signature: Type[Signature],+ demos: list[dict[str, Any]],+ inputs: dict[str, Any],+ ) -> list[dict[str, Any]]:+ result = self._json_adapter_call_common(lm, lm_kwargs, signature, demos, inputs, super().acall)+ if result:+ return await result- # Try structured output first, fall back to basic JSON if it fails.try:structured_output_model = _get_structured_outputs_response_format(signature)lm_kwargs["response_format"] = structured_output_model- return super().__call__(lm, lm_kwargs, signature, demos, inputs)+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)except Exception:logger.warning("Failed to use structured output format, falling back to JSON mode.")try:lm_kwargs["response_format"] = {"type": "json_object"}- return super().__call__(lm, lm_kwargs, signature, demos, inputs)+ return await super().acall(lm, lm_kwargs, signature, demos, inputs)except AdapterParseError as e:- # On AdapterParseError, we raise the original error.raise eexcept Exception as e:- # On any other error, we raise a RuntimeError with the original error message.- raise RuntimeError(- "Both structured output format and JSON mode failed. Please choose a model that supports "- f"`response_format` argument. Original error: {e}"- ) from e+ raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from edef _call_preprocess(self,diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.pyindex 969a8f177..e599da585 100644--- a/tests/adapters/test_chat_adapter.py+++ b/tests/adapters/test_chat_adapter.py@@ -3,6 +3,7 @@ from unittest import mockimport pydanticimport pytest+from litellm.utils import Choices, Message, ModelResponseimport dspy@@ -376,3 +377,48 @@ def test_chat_adapter_formats_conversation_history():assert messages[2]["content"] == "[[ ## answer ## ]]\nParis\n\n[[ ## completed ## ]]\n"assert messages[3]["content"] == "[[ ## question ## ]]\nWhat is the capital of Germany?"assert messages[4]["content"] == "[[ ## answer ## ]]\nBerlin\n\n[[ ## completed ## ]]\n"+++def test_chat_adapter_fallback_to_json_adapter_on_exception():+ signature = dspy.make_signature("question->answer")+ adapter = dspy.ChatAdapter()++ with mock.patch("litellm.completion") as mock_completion:+ # Mock returning a response compatible with JSONAdapter but not ChatAdapter+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],+ model="openai/gpt-4o-mini",+ )++ lm = dspy.LM("openai/gpt-4o-mini", cache=False)++ with mock.patch("dspy.adapters.json_adapter.JSONAdapter.__call__") as mock_json_adapter_call:+ adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})+ mock_json_adapter_call.assert_called_once()++ # The parse should succeed+ result = adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})+ assert result == [{"answer": "Paris"}]+++async def test_chat_adapter_fallback_to_json_adapter_on_exception_async():+ signature = dspy.make_signature("question->answer")+ adapter = dspy.ChatAdapter()++ with mock.patch("litellm.acompletion") as mock_completion:+ # Mock returning a response compatible with JSONAdapter but not ChatAdapter+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content="{'answer': 'Paris'}"))],+ model="openai/gpt-4o-mini",+ )++ lm = dspy.LM("openai/gpt-4o-mini", cache=False)++ with mock.patch("dspy.adapters.json_adapter.JSONAdapter.acall") as mock_json_adapter_acall:+ await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})+ mock_json_adapter_acall.assert_called_once()++ # The parse should succeed+ result = await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})+ assert result == [{"answer": "Paris"}]diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.pyindex 7db4a3202..f688af43f 100644--- a/tests/adapters/test_json_adapter.py+++ b/tests/adapters/test_json_adapter.py@@ -5,6 +5,7 @@ import pytestfrom litellm.utils import Choices, Message, ModelResponseimport dspy+from dspy.adapters.json_adapter import ERROR_MESSAGE_ON_JSON_ADAPTER_FAILUREdef test_json_adapter_passes_structured_output_when_supported_by_model():@@ -22,7 +23,7 @@ def test_json_adapter_passes_structured_output_when_supported_by_model():program = dspy.Predict(TestSignature)# Configure DSPy to use an OpenAI LM that supports structured outputs- dspy.configure(lm=dspy.LM(model="openai/gpt4o"), adapter=dspy.JSONAdapter())+ dspy.configure(lm=dspy.LM(model="openai/gpt-4o"), adapter=dspy.JSONAdapter())with mock.patch("litellm.completion") as mock_completion:program(input1="Test input")@@ -43,14 +44,28 @@ def test_json_adapter_passes_structured_output_when_supported_by_model():assert issubclass(response_format, pydantic.BaseModel)assert response_format.model_fields.keys() == {"output1", "output2", "output3", "output4_unannotated"}++def test_json_adapter_not_using_structured_outputs_when_not_supported_by_model():+ class TestSignature(dspy.Signature):+ input1: str = dspy.InputField()+ output1: str = dspy.OutputField()+ output2: bool = dspy.OutputField()++ program = dspy.Predict(TestSignature)+# Configure DSPy to use a model from a fake provider that doesn't support structured outputs- dspy.configure(lm=dspy.LM(model="fakeprovider/fakemodel"), adapter=dspy.JSONAdapter())+ dspy.configure(lm=dspy.LM(model="fakeprovider/fakemodel", cache=False), adapter=dspy.JSONAdapter())with mock.patch("litellm.completion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[Choices(message=Message(content=("{'output1': 'Test output', 'output2': True}")))],+ model="openai/gpt-4o",+ )+program(input1="Test input")mock_completion.assert_called_once()_, call_kwargs = mock_completion.call_args- assert response_format not in call_kwargs+ assert "response_format" not in call_kwargsdef test_json_adapter_falls_back_when_structured_outputs_fails():@@ -442,3 +457,186 @@ def test_json_adapter_formats_conversation_history():assert messages[2]["content"] == '{\n "answer": "Paris"\n}'assert messages[3]["content"] == "[[ ## question ## ]]\nWhat is the capital of Germany?"assert messages[4]["content"] == '{\n "answer": "Berlin"\n}'+++async def test_json_adapter_on_pydantic_model_async():+ from litellm.utils import Choices, Message, ModelResponse++ class User(pydantic.BaseModel):+ id: int+ name: str+ email: str++ class Answer(pydantic.BaseModel):+ analysis: str+ result: str++ class TestSignature(dspy.Signature):+ user: User = dspy.InputField(desc="The user who asks the question")+ question: str = dspy.InputField(desc="Question the user asks")+ answer: Answer = dspy.OutputField(desc="Answer to this question")++ program = dspy.Predict(TestSignature)++ with mock.patch("litellm.acompletion") as mock_completion:+ mock_completion.return_value = ModelResponse(+ choices=[+ Choices(+ message=Message(+ content="{'answer': {'analysis': 'Paris is the captial of France', 'result': 'Paris'}}"+ )+ )+ ],+ model="openai/gpt4o",+ )++ with dspy.context(lm=dspy.LM(model="openai/gpt4o", cache=False), adapter=dspy.JSONAdapter()):+ result = await program.acall(+ user={"id": 5, "name": "name_test", "email": "email_test"}, question="What is the capital of France?"+ )++ # Check that litellm.acompletion was called exactly once+ mock_completion.assert_called_once()++ _, call_kwargs = mock_completion.call_args+ # Assert that there are exactly 2 messages (system + user)+ assert len(call_kwargs["messages"]) == 2++ assert call_kwargs["messages"][0]["role"] == "system"+ content = call_kwargs["messages"][0]["content"]+ assert content is not None++ # Assert that system prompt includes correct input field descriptions+ expected_input_fields = (+ "1. `user` (User): The user who asks the question\n2. `question` (str): Question the user asks\n"+ )+ assert expected_input_fields in content++ # Assert that system prompt includes correct output field description+ expected_output_fields = "1. `answer` (Answer): Answer to this question\n"+ assert expected_output_fields in content++ # Assert that system prompt includes input formatting structure+ expected_input_structure = "[[ ## user ## ]]\n{user}\n\n[[ ## question ## ]]\n{question}\n\n"+ assert expected_input_structure in content++ # Assert that system prompt includes output formatting structure+ expected_output_structure = (+ "Outputs will be a JSON object with the following fields.\n\n{\n "+ '"answer": "{answer} # note: the value you produce must adhere to the JSON schema: '+ '{\\"type\\": \\"object\\", \\"properties\\": {\\"analysis\\": {\\"type\\": \\"string\\", \\"title\\": '+ '\\"Analysis\\"}, \\"result\\": {\\"type\\": \\"string\\", \\"title\\": \\"Result\\"}}, \\"required\\": '+ '[\\"analysis\\", \\"result\\"], \\"title\\": \\"Answer\\"}"\n}'+ )+ assert expected_output_structure in content++ assert call_kwargs["messages"][1]["role"] == "user"+ user_message_content = call_kwargs["messages"][1]["content"]+ assert user_message_content is not None++ # Assert that the user input data is formatted correctly+ expected_input_data = (+ '[[ ## user ## ]]\n{"id": 5, "name": "name_test", "email": "email_test"}\n\n[[ ## question ## ]]\n'+ "What is the capital of France?\n\n"+ )+ assert expected_input_data in user_message_content++ # Assert that the adapter output has expected fields and values+ assert result.answer.analysis == "Paris is the captial of France"+ assert result.answer.result == "Paris"+++def test_json_adapter_fallback_to_json_mode_on_structured_output_failure():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField(desc="String output field")++ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())+ program = dspy.Predict(TestSignature)++ with mock.patch("litellm.completion") as mock_completion:+ # First call raises error to simulate structured output failure, second call returns a valid response+ mock_completion.side_effect = [+ RuntimeError("Structured output failed!"),+ ModelResponse(choices=[Choices(message=Message(content="{'answer': 'Test output'}"))]),+ ]++ result = program(question="Dummy question!")+ # The parse should succeed on the second call+ assert mock_completion.call_count == 2+ assert result.answer == "Test output"++ # The first call should have tried structured output+ _, first_call_kwargs = mock_completion.call_args_list[0]+ assert issubclass(first_call_kwargs.get("response_format"), pydantic.BaseModel)++ # The second call should have used JSON mode+ _, second_call_kwargs = mock_completion.call_args_list[1]+ assert second_call_kwargs.get("response_format") == {"type": "json_object"}+++async def test_json_adapter_fallback_to_json_mode_on_structured_output_failure_async():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField(desc="String output field")++ program = dspy.Predict(TestSignature)++ with mock.patch("litellm.acompletion") as mock_acompletion:+ # First call raises error to simulate structured output failure, second call returns a valid response+ mock_acompletion.side_effect = [+ RuntimeError("Structured output failed!"),+ ModelResponse(choices=[Choices(message=Message(content="{'answer': 'Test output'}"))]),+ ]++ with dspy.context(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):+ result = await program.acall(question="Dummy question!")+ # The parse should succeed on the second call+ assert mock_acompletion.call_count == 2+ assert result.answer == "Test output"++ # The first call should have tried structured output+ _, first_call_kwargs = mock_acompletion.call_args_list[0]+ assert issubclass(first_call_kwargs.get("response_format"), pydantic.BaseModel)++ # The second call should have used JSON mode+ _, second_call_kwargs = mock_acompletion.call_args_list[1]+ assert second_call_kwargs.get("response_format") == {"type": "json_object"}+++def test_error_message_on_json_adapter_failure():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField(desc="String output field")++ program = dspy.Predict(TestSignature)++ dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())++ with mock.patch("litellm.completion") as mock_completion:+ mock_completion.side_effect = RuntimeError("Failed!")++ with pytest.raises(RuntimeError) as error:+ program(question="Dummy question!")++ assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)+++async def test_error_message_on_json_adapter_failure_async():+ class TestSignature(dspy.Signature):+ question: str = dspy.InputField()+ answer: str = dspy.OutputField(desc="String output field")++ program = dspy.Predict(TestSignature)++ with mock.patch("litellm.acompletion") as mock_acompletion:+ mock_acompletion.side_effect = RuntimeError("Failed!")++ with dspy.context(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):+ with pytest.raises(RuntimeError) as error:+ await program.acall(question="Dummy question!")++ assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)