DSPy: The framework for programming—not prompting—language models
Python
Fix content input conversion for OpenAI Responses API (#8993)
Copilot7e44517fdfb7
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 71a8934cc..19085bea8 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -468,7 +468,9 @@ def _convert_chat_request_to_responses_request(request: dict[str, Any]):if isinstance(c, str):content_blocks.append({"type": "input_text", "text": c})elif isinstance(c, list):- content_blocks.extend(c)+ # Convert each content item from Chat API format to Responses API format+ for item in c:+ content_blocks.append(_convert_content_item_to_responses_format(item))request["input"] = [{"role": msg.get("role", "user"), "content": content_blocks}]# Convert `response_format` to `text.format` for Responses API@@ -480,6 +482,38 @@ def _convert_chat_request_to_responses_request(request: dict[str, Any]):return request+def _convert_content_item_to_responses_format(item: dict[str, Any]) -> dict[str, Any]:+ """+ Convert a content item from Chat API format to Responses API format.++ For images, converts from:+ {"type": "image_url", "image_url": {"url": "..."}}+ To:+ {"type": "input_image", "image_url": "..."}++ For text, converts from:+ {"type": "text", "text": "..."}+ To:+ {"type": "input_text", "text": "..."}++ For other types, passes through as-is.+ """+ if item.get("type") == "image_url":+ image_url = item.get("image_url", {}).get("url", "")+ return {+ "type": "input_image",+ "image_url": image_url,+ }+ elif item.get("type") == "text":+ return {+ "type": "input_text",+ "text": item.get("text", ""),+ }++ # For other items, return as-is+ return item++def _get_headers(headers: dict[str, Any] | None = None):headers = headers or {}return {diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 336ccc41b..a24c09a9d 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -343,8 +343,8 @@ def test_reasoning_model_requirements(model_name):lm = dspy.LM(model=model_name,)- assert lm.kwargs["temperature"] == None- assert lm.kwargs["max_completion_tokens"] == None+ assert lm.kwargs["temperature"] is None+ assert lm.kwargs["max_completion_tokens"] is Nonedef test_dump_state():@@ -633,3 +633,127 @@ def test_api_key_not_saved_in_json():assert saved_state["lm"]["model"] == "openai/gpt-4o-mini"assert saved_state["lm"]["temperature"] == 1.0assert saved_state["lm"]["max_tokens"] == 100+++def test_responses_api_converts_images_correctly():+ from dspy.clients.lm import _convert_chat_request_to_responses_request++ # Test with base64 image+ request_with_base64_image = {+ "model": "openai/gpt-5-mini",+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "What's in this image?"},+ {+ "type": "image_url",+ "image_url": {+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+ }+ }+ ]+ }+ ]+ }++ result = _convert_chat_request_to_responses_request(request_with_base64_image)++ assert "input" in result+ assert len(result["input"]) == 1+ assert result["input"][0]["role"] == "user"++ content = result["input"][0]["content"]+ assert len(content) == 2++ # First item should be text converted to input_text format+ assert content[0]["type"] == "input_text"+ assert content[0]["text"] == "What's in this image?"++ # Second item should be converted to input_image format+ assert content[1]["type"] == "input_image"+ assert content[1]["image_url"] == "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="++ # Test with URL image+ request_with_url_image = {+ "model": "openai/gpt-5-mini",+ "messages": [+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.jpg"+ }+ }+ ]+ }+ ]+ }++ result = _convert_chat_request_to_responses_request(request_with_url_image)++ content = result["input"][0]["content"]+ assert len(content) == 1+ assert content[0]["type"] == "input_image"+ assert content[0]["image_url"] == "https://example.com/image.jpg"+++def test_responses_api_with_image_input():+ api_response = make_response(+ output_blocks=[+ ResponseOutputMessage(+ **{+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "status": "completed",+ "content": [+ {"type": "output_text", "text": "This is a test answer with image input.", "annotations": []}+ ],+ },+ ),+ ]+ )++ with mock.patch("litellm.responses", autospec=True, return_value=api_response) as dspy_responses:+ lm = dspy.LM(+ model="openai/gpt-5-mini",+ model_type="responses",+ cache=False,+ temperature=1.0,+ max_tokens=16000,+ )++ # Test with messages containing an image+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Describe this image"},+ {+ "type": "image_url",+ "image_url": {+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+ }+ }+ ]+ }+ ]++ lm_result = lm(messages=messages)++ assert lm_result == [{"text": "This is a test answer with image input."}]++ dspy_responses.assert_called_once()+ call_args = dspy_responses.call_args.kwargs++ # Verify the request was converted correctly+ assert "input" in call_args+ content = call_args["input"][0]["content"]++ # Check that image was converted to input_image format+ image_content = [c for c in content if c.get("type") == "input_image"]+ assert len(image_content) == 1+ assert image_content[0]["image_url"] == "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="