The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
core, openai: support standard multi-modal blocks in convert_to_openai_messages (#30968)
ccurme4bc70766b583
diff --git a/libs/core/langchain_core/messages/__init__.py b/libs/core/langchain_core/messages/__init__.pyindex d4e22138ce..a36042ba86 100644--- a/libs/core/langchain_core/messages/__init__.py+++ b/libs/core/langchain_core/messages/__init__.py@@ -33,6 +33,7 @@ if TYPE_CHECKING:)from langchain_core.messages.chat import ChatMessage, ChatMessageChunkfrom langchain_core.messages.content_blocks import (+ convert_to_openai_data_block,convert_to_openai_image_block,is_data_content_block,)@@ -83,6 +84,7 @@ __all__ = ("ToolMessageChunk","RemoveMessage","_message_from_dict",+ "convert_to_openai_data_block","convert_to_openai_image_block","convert_to_messages","get_buffer_string",@@ -124,6 +126,7 @@ _dynamic_imports = {"MessageLikeRepresentation": "utils","_message_from_dict": "utils","convert_to_messages": "utils",+ "convert_to_openai_data_block": "content_blocks","convert_to_openai_image_block": "content_blocks","convert_to_openai_messages": "utils","filter_messages": "utils",diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.pyindex 3e3ee11a7e..cbd9bfa2f0 100644--- a/libs/core/langchain_core/messages/content_blocks.py+++ b/libs/core/langchain_core/messages/content_blocks.py@@ -1,5 +1,6 @@"""Types for content blocks."""+import warningsfrom typing import Any, Literal, Unionfrom pydantic import TypeAdapter, ValidationError@@ -108,3 +109,47 @@ def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict:}error_message = "Unsupported source type. Only 'url' and 'base64' are supported."raise ValueError(error_message)+++def convert_to_openai_data_block(block: dict) -> dict:+ """Format standard data content block to format expected by OpenAI."""+ if block["type"] == "image":+ formatted_block = convert_to_openai_image_block(block)++ elif block["type"] == "file":+ if block["source_type"] == "base64":+ file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}+ if filename := block.get("filename"):+ file["filename"] = filename+ elif (metadata := block.get("metadata")) and ("filename" in metadata):+ file["filename"] = metadata["filename"]+ else:+ warnings.warn(+ "OpenAI may require a filename for file inputs. Specify a filename "+ "in the content block: {'type': 'file', 'source_type': 'base64', "+ "'mime_type': 'application/pdf', 'data': '...', "+ "'filename': 'my-pdf'}",+ stacklevel=1,+ )+ formatted_block = {"type": "file", "file": file}+ elif block["source_type"] == "id":+ formatted_block = {"type": "file", "file": {"file_id": block["id"]}}+ else:+ error_msg = "source_type base64 or id is required for file blocks."+ raise ValueError(error_msg)++ elif block["type"] == "audio":+ if block["source_type"] == "base64":+ format = block["mime_type"].split("/")[-1]+ formatted_block = {+ "type": "input_audio",+ "input_audio": {"data": block["data"], "format": format},+ }+ else:+ error_msg = "source_type base64 is required for audio blocks."+ raise ValueError(error_msg)+ else:+ error_msg = f"Block of type {block['type']} is not supported."+ raise ValueError(error_msg)++ return formatted_blockdiff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.pyindex 92a658afd2..6582f298ab 100644--- a/libs/core/langchain_core/messages/utils.py+++ b/libs/core/langchain_core/messages/utils.py@@ -30,6 +30,7 @@ from typing import (from pydantic import Discriminator, Field, Tagfrom langchain_core.exceptions import ErrorCode, create_message+from langchain_core.messages import convert_to_openai_data_block, is_data_content_blockfrom langchain_core.messages.ai import AIMessage, AIMessageChunkfrom langchain_core.messages.base import BaseMessage, BaseMessageChunkfrom langchain_core.messages.chat import ChatMessage, ChatMessageChunk@@ -1067,6 +1068,9 @@ def convert_to_openai_messages("image_url": block["image_url"],})+ # Standard multi-modal content block+ elif is_data_content_block(block):+ content.append(convert_to_openai_data_block(block))# Anthropic and Bedrock converse formatelif (block.get("type") == "image") or "image" in block:# Anthropicdiff --git a/libs/core/tests/unit_tests/messages/test_imports.py b/libs/core/tests/unit_tests/messages/test_imports.pyindex 1b97e647e0..ff9fbf92fc 100644--- a/libs/core/tests/unit_tests/messages/test_imports.py+++ b/libs/core/tests/unit_tests/messages/test_imports.py@@ -33,6 +33,7 @@ EXPECTED_ALL = ["filter_messages","merge_message_runs","trim_messages",+ "convert_to_openai_data_block","convert_to_openai_image_block","convert_to_openai_messages",]diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.pyindex 9bc7b35bbf..9031e1be5d 100644--- a/libs/core/tests/unit_tests/messages/test_utils.py+++ b/libs/core/tests/unit_tests/messages/test_utils.py@@ -1186,6 +1186,55 @@ def test_convert_to_openai_messages_developer() -> None:assert result == [{"role": "developer", "content": "a"}] * 2+def test_convert_to_openai_messages_multimodal() -> None:+ messages = [+ HumanMessage(+ content=[+ {"type": "text", "text": "Text message"},+ {+ "type": "image",+ "source_type": "url",+ "url": "https://example.com/test.png",+ },+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 string>",+ "mime_type": "image/png",+ },+ {+ "type": "file",+ "source_type": "base64",+ "data": "<base64 string>",+ "mime_type": "application/pdf",+ },+ {+ "type": "file",+ "source_type": "base64",+ "data": "<base64 string>",+ "mime_type": "application/pdf",+ "filename": "test.pdf",+ },+ {+ "type": "file",+ "source_type": "id",+ "id": "file-abc123",+ },+ {+ "type": "audio",+ "source_type": "base64",+ "data": "<base64 string>",+ "mime_type": "audio/wav",+ },+ ]+ )+ ]+ result = convert_to_openai_messages(messages, text_format="block")+ assert len(result) == 1+ message = result[0]+ assert len(message["content"]) == 7++def test_count_tokens_approximately_empty_messages() -> None:# Test with empty message listassert count_tokens_approximately([]) == 0diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex 16afa1a102..3c6a9ba687 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -61,7 +61,7 @@ from langchain_core.messages import (ToolCall,ToolMessage,ToolMessageChunk,- convert_to_openai_image_block,+ convert_to_openai_data_block,is_data_content_block,)from langchain_core.messages.ai import (@@ -186,45 +186,6 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:return ChatMessage(content=_dict.get("content", ""), role=role, id=id_) # type: ignore[arg-type]-def _format_data_content_block(block: dict) -> dict:- """Format standard data content block to format expected by OpenAI."""- if block["type"] == "image":- formatted_block = convert_to_openai_image_block(block)-- elif block["type"] == "file":- if block["source_type"] == "base64":- file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}- if filename := block.get("filename"):- file["filename"] = filename- elif (metadata := block.get("metadata")) and ("filename" in metadata):- file["filename"] = metadata["filename"]- else:- warnings.warn(- "OpenAI may require a filename for file inputs. Specify a filename "- "in the content block: {'type': 'file', 'source_type': 'base64', "- "'mime_type': 'application/pdf', 'data': '...', "- "'filename': 'my-pdf'}"- )- formatted_block = {"type": "file", "file": file}- elif block["source_type"] == "id":- formatted_block = {"type": "file", "file": {"file_id": block["id"]}}- else:- raise ValueError("source_type base64 or id is required for file blocks.")- elif block["type"] == "audio":- if block["source_type"] == "base64":- format = block["mime_type"].split("/")[-1]- formatted_block = {- "type": "input_audio",- "input_audio": {"data": block["data"], "format": format},- }- else:- raise ValueError("source_type base64 is required for audio blocks.")- else:- raise ValueError(f"Block of type {block['type']} is not supported.")-- return formatted_block--def _format_message_content(content: Any) -> Any:"""Format message content."""if content and isinstance(content, list):@@ -238,7 +199,7 @@ def _format_message_content(content: Any) -> Any:):continueelif isinstance(block, dict) and is_data_content_block(block):- formatted_content.append(_format_data_content_block(block))+ formatted_content.append(convert_to_openai_data_block(block))# Anthropic image blockselif (isinstance(block, dict)