The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
multiple: multi-modal content blocks (#30746)
ccurme9cfe6bcacd5a
diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.pyindex a62e06f460..93fdc32589 100644--- a/libs/core/langchain_core/language_models/chat_models.py+++ b/libs/core/langchain_core/language_models/chat_models.py@@ -53,6 +53,8 @@ from langchain_core.messages import (BaseMessageChunk,HumanMessage,convert_to_messages,+ convert_to_openai_image_block,+ is_data_content_block,message_chunk_to_message,)from langchain_core.outputs import (@@ -103,6 +105,41 @@ def _generate_response_from_error(error: BaseException) -> list[ChatGeneration]:return generations+def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]:+ """Format messages for tracing in on_chat_model_start.++ For backward compatibility, we update image content blocks to OpenAI Chat+ Completions format.++ Args:+ messages: List of messages to format.++ Returns:+ List of messages formatted for tracing.+ """+ messages_to_trace = []+ for message in messages:+ message_to_trace = message+ if isinstance(message.content, list):+ for idx, block in enumerate(message.content):+ if (+ isinstance(block, dict)+ and block.get("type") == "image"+ and is_data_content_block(block)+ ):+ if message_to_trace is message:+ message_to_trace = message.model_copy()+ # Also shallow-copy content+ message_to_trace.content = list(message_to_trace.content)++ message_to_trace.content[idx] = ( # type: ignore[index] # mypy confused by .model_copy+ convert_to_openai_image_block(block)+ )+ messages_to_trace.append(message_to_trace)++ return messages_to_trace++def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:"""Generate from a stream.@@ -439,7 +476,7 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):)(run_manager,) = callback_manager.on_chat_model_start(self._serialized,- [messages],+ [_format_for_tracing(messages)],invocation_params=params,options=options,name=config.get("run_name"),@@ -524,7 +561,7 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):)(run_manager,) = await callback_manager.on_chat_model_start(self._serialized,- [messages],+ [_format_for_tracing(messages)],invocation_params=params,options=options,name=config.get("run_name"),@@ -703,9 +740,12 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):inheritable_metadata,self.metadata,)+ messages_to_trace = [+ _format_for_tracing(message_list) for message_list in messages+ ]run_managers = callback_manager.on_chat_model_start(self._serialized,- messages,+ messages_to_trace,invocation_params=params,options=options,name=run_name,@@ -812,9 +852,12 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):self.metadata,)+ messages_to_trace = [+ _format_for_tracing(message_list) for message_list in messages+ ]run_managers = await callback_manager.on_chat_model_start(self._serialized,- messages,+ messages_to_trace,invocation_params=params,options=options,name=run_name,diff --git a/libs/core/langchain_core/messages/__init__.py b/libs/core/langchain_core/messages/__init__.pyindex 4c28efd71d..a7f8db60f6 100644--- a/libs/core/langchain_core/messages/__init__.py+++ b/libs/core/langchain_core/messages/__init__.py@@ -31,6 +31,10 @@ if TYPE_CHECKING:messages_to_dict,)from langchain_core.messages.chat import ChatMessage, ChatMessageChunk+ from langchain_core.messages.content_blocks import (+ convert_to_openai_image_block,+ is_data_content_block,+ )from langchain_core.messages.function import FunctionMessage, FunctionMessageChunkfrom langchain_core.messages.human import HumanMessage, HumanMessageChunkfrom langchain_core.messages.modifier import RemoveMessage@@ -78,8 +82,10 @@ __all__ = ["ToolMessageChunk","RemoveMessage","_message_from_dict",+ "convert_to_openai_image_block","convert_to_messages","get_buffer_string",+ "is_data_content_block","merge_content","message_chunk_to_message","message_to_dict",@@ -117,9 +123,11 @@ _dynamic_imports = {"MessageLikeRepresentation": "utils","_message_from_dict": "utils","convert_to_messages": "utils",+ "convert_to_openai_image_block": "content_blocks","convert_to_openai_messages": "utils","filter_messages": "utils","get_buffer_string": "utils",+ "is_data_content_block": "content_blocks","merge_message_runs": "utils","message_chunk_to_message": "utils","messages_from_dict": "utils",diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.pynew file mode 100644index 0000000000..889e25b421--- /dev/null+++ b/libs/core/langchain_core/messages/content_blocks.py@@ -0,0 +1,112 @@+"""Types for content blocks."""++from typing import Any, Literal, Union++from pydantic import TypeAdapter, ValidationError+from typing_extensions import NotRequired, TypedDict+++class BaseDataContentBlock(TypedDict):+ """Base class for data content blocks."""++ mime_type: NotRequired[str]+ """MIME type of the content block (if needed)."""+ metadata: NotRequired[dict]+ """Provider-specific metadata such as citations or filenames."""+++class URLContentBlock(BaseDataContentBlock):+ """Content block for data from a URL."""++ type: Literal["image", "audio", "file"]+ """Type of the content block."""+ source_type: Literal["url"]+ """Source type (url)."""+ url: str+ """URL for data."""+++class Base64ContentBlock(BaseDataContentBlock):+ """Content block for inline data from a base64 string."""++ type: Literal["image", "audio", "file"]+ """Type of the content block."""+ source_type: Literal["base64"]+ """Source type (base64)."""+ data: str+ """Data as a base64 string."""+++class PlainTextContentBlock(BaseDataContentBlock):+ """Content block for plain text data (e.g., from a document)."""++ type: Literal["file"]+ """Type of the content block."""+ source_type: Literal["text"]+ """Source type (text)."""+ text: str+ """Text data."""+++class IDContentBlock(TypedDict):+ """Content block for data specified by an identifier."""++ type: Literal["image", "audio", "file"]+ """Type of the content block."""+ source_type: Literal["id"]+ """Source type (id)."""+ id: str+ """Identifier for data source."""+++DataContentBlock = Union[+ URLContentBlock,+ Base64ContentBlock,+ PlainTextContentBlock,+ IDContentBlock,+]++_DataContentBlockAdapter: TypeAdapter[DataContentBlock] = TypeAdapter(DataContentBlock)+++def is_data_content_block(+ content_block: dict,+) -> bool:+ """Check if the content block is a standard data content block.++ Args:+ content_block: The content block to check.++ Returns:+ True if the content block is a data content block, False otherwise.+ """+ try:+ _ = _DataContentBlockAdapter.validate_python(content_block)+ except ValidationError:+ return False+ else:+ return True+++def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict:+ """Convert image content block to format expected by OpenAI Chat Completions API."""+ if content_block["source_type"] == "url":+ return {+ "type": "image_url",+ "image_url": {+ "url": content_block["url"],+ },+ }+ if content_block["source_type"] == "base64":+ if "mime_type" not in content_block:+ error_message = "mime_type key is required for base64 data."+ raise ValueError(error_message)+ mime_type = content_block["mime_type"]+ return {+ "type": "image_url",+ "image_url": {+ "url": f"data:{mime_type};base64,{content_block['data']}",+ },+ }+ error_message = "Unsupported source type. Only 'url' and 'base64' are supported."+ raise ValueError(error_message)diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py b/libs/core/tests/unit_tests/language_models/chat_models/test_base.pyindex 04fd163726..eb1a596054 100644--- a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py+++ b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py@@ -8,7 +8,11 @@ import pytestfrom typing_extensions import overridefrom langchain_core.callbacks import CallbackManagerForLLMRun-from langchain_core.language_models import BaseChatModel, FakeListChatModel+from langchain_core.language_models import (+ BaseChatModel,+ FakeListChatModel,+ ParrotFakeChatModel,+)from langchain_core.language_models.fake_chat_models import FakeListChatModelErrorfrom langchain_core.messages import (AIMessage,@@ -396,3 +400,58 @@ async def test_disable_streaming_no_streaming_model_async(async for c in model.astream([], tools=[{}]):assert c.content == "invoke"break+++class FakeChatModelStartTracer(FakeTracer):+ def __init__(self) -> None:+ super().__init__()+ self.messages: list = []++ def on_chat_model_start(self, *args: Any, **kwargs: Any) -> Run:+ _, messages = args+ self.messages.append(messages)+ return super().on_chat_model_start(+ *args,+ **kwargs,+ )+++def test_trace_images_in_openai_format() -> None:+ """Test that images are traced in OpenAI format."""+ llm = ParrotFakeChatModel()+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "source_type": "url",+ "url": "https://example.com/image.png",+ }+ ],+ }+ ]+ tracer = FakeChatModelStartTracer()+ response = llm.invoke(messages, config={"callbacks": [tracer]})+ assert tracer.messages == [+ [+ [+ HumanMessage(+ content=[+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ }+ ]+ )+ ]+ ]+ ]+ # Test no mutation+ assert response.content == [+ {+ "type": "image",+ "source_type": "url",+ "url": "https://example.com/image.png",+ }+ ]diff --git a/libs/core/tests/unit_tests/messages/test_imports.py b/libs/core/tests/unit_tests/messages/test_imports.pyindex 18d8cd5243..1b97e647e0 100644--- a/libs/core/tests/unit_tests/messages/test_imports.py+++ b/libs/core/tests/unit_tests/messages/test_imports.py@@ -24,6 +24,7 @@ EXPECTED_ALL = ["RemoveMessage","convert_to_messages","get_buffer_string",+ "is_data_content_block","merge_content","message_chunk_to_message","message_to_dict",@@ -32,6 +33,7 @@ EXPECTED_ALL = ["filter_messages","merge_message_runs","trim_messages",+ "convert_to_openai_image_block","convert_to_openai_messages",]diff --git a/libs/core/tests/unit_tests/test_messages.py b/libs/core/tests/unit_tests/test_messages.pyindex ecd940ec49..5ba3fc8ac5 100644--- a/libs/core/tests/unit_tests/test_messages.py+++ b/libs/core/tests/unit_tests/test_messages.py@@ -21,7 +21,9 @@ from langchain_core.messages import (SystemMessage,ToolMessage,convert_to_messages,+ convert_to_openai_image_block,get_buffer_string,+ is_data_content_block,merge_content,message_chunk_to_message,message_to_dict,@@ -1087,3 +1089,86 @@ def test_message_text() -> None:).text()== "")+++def test_is_data_content_block() -> None:+ assert is_data_content_block(+ {+ "type": "image",+ "source_type": "url",+ "url": "https://...",+ }+ )+ assert is_data_content_block(+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "image/jpeg",+ }+ )+ assert is_data_content_block(+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "image/jpeg",+ "metadata": {"cache_control": {"type": "ephemeral"}},+ }+ )++ assert not is_data_content_block(+ {+ "type": "text",+ "text": "foo",+ }+ )+ assert not is_data_content_block(+ {+ "type": "image_url",+ "image_url": {"url": "https://..."},+ }+ )+ assert not is_data_content_block(+ {+ "type": "image",+ "source_type": "base64",+ }+ )+ assert not is_data_content_block(+ {+ "type": "image",+ "source": "<base64 data>",+ }+ )+++def test_convert_to_openai_image_block() -> None:+ input_block = {+ "type": "image",+ "source_type": "url",+ "url": "https://...",+ "metadata": {"cache_control": {"type": "ephemeral"}},+ }+ expected = {+ "type": "image_url",+ "image_url": {"url": "https://..."},+ }+ result = convert_to_openai_image_block(input_block)+ assert result == expected++ input_block = {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "image/jpeg",+ "metadata": {"cache_control": {"type": "ephemeral"}},+ }+ expected = {+ "type": "image_url",+ "image_url": {+ "url": "data:image/jpeg;base64,<base64 data>",+ },+ }+ result = convert_to_openai_image_block(input_block)+ assert result == expecteddiff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.pyindex d998f94c5d..16d13a9cb6 100644--- a/libs/partners/anthropic/langchain_anthropic/chat_models.py+++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py@@ -35,6 +35,7 @@ from langchain_core.messages import (SystemMessage,ToolCall,ToolMessage,+ is_data_content_block,)from langchain_core.messages.ai import InputTokenDetails, UsageMetadatafrom langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk@@ -177,8 +178,78 @@ def _merge_messages(return merged+def _format_data_content_block(block: dict) -> dict:+ """Format standard data content block to format expected by Anthropic."""+ if block["type"] == "image":+ if block["source_type"] == "url":+ if block["url"].startswith("data:"):+ # Data URI+ formatted_block = {+ "type": "image",+ "source": _format_image(block["url"]),+ }+ else:+ formatted_block = {+ "type": "image",+ "source": {"type": "url", "url": block["url"]},+ }+ elif block["source_type"] == "base64":+ formatted_block = {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": block["mime_type"],+ "data": block["data"],+ },+ }+ else:+ raise ValueError(+ "Anthropic only supports 'url' and 'base64' source_type for image "+ "content blocks."+ )++ elif block["type"] == "file":+ if block["source_type"] == "url":+ formatted_block = {+ "type": "document",+ "source": {+ "type": "url",+ "url": block["url"],+ },+ }+ elif block["source_type"] == "base64":+ formatted_block = {+ "type": "document",+ "source": {+ "type": "base64",+ "media_type": block.get("mime_type") or "application/pdf",+ "data": block["data"],+ },+ }+ elif block["source_type"] == "text":+ formatted_block = {+ "type": "document",+ "source": {+ "type": "text",+ "media_type": block.get("mime_type") or "text/plain",+ "data": block["text"],+ },+ }++ else:+ raise ValueError(f"Block of type {block['type']} is not supported.")++ if formatted_block and (metadata := block.get("metadata")):+ if "cache_control" in metadata:+ formatted_block["cache_control"] = metadata["cache_control"]+ if "citations" in metadata:+ formatted_block["citations"] = metadata["citations"]++ return formatted_block++def _format_messages(- messages: list[BaseMessage],+ messages: Sequence[BaseMessage],) -> tuple[Union[str, list[dict], None], list[dict]]:"""Format messages for anthropic."""@@ -233,6 +304,8 @@ def _format_messages(# convert formatsource = _format_image(block["image_url"]["url"])content.append({"type": "image", "source": source})+ elif is_data_content_block(block):+ content.append(_format_data_content_block(block))elif block["type"] == "tool_use":# If a tool_call with the same id as a tool_use content block# exists, the tool_call is preferred.diff --git a/libs/partners/anthropic/tests/integration_tests/test_standard.py b/libs/partners/anthropic/tests/integration_tests/test_standard.pyindex 1a11a4850a..b216141d26 100644--- a/libs/partners/anthropic/tests/integration_tests/test_standard.py+++ b/libs/partners/anthropic/tests/integration_tests/test_standard.py@@ -25,6 +25,14 @@ class TestAnthropicStandard(ChatModelIntegrationTests):def supports_image_inputs(self) -> bool:return True+ @property+ def supports_image_urls(self) -> bool:+ return True++ @property+ def supports_pdf_inputs(self) -> bool:+ return True+@propertydef supports_image_tool_message(self) -> bool:return Truediff --git a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py b/libs/partners/anthropic/tests/unit_tests/test_chat_models.pyindex 7281a104cb..87811a6807 100644--- a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py+++ b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py@@ -690,6 +690,85 @@ def test__format_messages_with_cache_control() -> None:assert expected_system == actual_systemassert expected_messages == actual_messages+ # Test standard multi-modal format+ messages = [+ HumanMessage(+ [+ {+ "type": "text",+ "text": "Summarize this document:",+ },+ {+ "type": "file",+ "source_type": "base64",+ "mime_type": "application/pdf",+ "data": "<base64 data>",+ "metadata": {"cache_control": {"type": "ephemeral"}},+ },+ ]+ )+ ]+ actual_system, actual_messages = _format_messages(messages)+ assert actual_system is None+ expected_messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": "Summarize this document:",+ },+ {+ "type": "document",+ "source": {+ "type": "base64",+ "media_type": "application/pdf",+ "data": "<base64 data>",+ },+ "cache_control": {"type": "ephemeral"},+ },+ ],+ }+ ]+ assert actual_messages == expected_messages+++def test__format_messages_with_citations() -> None:+ input_messages = [+ HumanMessage(+ content=[+ {+ "type": "file",+ "source_type": "text",+ "text": "The grass is green. The sky is blue.",+ "mime_type": "text/plain",+ "metadata": {"citations": {"enabled": True}},+ },+ {"type": "text", "text": "What color is the grass and sky?"},+ ]+ )+ ]+ expected_messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "document",+ "source": {+ "type": "text",+ "media_type": "text/plain",+ "data": "The grass is green. The sky is blue.",+ },+ "citations": {"enabled": True},+ },+ {"type": "text", "text": "What color is the grass and sky?"},+ ],+ }+ ]+ actual_system, actual_messages = _format_messages(input_messages)+ assert actual_system is None+ assert actual_messages == expected_messages+def test__format_messages_with_multiple_system() -> None:messages = [diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex c3d5fa4bc0..6670a3b766 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -61,6 +61,8 @@ from langchain_core.messages import (ToolCall,ToolMessage,ToolMessageChunk,+ convert_to_openai_image_block,+ is_data_content_block,)from langchain_core.messages.ai import (InputTokenDetails,@@ -184,6 +186,32 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:return ChatMessage(content=_dict.get("content", ""), role=role, id=id_) # type: ignore[arg-type]+def _format_data_content_block(block: dict) -> dict:+ """Format standard data content block to format expected by OpenAI."""+ if block["type"] == "image":+ formatted_block = convert_to_openai_image_block(block)++ elif block["type"] == "file":+ if block["source_type"] == "base64":+ file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}+ if (metadata := block.get("metadata")) and ("filename" in metadata):+ file["filename"] = metadata["filename"]+ else:+ warnings.warn(+ "OpenAI may require a filename for file inputs. Specify a filename "+ "in the metadata: {'type': 'file', 'source_type': 'base64', "+ "'mime_type': 'application/pdf', 'data': '...', "+ "'metadata': {'filename': 'my-pdf'}}"+ )+ formatted_block = {"type": "file", "file": file}+ elif block["source_type"] == "id":+ formatted_block = {"type": "file", "file": {"file_id": block["id"]}}+ else:+ raise ValueError(f"Block of type {block['type']} is not supported.")++ return formatted_block++def _format_message_content(content: Any) -> Any:"""Format message content."""if content and isinstance(content, list):@@ -196,6 +224,8 @@ def _format_message_content(content: Any) -> Any:and block["type"] in ("tool_use", "thinking")):continue+ elif isinstance(block, dict) and is_data_content_block(block):+ formatted_content.append(_format_data_content_block(block))# Anthropic image blockselif (isinstance(block, dict)@@ -3122,6 +3152,9 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list:if block["image_url"].get("detail"):new_block["detail"] = block["image_url"]["detail"]new_blocks.append(new_block)+ elif block["type"] == "file":+ new_block = {"type": "input_file", **block["file"]}+ new_blocks.append(new_block)elif block["type"] in ("input_text", "input_image", "input_file"):new_blocks.append(block)else:diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.pyindex 1e7d6f0cdc..41e6aec4bd 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py@@ -30,6 +30,10 @@ class TestAzureOpenAIStandard(ChatModelIntegrationTests):def supports_image_inputs(self) -> bool:return True+ @property+ def supports_image_urls(self) -> bool:+ return True+@propertydef supports_json_mode(self) -> bool:return Truediff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.pyindex e23bbffa7c..0bbf3ce7a1 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py@@ -1,10 +1,12 @@"""Standard LangChain interface tests"""+import base64from pathlib import Pathfrom typing import Literal, cast+import httpxfrom langchain_core.language_models import BaseChatModel-from langchain_core.messages import AIMessage+from langchain_core.messages import AIMessage, HumanMessagefrom langchain_tests.integration_tests import ChatModelIntegrationTestsfrom langchain_openai import ChatOpenAI@@ -25,6 +27,10 @@ class TestOpenAIStandard(ChatModelIntegrationTests):def supports_image_inputs(self) -> bool:return True+ @property+ def supports_image_urls(self) -> bool:+ return True+@propertydef supports_json_mode(self) -> bool:return True@@ -71,6 +77,31 @@ class TestOpenAIStandard(ChatModelIntegrationTests):)return _invoke(llm, input_, stream)+ @property+ def supports_pdf_inputs(self) -> bool:+ # OpenAI requires a filename for PDF inputs+ # For now, we test with filename in OpenAI-specific tests+ return False++ def test_openai_pdf_inputs(self, model: BaseChatModel) -> None:+ """Test that the model can process PDF inputs."""+ url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"+ pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")++ message = HumanMessage(+ [+ {"type": "text", "text": "Summarize this document:"},+ {+ "type": "file",+ "source_type": "base64",+ "mime_type": "application/pdf",+ "data": pdf_data,+ "metadata": {"filename": "my-pdf"}, # OpenAI requires a filename+ },+ ]+ )+ _ = model.invoke([message])+def _invoke(llm: ChatOpenAI, input_: str, stream: bool) -> AIMessage:if stream:diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.pyindex 5b8d219d92..ded98f4d63 100644--- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py+++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py@@ -649,6 +649,51 @@ def test_format_message_content() -> None:]assert [{"type": "text", "text": "hello"}] == _format_message_content(content)+ # Standard multi-modal inputs+ content = [{"type": "image", "source_type": "url", "url": "https://..."}]+ expected = [{"type": "image_url", "image_url": {"url": "https://..."}}]+ assert expected == _format_message_content(content)++ content = [+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "image/png",+ }+ ]+ expected = [+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,<base64 data>"},+ }+ ]+ assert expected == _format_message_content(content)++ content = [+ {+ "type": "file",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "application/pdf",+ "metadata": {"filename": "my_file"},+ }+ ]+ expected = [+ {+ "type": "file",+ "file": {+ "filename": "my_file",+ "file_data": "data:application/pdf;base64,<base64 data>",+ },+ }+ ]+ assert expected == _format_message_content(content)++ content = [{"type": "file", "source_type": "id", "id": "file-abc123"}]+ expected = [{"type": "file", "file": {"file_id": "file-abc123"}}]+ assert expected == _format_message_content(content)+class GenerateUsername(BaseModel):"Get a username based on someone's name and hair color."diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.pyindex b5e294ffc8..ae083c5bf3 100644--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py@@ -298,13 +298,21 @@ class ChatModelIntegrationTests(ChatModelTests):.. code-block:: python- [- {"type": "text", "text": "describe the weather in this image"},- {- "type": "image_url",- "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},- },- ]+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 image data>",+ "mime_type": "image/jpeg", # or appropriate mime-type+ }++ In addition to OpenAI-style content blocks:++ .. code-block:: python++ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},+ }See https://python.langchain.com/docs/concepts/multimodality/@@ -316,6 +324,59 @@ class ChatModelIntegrationTests(ChatModelTests):def supports_image_inputs(self) -> bool:return True+ .. dropdown:: supports_image_urls++ Boolean property indicating whether the chat model supports image inputs from+ URLs. Defaults to ``False``.++ If set to ``True``, the chat model will be tested using content blocks of the+ form++ .. code-block:: python++ {+ "type": "image",+ "source_type": "url",+ "url": "https://...",+ }++ See https://python.langchain.com/docs/concepts/multimodality/++ Example:++ .. code-block:: python++ @property+ def supports_image_urls(self) -> bool:+ return True++ .. dropdown:: supports_pdf_inputs++ Boolean property indicating whether the chat model supports PDF inputs.+ Defaults to ``False``.++ If set to ``True``, the chat model will be tested using content blocks of the+ form++ .. code-block:: python++ {+ "type": "file",+ "source_type": "base64",+ "data": "<base64 file data>",+ "mime_type": "application/pdf",+ }++ See https://python.langchain.com/docs/concepts/multimodality/++ Example:++ .. code-block:: python++ @property+ def supports_pdf_inputs(self) -> bool:+ return True+.. dropdown:: supports_video_inputsBoolean property indicating whether the chat model supports image inputs.@@ -1891,11 +1952,79 @@ class ChatModelIntegrationTests(ChatModelTests):result = model_with_tools.invoke(messages)assert isinstance(result, AIMessage)+ def test_pdf_inputs(self, model: BaseChatModel) -> None:+ """Test that the model can process PDF inputs.++ This test should be skipped (see Configuration below) if the model does not+ support PDF inputs. These will take the form:++ .. code-block:: python++ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 image data>",+ "mime_type": "application/pdf",+ }++ See https://python.langchain.com/docs/concepts/multimodality/++ .. dropdown:: Configuration++ To disable this test, set ``supports_pdf_inputs`` to False in your+ test class:++ .. code-block:: python++ class TestMyChatModelIntegration(ChatModelIntegrationTests):++ @property+ def supports_pdf_inputs(self) -> bool:+ return False++ .. dropdown:: Troubleshooting++ If this test fails, check that the model can correctly handle messages+ with pdf content blocks, including base64-encoded files. Otherwise, set+ the ``supports_pdf_inputs`` property to False.+ """+ if not self.supports_pdf_inputs:+ pytest.skip("Model does not support PDF inputs.")+ url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"+ pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")++ message = HumanMessage(+ [+ {+ "type": "text",+ "text": "Summarize this document:",+ },+ {+ "type": "file",+ "source_type": "base64",+ "mime_type": "application/pdf",+ "data": pdf_data,+ },+ ]+ )+ _ = model.invoke([message])+def test_image_inputs(self, model: BaseChatModel) -> None:"""Test that the model can process image inputs.This test should be skipped (see Configuration below) if the model does not- support image inputs These will take the form of messages with OpenAI-style+ support image inputs. These will take the form:++ .. code-block:: python++ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 image data>",+ "mime_type": "image/jpeg", # or appropriate mime-type+ }++ For backward-compatibility, we must also support OpenAI-styleimage content blocks:.. code-block:: python@@ -1910,6 +2039,17 @@ class ChatModelIntegrationTests(ChatModelTests):See https://python.langchain.com/docs/concepts/multimodality/+ If the property ``supports_image_urls`` is set to True, the test will also+ check that we can process content blocks of the form:++ .. code-block:: python++ {+ "type": "image",+ "source_type": "url",+ "url": "<url>",+ }+.. dropdown:: ConfigurationTo disable this test, set ``supports_image_inputs`` to False in your@@ -1922,16 +2062,23 @@ class ChatModelIntegrationTests(ChatModelTests):def supports_image_inputs(self) -> bool:return False+ # Can also explicitly disable testing image URLs:+ @property+ def supports_image_urls(self) -> bool:+ return False+.. dropdown:: TroubleshootingIf this test fails, check that the model can correctly handle messages- with image content blocks in OpenAI format, including base64-encoded- images. Otherwise, set the ``supports_image_inputs`` property to False.+ with image content blocks, including base64-encoded images. Otherwise, set+ the ``supports_image_inputs`` property to False."""if not self.supports_image_inputs:pytest.skip("Model does not support image message.")image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"image_data = base64.b64encode(httpx.get(image_url).content).decode("utf-8")++ # OpenAI format, base64 datamessage = HumanMessage(content=[{"type": "text", "text": "describe the weather in this image"},@@ -1941,7 +2088,35 @@ class ChatModelIntegrationTests(ChatModelTests):},],)- model.invoke([message])+ _ = model.invoke([message])++ # Standard format, base64 data+ message = HumanMessage(+ content=[+ {"type": "text", "text": "describe the weather in this image"},+ {+ "type": "image",+ "source_type": "base64",+ "mime_type": "image/jpeg",+ "data": image_data,+ },+ ],+ )+ _ = model.invoke([message])++ # Standard format, URL+ if self.supports_image_urls:+ message = HumanMessage(+ content=[+ {"type": "text", "text": "describe the weather in this image"},+ {+ "type": "image",+ "source_type": "url",+ "url": image_url,+ },+ ],+ )+ _ = model.invoke([message])def test_image_tool_message(self, model: BaseChatModel) -> None:"""Test that the model can process ToolMessages with image inputs.diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.pyindex beec0b98cb..f3e22d04ad 100644--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py@@ -160,6 +160,17 @@ class ChatModelTests(BaseStandardTests):``False``."""return False+ @property+ def supports_image_urls(self) -> bool:+ """(bool) whether the chat model supports image inputs from URLs, defaults to+ ``False``."""+ return False++ @property+ def supports_pdf_inputs(self) -> bool:+ """(bool) whether the chat model supports PDF inputs, defaults to ``False``."""+ return False+@propertydef supports_video_inputs(self) -> bool:"""(bool) whether the chat model supports video inputs, efaults to ``False``.@@ -373,13 +384,21 @@ class ChatModelUnitTests(ChatModelTests):.. code-block:: python- [- {"type": "text", "text": "describe the weather in this image"},- {- "type": "image_url",- "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},- },- ]+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 image data>",+ "mime_type": "image/jpeg", # or appropriate mime-type+ }++ In addition to OpenAI-style content blocks:++ .. code-block:: python++ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},+ }See https://python.langchain.com/docs/concepts/multimodality/@@ -391,6 +410,59 @@ class ChatModelUnitTests(ChatModelTests):def supports_image_inputs(self) -> bool:return True+ .. dropdown:: supports_image_urls++ Boolean property indicating whether the chat model supports image inputs from+ URLs. Defaults to ``False``.++ If set to ``True``, the chat model will be tested using content blocks of the+ form++ .. code-block:: python++ {+ "type": "image",+ "source_type": "url",+ "url": "https://...",+ }++ See https://python.langchain.com/docs/concepts/multimodality/++ Example:++ .. code-block:: python++ @property+ def supports_image_urls(self) -> bool:+ return True++ .. dropdown:: supports_pdf_inputs++ Boolean property indicating whether the chat model supports PDF inputs.+ Defaults to ``False``.++ If set to ``True``, the chat model will be tested using content blocks of the+ form++ .. code-block:: python++ {+ "type": "file",+ "source_type": "base64",+ "data": "<base64 file data>",+ "mime_type": "application/pdf",+ }++ See https://python.langchain.com/docs/concepts/multimodality/++ Example:++ .. code-block:: python++ @property+ def supports_pdf_inputs(self) -> bool:+ return True+.. dropdown:: supports_video_inputsBoolean property indicating whether the chat model supports image inputs.