The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
multiple: permit optional fields on multimodal content blocks (#30887)
ccurme86d51f6be6b2
diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.pyindex 889e25b421..3e3ee11a7e 100644--- a/libs/core/langchain_core/messages/content_blocks.py+++ b/libs/core/langchain_core/messages/content_blocks.py@@ -6,13 +6,11 @@ from pydantic import TypeAdapter, ValidationErrorfrom typing_extensions import NotRequired, TypedDict-class BaseDataContentBlock(TypedDict):+class BaseDataContentBlock(TypedDict, total=False):"""Base class for data content blocks."""mime_type: NotRequired[str]"""MIME type of the content block (if needed)."""- metadata: NotRequired[dict]- """Provider-specific metadata such as citations or filenames."""class URLContentBlock(BaseDataContentBlock):diff --git a/libs/core/tests/unit_tests/test_messages.py b/libs/core/tests/unit_tests/test_messages.pyindex 5ba3fc8ac5..5899468537 100644--- a/libs/core/tests/unit_tests/test_messages.py+++ b/libs/core/tests/unit_tests/test_messages.py@@ -1107,6 +1107,15 @@ def test_is_data_content_block() -> None:"mime_type": "image/jpeg",})+ assert is_data_content_block(+ {+ "type": "image",+ "source_type": "base64",+ "data": "<base64 data>",+ "mime_type": "image/jpeg",+ "cache_control": {"type": "ephemeral"},+ }+ )assert is_data_content_block({"type": "image",@@ -1148,7 +1157,7 @@ def test_convert_to_openai_image_block() -> None:"type": "image","source_type": "url","url": "https://...",- "metadata": {"cache_control": {"type": "ephemeral"}},+ "cache_control": {"type": "ephemeral"},}expected = {"type": "image_url",@@ -1162,7 +1171,7 @@ def test_convert_to_openai_image_block() -> None:"source_type": "base64","data": "<base64 data>","mime_type": "image/jpeg",- "metadata": {"cache_control": {"type": "ephemeral"}},+ "cache_control": {"type": "ephemeral"},}expected = {"type": "image_url",diff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.pyindex 16d13a9cb6..c1e952997d 100644--- a/libs/partners/anthropic/langchain_anthropic/chat_models.py+++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py@@ -239,11 +239,12 @@ def _format_data_content_block(block: dict) -> dict:else:raise ValueError(f"Block of type {block['type']} is not supported.")- if formatted_block and (metadata := block.get("metadata")):- if "cache_control" in metadata:- formatted_block["cache_control"] = metadata["cache_control"]- if "citations" in metadata:- formatted_block["citations"] = metadata["citations"]+ if formatted_block:+ for key in ["cache_control", "citations", "title", "context"]:+ if key in block:+ formatted_block[key] = block[key]+ elif (metadata := block.get("metadata")) and key in metadata:+ formatted_block[key] = metadata[key]return formatted_blockdiff --git a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py b/libs/partners/anthropic/tests/unit_tests/test_chat_models.pyindex 87811a6807..dc97ad10d0 100644--- a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py+++ b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py@@ -703,7 +703,7 @@ def test__format_messages_with_cache_control() -> None:"source_type": "base64","mime_type": "application/pdf","data": "<base64 data>",- "metadata": {"cache_control": {"type": "ephemeral"}},+ "cache_control": {"type": "ephemeral"},},])@@ -742,7 +742,7 @@ def test__format_messages_with_citations() -> None:"source_type": "text","text": "The grass is green. The sky is blue.","mime_type": "text/plain",- "metadata": {"citations": {"enabled": True}},+ "citations": {"enabled": True},},{"type": "text", "text": "What color is the grass and sky?"},]diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex 29fb8e9cde..119058038b 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -194,14 +194,16 @@ def _format_data_content_block(block: dict) -> dict:elif block["type"] == "file":if block["source_type"] == "base64":file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}- if (metadata := block.get("metadata")) and ("filename" in metadata):+ if filename := block.get("filename"):+ file["filename"] = filename+ elif (metadata := block.get("metadata")) and ("filename" in metadata):file["filename"] = metadata["filename"]else:warnings.warn("OpenAI may require a filename for file inputs. Specify a filename "- "in the metadata: {'type': 'file', 'source_type': 'base64', "+ "in the content block: {'type': 'file', 'source_type': 'base64', ""'mime_type': 'application/pdf', 'data': '...', "- "'metadata': {'filename': 'my-pdf'}}"+ "'filename': 'my-pdf'}")formatted_block = {"type": "file", "file": file}elif block["source_type"] == "id":diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.pyindex 0bbf3ce7a1..71e83fe0ea 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py@@ -96,7 +96,7 @@ class TestOpenAIStandard(ChatModelIntegrationTests):"source_type": "base64","mime_type": "application/pdf","data": pdf_data,- "metadata": {"filename": "my-pdf"}, # OpenAI requires a filename+ "filename": "my-pdf", # OpenAI requires a filename},])diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.pyindex ded98f4d63..a212d82540 100644--- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py+++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py@@ -676,7 +676,7 @@ def test_format_message_content() -> None:"source_type": "base64","data": "<base64 data>","mime_type": "application/pdf",- "metadata": {"filename": "my_file"},+ "filename": "my_file",}]expected = [