Skip to content

The agent engineering platform.

multiple: permit optional fields on multimodal content blocks (#30887)

ccurme86d51f6be6b2
diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.py
index 889e25b421..3e3ee11a7e 100644
--- a/libs/core/langchain_core/messages/content_blocks.py
+++ b/libs/core/langchain_core/messages/content_blocks.py
@@ -6,13 +6,11 @@ from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, TypedDict
-class BaseDataContentBlock(TypedDict):
+class BaseDataContentBlock(TypedDict, total=False):
"""Base class for data content blocks."""
mime_type: NotRequired[str]
"""MIME type of the content block (if needed)."""
- metadata: NotRequired[dict]
- """Provider-specific metadata such as citations or filenames."""
class URLContentBlock(BaseDataContentBlock):
diff --git a/libs/core/tests/unit_tests/test_messages.py b/libs/core/tests/unit_tests/test_messages.py
index 5ba3fc8ac5..5899468537 100644
--- a/libs/core/tests/unit_tests/test_messages.py
+++ b/libs/core/tests/unit_tests/test_messages.py
@@ -1107,6 +1107,15 @@ def test_is_data_content_block() -> None:
"mime_type": "image/jpeg",
}
)
+ assert is_data_content_block(
+ {
+ "type": "image",
+ "source_type": "base64",
+ "data": "<base64 data>",
+ "mime_type": "image/jpeg",
+ "cache_control": {"type": "ephemeral"},
+ }
+ )
assert is_data_content_block(
{
"type": "image",
@@ -1148,7 +1157,7 @@ def test_convert_to_openai_image_block() -> None:
"type": "image",
"source_type": "url",
"url": "https://...",
- "metadata": {"cache_control": {"type": "ephemeral"}},
+ "cache_control": {"type": "ephemeral"},
}
expected = {
"type": "image_url",
@@ -1162,7 +1171,7 @@ def test_convert_to_openai_image_block() -> None:
"source_type": "base64",
"data": "<base64 data>",
"mime_type": "image/jpeg",
- "metadata": {"cache_control": {"type": "ephemeral"}},
+ "cache_control": {"type": "ephemeral"},
}
expected = {
"type": "image_url",
diff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.py
index 16d13a9cb6..c1e952997d 100644
--- a/libs/partners/anthropic/langchain_anthropic/chat_models.py
+++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py
@@ -239,11 +239,12 @@ def _format_data_content_block(block: dict) -> dict:
else:
raise ValueError(f"Block of type {block['type']} is not supported.")
- if formatted_block and (metadata := block.get("metadata")):
- if "cache_control" in metadata:
- formatted_block["cache_control"] = metadata["cache_control"]
- if "citations" in metadata:
- formatted_block["citations"] = metadata["citations"]
+ if formatted_block:
+ for key in ["cache_control", "citations", "title", "context"]:
+ if key in block:
+ formatted_block[key] = block[key]
+ elif (metadata := block.get("metadata")) and key in metadata:
+ formatted_block[key] = metadata[key]
return formatted_block
diff --git a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py
index 87811a6807..dc97ad10d0 100644
--- a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py
+++ b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py
@@ -703,7 +703,7 @@ def test__format_messages_with_cache_control() -> None:
"source_type": "base64",
"mime_type": "application/pdf",
"data": "<base64 data>",
- "metadata": {"cache_control": {"type": "ephemeral"}},
+ "cache_control": {"type": "ephemeral"},
},
]
)
@@ -742,7 +742,7 @@ def test__format_messages_with_citations() -> None:
"source_type": "text",
"text": "The grass is green. The sky is blue.",
"mime_type": "text/plain",
- "metadata": {"citations": {"enabled": True}},
+ "citations": {"enabled": True},
},
{"type": "text", "text": "What color is the grass and sky?"},
]
diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index 29fb8e9cde..119058038b 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -194,14 +194,16 @@ def _format_data_content_block(block: dict) -> dict:
elif block["type"] == "file":
if block["source_type"] == "base64":
file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}
- if (metadata := block.get("metadata")) and ("filename" in metadata):
+ if filename := block.get("filename"):
+ file["filename"] = filename
+ elif (metadata := block.get("metadata")) and ("filename" in metadata):
file["filename"] = metadata["filename"]
else:
warnings.warn(
"OpenAI may require a filename for file inputs. Specify a filename "
- "in the metadata: {'type': 'file', 'source_type': 'base64', "
+ "in the content block: {'type': 'file', 'source_type': 'base64', "
"'mime_type': 'application/pdf', 'data': '...', "
- "'metadata': {'filename': 'my-pdf'}}"
+ "'filename': 'my-pdf'}"
)
formatted_block = {"type": "file", "file": file}
elif block["source_type"] == "id":
diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py
index 0bbf3ce7a1..71e83fe0ea 100644
--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py
+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py
@@ -96,7 +96,7 @@ class TestOpenAIStandard(ChatModelIntegrationTests):
"source_type": "base64",
"mime_type": "application/pdf",
"data": pdf_data,
- "metadata": {"filename": "my-pdf"}, # OpenAI requires a filename
+ "filename": "my-pdf", # OpenAI requires a filename
},
]
)
diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py
index ded98f4d63..a212d82540 100644
--- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py
+++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py
@@ -676,7 +676,7 @@ def test_format_message_content() -> None:
"source_type": "base64",
"data": "<base64 data>",
"mime_type": "application/pdf",
- "metadata": {"filename": "my_file"},
+ "filename": "my_file",
}
]
expected = [