Skip to content

The agent engineering platform.

feat: (core, standard-tests) support PDF inputs in ToolMessages (#33183)

ccurme002d623f2d5d
diff --git a/libs/core/langchain_core/tools/base.py b/libs/core/langchain_core/tools/base.py
index 7aa4b2532e..fddbb810e6 100644
--- a/libs/core/langchain_core/tools/base.py
+++ b/libs/core/langchain_core/tools/base.py
@@ -82,6 +82,7 @@ TOOL_MESSAGE_BLOCK_TYPES = (
"search_result",
"custom_tool_call_output",
"document",
+ "file",
)
diff --git a/libs/partners/anthropic/tests/integration_tests/test_standard.py b/libs/partners/anthropic/tests/integration_tests/test_standard.py
index 9443cc4460..c9c0492e12 100644
--- a/libs/partners/anthropic/tests/integration_tests/test_standard.py
+++ b/libs/partners/anthropic/tests/integration_tests/test_standard.py
@@ -41,6 +41,10 @@ class TestAnthropicStandard(ChatModelIntegrationTests):
def supports_image_tool_message(self) -> bool:
return True
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ return True
+
@property
def supports_anthropic_inputs(self) -> bool:
return True
diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_standard.py
index 26e2b3e588..81740c4aee 100644
--- a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_standard.py
+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_standard.py
@@ -1,11 +1,13 @@
"""Standard LangChain interface tests for Responses API"""
+import base64
from pathlib import Path
from typing import cast
+import httpx
import pytest
from langchain_core.language_models import BaseChatModel
-from langchain_core.messages import AIMessage
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_openai import ChatOpenAI
from tests.integration_tests.chat_models.test_base_standard import TestOpenAIStandard
@@ -52,6 +54,55 @@ class TestOpenAIResponses(TestOpenAIStandard):
input_ = "What was the 3rd highest building in 2000?"
return _invoke(llm, input_, stream)
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ # OpenAI requires a filename for PDF inputs
+ # For now, we test with filename in OpenAI-specific tests
+ return False
+
+ def test_openai_pdf_tool_messages(self, model: BaseChatModel) -> None:
+ """Test that the model can process PDF inputs in ToolMessages."""
+ url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
+ pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")
+
+ tool_message = ToolMessage(
+ content=[
+ {
+ "type": "file",
+ "source_type": "base64",
+ "data": pdf_data,
+ "mime_type": "application/pdf",
+ "filename": "my-pdf", # specify filename
+ },
+ ],
+ tool_call_id="1",
+ name="random_pdf",
+ )
+
+ messages = [
+ HumanMessage(
+ "Get a random PDF using the tool and relay the title verbatim."
+ ),
+ AIMessage(
+ [],
+ tool_calls=[
+ {
+ "type": "tool_call",
+ "id": "1",
+ "name": "random_pdf",
+ "args": {},
+ }
+ ],
+ ),
+ tool_message,
+ ]
+
+ def random_pdf() -> str:
+ """Return a random PDF."""
+ return ""
+
+ _ = model.bind_tools([random_pdf]).invoke(messages)
+
def _invoke(llm: ChatOpenAI, input_: str, stream: bool) -> AIMessage:
if stream:
diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index 79e4b85693..cbb6b517ba 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -521,6 +521,39 @@ class ChatModelIntegrationTests(ChatModelTests):
def supports_image_tool_message(self) -> bool:
return False
+ .. dropdown:: supports_pdf_tool_message
+
+ Boolean property indicating whether the chat model supports ToolMessages
+ that include PDF content, i.e.,
+
+ .. code-block:: python
+
+ ToolMessage(
+ content=[
+ {
+ "type": "file",
+ "source_type": "base64",
+ "data": pdf_data,
+ "mime_type": "application/pdf",
+ },
+ ],
+ tool_call_id="1",
+ name="random_pdf",
+ )
+
+ (standard format).
+
+ If set to ``True``, the chat model will be tested with message sequences that
+ include ToolMessages of this form.
+
+ Example:
+
+ .. code-block:: python
+
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ return False
+
.. dropdown:: supported_usage_metadata_details
Property controlling what usage metadata details are emitted in both invoke
@@ -2707,6 +2740,95 @@ class ChatModelIntegrationTests(ChatModelTests):
_ = model.bind_tools([random_image]).invoke(messages)
+ def test_pdf_tool_message(self, model: BaseChatModel) -> None:
+ """Test that the model can process ToolMessages with PDF inputs.
+
+ This test should be skipped if the model does not support messages of the
+ form:
+
+ .. code-block:: python
+
+ ToolMessage(
+ content=[
+ {
+ "type": "file",
+ "source_type": "base64",
+ "data": pdf_data,
+ "mime_type": "application/pdf",
+ },
+ ],
+ tool_call_id="1",
+ name="random_pdf",
+ )
+
+ containing PDF content blocks in standard format.
+
+ This test can be skipped by setting the ``supports_pdf_tool_message`` property
+ to False (see Configuration below).
+
+ .. dropdown:: Configuration
+
+ To disable this test, set ``supports_pdf_tool_message`` to False in your
+ test class:
+
+ .. code-block:: python
+
+ class TestMyChatModelIntegration(ChatModelIntegrationTests):
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ return False
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that the model can correctly handle messages
+ with PDF content blocks in ToolMessages, specifically base64-encoded
+ PDFs. Otherwise, set the ``supports_pdf_tool_message`` property to
+ False.
+
+ """
+ if not self.supports_pdf_tool_message:
+ pytest.skip("Model does not support PDF tool message.")
+
+ url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
+ pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")
+
+ tool_message = ToolMessage(
+ content=[
+ {
+ "type": "file",
+ "source_type": "base64",
+ "data": pdf_data,
+ "mime_type": "application/pdf",
+ },
+ ],
+ tool_call_id="1",
+ name="random_pdf",
+ )
+
+ messages = [
+ HumanMessage(
+ "Get a random PDF using the tool and relay the title verbatim."
+ ),
+ AIMessage(
+ [],
+ tool_calls=[
+ {
+ "type": "tool_call",
+ "id": "1",
+ "name": "random_pdf",
+ "args": {},
+ }
+ ],
+ ),
+ tool_message,
+ ]
+
+ def random_pdf() -> str:
+ """Return a random PDF."""
+ return ""
+
+ _ = model.bind_tools([random_pdf]).invoke(messages)
+
def test_anthropic_inputs(self, model: BaseChatModel) -> None:
"""Test that model can process Anthropic-style message histories.
diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
index bb5d7ce4af..a18a069ddf 100644
--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
@@ -231,6 +231,16 @@ class ChatModelTests(BaseStandardTests):
"""
return False
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ """Supports PDF ToolMessages.
+
+ (bool) whether the chat model supports ToolMessages that include PDF
+ content.
+
+ """
+ return False
+
@property
def enable_vcr_tests(self) -> bool:
"""(bool) whether to enable VCR tests for the chat model.
@@ -645,6 +655,39 @@ class ChatModelUnitTests(ChatModelTests):
def supports_image_tool_message(self) -> bool:
return False
+ .. dropdown:: supports_pdf_tool_message
+
+ Boolean property indicating whether the chat model supports ToolMessages
+ that include PDF content, i.e.,
+
+ .. code-block:: python
+
+ ToolMessage(
+ content=[
+ {
+ "type": "file",
+ "source_type": "base64",
+ "data": pdf_data,
+ "mime_type": "application/pdf",
+ },
+ ],
+ tool_call_id="1",
+ name="random_pdf",
+ )
+
+ (standard format).
+
+ If set to ``True``, the chat model will be tested with message sequences that
+ include ToolMessages of this form.
+
+ Example:
+
+ .. code-block:: python
+
+ @property
+ def supports_pdf_tool_message(self) -> bool:
+ return False
+
.. dropdown:: supported_usage_metadata_details
Property controlling what usage metadata details are emitted in both ``invoke``