Skip to content

The agent engineering platform.

standard-tests[patch]: use `has_structured_output` property to engage structured output tests (#29635)

ccurme9da06e6e94fe
diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
index f7371c6bcf..0b0c6a5404 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py
@@ -1202,20 +1202,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support
- tool calling (see Configuration below).
+ structured output (see Configuration below).
.. dropdown:: Configuration
- To disable tool calling tests, set ``has_tool_calling`` to False in your
- test class:
+ To disable structured output tests, set ``has_structured_output`` to False
+ in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
- def has_tool_calling(self) -> bool:
+ def has_structured_output(self) -> bool:
return False
+ By default, ``has_structured_output`` is True if a model overrides the
+ ``with_structured_output`` or ``bind_tools`` methods.
+
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@@ -1225,8 +1228,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501
- if not self.has_tool_calling:
- pytest.skip("Test requires tool calling.")
+ if not self.has_structured_output:
+ pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
chat = model.with_structured_output(schema, **self.structured_output_kwargs)
@@ -1277,20 +1280,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support
- tool calling (see Configuration below).
+ structured output (see Configuration below).
.. dropdown:: Configuration
- To disable tool calling tests, set ``has_tool_calling`` to False in your
- test class:
+ To disable structured output tests, set ``has_structured_output`` to False
+ in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
- def has_tool_calling(self) -> bool:
+ def has_structured_output(self) -> bool:
return False
+ By default, ``has_structured_output`` is True if a model overrides the
+ ``with_structured_output`` or ``bind_tools`` methods.
+
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@@ -1300,8 +1306,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501
- if not self.has_tool_calling:
- pytest.skip("Test requires tool calling.")
+ if not self.has_structured_output:
+ pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
@@ -1352,20 +1358,23 @@ class ChatModelIntegrationTests(ChatModelTests):
pydantic.v1.BaseModel is available in the pydantic 2 package.
This test is optional and should be skipped if the model does not support
- tool calling (see Configuration below).
+ structured output (see Configuration below).
.. dropdown:: Configuration
- To disable tool calling tests, set ``has_tool_calling`` to False in your
- test class:
+ To disable structured output tests, set ``has_structured_output`` to False
+ in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
- def has_tool_calling(self) -> bool:
+ def has_structured_output(self) -> bool:
return False
+ By default, ``has_structured_output`` is True if a model overrides the
+ ``with_structured_output`` or ``bind_tools`` methods.
+
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@@ -1375,8 +1384,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
"""
- if not self.has_tool_calling:
- pytest.skip("Test requires tool calling.")
+ if not self.has_structured_output:
+ pytest.skip("Test requires structured output.")
class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel
"""Joke to tell user."""
@@ -1410,20 +1419,23 @@ class ChatModelIntegrationTests(ChatModelTests):
parameters.
This test is optional and should be skipped if the model does not support
- tool calling (see Configuration below).
+ structured output (see Configuration below).
.. dropdown:: Configuration
- To disable tool calling tests, set ``has_tool_calling`` to False in your
- test class:
+ To disable structured output tests, set ``has_structured_output`` to False
+ in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
- def has_tool_calling(self) -> bool:
+ def has_structured_output(self) -> bool:
return False
+ By default, ``has_structured_output`` is True if a model overrides the
+ ``with_structured_output`` or ``bind_tools`` methods.
+
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@@ -1433,8 +1445,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
"""
- if not self.has_tool_calling:
- pytest.skip("Test requires tool calling.")
+ if not self.has_structured_output:
+ pytest.skip("Test requires structured output.")
# Pydantic
class Joke(BaseModel):
diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
index e2b58d2f05..65dea9079b 100644
--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py
@@ -133,7 +133,7 @@ class ChatModelTests(BaseStandardTests):
return (
self.chat_model_class.with_structured_output
is not BaseChatModel.with_structured_output
- )
+ ) or self.has_tool_calling
@property
def structured_output_kwargs(self) -> dict:
@@ -293,9 +293,9 @@ class ChatModelUnitTests(ChatModelTests):
Boolean property indicating whether the chat model supports structured
output.
- By default, this is determined by whether the chat model's
- ``with_structured_output`` method is overridden. If the base implementation is
- intended to be used, this method should be overridden.
+ By default, this is determined by whether the chat model overrides the
+ ``with_structured_output`` or ``bind_tools`` methods. If the base
+ implementations are intended to be used, this method should be overridden.
See: https://python.langchain.com/docs/concepts/structured_outputs/