The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
standard-tests: show right classes in api docs (#28591)
Erick Friis9e2abcd15216
diff --git a/libs/standard-tests/langchain_tests/integration_tests/base_store.py b/libs/standard-tests/langchain_tests/integration_tests/base_store.pyindex cc5fab8bcf..ff9e178aea 100644--- a/libs/standard-tests/langchain_tests/integration_tests/base_store.py+++ b/libs/standard-tests/langchain_tests/integration_tests/base_store.py@@ -1,3 +1,11 @@+"""+Standard tests for the BaseStore abstraction++We don't recommend implementing externally managed BaseStore abstractions at this time.++:private:+"""+from abc import abstractmethodfrom typing import AsyncGenerator, Generator, Generic, Tuple, TypeVardiff --git a/libs/standard-tests/langchain_tests/integration_tests/cache.py b/libs/standard-tests/langchain_tests/integration_tests/cache.pyindex 7087da8ea0..3d04731d16 100644--- a/libs/standard-tests/langchain_tests/integration_tests/cache.py+++ b/libs/standard-tests/langchain_tests/integration_tests/cache.py@@ -1,3 +1,11 @@+"""+Standard tests for the BaseCache abstraction++We don't recommend implementing externally managed BaseCache abstractions at this time.++:private:+"""+from abc import abstractmethodimport pytestdiff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.pyindex 4e6a811c45..0dd5334cc4 100644--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py@@ -16,7 +16,7 @@ from langchain_core.messages import ()from langchain_core.output_parsers import StrOutputParserfrom langchain_core.prompts import ChatPromptTemplate-from langchain_core.tools import tool+from langchain_core.tools import BaseTool, toolfrom langchain_core.utils.function_calling import tool_example_to_messagesfrom pydantic import BaseModel, Fieldfrom pydantic.v1 import BaseModel as BaseModelV1@@ -24,16 +24,29 @@ from pydantic.v1 import Field as FieldV1from langchain_tests.unit_tests.chat_models import (ChatModelTests,- my_adder_tool,)from langchain_tests.utils.pydantic import PYDANTIC_MAJOR_VERSION-class MagicFunctionSchema(BaseModel):+def _get_joke_class() -> type[BaseModel]:+ """+ :private:+ """++ class Joke(BaseModel):+ """Joke to tell user."""++ setup: str = Field(description="question to set up a joke")+ punchline: str = Field(description="answer to resolve the joke")++ return Joke+++class _MagicFunctionSchema(BaseModel):input: int = Field(..., gt=-1000, lt=1000)-@tool(args_schema=MagicFunctionSchema)+@tool(args_schema=_MagicFunctionSchema)def magic_function(input: int) -> int:"""Applies a magic function to an input."""return input + 2@@ -45,13 +58,6 @@ def magic_function_no_args() -> int:return 5-class Joke(BaseModel):- """Joke to tell user."""-- setup: str = Field(description="question to set up a joke")- punchline: str = Field(description="answer to resolve the joke")--def _validate_tool_call_message(message: BaseMessage) -> None:assert isinstance(message, AIMessage)assert len(message.tool_calls) == 1@@ -103,12 +109,201 @@ class ChatModelIntegrationTests(ChatModelTests):.. note::API references for individual test methods include troubleshooting tips.- .. note::- Test subclasses can control what features are tested (such as tool- calling or multi-modality) by selectively overriding the properties on the- class. Relevant properties are mentioned in the references for each method.- See this page for detail on all properties:- https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html++ Test subclasses must implement the following two properties:++ chat_model_class+ The chat model class to test, e.g., ``ChatParrotLink``.++ Example:++ .. code-block:: python++ @property+ def chat_model_class(self) -> Type[ChatParrotLink]:+ return ChatParrotLink++ chat_model_params+ Initialization parameters for the chat model.++ Example:++ .. code-block:: python++ @property+ def chat_model_params(self) -> dict:+ return {"model": "bird-brain-001", "temperature": 0}++ In addition, test subclasses can control what features are tested (such as tool+ calling or multi-modality) by selectively overriding the following properties.+ Expand to see details:++ .. dropdown:: has_tool_calling++ Boolean property indicating whether the chat model supports tool calling.++ By default, this is determined by whether the chat model's `bind_tools` method+ is overridden. It typically does not need to be overridden on the test class.++ .. dropdown:: tool_choice_value++ Value to use for tool choice when used in tests.++ Some tests for tool calling features attempt to force tool calling via a+ `tool_choice` parameter. A common value for this parameter is "any". Defaults+ to `None`.++ Note: if the value is set to "tool_name", the name of the tool used in each+ test will be set as the value for `tool_choice`.++ Example:++ .. code-block:: python++ @property+ def tool_choice_value(self) -> Optional[str]:+ return "any"++ .. dropdown:: has_structured_output++ Boolean property indicating whether the chat model supports structured+ output.++ By default, this is determined by whether the chat model's+ `with_structured_output` method is overridden. If the base implementation is+ intended to be used, this method should be overridden.++ See: https://python.langchain.com/docs/concepts/structured_outputs/++ Example:++ .. code-block:: python++ @property+ def has_structured_output(self) -> bool:+ return True++ .. dropdown:: supports_image_inputs++ Boolean property indicating whether the chat model supports image inputs.+ Defaults to ``False``.++ If set to ``True``, the chat model will be tested using content blocks of the+ form++ .. code-block:: python++ [+ {"type": "text", "text": "describe the weather in this image"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},+ },+ ]++ See https://python.langchain.com/docs/concepts/multimodality/++ Example:++ .. code-block:: python++ @property+ def supports_image_inputs(self) -> bool:+ return True++ .. dropdown:: supports_video_inputs++ Boolean property indicating whether the chat model supports image inputs.+ Defaults to ``False``. No current tests are written for this feature.++ .. dropdown:: returns_usage_metadata++ Boolean property indicating whether the chat model returns usage metadata+ on invoke and streaming responses.++ ``usage_metadata`` is an optional dict attribute on AIMessages that track input+ and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html++ Example:++ .. code-block:: python++ @property+ def returns_usage_metadata(self) -> bool:+ return False++ .. dropdown:: supports_anthropic_inputs++ Boolean property indicating whether the chat model supports Anthropic-style+ inputs.++ These inputs might feature "tool use" and "tool result" content blocks, e.g.,++ .. code-block:: python++ [+ {"type": "text", "text": "Hmm let me think about that"},+ {+ "type": "tool_use",+ "input": {"fav_color": "green"},+ "id": "foo",+ "name": "color_picker",+ },+ ]++ If set to ``True``, the chat model will be tested using content blocks of this+ form.++ Example:++ .. code-block:: python++ @property+ def supports_anthropic_inputs(self) -> bool:+ return False++ .. dropdown:: supports_image_tool_message++ Boolean property indicating whether the chat model supports ToolMessages+ that include image content, e.g.,++ .. code-block:: python++ ToolMessage(+ content=[+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},+ },+ ],+ tool_call_id="1",+ name="random_image",+ )++ If set to ``True``, the chat model will be tested with message sequences that+ include ToolMessages of this form.++ Example:++ .. code-block:: python++ @property+ def supports_image_tool_message(self) -> bool:+ return False++ .. dropdown:: supported_usage_metadata_details++ Property controlling what usage metadata details are emitted in both invoke+ and stream.++ ``usage_metadata`` is an optional dict attribute on AIMessages that track input+ and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html++ It includes optional keys ``input_token_details`` and ``output_token_details``+ that can track usage details associated with special types of tokens, such as+ cached, audio, or reasoning.++ Only needs to be overridden if these details are supplied."""@property@@ -908,6 +1103,7 @@ class ChatModelIntegrationTests(ChatModelTests):if not self.has_tool_calling:pytest.skip("Test requires tool calling.")+ Joke = _get_joke_class()# Pydantic class# Type ignoring since the interface only officially supports pydantic 1# or pydantic.v1.BaseModel but not pydantic.BaseModel from pydantic 2.@@ -960,6 +1156,8 @@ class ChatModelIntegrationTests(ChatModelTests):if not self.has_tool_calling:pytest.skip("Test requires tool calling.")+ Joke = _get_joke_class()+# Pydantic class# Type ignoring since the interface only officially supports pydantic 1# or pydantic.v1.BaseModel but not pydantic.BaseModel from pydantic 2.@@ -1089,7 +1287,9 @@ class ChatModelIntegrationTests(ChatModelTests):joke_result = chat.invoke("Give me a joke about cats, include the punchline.")assert isinstance(joke_result, Joke)- def test_tool_message_histories_string_content(self, model: BaseChatModel) -> None:+ def test_tool_message_histories_string_content(+ self, model: BaseChatModel, my_adder_tool: BaseTool+ ) -> None:"""Test that message histories are compatible with string tool contents(e.g. OpenAI format). If a model passes this test, it should be compatiblewith messages generated from providers following OpenAI format.@@ -1158,6 +1358,7 @@ class ChatModelIntegrationTests(ChatModelTests):def test_tool_message_histories_list_content(self,model: BaseChatModel,+ my_adder_tool: BaseTool,) -> None:"""Test that message histories are compatible with list tool contents(e.g. Anthropic format).@@ -1246,7 +1447,9 @@ class ChatModelIntegrationTests(ChatModelTests):result_list_content = model_with_tools.invoke(messages_list_content)assert isinstance(result_list_content, AIMessage)- def test_structured_few_shot_examples(self, model: BaseChatModel) -> None:+ def test_structured_few_shot_examples(+ self, model: BaseChatModel, my_adder_tool: BaseTool+ ) -> None:"""Test that the model can process few-shot examples with tool calls.These are represented as a sequence of messages of the following form:@@ -1557,7 +1760,9 @@ class ChatModelIntegrationTests(ChatModelTests):]model.bind_tools([color_picker]).invoke(messages)- def test_tool_message_error_status(self, model: BaseChatModel) -> None:+ def test_tool_message_error_status(+ self, model: BaseChatModel, my_adder_tool: BaseTool+ ) -> None:"""Test that ToolMessage with ``status="error"`` can be handled.These messages may take the form:diff --git a/libs/standard-tests/langchain_tests/integration_tests/indexer.py b/libs/standard-tests/langchain_tests/integration_tests/indexer.pyindex f1e5d9eee0..bdc0fc2e6b 100644--- a/libs/standard-tests/langchain_tests/integration_tests/indexer.py+++ b/libs/standard-tests/langchain_tests/integration_tests/indexer.py@@ -1,4 +1,12 @@-"""Test suite to check index implementations."""+"""Test suite to check index implementations.++Standard tests for the DocumentIndex abstraction++We don't recommend implementing externally managed DocumentIndex abstractions at this+time.++:private:+"""import inspectimport uuiddiff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.pyindex 71f3dd2169..19225c1f2a 100644--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py@@ -11,7 +11,7 @@ import pytestfrom langchain_core.language_models import BaseChatModelfrom langchain_core.load import dumpd, loadfrom langchain_core.runnables import RunnableBinding-from langchain_core.tools import tool+from langchain_core.tools import BaseTool, toolfrom pydantic import BaseModel, Field, SecretStrfrom pydantic.v1 import (BaseModel as BaseModelV1,@@ -28,15 +28,12 @@ from langchain_tests.base import BaseStandardTestsfrom langchain_tests.utils.pydantic import PYDANTIC_MAJOR_VERSION-class Person(BaseModel): # Used by some dependent tests. Should be deprecated.- """Record attributes of a person."""-- name: str = Field(..., description="The name of the person.")- age: int = Field(..., description="The age of the person.")--def generate_schema_pydantic_v1_from_2() -> Any:- """Use to generate a schema from v1 namespace in pydantic 2."""+ """+ Use to generate a schema from v1 namespace in pydantic 2.++ :private:+ """if PYDANTIC_MAJOR_VERSION != 2:raise AssertionError("This function is only compatible with Pydantic v2.")@@ -50,7 +47,11 @@ def generate_schema_pydantic_v1_from_2() -> Any:def generate_schema_pydantic() -> Any:- """Works with either pydantic 1 or 2"""+ """+ Works with either pydantic 1 or 2++ :private:+ """class PersonA(BaseModel):"""Record attributes of a person."""@@ -67,19 +68,152 @@ if PYDANTIC_MAJOR_VERSION == 2:TEST_PYDANTIC_MODELS.append(generate_schema_pydantic_v1_from_2())-@tool-def my_adder_tool(a: int, b: int) -> int:- """Takes two integers, a and b, and returns their sum."""- return a + b+class ChatModelTests(BaseStandardTests):+ """Base class for chat model tests.+ :private:+ """ # noqa: E501-def my_adder(a: int, b: int) -> int:- """Takes two integers, a and b, and returns their sum."""- return a + b+ @property+ @abstractmethod+ def chat_model_class(self) -> Type[BaseChatModel]:+ """The chat model class to test, e.g., `ChatParrotLink`."""+ ...+ @property+ def chat_model_params(self) -> dict:+ """Initialization parameters for the chat mobdel."""+ return {}++ @property+ def standard_chat_model_params(self) -> dict:+ """:meta private:"""+ return {+ "temperature": 0,+ "max_tokens": 100,+ "timeout": 60,+ "stop": [],+ "max_retries": 2,+ }++ @pytest.fixture+ def model(self) -> BaseChatModel:+ """Fixture that returns an instance of the chat model. Should not be+ overridden."""+ return self.chat_model_class(+ **{**self.standard_chat_model_params, **self.chat_model_params}+ )++ @pytest.fixture+ def my_adder_tool(self) -> BaseTool:+ @tool+ def my_adder_tool(a: int, b: int) -> int:+ """Takes two integers, a and b, and returns their sum."""+ return a + b++ return my_adder_tool++ @property+ def has_tool_calling(self) -> bool:+ """Boolean property indicating whether the model supports tool calling."""+ return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools++ @property+ def tool_choice_value(self) -> Optional[str]:+ """Value to use for tool choice when used in tests."""+ return None++ @property+ def has_structured_output(self) -> bool:+ """Boolean property indicating whether the chat model supports structured+ output."""+ return (+ self.chat_model_class.with_structured_output+ is not BaseChatModel.with_structured_output+ )++ @property+ def supports_image_inputs(self) -> bool:+ """Boolean property indicating whether the chat model supports image inputs.+ Defaults to ``False``."""+ return False++ @property+ def supports_video_inputs(self) -> bool:+ """Boolean property indicating whether the chat model supports image inputs.+ Defaults to ``False``. No current tests are written for this feature."""+ return False++ @property+ def returns_usage_metadata(self) -> bool:+ """Boolean property indicating whether the chat model returns usage metadata+ on invoke and streaming responses."""+ return True++ @property+ def supports_anthropic_inputs(self) -> bool:+ """Boolean property indicating whether the chat model supports Anthropic-style+ inputs."""+ return False++ @property+ def supports_image_tool_message(self) -> bool:+ """Boolean property indicating whether the chat model supports ToolMessages+ that include image content."""+ return False++ @property+ def supported_usage_metadata_details(+ self,+ ) -> Dict[+ Literal["invoke", "stream"],+ List[+ Literal[+ "audio_input",+ "audio_output",+ "reasoning_output",+ "cache_read_input",+ "cache_creation_input",+ ]+ ],+ ]:+ """Property controlling what usage metadata details are emitted in both invoke+ and stream. Only needs to be overridden if these details are returned by the+ model."""+ return {"invoke": [], "stream": []}+++class ChatModelUnitTests(ChatModelTests):+ """Base class for chat model unit tests.++ Test subclasses must implement the ``chat_model_class`` and+ ``chat_model_params`` properties to specify what model to test and its+ initialization parameters.++ Example:++ .. code-block:: python++ from typing import Type++ from langchain_tests.unit_tests import ChatModelUnitTests+ from my_package.chat_models import MyChatModel+++ class TestMyChatModelUnit(ChatModelUnitTests):+ @property+ def chat_model_class(self) -> Type[MyChatModel]:+ # Return the chat model class to test here+ return MyChatModel++ @property+ def chat_model_params(self) -> dict:+ # Return initialization parameters for the model.+ return {"model": "model-001", "temperature": 0}++ .. note::+ API references for individual test methods include troubleshooting tips.-class ChatModelTests(BaseStandardTests):- """Base class for chat model tests.Test subclasses must implement the following two properties:@@ -275,146 +409,6 @@ class ChatModelTests(BaseStandardTests):cached, audio, or reasoning.Only needs to be overridden if these details are supplied.- """ # noqa: E501-- @property- @abstractmethod- def chat_model_class(self) -> Type[BaseChatModel]:- """The chat model class to test, e.g., `ChatParrotLink`."""- ...-- @property- def chat_model_params(self) -> dict:- """Initialization parameters for the chat mobdel."""- return {}-- @property- def standard_chat_model_params(self) -> dict:- """:meta private:"""- return {- "temperature": 0,- "max_tokens": 100,- "timeout": 60,- "stop": [],- "max_retries": 2,- }-- @pytest.fixture- def model(self) -> BaseChatModel:- """Fixture that returns an instance of the chat model. Should not be- overridden."""- return self.chat_model_class(- **{**self.standard_chat_model_params, **self.chat_model_params}- )-- @property- def has_tool_calling(self) -> bool:- """Boolean property indicating whether the model supports tool calling."""- return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools-- @property- def tool_choice_value(self) -> Optional[str]:- """Value to use for tool choice when used in tests."""- return None-- @property- def has_structured_output(self) -> bool:- """Boolean property indicating whether the chat model supports structured- output."""- return (- self.chat_model_class.with_structured_output- is not BaseChatModel.with_structured_output- )-- @property- def supports_image_inputs(self) -> bool:- """Boolean property indicating whether the chat model supports image inputs.- Defaults to ``False``."""- return False-- @property- def supports_video_inputs(self) -> bool:- """Boolean property indicating whether the chat model supports image inputs.- Defaults to ``False``. No current tests are written for this feature."""- return False-- @property- def returns_usage_metadata(self) -> bool:- """Boolean property indicating whether the chat model returns usage metadata- on invoke and streaming responses."""- return True-- @property- def supports_anthropic_inputs(self) -> bool:- """Boolean property indicating whether the chat model supports Anthropic-style- inputs."""- return False-- @property- def supports_image_tool_message(self) -> bool:- """Boolean property indicating whether the chat model supports ToolMessages- that include image content."""- return False-- @property- def supported_usage_metadata_details(- self,- ) -> Dict[- Literal["invoke", "stream"],- List[- Literal[- "audio_input",- "audio_output",- "reasoning_output",- "cache_read_input",- "cache_creation_input",- ]- ],- ]:- """Property controlling what usage metadata details are emitted in both invoke- and stream. Only needs to be overridden if these details are returned by the- model."""- return {"invoke": [], "stream": []}---class ChatModelUnitTests(ChatModelTests):- """Base class for chat model unit tests.-- Test subclasses must implement the ``chat_model_class`` and- ``chat_model_params`` properties to specify what model to test and its- initialization parameters.-- Example:-- .. code-block:: python-- from typing import Type-- from langchain_tests.unit_tests import ChatModelUnitTests- from my_package.chat_models import MyChatModel--- class TestMyChatModelUnit(ChatModelUnitTests):- @property- def chat_model_class(self) -> Type[MyChatModel]:- # Return the chat model class to test here- return MyChatModel-- @property- def chat_model_params(self) -> dict:- # Return initialization parameters for the model.- return {"model": "model-001", "temperature": 0}-- .. note::- API references for individual test methods include troubleshooting tips.-- .. note::- Test subclasses can control what features are tested (such as tool- calling or multi-modality) by selectively overriding the properties on the- class. Relevant properties are mentioned in the references for each method.- See this page for detail on all properties:- https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html-Testing initialization from environment variablesSome unit tests may require testing initialization from environment variables.@@ -526,6 +520,7 @@ class ChatModelUnitTests(ChatModelTests):def test_bind_tool_pydantic(self,model: BaseChatModel,+ my_adder_tool: BaseTool,) -> None:"""Test that chat model correctly handles Pydantic models that are passedinto ``bind_tools``. Test is skipped if the ``has_tool_calling`` property@@ -542,6 +537,10 @@ class ChatModelUnitTests(ChatModelTests):if not self.has_tool_calling:return+ def my_adder(a: int, b: int) -> int:+ """Takes two integers, a and b, and returns their sum."""+ return a + b+tools = [my_adder_tool, my_adder]for pydantic_model in TEST_PYDANTIC_MODELS:diff --git a/libs/standard-tests/langchain_tests/unit_tests/embeddings.py b/libs/standard-tests/langchain_tests/unit_tests/embeddings.pyindex c580243b16..527be61e8b 100644--- a/libs/standard-tests/langchain_tests/unit_tests/embeddings.py+++ b/libs/standard-tests/langchain_tests/unit_tests/embeddings.py@@ -11,6 +11,10 @@ from langchain_tests.base import BaseStandardTestsclass EmbeddingsTests(BaseStandardTests):+ """+ :private:+ """+@property@abstractmethoddef embeddings_class(self) -> Type[Embeddings]: ...