The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(standard-tests): invocation model override (#34170)
Mason Daughertyb009ca4d2332
diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.pyindex 8c9c255103..f547ef3711 100644--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py@@ -880,6 +880,135 @@ class ChatModelIntegrationTests(ChatModelTests):assert len(full.content_blocks) == 1assert full.content_blocks[0]["type"] == "text"+ def test_invoke_with_model_override(self, model: BaseChatModel) -> None:+ """Test that model name can be overridden at invoke time via kwargs.++ This enables dynamic model selection without creating new instances,+ which is useful for fallback strategies, A/B testing, or cost optimization.++ Test is skipped if `supports_model_override` is `False`.++ ??? question "Troubleshooting"++ If this test fails, ensure that your `_generate` method passes+ `**kwargs` through to the API request payload in a way that allows+ the `model` parameter to be overridden.++ For example:+ ```python+ def _get_request_payload(self, ..., **kwargs) -> dict:+ return {+ "model": self.model,+ ...+ **kwargs, # kwargs should come last to allow overrides+ }+ ```+ """+ if not self.supports_model_override:+ pytest.skip("Model override not supported.")++ override_model = self.model_override_value+ if not override_model:+ pytest.skip("model_override_value not specified.")++ result = model.invoke("Hello", model=override_model)+ assert result is not None+ assert isinstance(result, AIMessage)++ # Verify the overridden model was used+ model_name = result.response_metadata.get("model_name")+ assert model_name is not None, "model_name not found in response_metadata"+ assert override_model in model_name, (+ f"Expected model '{override_model}' but got '{model_name}'"+ )++ async def test_ainvoke_with_model_override(self, model: BaseChatModel) -> None:+ """Test that model name can be overridden at ainvoke time via kwargs.++ Test is skipped if `supports_model_override` is `False`.++ ??? question "Troubleshooting"++ See troubleshooting for `test_invoke_with_model_override`.+ """+ if not self.supports_model_override:+ pytest.skip("Model override not supported.")++ override_model = self.model_override_value+ if not override_model:+ pytest.skip("model_override_value not specified.")++ result = await model.ainvoke("Hello", model=override_model)+ assert result is not None+ assert isinstance(result, AIMessage)++ # Verify the overridden model was used+ model_name = result.response_metadata.get("model_name")+ assert model_name is not None, "model_name not found in response_metadata"+ assert override_model in model_name, (+ f"Expected model '{override_model}' but got '{model_name}'"+ )++ def test_stream_with_model_override(self, model: BaseChatModel) -> None:+ """Test that model name can be overridden at stream time via kwargs.++ Test is skipped if `supports_model_override` is `False`.++ ??? question "Troubleshooting"++ See troubleshooting for `test_invoke_with_model_override`.+ """+ if not self.supports_model_override:+ pytest.skip("Model override not supported.")++ override_model = self.model_override_value+ if not override_model:+ pytest.skip("model_override_value not specified.")++ full: AIMessageChunk | None = None+ for chunk in model.stream("Hello", model=override_model):+ assert isinstance(chunk, AIMessageChunk)+ full = chunk if full is None else full + chunk++ assert full is not None++ # Verify the overridden model was used+ model_name = full.response_metadata.get("model_name")+ assert model_name is not None, "model_name not found in response_metadata"+ assert override_model in model_name, (+ f"Expected model '{override_model}' but got '{model_name}'"+ )++ async def test_astream_with_model_override(self, model: BaseChatModel) -> None:+ """Test that model name can be overridden at astream time via kwargs.++ Test is skipped if `supports_model_override` is `False`.++ ??? question "Troubleshooting"++ See troubleshooting for `test_invoke_with_model_override`.+ """+ if not self.supports_model_override:+ pytest.skip("Model override not supported.")++ override_model = self.model_override_value+ if not override_model:+ pytest.skip("model_override_value not specified.")++ full: AIMessageChunk | None = None+ async for chunk in model.astream("Hello", model=override_model):+ assert isinstance(chunk, AIMessageChunk)+ full = chunk if full is None else full + chunk++ assert full is not None++ # Verify the overridden model was used+ model_name = full.response_metadata.get("model_name")+ assert model_name is not None, "model_name not found in response_metadata"+ assert override_model in model_name, (+ f"Expected model '{override_model}' but got '{model_name}'"+ )+def test_batch(self, model: BaseChatModel) -> None:"""Test to verify that `model.batch([messages])` works.diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.pyindex e339e5b159..5460623bd9 100644--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py@@ -250,6 +250,28 @@ class ChatModelTests(BaseStandardTests):"""return {"invoke": [], "stream": []}+ @property+ def supports_model_override(self) -> bool:+ """Whether the model supports overriding the model name at runtime.++ Defaults to `True`.++ If `True`, the model accepts a `model` kwarg in `invoke()`, `stream()`,+ etc. that overrides the model specified at initialization.++ This enables dynamic model selection without creating new instances.+ """+ return True++ @property+ def model_override_value(self) -> str | None:+ """Alternative model name to use when testing model override.++ Should return a valid model name that differs from the default model.+ Required if `supports_model_override` is `True`.+ """+ return None+class ChatModelUnitTests(ChatModelTests):'''Base class for chat model unit tests.@@ -672,6 +694,36 @@ class ChatModelUnitTests(ChatModelTests):Only needs to be overridden if these details are supplied.+ ??? info "`supports_model_override`"++ Boolean property indicating whether the chat model supports overriding the+ model name at runtime via kwargs.++ If `True`, the model accepts a `model` kwarg in `invoke()`, `stream()`, etc.+ that overrides the model specified at initialization. This enables dynamic+ model selection without creating new chat model instances.++ Defaults to `False`.++ ```python+ @property+ def supports_model_override(self) -> bool:+ return True+ ```++ ??? info "`model_override_value`"++ Alternative model name to use when testing model override.++ Should return a valid model name that differs from the default model.+ Required if `supports_model_override` is `True`.++ ```python+ @property+ def model_override_value(self) -> str:+ return "gpt-4o-mini" # e.g. if default is "gpt-4o"+ ```+??? info "`enable_vcr_tests`"Property controlling whether to enable select tests that rely on