The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import logging
from itertools import chain
from typing import Dict, Final, List, Optional, Union
from unittest.mock import Mock
import pytest
import litellm
from litellm import Router
from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router_utils.common_utils import (
_deployment_supports_web_search,
add_model_file_id_mappings,
filter_team_based_models,
filter_web_search_deployments,
format_fallback_outcome_message,
provider_for_generic_call,
resolve_model_group_alias,
resolve_served_model,
truncate_fallback_error_detail,
PROVIDER_SCOPED_CREDENTIAL_PARAMS,
warn_on_provider_credential_mismatch,
)
from litellm.types.router import Deployment, FallbackSkipReason, LiteLLM_Params, ModelInfo, SkippedFallbackTarget
from litellm.types.utils import ProviderSpecificModelInfo
class TestFilterTeamBasedModels:
"""Test cases for filter_team_based_models function"""
@pytest.fixture
def sample_deployments_with_teams(self) -> List[Dict]:
"""Sample deployments where some have team_id and some don't"""
return [
{"model_info": {"id": "deployment-1", "team_id": "team-a"}},
{"model_info": {"id": "deployment-2", "team_id": "team-b"}},
{
"model_info": {
"id": "deployment-3"
# No team_id - should always be included
}
},
{"model_info": {"id": "deployment-4", "team_id": "team-a"}},
]
@pytest.fixture
def sample_deployments_no_teams(self) -> List[Dict]:
"""Sample deployments with no team_id restrictions"""
return [
{"model_info": {"id": "deployment-1"}},
{"model_info": {"id": "deployment-2"}},
]
def test_filter_team_based_models_none_request_kwargs(
self, sample_deployments_with_teams
):
"""Test that when request_kwargs is None, all deployments are returned unchanged"""
result = filter_team_based_models(sample_deployments_with_teams, None)
assert result == sample_deployments_with_teams
def test_filter_team_based_models_empty_request_kwargs(
self, sample_deployments_with_teams
):
"""Test with empty request_kwargs"""
result = filter_team_based_models(sample_deployments_with_teams, {})
# Should include all deployments since no team_id in request
assert len(result) == 1
def test_filter_team_based_models_no_metadata(self, sample_deployments_with_teams):
"""Test with request_kwargs that has no metadata"""
request_kwargs = {"some_other_key": "value"}
result = filter_team_based_models(sample_deployments_with_teams, request_kwargs)
# Should include only non-team based deployments
assert len(result) == 1
def test_filter_team_based_models_team_match_metadata(
self, sample_deployments_with_teams
):
"""Test filtering when team_id is in metadata"""
request_kwargs = {"metadata": {"user_api_key_team_id": "team-a"}}
result = filter_team_based_models(sample_deployments_with_teams, request_kwargs)
# Should include:
# - deployment-1 (team-a matches)
# - deployment-3 (no team_id restriction)
# - deployment-4 (team-a matches)
# Should exclude:
# - deployment-2 (team-b doesn't match)
expected_ids = ["deployment-1", "deployment-3", "deployment-4"]
result_ids = [d.get("model_info", {}).get("id") for d in result]
assert sorted(result_ids) == sorted(expected_ids)
def test_filter_team_based_models_team_match_litellm_metadata(
self, sample_deployments_with_teams
):
"""Test filtering when team_id is in litellm_metadata"""
request_kwargs = {"litellm_metadata": {"user_api_key_team_id": "team-b"}}
result = filter_team_based_models(sample_deployments_with_teams, request_kwargs)
# Should include:
# - deployment-2 (team-b matches)
# - deployment-3 (no team_id restriction)
# Should exclude:
# - deployment-1 (team-a doesn't match)
# - deployment-4 (team-a doesn't match)
expected_ids = ["deployment-2", "deployment-3"]
result_ids = [d.get("model_info", {}).get("id") for d in result]
assert sorted(result_ids) == sorted(expected_ids)
def test_filter_team_based_models_priority_metadata_over_litellm(
self, sample_deployments_with_teams
):
"""Test that metadata.user_api_key_team_id takes priority over litellm_metadata.user_api_key_team_id"""
request_kwargs = {
"metadata": {
"user_api_key_team_id": "team-a", # This should take priority
"litellm_metadata": {"user_api_key_team_id": "team-b"},
}
}
result = filter_team_based_models(sample_deployments_with_teams, request_kwargs)
# Should filter based on team-a (from metadata, not litellm_metadata)
expected_ids = ["deployment-1", "deployment-3", "deployment-4"]
result_ids = [d.get("model_info", {}).get("id") for d in result]
assert sorted(result_ids) == sorted(expected_ids)
def test_filter_team_based_models_no_matching_team(
self, sample_deployments_with_teams
):
"""Test when request team doesn't match any deployment teams"""
request_kwargs = {"metadata": {"user_api_key_team_id": "team-nonexistent"}}
result = filter_team_based_models(sample_deployments_with_teams, request_kwargs)
# Should only include deployment-3 (no team_id restriction)
expected_ids = ["deployment-3"]
result_ids = [d.get("model_info", {}).get("id") for d in result]
assert result_ids == expected_ids
def test_filter_team_based_models_no_team_restrictions(
self, sample_deployments_no_teams
):
"""Test with deployments that have no team restrictions"""
request_kwargs = {"metadata": {"user_api_key_team_id": "any-team"}}
result = filter_team_based_models(sample_deployments_no_teams, request_kwargs)
# Should include all deployments since none have team_id restrictions
assert result == sample_deployments_no_teams
def test_filter_team_based_models_missing_model_info(self):
"""Test with deployments missing model_info"""
deployments = [
{"model_info": {"id": "deployment-1", "team_id": "team-a"}},
{
# Missing model_info entirely
},
{
"model_info": {
# Missing id
"team_id": "team-b"
}
},
]
request_kwargs = {"metadata": {"user_api_key_team_id": "team-a"}}
result = filter_team_based_models(deployments, request_kwargs)
# Should handle missing model_info gracefully
# deployment-1 should be included (team matches)
# others should be included since they don't have proper team_id setup
assert len(result) >= 1 # At least deployment-1 should be included
def test_filter_team_based_models_dict_input(self):
"""Test with Dict input instead of List[Dict]"""
# Note: Based on the function signature, it accepts Union[List[Dict], Dict]
# But the implementation seems to expect List[Dict] for the filtering logic
# This test documents the current behavior
deployments_dict = {"key1": "value1", "key2": "value2"}
request_kwargs = {"metadata": {"user_api_key_team_id": "team-a"}}
# This should not crash, though the filtering logic won't apply to Dict input
result = filter_team_based_models(deployments_dict, request_kwargs)
# The function will likely return the dict unchanged or handle it differently
assert result is not None
def test_filter_team_based_models_empty_deployments(self):
"""Test with empty deployments list"""
result = filter_team_based_models(
[], {"metadata": {"user_api_key_team_id": "team-a"}}
)
assert result == []
def test_filter_team_based_models_none_team_id_in_deployment(self):
"""Test with explicit None team_id in deployment"""
deployments = [
{"model_info": {"id": "deployment-1", "team_id": None}},
{"model_info": {"id": "deployment-2", "team_id": "team-a"}},
]
request_kwargs = {"metadata": {"user_api_key_team_id": "team-a"}}
result = filter_team_based_models(deployments, request_kwargs)
# Both should be included:
# - deployment-1 (None team_id is treated as no restriction)
# - deployment-2 (team matches)
expected_ids = ["deployment-1", "deployment-2"]
result_ids = [d.get("model_info", {}).get("id") for d in result]
assert sorted(result_ids) == sorted(expected_ids)
class TestDeploymentSupportsWebSearch:
"""Test cases for _deployment_supports_web_search helper function"""
def test_model_info_true(self):
"""model_info.supports_web_search=True returns True"""
deployment = {"model_info": {"supports_web_search": True}}
assert _deployment_supports_web_search(deployment) is True
def test_model_info_false(self):
"""model_info.supports_web_search=False returns False"""
deployment = {"model_info": {"supports_web_search": False}}
assert _deployment_supports_web_search(deployment) is False
def test_no_config_defaults_to_true(self):
"""When no supports_web_search in config, default to True"""
deployment = {"litellm_params": {"model": "gpt-4"}, "model_info": {"id": "123"}}
assert _deployment_supports_web_search(deployment) is True
def test_empty_deployment_defaults_to_true(self):
"""Empty deployment defaults to True"""
assert _deployment_supports_web_search({}) is True
def test_missing_model_info_defaults_to_true(self):
"""When model_info missing, default to True"""
deployment = {"litellm_params": {"model": "gpt-4"}}
assert _deployment_supports_web_search(deployment) is True
class TestFilterWebSearchDeployments:
"""Test cases for filter_web_search_deployments function"""
@pytest.fixture
def sample_deployments(self) -> List[Dict]:
"""Sample deployments with varying web search support"""
return [
{"model_info": {"id": "deployment-1"}}, # default True
{"model_info": {"id": "deployment-2", "supports_web_search": True}},
{"model_info": {"id": "deployment-3", "supports_web_search": False}},
]
def test_no_request_kwargs_returns_all(self, sample_deployments):
"""When request_kwargs is None, return all deployments"""
result = filter_web_search_deployments(sample_deployments, None)
assert result == sample_deployments
def test_no_tools_returns_all(self, sample_deployments):
"""When no tools in request, return all deployments"""
result = filter_web_search_deployments(sample_deployments, {"other": "value"})
assert result == sample_deployments
def test_empty_tools_returns_all(self, sample_deployments):
"""When tools list is empty, return all deployments"""
result = filter_web_search_deployments(sample_deployments, {"tools": []})
assert result == sample_deployments
def test_none_tools_returns_all(self, sample_deployments):
"""When tools is explicitly None, return all deployments (regression test for #17672)"""
result = filter_web_search_deployments(sample_deployments, {"tools": None})
assert result == sample_deployments
def test_non_web_search_tools_returns_all(self, sample_deployments):
"""When tools don't include web_search, return all deployments"""
request_kwargs = {"tools": [{"type": "function", "function": {}}]}
result = filter_web_search_deployments(sample_deployments, request_kwargs)
assert result == sample_deployments
def test_web_search_filters_unsupported(self, sample_deployments):
"""When web_search tool present, filter out deployments that don't support it"""
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments(sample_deployments, request_kwargs)
# Should exclude deployment-3 (supports_web_search=False)
assert len(result) == 2
result_ids = [d["model_info"]["id"] for d in result]
assert "deployment-1" in result_ids
assert "deployment-2" in result_ids
assert "deployment-3" not in result_ids
def test_web_search_in_a_tuple_filters_unsupported(self, sample_deployments: list[dict[str, object]]) -> None:
result: Final = filter_web_search_deployments(sample_deployments, {"tools": ({"type": "web_search"},)})
assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"]
@pytest.mark.parametrize("tool_type", [{}, [], None, 5], ids=["object", "list", "null", "int"])
def test_a_non_string_tool_type_is_no_web_search_request(
self, sample_deployments: list[dict[str, object]], tool_type: object
) -> None:
result: Final = filter_web_search_deployments(
sample_deployments, {"tools": [{"type": tool_type}, {"type": "function"}]}
)
assert result == sample_deployments
def test_web_search_preview_filters_unsupported(self, sample_deployments):
"""web_search_preview type should also trigger filtering"""
request_kwargs = {"tools": [{"type": "web_search_preview"}]}
result = filter_web_search_deployments(sample_deployments, request_kwargs)
assert len(result) == 2
result_ids = [d["model_info"]["id"] for d in result]
assert "deployment-3" not in result_ids
def test_web_search_with_other_tools(self, sample_deployments):
"""Web search filtering works when mixed with other tools"""
request_kwargs = {
"tools": [
{"type": "function", "function": {"name": "get_weather"}},
{"type": "web_search"},
]
}
result = filter_web_search_deployments(sample_deployments, request_kwargs)
assert len(result) == 2
result_ids = [d["model_info"]["id"] for d in result]
assert "deployment-3" not in result_ids
def test_all_deployments_support_web_search(self):
"""When all deployments support web search, none are filtered"""
deployments = [
{"model_info": {"id": "d1", "supports_web_search": True}},
{"model_info": {"id": "d2", "supports_web_search": True}},
]
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments(deployments, request_kwargs)
assert len(result) == 2
def test_no_deployments_support_web_search(self):
"""When no deployments support web search, all are filtered out"""
deployments = [
{"model_info": {"id": "d1", "supports_web_search": False}},
{"model_info": {"id": "d2", "supports_web_search": False}},
]
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments(deployments, request_kwargs)
assert len(result) == 0
def test_missing_config_defaults_to_supported(self):
"""Deployments without supports_web_search config default to True"""
deployments = [
{"model_info": {"id": "d1"}}, # No supports_web_search - defaults to True
{"model_info": {"id": "d2"}}, # No supports_web_search - defaults to True
{
"model_info": {"id": "d3", "supports_web_search": False}
}, # Explicit False
]
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments(deployments, request_kwargs)
# d1 and d2 should be included (default True), d3 excluded (explicit False)
assert len(result) == 2
result_ids = [d["model_info"]["id"] for d in result]
assert "d1" in result_ids
assert "d2" in result_ids
assert "d3" not in result_ids
def test_empty_deployments_list(self):
"""Empty deployments list returns empty list"""
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments([], request_kwargs)
assert result == []
def test_dict_deployment_passthrough(self):
"""When deployment is a dict (single deployment), pass through unchanged"""
deployment = {"model_info": {"id": "d1", "supports_web_search": False}}
request_kwargs = {"tools": [{"type": "web_search"}]}
result = filter_web_search_deployments(deployment, request_kwargs)
# Should return the dict unchanged, not filter it
assert result == deployment
def test_invalidate_model_group_info_cache():
"""Test that _invalidate_model_group_info_cache clears the LRU cache."""
router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4", "api_key": "fake-key"},
}
]
)
# Populate the cache
router._cached_get_model_group_info("gpt-4")
assert router._cached_get_model_group_info.cache_info().currsize > 0
# Invalidate and verify cache is cleared
router._invalidate_model_group_info_cache()
assert router._cached_get_model_group_info.cache_info().currsize == 0
def test_filter_deployments_by_model_access_groups_access_group_only_key():
"""
Access-group-only keys should only route to deployments in allowed groups,
even when multiple deployments share the same public model name.
"""
router = Router(
model_list=[
{
"model_name": "gpt-5",
"litellm_params": {"model": "openai/gpt-5.1", "api_key": "key-1"},
"model_info": {"access_groups": ["AG1"]},
},
{
"model_name": "gpt-5",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "key-2"},
"model_info": {"access_groups": ["AG2"]},
},
]
)
scoped_key = UserAPIKeyAuth(
api_key="hashed-key",
team_id="team-2",
models=["AG2"],
team_models=["AG2"],
)
filtered = router._filter_deployments_by_model_access_groups(
model="gpt-5",
healthy_deployments=router.get_all_deployments(model_name="gpt-5"),
request_kwargs={
"metadata": {
"user_api_key_team_id": "team-2",
"user_api_key_auth": scoped_key,
}
},
request_team_id="team-2",
)
assert len(filtered) == 1
assert filtered[0].get("model_info", {}).get("access_groups") == ["AG2"]
class TestAddModelFileIdMappings:
"""Test cases for add_model_file_id_mappings.
The router may pass either a list of deployment dicts (multiple matched
deployments) or a single deployment dict (when a specific deployment was
resolved, e.g. because the requested model matched a `model_info.id`).
Both shapes must produce a `{model_id: file_id}` mapping by extracting
`model_info.id` from each deployment.
"""
@staticmethod
def _make_response(file_id: str):
response = Mock()
response.id = file_id
return response
def test_should_map_each_deployment_id_when_given_list(self):
deployments = [
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4"},
"model_info": {"id": "deployment-1"},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4"},
"model_info": {"id": "deployment-2"},
},
]
responses = [self._make_response("file-1"), self._make_response("file-2")]
result = add_model_file_id_mappings(deployments, responses)
assert result == {"deployment-1": "file-1", "deployment-2": "file-2"}
def test_should_extract_model_info_id_when_given_single_deployment_dict(self):
"""Regression test: when `_common_checks_available_deployment` resolves
a specific deployment (returned as a dict, not a list), the function
must still extract `model_info.id` rather than iterate over the
deployment's own keys (`model_name`, `litellm_params`, `model_info`).
"""
deployment = {
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4", "api_key": "sk-test"},
"model_info": {"id": "deployment-1", "mode": "chat"},
}
responses = [self._make_response("file-1")]
result = add_model_file_id_mappings(deployment, responses)
assert result == {"deployment-1": "file-1"}
assert all(isinstance(v, str) for v in result.values())
def test_should_handle_batch_model_when_id_matches_model_name(self):
"""Regression test for the batch-model case: when `model_info.id` is
intentionally set equal to `model_name`, the router resolves a single
deployment via `has_model_id` and returns it as a dict. The mapping
must contain only `{id: file_id}` with string values so the resulting
`LiteLLM_ManagedFileTable` Pydantic validation passes.
"""
deployment = {
"model_name": "openai/openai/gpt-5.5-batch",
"litellm_params": {
"model": "openai/gpt-5.5",
"api_key": "sk-test",
"tpm": 40000000,
"rpm": 15000,
},
"model_info": {
"id": "openai/openai/gpt-5.5-batch",
"mode": "batch",
"base_model": "gpt-5.5",
"access_groups": ["default-models"],
},
}
responses = [self._make_response("file-batch-1")]
result = add_model_file_id_mappings(deployment, responses)
# Bug case would have produced keys ["model_name", "litellm_params",
# "model_info"] with non-string values.
assert result == {"openai/openai/gpt-5.5-batch": "file-batch-1"}
assert "litellm_params" not in result
assert "model_info" not in result
def test_should_skip_deployment_when_model_info_id_missing(self):
deployments = [
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4"},
"model_info": {},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4"},
"model_info": {"id": "deployment-2"},
},
]
responses = [self._make_response("file-1"), self._make_response("file-2")]
result = add_model_file_id_mappings(deployments, responses)
assert result == {"deployment-2": "file-2"}
def test_should_return_empty_mapping_when_given_empty_list(self):
result = add_model_file_id_mappings([], [])
assert result == {}
class TestResolveModelGroupAlias:
"""``model_group_alias`` maps reach this helper from validated config and
from key/team rows, so both entry shapes must resolve and malformed entries
must not raise mid-request."""
@pytest.mark.parametrize(
"alias_map, expected",
[
({"group-a": "group-b"}, "group-b"),
({"group-a": {"model": "group-b", "hidden": True}}, "group-b"),
({"group-a": {"model": "group-b"}}, "group-b"),
({"other": "group-b"}, None),
({}, None),
(None, None),
("not-a-map", None),
({"group-a": {"hidden": True}}, None),
({"group-a": {"model": 5}}, None),
({"group-a": 5}, None),
({"group-a": None}, None),
({"group-a": ""}, None),
],
)
def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected):
assert resolve_model_group_alias(alias_map, "group-a") == expected
def test_router_alias_resolution_uses_the_shared_helper(self):
router = Router(
model_list=[
{
"model_name": "group-b",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
}
],
model_group_alias={"group-a": "group-b", "group-item": {"model": "group-b", "hidden": True}},
)
assert router.get_model_from_alias("group-a") == "group-b"
assert router.get_model_from_alias("group-item") == "group-b"
assert router.get_model_from_alias("group-b") is None
@pytest.mark.parametrize(
"model, model_alias_map, key_aliases, model_group_alias, expected",
[
pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d"}, "d", id="global-key-router-order"),
pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"),
pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"),
pytest.param("a", {}, {}, {}, "a", id="no-alias"),
],
)
def test_resolves_served_model_aliases_in_order(
model: str,
model_alias_map: object,
key_aliases: object,
model_group_alias: object,
expected: str,
) -> None:
assert resolve_served_model(model, model_alias_map, key_aliases, model_group_alias) == expected
class TestTruncateFallbackErrorDetail:
def test_short_detail_is_returned_unchanged(self):
assert truncate_fallback_error_detail("boom") == "boom"
def test_detail_at_the_limit_is_returned_unchanged(self):
detail = "x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
assert truncate_fallback_error_detail(detail) == detail
def test_long_detail_is_bounded_and_reports_what_was_dropped(self):
detail = "x" * (ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS + 500)
truncated = truncate_fallback_error_detail(detail)
assert truncated.startswith("x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS)
assert truncated.endswith("... [truncated 500 characters]")
assert len(truncated) < len(detail)
def test_a_megabyte_of_detail_comes_back_small(self):
"""The detail is what a fallback level records about the level below it, so it has
to stay small enough that a walk over many model groups cannot compound it into an
output volume that starves the process."""
assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000
def test_format_fallback_outcome_reports_unauthorized_skips_without_failure_detail() -> None:
message: Final = format_fallback_outcome_message(
"primary",
("restricted-model",),
"",
skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),),
)
assert "Fallback to restricted-model was skipped because the caller is not authorized to call it." in message
assert "also failed" not in message
def test_format_fallback_outcome_separates_attempted_failures_from_skipped_targets() -> None:
message: Final = format_fallback_outcome_message(
"primary",
("restricted-model", "dead-allowed"),
"boom",
skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),),
)
failed_sentence: Final = message.split("Fallback to dead-allowed also failed:", 1)[1].split(
" Fallback to restricted-model was skipped", 1
)[0]
assert "Fallback to dead-allowed also failed: boom" in message
assert "Fallback to restricted-model was skipped because the caller is not authorized" in message
assert "restricted-model" not in failed_sentence
def test_format_fallback_outcome_uses_plural_for_multiple_over_budget_skips() -> None:
message: Final = format_fallback_outcome_message(
"primary",
("paid-one", "paid-two"),
"",
skipped=(
SkippedFallbackTarget("paid-one", FallbackSkipReason.OVER_BUDGET),
SkippedFallbackTarget("paid-two", FallbackSkipReason.OVER_BUDGET),
),
)
assert "Fallback to paid-one, paid-two was skipped because the caller is over budget for them." in message
def test_format_fallback_outcome_preserves_attempted_failure_wording_without_skips() -> None:
assert (
format_fallback_outcome_message("primary", ("fallback",), "detail")
== "\n\nLiteLLM: model group 'primary' failed with the error above. "
"Fallback to fallback also failed: detail"
)
class TestWarnOnProviderCredentialMismatch:
"""A deployment that carries one provider's credentials while resolving to
another is silently broken: litellm ignores the credentials and sends the
request to the resolved provider, which 401s. The classic shape is a bedrock
model group where one entry lost its route prefix, which fails only on the
requests the router happens to send to that entry."""
def test_warns_when_aws_params_sit_on_an_anthropic_model(self):
warning = warn_on_provider_credential_mismatch(
model_name="claude-sonnet-5",
litellm_params={"model": "claude-sonnet-5", "aws_region_name": "eu-central-1"},
)
assert warning is not None
assert "aws_region_name" in warning
assert "anthropic" in warning
assert "bedrock/claude-sonnet-5" in warning
def test_silent_when_the_prefix_is_present(self):
assert (
warn_on_provider_credential_mismatch(
model_name="claude-sonnet-5",
litellm_params={
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_region_name": "eu-central-1",
},
)
is None
)
def test_silent_when_custom_llm_provider_supplies_the_route(self):
"""An operator may name the provider explicitly instead of prefixing the
model; that is consistent and must not warn."""
assert (
warn_on_provider_credential_mismatch(
model_name="claude-sonnet-5",
litellm_params={
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
"custom_llm_provider": "bedrock",
"aws_region_name": "eu-central-1",
},
)
is None
)
def test_silent_when_no_provider_scoped_credentials_are_set(self):
assert (
warn_on_provider_credential_mismatch(
model_name="gpt-5.5", litellm_params={"model": "gpt-5.5"}
)
is None
)
def test_vertex_params_name_vertex_not_bedrock(self):
"""The hint must follow the params that were actually set, otherwise it
sends the operator to the wrong prefix."""
warning = warn_on_provider_credential_mismatch(
model_name="claude-on-vertex",
litellm_params={"model": "claude-sonnet-5", "vertex_project": "my-project"},
)
assert warning is not None
assert "vertex_ai/claude-sonnet-5" in warning
assert "bedrock" not in warning
def test_silent_for_a_model_litellm_cannot_classify(self):
"""An unresolvable model must not warn and must not raise: this runs on
the router startup path, so a wrong guess would spam every boot."""
assert (
warn_on_provider_credential_mismatch(
model_name="mystery",
litellm_params={"model": "not-a-real-provider-model-xyz", "aws_region_name": "us-east-1"},
)
is None
)
def test_router_warns_for_a_config_shaped_model_list(self, caplog):
"""The whole point is that this fires where operators declare models, so
drive Router rather than the helper."""
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
Router(
model_list=[
{
"model_name": "claude-sonnet-5",
"litellm_params": {
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_region_name": "us-east-1",
},
},
{
"model_name": "claude-sonnet-5",
"litellm_params": {
"model": "claude-sonnet-5",
"aws_region_name": "us-east-1",
},
},
]
)
mismatch_warnings = [r for r in caplog.records if "resolves to provider" in r.getMessage()]
assert len(mismatch_warnings) == 1, (
"exactly the prefix-less deployment should warn; "
f"got {[r.getMessage() for r in mismatch_warnings]}"
)
assert "aws_region_name" in mismatch_warnings[0].getMessage()
@pytest.mark.parametrize(
"model",
[
"bedrock/mantle/anthropic.claude-sonnet-4-5-20250929-v1:0",
"bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"sagemaker/my-endpoint",
],
)
def test_silent_for_every_aws_family_route(self, model):
"""The AWS family is wider than 'bedrock': mantle, sagemaker and the
sagemaker variants all read aws_* legitimately. Warning on any of them
would tell an operator to 'fix' a working deployment, so the provider
set is derived from LlmProviders rather than hand-listed."""
assert (
warn_on_provider_credential_mismatch(
model_name="aws-deployment",
litellm_params={"model": model, "aws_region_name": "us-east-1"},
)
is None
)
def test_every_aws_family_provider_is_covered(self):
"""Pins the derivation itself: a newly added bedrock_*/sagemaker_* provider
must join the set automatically, or it starts drawing false warnings."""
from litellm.types.utils import LlmProviders
aws_family = {p.value for p in LlmProviders if p.value.startswith(("bedrock", "sagemaker"))}
assert aws_family <= PROVIDER_SCOPED_CREDENTIAL_PARAMS["aws_region_name"]
assert {"bedrock", "bedrock_mantle", "sagemaker", "sagemaker_chat", "sagemaker_nova"} <= aws_family
def test_silent_when_credentials_come_from_a_named_credential(self):
"""Named credentials resolve after registration, so the params are absent
here. Warning on that absence would fire on every such deployment."""
assert (
warn_on_provider_credential_mismatch(
model_name="claude-sonnet-5",
litellm_params={
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"litellm_credential_name": "my-aws-creds",
},
)
is None
)
@pytest.mark.parametrize("provider", ["bedrock_mantle", "sagemaker_nova"])
def test_silent_for_aws_providers_named_explicitly(self, provider):
"""The false-positive shape: an operator names a less common AWS provider
directly, so the model string carries no route prefix to key off. A
hand-listed provider set misses these and tells them to 'fix' a working
deployment by prefixing it with bedrock/."""
assert (
warn_on_provider_credential_mismatch(
model_name="aws-deployment",
litellm_params={
"model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
"custom_llm_provider": provider,
"aws_region_name": "us-east-1",
},
)
is None
)
@pytest.mark.parametrize(
("litellm_params", "expected"),
[
({"model": "azure_ai/gpt-5.4-mini", "custom_llm_provider": "azure"}, "azure"),
({"model": "azure_ai/gpt-5.4-mini", "api_base": "https://my-resource.openai.azure.com"}, "azure_ai"),
({"model": "cohere/command-r"}, "cohere"),
({"model": "gpt-5.4-mini"}, "openai"),
({"model": "no-provider-knows-this-model"}, None),
({"api_base": "https://my-resource.openai.azure.com"}, None),
],
ids=["declared_wins", "prefix_beats_host_flip", "prefix_beats_cohere_chat_flip", "unprefixed_inferred", "unknown", "no_model"],
)
def test_provider_for_generic_call(litellm_params, expected, monkeypatch):
monkeypatch.setenv("AZURE_AI_API_BASE", "https://unrelated.openai.azure.com")
assert provider_for_generic_call(litellm_params) == expected
def test_router_warns_only_for_a_capability_flag_set_under_litellm_params(caplog):
bedrock_model: Final = "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
Router(
model_list=[
{
"model_name": "flag-in-litellm-params",
"litellm_params": {"model": bedrock_model, "supports_max_reasoning_effort": False},
},
{
"model_name": "flag-in-model-info",
"litellm_params": {"model": bedrock_model},
"model_info": {"supports_max_reasoning_effort": False},
},
{
"model_name": "declared-litellm-param",
"litellm_params": {"model": bedrock_model, "supports_system_message": False},
},
]
)
capability_warnings: Final = [r.getMessage() for r in caplog.records if "model_info" in r.getMessage()]
assert len(capability_warnings) == 1, capability_warnings
assert "flag-in-litellm-params" in capability_warnings[0]
assert "supports_max_reasoning_effort" in capability_warnings[0]
def test_router_serves_deployments_without_model_capability_flags():
capability_flags: Final = frozenset(ProviderSpecificModelInfo.__annotations__) | frozenset(
key for key in chain.from_iterable(GetModelCostMap.loaded_model_cost_map().values()) if key.startswith("supports_")
)
declared_params: Final = {"supports_system_message": False, "supports_system_prompt": True}
router: Final = Router(
model_list=[
{
"model_name": "every-flag",
"litellm_params": {
"model": "replicate/meta/meta-llama-3-8b-instruct",
**dict.fromkeys(capability_flags - declared_params.keys(), True),
**declared_params,
},
}
]
)
served_params: Final = router.get_model_list()[0]["litellm_params"]
assert sorted(capability_flags & served_params.keys() - declared_params.keys()) == []
assert {key: served_params.get(key) for key in declared_params} == declared_params
def test_router_keeps_a_provider_param_that_another_deployments_model_info_names():
Router(
model_list=[
{
"model_name": "registers-the-name",
"litellm_params": {"model": "openai/litellm-qa-registered-capability"},
"model_info": {"supports_system_prompt": True},
}
]
)
router: Final = Router(
model_list=[
{
"model_name": "replicate-system-prompt",
"litellm_params": {"model": "replicate/meta/meta-llama-3-8b-instruct", "supports_system_prompt": True},
}
]
)
assert router.get_model_list()[0]["litellm_params"].get("supports_system_prompt") is True
def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_db(caplog):
router: Final = Router(model_list=[])
db_deployment: Final = Deployment(
model_name="db-model",
litellm_params=LiteLLM_Params(
model="bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
supports_max_reasoning_effort=False,
),
model_info=ModelInfo(id="db-model-id"),
)
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
first_sync: Final = router.upsert_deployment(db_deployment)
second_sync: Final = router.upsert_deployment(db_deployment)
capability_warnings: Final = [r.getMessage() for r in caplog.records if "model_info" in r.getMessage()]
assert first_sync is not None
assert second_sync is None
assert len(capability_warnings) == 1, capability_warnings
assert "db-model" in capability_warnings[0]
assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"]