The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from collections.abc import Iterator, Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import Request
import litellm
from litellm.proxy import proxy_server
from litellm.proxy._types import (
LiteLLM_TeamTableCachedObj,
LiteLLM_UserTable,
LitellmUserRoles,
ModelAccessDeniedProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.auth import auth_checks
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, common_checks
from litellm.proxy.utils import get_available_models_for_user
_TEAMS: Final[Mapping[str, LiteLLM_TeamTableCachedObj]] = {
"team-a": LiteLLM_TeamTableCachedObj(team_id="team-a", models=["gpt-a", "shared"]),
"team-b": LiteLLM_TeamTableCachedObj(team_id="team-b", models=["gpt-b", "shared"]),
"team-open": LiteLLM_TeamTableCachedObj(team_id="team-open", models=[]),
"team-blocked": LiteLLM_TeamTableCachedObj(team_id="team-blocked", models=["gpt-a"], blocked=True),
}
_ROUTER: Final = litellm.Router(
model_list=[
{"model_name": name, "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-fake"}}
for name in ("gpt-a", "gpt-b", "shared", "gpt-c")
]
)
@pytest.fixture
def teams(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTableCachedObj:
if team_id not in _TEAMS:
raise Exception(f"team {team_id} lookup failed")
return _TEAMS[team_id]
async def _no_membership(**_: object) -> None:
return None
monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object)
monkeypatch.setattr(auth_checks, "get_team_membership", _no_membership)
yield
def _enable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True)
async def _common_checks(
model: str | list[str],
user: LiteLLM_UserTable | None,
token: UserAPIKeyAuth,
fallbacks: tuple[str, ...] = (),
) -> bool:
request_body: Final = {"model": model} | ({"fallbacks": list(fallbacks)} if fallbacks else {})
return await common_checks(
request_body=request_body,
team_object=None,
user_object=user,
end_user_object=None,
global_proxy_spend=None,
general_settings={},
route="/chat/completions",
llm_router=_ROUTER,
proxy_logging_obj=MagicMock(),
valid_token=token,
request=MagicMock(spec=Request),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model, allowed",
[
("gpt-a", True),
("gpt-b", True),
("shared", True),
("gpt-c", False),
],
)
async def test_personal_key_inherits_the_union_of_its_owner_team_models(
monkeypatch: pytest.MonkeyPatch, teams: None, model: str, allowed: bool
) -> None:
_enable(monkeypatch)
user: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
if allowed:
assert await _common_checks(model, user, token) is True
else:
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks(model, user, token)
@pytest.mark.asyncio
async def test_personal_key_without_teams_gets_no_models(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks(
"gpt-a", LiteLLM_UserTable(user_id="u1", teams=[]), UserAPIKeyAuth(token="k1", user_id="u1")
)
@pytest.mark.asyncio
async def test_personal_key_denied_when_owner_not_loaded(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks("gpt-a", None, UserAPIKeyAuth(token="k1", user_id="u1"))
@pytest.mark.asyncio
async def test_proxy_admin_personal_key_is_not_capped(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
user: Final = LiteLLM_UserTable(user_id="admin", teams=[], user_role=LitellmUserRoles.PROXY_ADMIN.value)
token: Final = UserAPIKeyAuth(token="k1", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
assert await _common_checks("gpt-c", user, token) is True
@pytest.mark.asyncio
async def test_personal_key_unrestricted_when_flag_off(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", False)
user: Final = LiteLLM_UserTable(user_id="u1", teams=[])
assert await _common_checks("gpt-c", user, UserAPIKeyAuth(token="k1", user_id="u1")) is True
@pytest.mark.asyncio
async def test_personal_key_fallbacks_are_capped_by_owner_teams(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
with pytest.raises(ModelAccessDeniedProxyException) as exc_info:
await _common_checks("gpt-a", owner, token, fallbacks=("gpt-c",))
assert str(exc_info.value.code) == "403"
assert await _common_checks("gpt-a", owner, token, fallbacks=("shared",)) is True
@pytest.mark.asyncio
async def test_personal_key_list_models_can_be_allowed_by_different_teams(
monkeypatch: pytest.MonkeyPatch, teams: None
) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
assert await _common_checks(["gpt-a", "gpt-b"], owner, token) is True
@pytest.mark.asyncio
async def test_personal_key_list_models_denies_the_unavailable_model(
monkeypatch: pytest.MonkeyPatch, teams: None
) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
with pytest.raises(ModelAccessDeniedProxyException) as exc_info:
await _common_checks(["gpt-a", "gpt-c"], owner, token)
assert exc_info.value.code == "403"
assert "gpt-c" in exc_info.value.message
@pytest.mark.asyncio
async def test_personal_key_fallbacks_are_unchanged_when_flag_off(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", False)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
assert await _common_checks("gpt-a", owner, token, fallbacks=("gpt-c",)) is True
@pytest.mark.asyncio
async def test_personal_key_fallbacks_load_each_owner_team_once(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
get_team: Final = AsyncMock(wraps=auth_checks.get_team_object)
monkeypatch.setattr(auth_checks, "get_team_object", get_team)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
assert await _common_checks("gpt-a", owner, token, fallbacks=("shared", "gpt-a")) is True
assert sorted(call.kwargs["team_id"] for call in get_team.call_args_list) == ["team-a", "team-b"]
@pytest.mark.asyncio
async def test_team_key_is_not_capped_by_owner_teams(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
user: Final = LiteLLM_UserTable(user_id="u1", teams=[])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1", team_id="team-open")
assert await _common_checks("gpt-c", user, token, fallbacks=("gpt-c",)) is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
"teams_of_user, allowed",
[
(["team-missing", "team-a"], True),
(["team-missing"], False),
(["team-blocked"], False),
(["team-blocked", "team-a"], True),
],
)
async def test_unloadable_or_blocked_team_grants_nothing(
monkeypatch: pytest.MonkeyPatch, teams: None, teams_of_user: list[str], allowed: bool
) -> None:
_enable(monkeypatch)
user: Final = LiteLLM_UserTable(user_id="u1", teams=teams_of_user)
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
if allowed:
assert await _common_checks("gpt-a", user, token) is True
else:
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks("gpt-a", user, token)
@pytest.mark.asyncio
async def test_membership_change_applies_to_existing_key(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
assert await _common_checks("gpt-a", LiteLLM_UserTable(user_id="u1", teams=["team-a"]), token) is True
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks("gpt-a", LiteLLM_UserTable(user_id="u1", teams=["team-b"]), token)
@pytest.mark.asyncio
async def test_resolved_model_check_caps_personal_key(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a"])
async def _get_user_object(**_: object) -> LiteLLM_UserTable:
return owner
monkeypatch.setattr(auth_checks, "get_user_object", _get_user_object)
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
await can_key_call_resolved_model(model="gpt-a", llm_model_list=None, valid_token=token, llm_router=_ROUTER)
with pytest.raises(ModelAccessDeniedProxyException):
await can_key_call_resolved_model(model="gpt-b", llm_model_list=None, valid_token=token, llm_router=_ROUTER)
@pytest.mark.asyncio
async def test_resolved_model_check_denies_when_personal_key_owner_lookup_fails(
monkeypatch: pytest.MonkeyPatch, teams: None
) -> None:
_enable(monkeypatch)
monkeypatch.setattr(proxy_server, "prisma_client", None)
token: Final = UserAPIKeyAuth(token="k1", user_id="u-no-db")
await proxy_server.user_api_key_cache.async_delete_cache("u-no-db")
assert await proxy_server.user_api_key_cache.async_get_cache(key="u-no-db", model_type=LiteLLM_UserTable) is None
with pytest.raises(ModelAccessDeniedProxyException) as exc_info:
await can_key_call_resolved_model(model="gpt-a", llm_model_list=None, valid_token=token, llm_router=_ROUTER)
assert str(exc_info.value.code) == "403"
async def _list_models(monkeypatch: pytest.MonkeyPatch, owner: LiteLLM_UserTable, token: UserAPIKeyAuth) -> list[str]:
async def _get_user_object(**_: object) -> LiteLLM_UserTable:
return owner
monkeypatch.setattr(auth_checks, "get_user_object", _get_user_object)
return await get_available_models_for_user(
user_api_key_dict=token,
llm_router=_ROUTER,
general_settings={},
user_model=None,
prisma_client=MagicMock(),
proxy_logging_obj=MagicMock(),
user_api_key_cache=MagicMock(),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"key_models, expected",
[
([], {"gpt-a", "gpt-b", "shared"}),
(["gpt-a", "gpt-c"], {"gpt-a"}),
],
)
async def test_model_listing_matches_team_derived_access(
monkeypatch: pytest.MonkeyPatch, teams: None, key_models: list[str], expected: set[str]
) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
listed: Final = await _list_models(monkeypatch, owner, UserAPIKeyAuth(token="k1", user_id="u1", models=key_models))
assert set(listed) == expected
@pytest.mark.asyncio
async def test_model_listing_empty_without_teams(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=[])
assert await _list_models(monkeypatch, owner, UserAPIKeyAuth(token="k1", user_id="u1")) == []
@pytest.mark.asyncio
async def test_model_listing_unchanged_when_flag_off(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", False)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=[])
listed: Final = await _list_models(monkeypatch, owner, UserAPIKeyAuth(token="k1", user_id="u1"))
assert set(listed) == {"gpt-a", "gpt-b", "shared", "gpt-c"}
@pytest.mark.asyncio
async def test_model_listing_loads_each_team_once(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
get_team: Final = AsyncMock(wraps=auth_checks.get_team_object)
monkeypatch.setattr(auth_checks, "get_team_object", get_team)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b", "team-a"])
listed: Final = await _list_models(monkeypatch, owner, UserAPIKeyAuth(token="k1", user_id="u1"))
assert set(listed) == {"gpt-a", "gpt-b", "shared"}
assert sorted(call.kwargs["team_id"] for call in get_team.call_args_list) == ["team-a", "team-b"]
@pytest.mark.asyncio
async def test_team_check_error_grants_nothing_for_that_team(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
check_team: Final = auth_checks.can_team_access_model
async def _failing_for_team_b(model: str, team_object: LiteLLM_TeamTableCachedObj, **_: object) -> bool:
if team_object.team_id == "team-b":
raise RuntimeError("access group lookup failed")
return await check_team(model=model, team_object=team_object, llm_router=_ROUTER)
monkeypatch.setattr(auth_checks, "can_team_access_model", _failing_for_team_b)
owner: Final = LiteLLM_UserTable(user_id="u1", teams=["team-a", "team-b"])
token: Final = UserAPIKeyAuth(token="k1", user_id="u1")
assert await _common_checks("gpt-a", owner, token) is True
with pytest.raises(ModelAccessDeniedProxyException):
await _common_checks("gpt-b", owner, token)
async def _owner_lookup_fails(**_: object) -> LiteLLM_UserTable:
raise RuntimeError("database unavailable")
async def _owner_missing(**_: object) -> None:
return None
@pytest.mark.asyncio
@pytest.mark.parametrize("get_owner", [_owner_lookup_fails, _owner_missing])
async def test_model_listing_empty_when_owner_not_loaded(
monkeypatch: pytest.MonkeyPatch, teams: None, get_owner: object
) -> None:
_enable(monkeypatch)
monkeypatch.setattr(auth_checks, "get_user_object", get_owner)
listed: Final = await get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(token="k1", user_id="u1"),
llm_router=_ROUTER,
general_settings={},
user_model=None,
prisma_client=MagicMock(),
proxy_logging_obj=MagicMock(),
user_api_key_cache=MagicMock(),
)
assert listed == []
@pytest.mark.asyncio
async def test_model_listing_empty_without_database(monkeypatch: pytest.MonkeyPatch, teams: None) -> None:
_enable(monkeypatch)
listed: Final = await get_available_models_for_user(
user_api_key_dict=UserAPIKeyAuth(token="k1", user_id="u1"),
llm_router=_ROUTER,
general_settings={},
user_model=None,
prisma_client=None,
proxy_logging_obj=MagicMock(),
user_api_key_cache=MagicMock(),
)
assert listed == []