The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from typing import Final
import pytest
from litellm._internal_context import current_service_target
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.openai_files_endpoints.file_usage_caps import (
FileUsageLimit,
ScopedFileUsageLimit,
batch_file_record_limit,
consume_file_usage,
enforce_batch_file_upload_limit,
enforce_file_download_limit,
resolve_scoped_limits,
)
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
DAY: Final = 86400
MIDDAY: Final = 20_000 * DAY + DAY / 2
def _cache() -> InternalUsageCache:
return InternalUsageCache(dual_cache=DualCache())
def _scoped(scope, scope_id, value, source="key", setting="max_batch_file_uploads_per_day"):
return ScopedFileUsageLimit(
scope=scope, scope_id=scope_id, limit=FileUsageLimit(setting=setting, value=value, source=source)
)
@pytest.mark.parametrize(
"key_metadata, team_metadata, general_settings, expected",
[
({}, {}, {}, None),
({}, {}, {"max_batch_file_records": 50}, FileUsageLimit("max_batch_file_records", 50, "general_settings")),
(
{"max_batch_file_records": 80},
{},
{"max_batch_file_records": 50},
FileUsageLimit("max_batch_file_records", 80, "key"),
),
(
{"max_batch_file_records": 80},
{"max_batch_file_records": 30},
{},
FileUsageLimit("max_batch_file_records", 30, "team"),
),
(
{},
{"max_batch_file_records": 90},
{"max_batch_file_records": 50},
FileUsageLimit("max_batch_file_records", 50, "general_settings"),
),
({"max_batch_file_records": 0}, {"max_batch_file_records": "lots"}, {}, None),
({"max_batch_file_records": "40"}, {}, {}, FileUsageLimit("max_batch_file_records", 40, "key")),
],
)
def test_record_limit_is_the_lowest_applicable_limit(key_metadata, team_metadata, general_settings, expected):
caller: Final = UserAPIKeyAuth(api_key="hashed", team_id="t1", metadata=key_metadata, team_metadata=team_metadata)
assert batch_file_record_limit(caller, general_settings) == expected
@pytest.mark.parametrize(
"caller, general_settings, expected",
[
(UserAPIKeyAuth(api_key="hashed", team_id="t1"), {}, ()),
(
UserAPIKeyAuth(api_key="hashed", team_id="t1"),
{"max_batch_file_uploads_per_day": 5},
(_scoped("key", "hashed", 5, "general_settings"),),
),
(
UserAPIKeyAuth(
api_key="hashed",
team_id="t1",
metadata={"max_batch_file_uploads_per_day": 9},
team_metadata={"max_batch_file_uploads_per_day": 20},
),
{"max_batch_file_uploads_per_day": 5},
(_scoped("key", "hashed", 9, "key"), _scoped("team", "t1", 20, "team")),
),
(
UserAPIKeyAuth(api_key="hashed", team_metadata={"max_batch_file_uploads_per_day": 20}),
{},
(),
),
(
UserAPIKeyAuth(api_key=None, user_id="u1", team_id="t1"),
{"max_batch_file_uploads_per_day": 5},
(_scoped("user", "u1", 5, "general_settings"),),
),
(UserAPIKeyAuth(api_key=None), {"max_batch_file_uploads_per_day": 5}, ()),
],
)
def test_counter_scopes_pair_each_limit_with_its_own_counter(caller, general_settings, expected):
assert resolve_scoped_limits(caller, general_settings, "max_batch_file_uploads_per_day") == expected
async def test_a_scope_admits_exactly_its_limit_per_window_and_resets_on_the_next():
cache: Final = _cache()
limits: Final = (_scoped("key", "hashed", 2),)
outcomes: Final = [await consume_file_usage(cache, limits, DAY, "", MIDDAY + i) for i in range(3)]
next_window: Final = await consume_file_usage(cache, limits, DAY, "", MIDDAY + DAY)
assert outcomes[:2] == [None, None]
assert outcomes[2] is not None
assert outcomes[2].limit == limits[0]
assert outcomes[2].retry_after_seconds == DAY / 2 - 2
assert next_window is None
async def test_a_rejection_on_one_scope_gives_back_the_slot_it_took_on_the_other():
cache: Final = _cache()
key_scope: Final = _scoped("key", "hashed", 3)
team_scope: Final = _scoped("team", "t1", 1, "team")
first: Final = await consume_file_usage(cache, (key_scope, team_scope), DAY, "", MIDDAY)
rejected: Final = [await consume_file_usage(cache, (key_scope, team_scope), DAY, "", MIDDAY) for _ in range(5)]
key_only: Final = [await consume_file_usage(cache, (key_scope,), DAY, "", MIDDAY) for _ in range(3)]
assert first is None
assert {outcome.limit for outcome in rejected if outcome is not None} == {team_scope}
assert len([outcome for outcome in rejected if outcome is not None]) == 5
assert key_only[:2] == [None, None]
assert key_only[2] is not None and key_only[2].limit == key_scope
async def test_a_rejection_does_not_use_up_the_scope_that_rejected_it():
cache: Final = _cache()
limits: Final = (_scoped("key", "hashed", 1),)
await consume_file_usage(cache, limits, DAY, "", MIDDAY)
for _ in range(4):
await consume_file_usage(cache, limits, DAY, "", MIDDAY)
raised_limit: Final = (_scoped("key", "hashed", 2),)
assert await consume_file_usage(cache, raised_limit, DAY, "", MIDDAY) is None
class _ServiceTargetSeen(Exception):
pass
class _TargetReportingCache(InternalUsageCache):
async def async_increment_cache(self, key, value, litellm_parent_otel_span, local_only=False, **kwargs):
raise _ServiceTargetSeen(current_service_target())
async def test_counter_writes_are_declared_as_rate_limit_calls_so_their_redis_spans_are_named():
cache: Final = _TargetReportingCache(dual_cache=DualCache())
with pytest.raises(_ServiceTargetSeen) as seen:
await consume_file_usage(cache, (_scoped("key", "hashed", 1),), DAY, "", MIDDAY)
assert seen.value.args == ("rate_limits",)
async def test_download_counters_are_per_file():
cache: Final = _cache()
limits: Final = (_scoped("key", "hashed", 1, setting="max_file_downloads_per_minute"),)
assert await consume_file_usage(cache, limits, 60, "file-a", MIDDAY) is None
assert await consume_file_usage(cache, limits, 60, "file-a", MIDDAY) is not None
assert await consume_file_usage(cache, limits, 60, "file-b", MIDDAY) is None
async def test_the_proxy_counter_store_keeps_a_counter_while_more_counters_are_live_than_a_default_cache_holds():
cache: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()).file_usage_cache
limits: Final = (_scoped("key", "hashed", 1, setting="max_file_downloads_per_minute"),)
other_files: Final = DEFAULT_MAX_SIZE_IN_MEMORY + 10
first: Final = await consume_file_usage(cache, limits, 60, "file-first", MIDDAY)
others: Final = [
await consume_file_usage(cache, limits, 60, f"file-{index}", MIDDAY) for index in range(other_files)
]
assert first is None
assert others == [None] * other_files
assert await consume_file_usage(cache, limits, 60, "file-first", MIDDAY) is not None
async def test_upload_limit_error_names_the_setting_value_scope_and_reset():
cache: Final = _cache()
caller: Final = UserAPIKeyAuth(api_key="hashed", team_id="t1", team_metadata={"max_batch_file_uploads_per_day": 1})
await enforce_batch_file_upload_limit(cache, caller, {}, clock=lambda: MIDDAY)
with pytest.raises(ProxyException) as exc:
await enforce_batch_file_upload_limit(cache, caller, {}, clock=lambda: MIDDAY + 100)
assert exc.value.code == "429"
assert exc.value.type == "rate_limit_error"
assert exc.value.headers == {"retry-after": str(DAY // 2 - 100)}
assert "max_batch_file_uploads_per_day is 1 for team t1" in exc.value.message
assert "this team's metadata" in exc.value.message
async def test_download_limit_error_names_the_file_and_general_settings_default():
cache: Final = _cache()
caller: Final = UserAPIKeyAuth(api_key="hashed")
general_settings: Final = {"max_file_downloads_per_minute": 2}
for _ in range(2):
await enforce_file_download_limit(cache, caller, general_settings, "file-a", clock=lambda: MIDDAY + 15)
with pytest.raises(ProxyException) as exc:
await enforce_file_download_limit(cache, caller, general_settings, "file-a", clock=lambda: MIDDAY + 15)
assert exc.value.code == "429"
assert exc.value.headers == {"retry-after": "45"}
assert "file-a" in exc.value.message
assert "max_file_downloads_per_minute is 2 for this key (set in general_settings)" in exc.value.message
async def test_no_configured_limit_never_rejects():
cache: Final = _cache()
caller: Final = UserAPIKeyAuth(api_key="hashed", team_id="t1")
for _ in range(50):
await enforce_batch_file_upload_limit(cache, caller, {}, clock=lambda: MIDDAY)
await enforce_file_download_limit(cache, caller, {}, "file-a", clock=lambda: MIDDAY)
async def test_a_caller_without_a_key_is_counted_per_user():
cache: Final = _cache()
general_settings: Final = {"max_file_downloads_per_minute": 1}
jwt_caller: Final = UserAPIKeyAuth(api_key=None, user_id="u1")
await enforce_file_download_limit(cache, jwt_caller, general_settings, "file-a", clock=lambda: MIDDAY)
await enforce_file_download_limit(
cache, UserAPIKeyAuth(api_key=None, user_id="u2"), general_settings, "file-a", clock=lambda: MIDDAY
)
with pytest.raises(ProxyException) as exc:
await enforce_file_download_limit(cache, jwt_caller, general_settings, "file-a", clock=lambda: MIDDAY)
assert "max_file_downloads_per_minute is 1 for user u1 (set in general_settings)" in exc.value.message