The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import os
import traceback
from litellm._uuid import uuid
from datetime import datetime, timezone
from collections.abc import Coroutine
from typing import Any, Final
from dotenv import load_dotenv
from fastapi import Request
from fastapi.routing import APIRoute
import contextlib
import io
import logging
import time
# this file is to test litellm/proxy
import asyncio
load_dotenv()
import pytest
import litellm
from litellm.proxy.proxy_server import (
LitellmUserRoles,
audio_transcriptions,
chat_completion,
completion,
embeddings,
model_list,
moderations,
user_api_key_auth,
)
from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend
from starlette.datastructures import URL
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
drain_audit_tasks,
get_audit_log_changed_by,
track_audit_task,
)
from litellm.proxy._types import LiteLLM_AuditLogs, LitellmTableNames, UserAPIKeyAuth
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import StandardAuditLogPayload
from unittest.mock import AsyncMock, MagicMock, patch
proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
import json
from tests._master_key import MASTER_KEY
def test_get_audit_log_changed_by_prefers_authenticated_user():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="authenticated-user",
)
assert (
get_audit_log_changed_by(
litellm_changed_by="spoofed-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "authenticated-user"
)
def test_get_audit_log_changed_by_honors_header_with_admin_opt_in():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
def test_get_audit_log_changed_by_honors_header_with_team_opt_in():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
team_metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
def test_get_audit_log_changed_by_ignores_header_without_opt_in_when_user_id_missing():
user_api_key_dict = UserAPIKeyAuth(api_key="test-key")
assert (
get_audit_log_changed_by(
litellm_changed_by="spoofed-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "proxy-admin"
)
def test_get_audit_log_changed_by_honors_header_with_opt_in_when_user_id_missing():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
@pytest.mark.asyncio
async def test_create_internal_user_audit_log_uses_changed_by_helper():
from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
metadata={"allow_litellm_changed_by_header": True},
)
with (
patch("litellm.store_audit_logs", True),
patch(
"litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update",
new_callable=AsyncMock,
) as mock_create_audit_log_for_update,
):
await UserManagementEventHooks.create_internal_user_audit_log(
user_id="target-user",
action="updated",
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
before_value='{"before": true}',
after_value='{"after": true}',
)
request_data = mock_create_audit_log_for_update.await_args.kwargs["request_data"]
assert request_data.changed_by == "delegated-user"
assert request_data.changed_by_api_key == "test-key"
assert request_data.object_id == "target-user"
assert request_data.action == "updated"
@pytest.mark.asyncio
async def test_create_audit_log_for_update_premium_user():
"""
Basic unit test for create_audit_log_for_update
Test that the audit log is created when a premium user updates a team
"""
with (
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma,
):
mock_prisma.db.litellm_auditlog.create = AsyncMock()
request_data = LiteLLM_AuditLogs(
id="test_id",
updated_at=datetime.now(),
changed_by="test_changed_by",
action="updated",
table_name=LitellmTableNames.TEAM_TABLE_NAME,
object_id="test_object_id",
updated_values=json.dumps({"key": "value"}),
before_value=json.dumps({"old_key": "old_value"}),
)
await create_audit_log_for_update(request_data)
mock_prisma.db.litellm_auditlog.create.assert_called_once_with(
data={
"id": "test_id",
"updated_at": request_data.updated_at,
"changed_by": request_data.changed_by,
"action": request_data.action,
"table_name": request_data.table_name,
"object_id": request_data.object_id,
"updated_values": request_data.updated_values,
"before_value": request_data.before_value,
}
)
@pytest.fixture
def prisma_client():
from litellm.proxy.proxy_cli import append_query_params
### add connection pool + pool timeout args
params = {"connection_limit": 100, "pool_timeout": 60}
database_url = os.getenv("DATABASE_URL")
modified_url = append_query_params(database_url, params)
os.environ["DATABASE_URL"] = modified_url
# Assuming PrismaClient is a class that needs to be instantiated
prisma_client = PrismaClient(database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj)
return prisma_client
@pytest.mark.skip(reason="Requires reliable external DB connection (prisma).")
@pytest.mark.asyncio()
async def test_create_audit_log_in_db(prisma_client):
print("prisma client=", prisma_client)
setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY)
setattr(litellm.proxy.proxy_server, "premium_user", True)
setattr(litellm, "store_audit_logs", True)
await litellm.proxy.proxy_server.prisma_client.connect()
audit_log_id = f"audit_log_id_{uuid.uuid4()}"
# create a audit log for /key/generate
request_data = LiteLLM_AuditLogs(
id=audit_log_id,
updated_at=datetime.now(),
changed_by="test_changed_by",
action="updated",
table_name=LitellmTableNames.TEAM_TABLE_NAME,
object_id="test_object_id",
updated_values=json.dumps({"key": "value"}),
before_value=json.dumps({"old_key": "old_value"}),
)
await create_audit_log_for_update(request_data)
await asyncio.sleep(1)
# now read the last log from the db
last_log = await prisma_client.db.litellm_auditlog.find_first(where={"id": audit_log_id})
assert last_log.id == audit_log_id
setattr(litellm, "store_audit_logs", False)
@pytest.mark.asyncio
async def test_track_audit_task_lifecycle():
task_completed: Final = asyncio.Event()
async def _sample_coroutine() -> None:
await asyncio.sleep(0)
task_completed.set()
task: Final = track_audit_task(asyncio.create_task(_sample_coroutine()))
assert not task.done()
await drain_audit_tasks(timeout=1.0)
assert task_completed.is_set()
assert task.done()
completed_task: Final = asyncio.create_task(asyncio.sleep(0))
await completed_task
assert track_audit_task(completed_task) is completed_task
@pytest.mark.asyncio
async def test_drain_audit_tasks_waits_for_all_tasks():
event1: Final = asyncio.Event()
event2: Final = asyncio.Event()
async def _worker(event: asyncio.Event) -> None:
await asyncio.sleep(0)
event.set()
task1: Final = track_audit_task(asyncio.create_task(_worker(event1)))
task2: Final = track_audit_task(asyncio.create_task(_worker(event2)))
assert not task1.done()
assert not task2.done()
await drain_audit_tasks(timeout=1.0)
assert event1.is_set()
assert event2.is_set()
assert task1.done()
assert task2.done()
@pytest.mark.asyncio
async def test_drain_audit_tasks_handles_failing_task_cleanly():
async def _failing_worker() -> None:
await asyncio.sleep(0)
raise RuntimeError("test audit log write failure")
task: Final = track_audit_task(asyncio.create_task(_failing_worker()))
assert not task.done()
await drain_audit_tasks(timeout=1.0)
assert task.done()
assert isinstance(task.exception(), RuntimeError)
@pytest.mark.asyncio
async def test_drain_audit_tasks_timeout_does_not_cancel_pending_tasks():
async def _long_running_worker() -> None:
await asyncio.Event().wait()
task: Final = track_audit_task(asyncio.create_task(_long_running_worker()))
try:
assert not task.done()
await drain_audit_tasks(timeout=0)
assert not task.cancelled()
assert not task.done()
finally:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_drain_audit_tasks_persists_management_audit_write_before_db_disconnect():
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
is_disconnected: Final = [False]
async def _mock_create(data: dict[str, Any]) -> None:
if is_disconnected[0]:
raise RuntimeError("Database already disconnected")
await asyncio.sleep(0)
writes.append(data)
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=_mock_create)
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
):
request_data: Final = LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by="admin-user",
changed_by_api_key="sk-admin",
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id="token-123",
action="blocked",
updated_values="{}",
before_value='{"key": "test"}',
)
task: Final = track_audit_task(asyncio.create_task(create_audit_log_for_update(request_data=request_data)))
assert not task.done()
await drain_audit_tasks(timeout=2.0)
is_disconnected[0] = True
assert task.done()
assert len(writes) == 1
assert writes[0]["object_id"] == "token-123"
assert writes[0]["action"] == "blocked"
@pytest.mark.asyncio
async def test_create_audit_log_auto_registers_in_drain():
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
async def _mock_create(data: dict[str, Any]) -> None:
await asyncio.sleep(0)
writes.append(data)
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=_mock_create)
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
):
request_data: Final = LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by="admin-user",
changed_by_api_key="sk-admin",
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id="token-456",
action="deleted",
updated_values="{}",
before_value='{"key": "test-deleted"}',
)
task: Final = asyncio.create_task(create_audit_log_for_update(request_data=request_data))
await asyncio.sleep(0)
await drain_audit_tasks(timeout=2.0)
assert task.done()
assert len(writes) == 1
assert writes[0]["object_id"] == "token-456"
assert writes[0]["action"] == "deleted"
@pytest.mark.asyncio
async def test_drain_audit_tasks_discards_timed_out_tasks_on_timeout():
async def _hung_worker() -> None:
await asyncio.Event().wait()
hung_task: Final = track_audit_task(asyncio.create_task(_hung_worker()))
try:
await drain_audit_tasks(timeout=0)
assert not hung_task.done()
second_drain: Final = asyncio.create_task(drain_audit_tasks(timeout=60))
for _ in range(3):
await asyncio.sleep(0)
assert second_drain.done()
finally:
hung_task.cancel()
try:
await hung_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_drain_audit_tasks_captures_tasks_queued_during_drain():
finished_tasks: Final[list[str]] = []
async def _first_worker() -> None:
await asyncio.sleep(0)
track_audit_task(asyncio.create_task(_second_worker()))
finished_tasks.append("first")
async def _second_worker() -> None:
await asyncio.sleep(0)
finished_tasks.append("second")
track_audit_task(asyncio.create_task(_first_worker()))
await drain_audit_tasks(timeout=2.0)
assert finished_tasks == ["first", "second"]
class _AuditCallbackBlockedUntilReleased(CustomLogger):
def __init__(self, release: asyncio.Event) -> None:
super().__init__()
self.release: Final = release
self.logged_object_ids: Final[list[str]] = []
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
await self.release.wait()
self.logged_object_ids.append(audit_log["object_id"])
class _WarningRecorder(logging.Handler):
def __init__(self) -> None:
super().__init__(level=logging.WARNING)
self.messages: Final[list[str]] = []
def emit(self, record: logging.LogRecord) -> None:
self.messages.append(record.getMessage())
def _deleted_key_audit_log(object_id: str) -> LiteLLM_AuditLogs:
return LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by="admin-user",
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=object_id,
action="deleted",
updated_values="{}",
before_value="{}",
)
@pytest.mark.asyncio
async def test_drain_waits_for_a_slow_audit_log_callback():
release: Final = asyncio.Event()
callback: Final = _AuditCallbackBlockedUntilReleased(release)
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", None),
patch("litellm.audit_log_callbacks", [callback]),
):
await create_audit_log_for_update(request_data=_deleted_key_audit_log("token-callback"))
drain: Final = asyncio.create_task(drain_audit_tasks(timeout=60))
for _ in range(3):
await asyncio.sleep(0)
drain_returned_before_the_callback_finished: Final = drain.done()
release.set()
await drain
assert not drain_returned_before_the_callback_finished
assert callback.logged_object_ids == ["token-callback"]
@pytest.mark.asyncio
async def test_drain_timeout_warning_names_each_dropped_audit_row():
mock_prisma: Final = MagicMock()
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=_io_that_never_finishes)
recorder: Final = _WarningRecorder()
verbose_proxy_logger.addHandler(recorder)
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
):
writer: Final = asyncio.create_task(
create_audit_log_for_update(request_data=_deleted_key_audit_log("token-dropped"))
)
try:
for _ in range(3):
await asyncio.sleep(0)
await drain_audit_tasks(timeout=0.05)
finally:
verbose_proxy_logger.removeHandler(recorder)
writer.cancel()
with contextlib.suppress(asyncio.CancelledError):
await writer
assert recorder.messages == [
"Timed out draining 1 audit tasks on shutdown, dropping: "
f"deleted {LitellmTableNames.KEY_TABLE_NAME.value} token-dropped to the database"
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("store_audit_logs", "premium_user", "expected_write_count"),
[(True, True, 1), (False, True, 0), (True, False, 0)],
ids=["row_written", "logging_disabled", "not_premium"],
)
async def test_drain_does_not_wait_for_code_that_runs_after_an_inline_audit_write(
store_audit_logs: bool, premium_user: bool, expected_write_count: int
):
release_caller: Final = asyncio.Event()
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=lambda data: writes.append(data))
async def _caller_that_outlives_the_audit_call() -> None:
await create_audit_log_for_update(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by="test-user",
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id="test-obj",
action="updated",
updated_values="{}",
before_value="{}",
)
)
await release_caller.wait()
with (
patch("litellm.store_audit_logs", store_audit_logs),
patch("litellm.proxy.proxy_server.premium_user", premium_user),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
):
caller: Final = asyncio.create_task(_caller_that_outlives_the_audit_call())
for _ in range(3):
await asyncio.sleep(0)
drain: Final = asyncio.create_task(drain_audit_tasks(timeout=60))
for _ in range(3):
await asyncio.sleep(0)
drain_returned_while_caller_was_running: Final = drain.done()
release_caller.set()
await caller
await drain
assert drain_returned_while_caller_was_running
assert len(writes) == expected_write_count
async def _io_that_never_finishes(*args: object, **kwargs: object) -> None:
await asyncio.Event().wait()
async def _drain_while_hook_io_hangs(hook: Coroutine[object, object, None]) -> None:
hook_task: Final = asyncio.create_task(hook)
for _ in range(3):
await asyncio.sleep(0)
await drain_audit_tasks(timeout=2.0)
hook_task.cancel()
@pytest.mark.asyncio
async def test_key_generated_audit_write_is_drained_while_the_invite_email_hangs():
from litellm.proxy._types import GenerateKeyRequest, GenerateKeyResponse
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=lambda data: writes.append(data))
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch.object(KeyManagementEventHooks, "_send_key_created_email", side_effect=_io_that_never_finishes),
):
await _drain_while_hook_io_hangs(
KeyManagementEventHooks.async_key_generated_hook(
data=GenerateKeyRequest(send_invite_email=True),
response=GenerateKeyResponse(key="sk-new", token_id="hashed-new"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin", user_id="admin"),
)
)
assert [(write["object_id"], write["action"]) for write in writes] == [("hashed-new", "created")]
@pytest.mark.asyncio
async def test_key_updated_audit_write_is_drained_while_the_secret_manager_rename_hangs():
from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=lambda data: writes.append(data))
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch.object(
KeyManagementEventHooks, "_rename_virtual_key_in_secret_manager", side_effect=_io_that_never_finishes
),
):
await _drain_while_hook_io_hangs(
KeyManagementEventHooks.async_key_updated_hook(
data=UpdateKeyRequest(key="sk-existing", key_alias="new-alias"),
existing_key_row=LiteLLM_VerificationToken(token="hashed-existing", key_alias="old-alias"),
response=None,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin", user_id="admin"),
)
)
assert [(write["object_id"], write["action"]) for write in writes] == [(hash_token("sk-existing"), "updated")]
@pytest.mark.asyncio
async def test_key_rotated_audit_write_is_drained_while_the_secret_manager_rotation_hangs():
from litellm.proxy._types import GenerateKeyResponse, LiteLLM_VerificationToken
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=lambda data: writes.append(data))
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch.object(
KeyManagementEventHooks, "_rotate_virtual_key_in_secret_manager", side_effect=_io_that_never_finishes
),
):
await _drain_while_hook_io_hangs(
KeyManagementEventHooks.async_key_rotated_hook(
data=None,
existing_key_row=LiteLLM_VerificationToken(token="hashed-old"),
response=GenerateKeyResponse(key="sk-rotated", token_id="hashed-new"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin", user_id="admin"),
)
)
assert [(write["object_id"], write["action"]) for write in writes] == [("hashed-old", "rotated")]
@pytest.mark.asyncio
async def test_user_created_audit_write_is_drained_while_its_user_read_is_still_running():
from litellm.proxy._types import NewUserRequest, NewUserResponse
from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
mock_prisma: Final = MagicMock()
writes: Final[list[dict[str, Any]]] = []
release_user_read: Final = asyncio.Event()
async def _slow_user_read(*args: object, **kwargs: object) -> dict[str, str]:
await release_user_read.wait()
return {"user_id": "new-user"}
mock_prisma.db.litellm_auditlog.create = AsyncMock(side_effect=lambda data: writes.append(data))
mock_prisma.db.litellm_usertable.find_unique = AsyncMock(side_effect=_slow_user_read)
with (
patch("litellm.store_audit_logs", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch.object(UserManagementEventHooks, "async_send_user_invitation_email", side_effect=_io_that_never_finishes),
):
hook: Final = asyncio.create_task(
UserManagementEventHooks.async_user_created_hook(
data=NewUserRequest(user_id="new-user", send_invite_email=True),
response=NewUserResponse(key="sk-new", user_id="new-user"),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin", user_id="admin"),
)
)
for _ in range(3):
await asyncio.sleep(0)
drain: Final = asyncio.create_task(drain_audit_tasks(timeout=60))
for _ in range(3):
await asyncio.sleep(0)
drain_waited_for_the_user_read: Final = not drain.done()
release_user_read.set()
await drain
hook.cancel()
assert drain_waited_for_the_user_read
assert [(write["object_id"], write["action"]) for write in writes] == [("new-user", "created")]