The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import inspect
import json
from collections import Counter
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from typing import Final, Literal, NamedTuple, TypedDict
from typing_extensions import ReadOnly
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm.caching.caching import Cache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router import Router
_PRIMARY: Final = "https://hooks-primary.openai.azure.com"
_FALLBACK: Final = "https://hooks-fallback.openai.azure.com"
_API_VERSION: Final = "2024-10-21"
_MESSAGES: Final = [{"role": "user", "content": "Hi - i'm openai"}]
_USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}
_COMPLETION: Final = {
"id": "chatcmpl-hooks",
"object": "chat.completion",
"created": 1700000000,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}],
"usage": _USAGE,
}
_EMBEDDING_VECTOR: Final = [0.1, 0.2, 0.3]
_EMBEDDING: Final = {
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": _EMBEDDING_VECTOR}],
"model": "text-embedding-3-small",
"usage": {"prompt_tokens": 3, "total_tokens": 3},
}
_STREAM: Final = (
"".join(
f"data: {json.dumps(chunk)}\n\n"
for chunk in (
{
"id": "chatcmpl-hooks",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "hel"}, "finish_reason": None}],
},
{
"id": "chatcmpl-hooks",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "delta": {"content": "lo"}, "finish_reason": "stop"}],
},
)
)
+ "data: [DONE]\n\n"
)
_AUTH_ERROR: Final = httpx.Response(
401,
json={
"error": {"message": "Incorrect API key provided", "type": "invalid_request_error", "code": "invalid_api_key"}
},
)
_OUR_MODEL_GROUPS: Final = frozenset({"hooks-group", "primary-group", "fallback-group"})
_State = Literal[
"sync_pre_api_call",
"post_api_call",
"async_stream",
"sync_success",
"async_success",
"sync_failure",
"async_failure",
]
class _HookEvent(NamedTuple):
state: _State
model: object
kwargs: Mapping[str, object]
response: object
def _router_context_problems(kwargs: Mapping[str, object]) -> tuple[str, ...]:
litellm_params: Final = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
return ("litellm_params",)
metadata: Final = litellm_params.get("metadata")
model_info: Final = litellm_params.get("model_info")
checks: Final = {
"metadata": isinstance(metadata, dict),
"model_group": isinstance(metadata, dict) and isinstance(metadata.get("model_group"), str),
"deployment": isinstance(metadata, dict) and isinstance(metadata.get("deployment"), str),
"model_info": isinstance(model_info, dict),
"model_info id": isinstance(model_info, dict) and isinstance(model_info.get("id"), str),
"proxy_server_request": isinstance(litellm_params.get("proxy_server_request"), (str, type(None))),
"preset_cache_key": isinstance(litellm_params.get("preset_cache_key"), (str, type(None))),
"stream_response": isinstance(litellm_params.get("stream_response"), dict),
}
return tuple(name for name, ok in checks.items() if not ok)
def _request_problems(kwargs: Mapping[str, object]) -> tuple[str, ...]:
checks: Final = {
"model": isinstance(kwargs.get("model"), str),
"messages": isinstance(kwargs.get("messages"), list),
"optional_params": isinstance(kwargs.get("optional_params"), dict),
"start_time": isinstance(kwargs.get("start_time"), (datetime, type(None))),
"stream": isinstance(kwargs.get("stream"), bool),
"user": isinstance(kwargs.get("user"), (str, type(None))),
}
return (*(name for name, ok in checks.items() if not ok), *_router_context_problems(kwargs))
def _call_detail_problems(kwargs: Mapping[str, object]) -> tuple[str, ...]:
original_response: Final = kwargs.get("original_response")
checks: Final = {
"input": isinstance(kwargs.get("input"), (list, dict, str)),
"api_key": isinstance(kwargs.get("api_key"), (str, type(None))),
"original_response": isinstance(original_response, (str, litellm.CustomStreamWrapper, type(None)))
or inspect.iscoroutine(original_response)
or inspect.isasyncgen(original_response),
"additional_args": isinstance(kwargs.get("additional_args"), (dict, type(None))),
"log_event_type": isinstance(kwargs.get("log_event_type"), str),
}
return tuple(name for name, ok in checks.items() if not ok)
def _is_from_this_test(kwargs: Mapping[str, object]) -> bool:
litellm_params: Final = kwargs.get("litellm_params")
metadata: Final = litellm_params.get("metadata") if isinstance(litellm_params, dict) else None
return isinstance(metadata, dict) and metadata.get("model_group") in _OUR_MODEL_GROUPS
class _HookRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.events: Final[list[_HookEvent]] = []
self.errors: Final[list[str]] = []
self.loop: asyncio.AbstractEventLoop | None = None
self.waiters: Final[list[tuple[Callable[[Sequence[_State]], bool], asyncio.Event]]] = []
@property
def states(self) -> list[_State]:
return [event.state for event in self.events]
def _record(
self, state: _State, model: object, kwargs: Mapping[str, object], response: object, problems: Sequence[str]
) -> None:
if not _is_from_this_test(kwargs):
return
self.errors.extend(f"{state}: {problem}" for problem in problems)
self.events.append(_HookEvent(state, model, kwargs, response))
if self.loop is not None:
self.loop.call_soon_threadsafe(self._notify)
def _notify(self) -> None:
for predicate, event in self.waiters:
if predicate(tuple(self.states)):
event.set()
async def until(self, predicate: Callable[[Sequence[_State]], bool]) -> tuple[_State, ...]:
self.loop = asyncio.get_running_loop()
if not predicate(tuple(self.states)):
event: Final = asyncio.Event()
self.waiters.append((predicate, event))
await asyncio.wait_for(event.wait(), timeout=10)
return tuple(self.states)
def log_pre_api_call(self, model: object, messages: object, kwargs: Mapping[str, object]) -> None:
problems: Final = (
*(("model",) if not isinstance(model, str) else ()),
*(("messages",) if not isinstance(messages, list) else ()),
*_request_problems(kwargs),
)
self._record("sync_pre_api_call", model, kwargs, messages, problems)
def log_post_api_call(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
problems: Final = (
*(("start_time",) if not isinstance(start_time, datetime) else ()),
*(("end_time",) if end_time is not None else ()),
*(("response_obj",) if response_obj is not None else ()),
*_request_problems(kwargs),
*_call_detail_problems(kwargs),
)
self._record("post_api_call", kwargs.get("model"), kwargs, response_obj, problems)
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record("sync_success", kwargs.get("model"), kwargs, response_obj, ())
def log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record("sync_failure", kwargs.get("model"), kwargs, response_obj, ())
async def async_log_stream_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record("async_stream", kwargs.get("model"), kwargs, response_obj, ())
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
problems: Final = (
*(("times",) if not (isinstance(start_time, datetime) and isinstance(end_time, datetime)) else ()),
*(
("response_obj",)
if not isinstance(response_obj, (litellm.ModelResponse, litellm.EmbeddingResponse))
else ()
),
*(("cache_hit",) if not isinstance(kwargs.get("cache_hit"), (bool, type(None))) else ()),
*_request_problems(kwargs),
*_call_detail_problems(kwargs),
)
self._record("async_success", kwargs.get("model"), kwargs, response_obj, problems)
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
problems: Final = (
*(("times",) if not (isinstance(start_time, datetime) and isinstance(end_time, datetime)) else ()),
*(("response_obj",) if response_obj is not None else ()),
*(("exception",) if not isinstance(kwargs.get("exception"), Exception) else ()),
*_request_problems(kwargs),
*_call_detail_problems(kwargs),
)
self._record("async_failure", kwargs.get("model"), kwargs, response_obj, problems)
def _settled(terminal: int, posts: int) -> Callable[[Sequence[_State]], bool]:
def satisfied(states: Sequence[_State]) -> bool:
terminals: Final = sum(state in ("async_success", "async_failure") for state in states)
return terminals >= terminal and states.count("post_api_call") >= posts
return satisfied
@pytest.fixture
def recorder(monkeypatch: pytest.MonkeyPatch) -> _HookRecorder:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
hook_recorder: Final = _HookRecorder()
monkeypatch.setattr(litellm, "callbacks", [hook_recorder])
return hook_recorder
def _router(model: str, api_base: str = _PRIMARY) -> Router:
return Router(
model_list=[
{
"model_name": "hooks-group",
"litellm_params": {
"model": model,
"api_key": "sk-unit-test",
"api_base": api_base,
"api_version": _API_VERSION,
},
"model_info": {"base_model": model},
}
],
num_retries=0,
)
class _RouterMetadata(TypedDict):
model_group: ReadOnly[str]
deployment: ReadOnly[str]
class _RouterModelInfo(TypedDict):
id: ReadOnly[str]
class _RouterParams(TypedDict):
metadata: ReadOnly[_RouterMetadata]
model_info: ReadOnly[_RouterModelInfo]
def _router_params(event: _HookEvent) -> _RouterParams:
return TypeAdapter(_RouterParams).validate_python(event.kwargs["litellm_params"])
def _model_group(event: _HookEvent) -> str:
return _router_params(event)["metadata"]["model_group"]
def _model_id(event: _HookEvent) -> str:
return _router_params(event)["model_info"]["id"]
def _of_state(recorder: _HookRecorder, state: _State) -> list[_HookEvent]:
return [event for event in recorder.events if event.state == state]
def _completion(event: _HookEvent) -> litellm.ModelResponse:
assert isinstance(event.response, litellm.ModelResponse)
return event.response
@pytest.mark.asyncio
async def test_router_chat_success_streaming_and_failure_fire_the_hooks_in_order(
recorder: _HookRecorder, respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post(url__startswith=f"{_PRIMARY}/openai/deployments/gpt-4.1-mini/chat/completions")
router: Final = _router("azure/gpt-4.1-mini")
route.mock(return_value=httpx.Response(200, json=_COMPLETION))
await router.acompletion(model="hooks-group", messages=_MESSAGES)
await recorder.until(_settled(1, posts=1))
assert recorder.states == ["sync_pre_api_call", "post_api_call", "async_success"]
pre, post, success = recorder.events
assert pre.model == "gpt-4.1-mini"
assert pre.response == _MESSAGES
assert {_model_group(event) for event in recorder.events} == {"hooks-group"}
assert len({_model_id(event) for event in recorder.events}) == 1
assert post.kwargs["messages"] == _MESSAGES
assert success.kwargs["stream"] is False
assert _completion(success).choices[0].message.content == "hello"
assert _completion(success).usage.total_tokens == _USAGE["total_tokens"]
route.mock(return_value=httpx.Response(200, text=_STREAM, headers={"content-type": "text/event-stream"}))
stream: Final = await router.acompletion(model="hooks-group", messages=_MESSAGES, stream=True)
assert "".join([chunk.choices[0].delta.content or "" async for chunk in stream]) == "hello"
await recorder.until(_settled(2, posts=2))
streamed: Final = recorder.events[3:]
assert sorted(event.state for event in streamed[:2]) == ["post_api_call", "sync_pre_api_call"]
assert [event.state for event in streamed[2:]] == ["async_success"]
assert all(event.kwargs["stream"] is True for event in streamed)
assert _completion(streamed[2]).choices[0].message.content == "hello"
route.mock(return_value=_AUTH_ERROR)
with pytest.raises(litellm.AuthenticationError):
await router.acompletion(model="hooks-group", messages=_MESSAGES)
await recorder.until(_settled(3, posts=3))
failed: Final = recorder.events[6:]
assert [event.state for event in failed] == ["sync_pre_api_call", "post_api_call", "async_failure"]
assert failed[2].response is None
assert isinstance(failed[2].kwargs["exception"], litellm.AuthenticationError)
assert recorder.errors == []
@pytest.mark.asyncio
async def test_router_embedding_success_and_failure_fire_the_hooks_in_order(
recorder: _HookRecorder, respx_mock: respx.MockRouter
) -> None:
route: Final = respx_mock.post(url__startswith=f"{_PRIMARY}/openai/deployments/text-embedding-3-small/embeddings")
router: Final = _router("azure/text-embedding-3-small")
route.mock(return_value=httpx.Response(200, json=_EMBEDDING))
await router.aembedding(model="hooks-group", input=["hello"])
await recorder.until(_settled(1, posts=1))
assert recorder.states == ["sync_pre_api_call", "post_api_call", "async_success"]
assert {event.model for event in recorder.events} == {"text-embedding-3-small"}
assert {_model_group(event) for event in recorder.events} == {"hooks-group"}
embedding: Final = recorder.events[2].response
assert isinstance(embedding, litellm.EmbeddingResponse)
assert embedding.model_dump()["data"][0]["embedding"] == _EMBEDDING_VECTOR
assert embedding.usage.prompt_tokens == 3
route.mock(return_value=_AUTH_ERROR)
with pytest.raises(litellm.AuthenticationError):
await router.aembedding(model="hooks-group", input=["hello"])
await recorder.until(_settled(2, posts=2))
assert recorder.states[3:] == ["sync_pre_api_call", "post_api_call", "async_failure"]
assert recorder.events[5].response is None
assert isinstance(recorder.events[5].kwargs["exception"], litellm.AuthenticationError)
assert recorder.errors == []
@pytest.mark.asyncio
async def test_router_fallback_fires_failure_then_success_hooks(
recorder: _HookRecorder, respx_mock: respx.MockRouter
) -> None:
respx_mock.post(url__startswith=f"{_PRIMARY}/openai/deployments/gpt-4.1-mini/chat/completions").mock(
return_value=_AUTH_ERROR
)
fallback: Final = respx_mock.post(
url__startswith=f"{_FALLBACK}/openai/deployments/gpt-4.1-mini/chat/completions"
).mock(return_value=httpx.Response(200, json=_COMPLETION))
router: Final = Router(
model_list=[
{
"model_name": "primary-group",
"litellm_params": {
"model": "azure/gpt-4.1-mini",
"api_key": "my-bad-key",
"api_base": _PRIMARY,
"api_version": _API_VERSION,
},
},
{
"model_name": "fallback-group",
"litellm_params": {
"model": "azure/gpt-4.1-mini",
"api_key": "sk-unit-test",
"api_base": _FALLBACK,
"api_version": _API_VERSION,
},
},
],
fallbacks=[{"primary-group": ["fallback-group"]}],
num_retries=0,
)
await router.acompletion(model="primary-group", messages=_MESSAGES)
await recorder.until(_settled(2, posts=2))
assert fallback.call_count == 1
assert recorder.states == [
"sync_pre_api_call",
"post_api_call",
"async_failure",
"sync_pre_api_call",
"post_api_call",
"async_success",
]
assert [_model_group(event) for event in recorder.events] == ["primary-group"] * 3 + ["fallback-group"] * 3
assert _model_id(recorder.events[0]) != _model_id(recorder.events[3])
assert isinstance(recorder.events[2].kwargs["exception"], litellm.AuthenticationError)
assert _completion(recorder.events[5]).choices[0].message.content == "hello"
assert recorder.errors == []
@pytest.mark.asyncio
async def test_router_completion_cache_hit_fires_a_second_success_hook(
recorder: _HookRecorder, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "cache", Cache())
route: Final = respx_mock.post(url__startswith=f"{_PRIMARY}/openai/deployments/gpt-4.1-mini/chat/completions").mock(
return_value=httpx.Response(200, json=_COMPLETION)
)
router: Final = _router("azure/gpt-4.1-mini")
await router.acompletion(model="hooks-group", messages=_MESSAGES, caching=True)
await recorder.until(_settled(1, posts=1))
await router.acompletion(model="hooks-group", messages=_MESSAGES, caching=True)
await recorder.until(_settled(2, posts=1))
assert route.call_count == 1
assert recorder.states == ["sync_pre_api_call", "post_api_call", "async_success", "async_success"]
first, second = _of_state(recorder, "async_success")
assert first.kwargs.get("cache_hit") is not True
assert second.kwargs.get("cache_hit") is True
assert _completion(second).choices[0].message.content == "hello"
assert recorder.errors == []
@pytest.mark.asyncio
async def test_router_streaming_cache_hit_still_fires_the_success_hook(
recorder: _HookRecorder, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "cache", Cache())
route: Final = respx_mock.post(url__startswith=f"{_PRIMARY}/openai/deployments/gpt-4.1-mini/chat/completions").mock(
return_value=httpx.Response(200, text=_STREAM, headers={"content-type": "text/event-stream"})
)
router: Final = _router("azure/gpt-4.1-mini")
first: Final = await router.acompletion(model="hooks-group", messages=_MESSAGES, stream=True, caching=True)
first_text: Final = "".join([chunk.choices[0].delta.content or "" async for chunk in first])
await recorder.until(_settled(1, posts=1))
states_after_first: Final = len(recorder.states)
second: Final = await router.acompletion(model="hooks-group", messages=_MESSAGES, stream=True, caching=True)
second_text: Final = "".join([chunk.choices[0].delta.content or "" async for chunk in second])
await recorder.until(_settled(2, posts=1))
assert route.call_count == 1
assert first_text == second_text == "hello"
assert sorted(recorder.states[:2]) == ["post_api_call", "sync_pre_api_call"]
assert recorder.states[2:states_after_first] == ["async_success"]
assert recorder.states[states_after_first:] == ["async_success"]
first_success, second_success = _of_state(recorder, "async_success")
assert first_success.kwargs.get("cache_hit") is not True
assert second_success.kwargs.get("cache_hit") is True
assert _completion(second_success).choices[0].message.content == "hello"
assert recorder.errors == []
@pytest.mark.asyncio
async def test_router_embedding_cache_hit_fires_a_second_success_hook(
recorder: _HookRecorder, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "cache", Cache())
route: Final = respx_mock.post(
url__startswith=f"{_PRIMARY}/openai/deployments/text-embedding-3-small/embeddings"
).mock(return_value=httpx.Response(200, json=_EMBEDDING))
router: Final = _router("azure/text-embedding-3-small")
await router.aembedding(model="hooks-group", input=["hello"], caching=True)
await recorder.until(_settled(1, posts=1))
await router.aembedding(model="hooks-group", input=["hello"], caching=True)
await recorder.until(_settled(2, posts=1))
assert route.call_count == 1
assert recorder.states == ["sync_pre_api_call", "post_api_call", "async_success", "async_success"]
first, second = _of_state(recorder, "async_success")
assert first.kwargs.get("cache_hit") is not True
assert second.kwargs.get("cache_hit") is True
assert isinstance(second.response, litellm.EmbeddingResponse)
assert second.response.model_dump()["data"][0]["embedding"] == _EMBEDDING_VECTOR
assert recorder.errors == []