The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import uuid
from typing import Final
import pytest
from integration._support.responses_stream import (
AZURE_TARGET,
RATE_LIMIT_MESSAGE,
function_tools,
healthy_stream,
openai_error_stream,
rate_limited_stream,
serve,
)
from integration._support.wire import Wire, wire_server
import litellm
from litellm import Router
from litellm.exceptions import APIError, MidStreamFallbackError, RateLimitError
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
_MODEL: Final = "azure/gpt-6"
_GROUP: Final = "bridged-gpt-6"
_API_KEY: Final = "synthetic-azure-key"
_SENTINEL_PREFIX: Final = "litellm.MidStreamFallbackError: "
_TOOLS: Final = function_tools()
def _messages(marker: str) -> list[dict[str, str]]:
return [{"role": "user", "content": marker}]
def _router(wire: Wire) -> Router:
return Router(
model_list=[
{"model_name": _GROUP, "litellm_params": {"model": _MODEL, "api_base": wire.url, "api_key": _API_KEY}}
],
num_retries=0,
)
def _assert_one_attempt(wire: Wire, marker: str) -> None:
received: Final = wire.drain()
assert len(received) == 1 and marker.encode() in received[0].body, [request.target for request in received]
def _assert_wraps_the_provider_exception_once(raised: MidStreamFallbackError, wire: Wire, marker: str) -> None:
inner: Final = raised.original_exception
assert isinstance(inner, RateLimitError), repr(inner)
assert inner.status_code == 429 and RATE_LIMIT_MESSAGE in str(inner), str(inner)
assert raised.status_code == 429, raised.status_code
assert raised.is_pre_first_chunk and raised.generated_content == "", (
raised.is_pre_first_chunk,
raised.generated_content,
)
assert str(raised).count(_SENTINEL_PREFIX) == 1, str(raised)
_assert_one_attempt(wire, marker)
def _assert_surfaces_the_provider_exception(raised: RateLimitError, wire: Wire, marker: str) -> None:
assert type(raised) is RateLimitError, type(raised)
assert raised.status_code == 429 and RATE_LIMIT_MESSAGE in str(raised), str(raised)
assert _SENTINEL_PREFIX not in str(raised), str(raised)
_assert_one_attempt(wire, marker)
def test_sync_completion_stream_in_stream_rate_limit_wraps_the_provider_exception_once() -> None:
marker: Final = uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(f"resp_{marker}"), AZURE_TARGET)) as wire:
response: Final = litellm.completion(
model=_MODEL,
messages=_messages(marker),
tools=_TOOLS,
stream=True,
num_retries=0,
api_base=wire.url,
api_key=_API_KEY,
)
assert isinstance(response, CustomStreamWrapper), type(response)
with pytest.raises(MidStreamFallbackError) as raised:
for _ in response:
pass
_assert_wraps_the_provider_exception_once(raised.value, wire, marker)
async def test_async_completion_stream_in_stream_rate_limit_wraps_the_provider_exception_once() -> None:
marker: Final = uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(f"resp_{marker}"), AZURE_TARGET)) as wire:
response: Final = await litellm.acompletion(
model=_MODEL,
messages=_messages(marker),
tools=_TOOLS,
stream=True,
num_retries=0,
api_base=wire.url,
api_key=_API_KEY,
)
assert isinstance(response, CustomStreamWrapper), type(response)
with pytest.raises(MidStreamFallbackError) as raised:
async for _ in response:
pass
_assert_wraps_the_provider_exception_once(raised.value, wire, marker)
def test_router_sync_stream_in_stream_rate_limit_with_fallbacks_disabled_surfaces_the_provider_exception() -> None:
marker: Final = uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(f"resp_{marker}"), AZURE_TARGET)) as wire:
with pytest.raises(RateLimitError) as raised:
_router(wire).completion(
model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True, disable_fallbacks=True
)
_assert_surfaces_the_provider_exception(raised.value, wire, marker)
async def test_router_async_stream_in_stream_rate_limit_without_fallbacks_surfaces_the_provider_exception() -> None:
marker: Final = uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(f"resp_{marker}"), AZURE_TARGET)) as wire:
with pytest.raises(RateLimitError) as raised:
await _router(wire).acompletion(model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True)
_assert_surfaces_the_provider_exception(raised.value, wire, marker)
async def test_router_async_stream_in_stream_rate_limit_with_fallbacks_disabled_surfaces_the_provider_exception() -> (
None
):
marker: Final = uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(f"resp_{marker}"), AZURE_TARGET)) as wire:
with pytest.raises(RateLimitError) as raised:
await _router(wire).acompletion(
model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True, disable_fallbacks=True
)
_assert_surfaces_the_provider_exception(raised.value, wire, marker)
def _openai_error_stream(marker: str, code: str, message: str) -> tuple[bytes, ...]:
return openai_error_stream(f"resp_{marker}", code, message)
def test_sync_completion_stream_openai_error_event_raises_the_provider_code_and_message() -> None:
marker: Final = uuid.uuid4().hex
message: Final = f"The server had an error processing your request {marker}"
with wire_server(serve(_openai_error_stream(marker, "server_error", message), AZURE_TARGET)) as wire:
response: Final = litellm.completion(
model=_MODEL,
messages=_messages(marker),
tools=_TOOLS,
stream=True,
num_retries=0,
api_base=wire.url,
api_key=_API_KEY,
)
with pytest.raises(MidStreamFallbackError) as raised:
for _ in response:
pass
assert type(raised.value.original_exception) is APIError, repr(raised.value.original_exception)
assert raised.value.status_code == 500 and message in str(raised.value), str(raised.value)
_assert_one_attempt(wire, marker)
async def test_router_async_stream_openai_error_event_rate_limit_surfaces_the_provider_exception() -> None:
marker: Final = uuid.uuid4().hex
stream: Final = _openai_error_stream(marker, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire:
with pytest.raises(RateLimitError) as raised:
await _router(wire).acompletion(model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True)
_assert_surfaces_the_provider_exception(raised.value, wire, marker)
async def test_async_completion_stream_top_level_rate_limit_wraps_the_provider_exception_once() -> None:
marker: Final = uuid.uuid4().hex
stream: Final = _openai_error_stream(marker, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire:
response: Final = await litellm.acompletion(
model=_MODEL,
messages=_messages(marker),
tools=_TOOLS,
stream=True,
num_retries=0,
api_base=wire.url,
api_key=_API_KEY,
)
with pytest.raises(MidStreamFallbackError) as raised:
async for _ in response:
pass
_assert_wraps_the_provider_exception_once(raised.value, wire, marker)
def test_router_sync_stream_top_level_rate_limit_with_fallbacks_disabled_surfaces_the_provider_exception() -> None:
marker: Final = uuid.uuid4().hex
stream: Final = _openai_error_stream(marker, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire:
with pytest.raises(RateLimitError) as raised:
_router(wire).completion(
model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True, disable_fallbacks=True
)
_assert_surfaces_the_provider_exception(raised.value, wire, marker)
async def test_router_async_stream_top_level_rate_limit_falls_back_to_the_healthy_deployment() -> None:
marker: Final = uuid.uuid4().hex
limited: Final = _openai_error_stream(marker, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with (
wire_server(serve(limited, AZURE_TARGET)) as primary,
wire_server(serve(healthy_stream(f"resp_{marker}", "fallback answer"), AZURE_TARGET)) as spare,
):
router: Final = Router(
model_list=[
{"model_name": name, "litellm_params": {"model": _MODEL, "api_base": wire.url, "api_key": _API_KEY}}
for name, wire in ((_GROUP, primary), ("bridged-spare", spare))
],
num_retries=0,
disable_cooldowns=True,
fallbacks=[{_GROUP: ["bridged-spare"]}],
)
response: Final = await router.acompletion(model=_GROUP, messages=_messages(marker), tools=_TOOLS, stream=True)
content: Final = "".join([chunk.choices[0].delta.content or "" async for chunk in response if chunk.choices])
assert content == "fallback answer", content
_assert_one_attempt(primary, marker)
_assert_one_attempt(spare, marker)
def _provider_rate_limit(raised: Exception, message: str) -> None:
inner: Final = raised.original_exception if isinstance(raised, MidStreamFallbackError) else raised
assert type(inner) is RateLimitError, repr(inner)
assert getattr(raised, "status_code", None) == 429 and message in str(raised), str(raised)
def test_sync_responses_stream_top_level_rate_limit_raises_the_provider_rate_limit() -> None:
marker: Final = uuid.uuid4().hex
stream: Final = _openai_error_stream(marker, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire:
response: Final = litellm.responses(
model=_MODEL, input=marker, stream=True, api_base=wire.url, api_key=_API_KEY
)
with pytest.raises((MidStreamFallbackError, RateLimitError)) as raised:
for _ in response:
pass
_provider_rate_limit(raised.value, RATE_LIMIT_MESSAGE)
_assert_one_attempt(wire, marker)
async def test_async_responses_stream_top_level_insufficient_quota_raises_a_rate_limit() -> None:
marker: Final = uuid.uuid4().hex
quota: Final = "You exceeded your current quota, please check your plan and billing details."
with wire_server(serve(_openai_error_stream(marker, "insufficient_quota", quota), AZURE_TARGET)) as wire:
response: Final = await litellm.aresponses(
model=_MODEL, input=marker, stream=True, api_base=wire.url, api_key=_API_KEY
)
with pytest.raises((MidStreamFallbackError, RateLimitError)) as raised:
async for _ in response:
pass
_provider_rate_limit(raised.value, quota)
_assert_one_attempt(wire, marker)