The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import uuid
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import anthropic
import httpx
import openai
import pytest
import yaml
from integration._support.client import (
Gateway,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.responses_stream import (
AZURE_TARGET,
OPENAI_TARGET,
RATE_LIMIT_MESSAGE,
RATE_LIMIT_PREFIX,
SENTINEL_PREFIX,
assert_provider_typed_rate_limit,
azure_rate_limit,
chat_body,
chat_content,
created,
data_frames,
delta,
error_body,
error_event,
failed,
frame,
function_tools,
healthy_stream,
messages_body,
openai_error_event,
openai_error_stream,
rate_limited_stream,
serve,
spend_row,
sse_events,
)
from integration._support.wire import Reply, Request, Wire, wire_server
from integration._support.ports import allocate_port
from pydantic import JsonValue
_PRIMARY: Final = "bridged-primary"
_SPARE: Final = "bridged-spare"
pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 120), pytest.mark.shares_worker_state]
def assert_failed_once(wire: Wire, call_id: str, model: str, attempts: int = 1) -> tuple[Request, ...]:
received: Final = wire.drain()
assert len(received) == attempts, [request.target for request in received]
row: Final = spend_row(call_id)
assert row["status"] == "failure" and row["model_group"] == model, row
return received
def test_bridged_azure_in_stream_rate_limit_reaches_the_openai_sdk_as_a_throttling_error(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(identity), AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
with pytest.raises(openai.RateLimitError) as raised:
client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": identity}],
tools=function_tools(),
stream=True,
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
)
assert raised.value.status_code == 429
assert_provider_typed_rate_limit(object_value(raised.value.body))
assert_failed_once(wire, raised.value.response.headers["x-litellm-call-id"], model)
async def test_bridged_openai_in_stream_rate_limit_reaches_the_async_openai_sdk_as_a_throttling_error(
gateway: Gateway,
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(identity), OPENAI_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="openai/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key")
client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
with pytest.raises(openai.RateLimitError) as raised:
await client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": identity}],
stream=True,
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
)
assert raised.value.status_code == 429
assert_provider_typed_rate_limit(object_value(raised.value.body))
assert_failed_once(wire, raised.value.response.headers["x-litellm-call-id"], model)
def _chat(gateway: Gateway, body: Mapping[str, JsonValue]) -> httpx.Response:
return gateway.request("POST", "/v1/chat/completions", body)
@dataclass(frozen=True, slots=True)
class FallbackProxy:
gateway: Gateway
primary_port: int
spare_port: int
def _fallback_config(
directory: Path, primary_port: int, spare_port: int, extra_router_settings: Mapping[str, JsonValue]
) -> Path:
base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
deployments: Final = [
{
"model_name": name,
"litellm_params": {
"model": "azure/gpt-6",
"api_base": f"http://127.0.0.1:{port}",
"api_key": "synthetic-azure-key",
},
}
for name, port in ((_PRIMARY, primary_port), (_SPARE, spare_port))
]
router_settings: Final = {
"num_retries": 0,
"disable_cooldowns": True,
"fallbacks": [{_PRIMARY: [_SPARE]}],
**extra_router_settings,
}
path: Final = directory / "bridged-fallbacks.yaml"
path.write_text(yaml.safe_dump({**base, "model_list": deployments, "router_settings": router_settings}))
return path
def _fallback_proxy(
tmp_path_factory: pytest.TempPathFactory, extra_router_settings: Mapping[str, JsonValue]
) -> Iterator[FallbackProxy]:
directory: Final = tmp_path_factory.mktemp("bridged-fallbacks")
primary_port, spare_port = allocate_port(), allocate_port()
config: Final = _fallback_config(directory, primary_port, spare_port, extra_router_settings)
with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned:
yield FallbackProxy(owned, primary_port, spare_port)
@pytest.fixture(scope="module")
def fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]:
yield from _fallback_proxy(tmp_path_factory, {})
@pytest.fixture(scope="module")
def bad_request_opt_out_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]:
yield from _fallback_proxy(tmp_path_factory, {"fallback_on_unsupported_params": False})
def test_bridged_in_stream_rate_limit_falls_back_to_the_healthy_deployment(fallback_proxy: FallbackProxy) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
wire_server(serve(rate_limited_stream(identity), AZURE_TARGET), port=fallback_proxy.primary_port) as primary,
wire_server(
serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET), port=fallback_proxy.spare_port
) as spare,
):
response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 200, response.text
content: Final = chat_content(data_frames(response.text))
assert content == "fallback answer", response.text
assert len(primary.drain()) == 1 and len(spare.drain()) == 1
row: Final = spend_row(response.headers["x-litellm-call-id"])
assert row["status"] == "success" and row["model_group"] == _SPARE, row
def test_bridged_in_stream_rate_limit_whose_fallback_is_also_rate_limited_answers_a_throttling_error(
fallback_proxy: FallbackProxy,
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
wire_server(serve(rate_limited_stream(identity), AZURE_TARGET), port=fallback_proxy.primary_port) as primary,
wire_server(serve(rate_limited_stream(identity), AZURE_TARGET), port=fallback_proxy.spare_port) as spare,
):
response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response))
assert len(primary.drain()) == 1 and len(spare.drain()) == 1
assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
def test_bridged_top_level_rate_limit_falls_back_to_the_healthy_deployment(fallback_proxy: FallbackProxy) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
limited: Final = openai_error_stream(identity, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with (
wire_server(serve(limited, AZURE_TARGET), port=fallback_proxy.primary_port) as primary,
wire_server(
serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET), port=fallback_proxy.spare_port
) as spare,
):
response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 200, response.text
assert chat_content(data_frames(response.text)) == "fallback answer", response.text
assert len(primary.drain()) == 1 and len(spare.drain()) == 1
row: Final = spend_row(response.headers["x-litellm-call-id"])
assert row["status"] == "success" and row["model_group"] == _SPARE, row
_INVALID_PROMPT: Final = "Invalid prompt: your prompt was flagged."
_INVALID_PROMPT_REJECTIONS: Final = pytest.mark.parametrize(
"rejection",
[
pytest.param(
error_event({"type": "invalid_request_error", "code": "invalid_prompt", "message": _INVALID_PROMPT}),
id="nested",
),
pytest.param(openai_error_event("invalid_prompt", _INVALID_PROMPT), id="top_level"),
],
)
@_INVALID_PROMPT_REJECTIONS
def test_bridged_invalid_prompt_before_any_token_falls_back_to_the_healthy_deployment(
fallback_proxy: FallbackProxy, rejection: Mapping[str, JsonValue]
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
wire_server(
serve((frame(created(identity)), frame(rejection)), AZURE_TARGET), port=fallback_proxy.primary_port
) as primary,
wire_server(
serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET), port=fallback_proxy.spare_port
) as spare,
):
response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 200, response.text
assert chat_content(data_frames(response.text)) == "fallback answer", response.text
assert len(primary.drain()) == 1 and len(spare.drain()) == 1
row: Final = spend_row(response.headers["x-litellm-call-id"])
assert row["status"] == "success" and row["model_group"] == _SPARE, row
@_INVALID_PROMPT_REJECTIONS
def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallback_when_bad_requests_opt_out(
bad_request_opt_out_proxy: FallbackProxy, rejection: Mapping[str, JsonValue]
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
wire_server(
serve((frame(created(identity)), frame(rejection)), AZURE_TARGET),
port=bad_request_opt_out_proxy.primary_port,
) as primary,
wire_server(
serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET),
port=bad_request_opt_out_proxy.spare_port,
) as spare,
):
response: Final = _chat(bad_request_opt_out_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 400, response.text
message: Final = string_value(error_body(response)["message"])
assert message.startswith("litellm.BadRequestError: ") and _INVALID_PROMPT in message, message
assert len(primary.drain()) == 1 and len(spare.drain()) == 0
def _in_stream_error_status(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[httpx.Response, str]:
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
response: Final = _chat(gateway, chat_body(model, uuid.uuid4().hex))
assert_failed_once(wire, response.headers["x-litellm-call-id"], model)
return response, model
def test_bridged_in_stream_server_error_reaches_the_client_as_the_provider_error(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (
frame(created(identity)),
frame(error_event({"type": "server_error", "code": "server_error", "message": "The server had an error"})),
frame(failed(identity, "server_error", "The server had an error")),
)
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 500, response.text
error: Final = error_body(response)
message: Final = string_value(error["message"])
assert str(error["code"]) == "500", error
assert message.startswith("litellm.APIError: ") and "The server had an error" in message, message
assert SENTINEL_PREFIX not in message, message
def test_bridged_in_stream_invalid_prompt_is_a_bad_request_on_both_legs(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (
frame(created(identity)),
frame(error_event({"type": "invalid_request_error", "code": "invalid_prompt", "message": "Invalid prompt"})),
frame(failed(identity, "invalid_prompt", "Invalid prompt")),
)
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 400, response.text
error: Final = error_body(response)
message: Final = string_value(error["message"])
assert str(error["code"]) == "400", error
assert message.startswith("litellm.BadRequestError: ") and "Invalid prompt" in message, message
assert SENTINEL_PREFIX not in message, message
def test_bridged_error_event_without_an_error_object_is_a_provider_typed_internal_error(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (frame(created(identity)), frame(error_event(None)))
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 500, response.text
error: Final = error_body(response)
message: Final = string_value(error["message"])
assert str(error["code"]) == "500", error
assert message.startswith("litellm.APIError: ") and "Response API in-stream error" in message, message
assert SENTINEL_PREFIX not in message, message
def test_bridged_error_event_with_a_numeric_code_is_a_throttling_error(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (frame(created(identity)), frame(error_event({"code": "429", "message": RATE_LIMIT_MESSAGE})))
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response))
def test_bridged_openai_error_event_keeps_the_provider_code_and_message(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (frame(created(identity)), frame(openai_error_event("rate_limit_exceeded", RATE_LIMIT_MESSAGE)))
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response))
def test_bridged_response_failed_without_an_error_event_is_a_throttling_error(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (frame(created(identity)), frame(failed(identity, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)))
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response))
def test_bridged_rate_limit_after_output_is_a_provider_typed_error_frame_behind_the_text(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
stream: Final = (
frame(created(identity)),
frame(delta(identity, "Hello")),
frame(delta(identity, " there")),
frame(error_event(azure_rate_limit())),
frame(failed(identity, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)),
)
response, _ = _in_stream_error_status(gateway, stream)
assert response.status_code == 200, response.text
assert response.text.rstrip().endswith("data: [DONE]") and response.text.count("data: [DONE]") == 1, response.text
frames: Final = data_frames(response.text)
content: Final = chat_content(frames)
assert content == "Hello there", response.text
error: Final = object_value(frames[-1]["error"])
assert str(error["code"]) == "429", error
assert error["type"] == "throttling_error", error
message: Final = string_value(error["message"])
assert message.startswith(RATE_LIMIT_PREFIX) and RATE_LIMIT_MESSAGE in message, message
assert SENTINEL_PREFIX not in message, message
def test_bridged_transport_drop_after_response_created_is_a_500_without_the_sentinel(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
def respond(request: Request) -> Reply:
assert request.target.startswith(AZURE_TARGET), request.target
return Reply(
content_type="text/event-stream",
chunks=(frame(created(identity)), frame(delta(identity, "never sent"))),
abort_after=1,
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
response: Final = _chat(gateway, chat_body(model, identity))
assert response.status_code == 500, response.text
error: Final = error_body(response)
message: Final = string_value(error["message"])
assert str(error["code"]) == "500", error
assert "never sent" not in response.text
assert SENTINEL_PREFIX not in message, message
assert_failed_once(wire, response.headers["x-litellm-call-id"], model)
def test_plain_chat_http_rate_limit_is_a_throttling_error_on_both_legs(gateway: Gateway) -> None:
identity: Final = uuid.uuid4().hex
denial: Final = {"error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"}}
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/chat/completions", request.target
return Reply(status=429, body=json.dumps(denial).encode())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
with pytest.raises(openai.RateLimitError) as raised:
client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": identity}],
stream=True,
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
)
assert raised.value.status_code == 429
error: Final = object_value(raised.value.body)
assert error["type"] == "throttling_error" and str(error["code"]) == "429", error
message: Final = string_value(error["message"])
assert message.startswith(RATE_LIMIT_PREFIX) and "Rate limit reached" in message, message
assert_failed_once(wire, raised.value.response.headers["x-litellm-call-id"], model)
def assert_messages_errorframe(events: Sequence[tuple[str, Mapping[str, JsonValue]]]) -> None:
assert events[0][0] == "message_start", events
assert events[-1][0] == "error", events
error: Final = object_value(events[-1][1]["error"])
assert error["type"] == "rate_limit_error", error
message: Final = string_value(error["message"])
assert message.startswith(RATE_LIMIT_PREFIX) and RATE_LIMIT_MESSAGE in message, message
assert message.count(RATE_LIMIT_PREFIX) == 1 and SENTINEL_PREFIX not in message, message
def test_messages_over_the_bridged_stream_carry_the_provider_error_once_in_the_errorframe(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(identity), AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
response: Final = gateway.request(
"POST", "/v1/messages", messages_body(model, identity), headers={"anthropic-version": "2023-06-01"}
)
assert response.status_code == 200, response.text
assert_messages_errorframe(sse_events(response.text))
assert len(wire.drain()) == 1
async def _consume_anthropic_stream(client: anthropic.AsyncAnthropic, model: str, identity: str) -> None:
async with client.messages.stream(
model=model,
max_tokens=64,
messages=[{"role": "user", "content": identity}],
tools=[
{
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
}
],
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
) as stream:
async for _ in stream:
pass
async def test_messages_over_the_bridged_stream_raise_the_error_frame_in_the_anthropic_sdk(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(identity), AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
client: Final = anthropic.AsyncAnthropic(
base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0
)
with pytest.raises(anthropic.APIStatusError) as raised:
await _consume_anthropic_stream(client, model, identity)
body: Final = object_value(raised.value.body)
assert_messages_errorframe((("message_start", {}), ("error", body)))
assert len(wire.drain()) == 1
def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with wire_server(serve(rate_limited_stream(identity), AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="azure/gpt-6", api_base=wire.url, api_key="synthetic-azure-key")
response: Final = gateway.request(
"POST",
"/v1/responses",
{"model": model, "input": identity, "stream": True, "cache": {"no-cache": True}},
)
assert response.status_code == 200, response.text
frames: Final = data_frames(response.text)
assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text
failed: Final = object_value(frames[-1]["response"])
assert failed["status"] == "failed", failed
assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed
assert response.text.rstrip().endswith("data: [DONE]"), response.text
assert len(wire.drain()) == 1
assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"