The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import uuid
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
import anthropic
import httpx
import openai
import pytest
from integration._support.client import Gateway, Scenario, object_value, string_value
from integration._support.responses_stream import (
AZURE_TARGET,
OPENAI_TARGET,
RATE_LIMIT_MESSAGE,
RATE_LIMIT_PREFIX,
SENTINEL_PREFIX,
assert_provider_typed_rate_limit,
chat_body,
chat_content,
created,
data_frames,
delta,
error_body,
frame,
function_tools,
healthy_stream,
messages_body,
openai_error_event,
openai_error_stream,
serve,
spend_row,
sse_events,
)
from integration._support.wire import Wire, wire_server
from openai.types.responses import ResponseFailedEvent, ResponseStreamEvent
from pydantic import JsonValue
_AZURE_KEY: Final = "synthetic-azure-key"
_QUOTA_MESSAGE: Final = "You exceeded your current quota, please check your plan and billing details."
_SERVER_MESSAGE: Final = "The server had an error while processing your request. Sorry about that!"
_ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"}
@dataclass(frozen=True, slots=True)
class _Served:
response: httpx.Response
wire: Wire
def _azure_model(scenario: Scenario, wire: Wire) -> str:
return scenario.model(model="azure/gpt-6", api_base=wire.url, api_key=_AZURE_KEY)
def _post(gateway: Gateway, path: str, body: Mapping[str, JsonValue], stream: tuple[bytes, ...]) -> _Served:
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
response: Final = gateway.request("POST", path, {**body, "model": model}, headers=_ANTHROPIC_HEADERS)
return _Served(response, wire)
def _chat(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[httpx.Response, str]:
marker: Final = uuid.uuid4().hex
served: Final = _post(gateway, "/v1/chat/completions", chat_body("", marker), stream)
_assert_one_attempt(served.wire, marker)
return served.response, marker
def _assert_one_attempt(wire: Wire, marker: str) -> None:
received: Final = wire.drain()
assert len(received) == 1 and marker.encode() in received[0].body, [request.target for request in received]
def _assert_failure_logged(call_id: str, logged_code: str, provider_message: str) -> None:
row: Final = spend_row(call_id)
assert row["status"] == "failure", row
error: Final = object_value(object_value(row["metadata"])["error_information"])
assert str(error["error_code"]) == logged_code, error
assert provider_message in string_value(error["error_message"]), error
def _identity() -> str:
return "resp_" + uuid.uuid4().hex
def test_openai_sdk_chat_stream_top_level_rate_limit_is_a_throttling_error_with_the_provider_message(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
with pytest.raises(openai.RateLimitError) as raised:
client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": marker}],
tools=function_tools(),
stream=True,
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
)
assert raised.value.status_code == 429
assert_provider_typed_rate_limit(object_value(raised.value.body))
_assert_one_attempt(wire, marker)
_assert_failure_logged(raised.value.response.headers["x-litellm-call-id"], "429", RATE_LIMIT_MESSAGE)
async def test_async_openai_sdk_codex_chat_stream_top_level_rate_limit_is_a_throttling_error(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, OPENAI_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="openai/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key")
client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
with pytest.raises(openai.RateLimitError) as raised:
await client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": marker}],
stream=True,
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
)
assert raised.value.status_code == 429
assert_provider_typed_rate_limit(object_value(raised.value.body))
_assert_one_attempt(wire, marker)
@pytest.mark.parametrize(
("code", "message", "status"),
[
pytest.param("insufficient_quota", _QUOTA_MESSAGE, 429, id="insufficient_quota"),
pytest.param("server_error", _SERVER_MESSAGE, 500, id="server_error"),
pytest.param("invalid_prompt", "Invalid prompt: your prompt was flagged.", 400, id="invalid_prompt"),
pytest.param(
"context_length_exceeded",
"Your input exceeds the context window of this model.",
400,
id="context_length_exceeded",
),
pytest.param("503", "The engine is currently overloaded.", 503, id="decimal_503"),
pytest.param("brand_new_error_code", "Something new went wrong upstream.", 500, id="unknown_code"),
pytest.param("", "An error with an empty code.", 500, id="empty_code"),
],
)
def test_top_level_error_code_sets_the_chat_status_and_keeps_the_provider_message(
gateway: Gateway, code: str, message: str, status: int
) -> None:
response, _ = _chat(gateway, openai_error_stream(_identity(), code, message))
assert response.status_code == status, response.text
error: Final = error_body(response)
assert str(error["code"]) == str(status), error
client_message: Final = string_value(error["message"])
assert message in client_message and SENTINEL_PREFIX not in client_message, client_message
logged_code: Final = code if status == 400 else str(status)
_assert_failure_logged(response.headers["x-litellm-call-id"], logged_code, message)
def test_top_level_rate_limit_after_text_is_a_throttling_error_frame_behind_the_text(gateway: Gateway) -> None:
identity: Final = _identity()
stream: Final = (
frame(created(identity)),
frame(delta(identity, "Hello")),
frame(delta(identity, " there")),
frame(openai_error_event("rate_limit_exceeded", RATE_LIMIT_MESSAGE)),
)
response, _ = _chat(gateway, stream)
assert response.status_code == 200, response.text
frames: Final = data_frames(response.text)
assert chat_content(frames) == "Hello there", response.text
assert_provider_typed_rate_limit(object_value(frames[-1]["error"]))
def test_nested_error_object_wins_over_top_level_fields(gateway: Gateway) -> None:
event: Final = {
**openai_error_event("rate_limit_exceeded", "top-level says slow down"),
"error": {"type": "server_error", "code": "server_error", "message": "nested says the server broke"},
}
response, _ = _chat(gateway, (frame(created(_identity())), frame(event)))
assert response.status_code == 500, response.text
message: Final = string_value(error_body(response)["message"])
assert "nested says the server broke" in message and "top-level says" not in message, message
def _raw_event(body: str) -> bytes:
return f"event: error\ndata: {body}\n\n".encode()
_FIVE_KB_MESSAGE: Final = "Rate limit reached for gpt-6. " + "x" * 5120
@pytest.mark.parametrize(
("event", "provider_message"),
[
pytest.param(
frame({**openai_error_event("rate_limit_exceeded", RATE_LIMIT_MESSAGE), "error": None}),
RATE_LIMIT_MESSAGE,
id="error_null",
),
pytest.param(
frame({"type": "error", "sequence_number": 1, "code": 429, "message": RATE_LIMIT_MESSAGE, "param": None}),
RATE_LIMIT_MESSAGE,
id="integer_code",
),
pytest.param(frame(openai_error_event("rate_limit_exceeded", "")), "", id="empty_message"),
pytest.param(
frame({"type": "error", "sequence_number": 1, "code": "rate_limit_exceeded", "message": {"text": "x"}}),
"",
id="object_message",
),
pytest.param(frame(openai_error_event("rate_limit_exceeded", _FIVE_KB_MESSAGE)), _FIVE_KB_MESSAGE, id="5kb"),
pytest.param(
_raw_event(
'{"type": "error", "sequence_number": 1, "code": "rate_limit_exceeded", '
f'"code": "rate_limit_exceeded", "message": {json.dumps(RATE_LIMIT_MESSAGE)}, "param": null}}'
),
RATE_LIMIT_MESSAGE,
id="duplicated_code",
),
],
)
def test_malformed_top_level_fields_still_answer_a_throttling_error(
gateway: Gateway, event: bytes, provider_message: str
) -> None:
response, _ = _chat(gateway, (frame(created(_identity())), event))
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response), provider_message)
healthy: Final = _post(
gateway, "/v1/chat/completions", chat_body("", "after"), healthy_stream(_identity(), "still serving")
)
assert chat_content(data_frames(healthy.response.text)) == "still serving", healthy.response.text
def _cached_chat(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[tuple[httpx.Response, ...], int]:
marker: Final = uuid.uuid4().hex
body: Final = {key: value for key, value in chat_body("", marker).items() if key != "cache"}
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
responses: Final = tuple(
gateway.request("POST", "/v1/chat/completions", {**body, "model": model}) for _ in range(2)
)
return responses, len(wire.drain())
def test_a_top_level_error_stream_is_never_served_from_the_response_cache(gateway: Gateway) -> None:
responses, attempts = _cached_chat(
gateway, openai_error_stream(_identity(), "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
)
assert attempts == 2, attempts
for response in responses:
assert response.status_code == 429, response.text
assert_provider_typed_rate_limit(error_body(response))
def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[httpx.Response, Mapping[str, JsonValue]]:
marker: Final = uuid.uuid4().hex
served: Final = _post(
gateway, "/v1/responses", {"input": marker, "stream": True, "cache": {"no-cache": True}}, stream
)
_assert_one_attempt(served.wire, marker)
assert served.response.status_code == 200, served.response.text
assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text
frames: Final = data_frames(served.response.text)
assert frames[-1]["type"] == "response.failed", served.response.text
return served.response, object_value(object_value(frames[-1]["response"])["error"])
def _assert_rate_limit_failure(error: Mapping[str, JsonValue]) -> None:
assert error["code"] == "rate_limit_exceeded", error
assert RATE_LIMIT_MESSAGE in string_value(error["message"]), error
def test_native_responses_stream_fails_with_the_provider_code_and_message(gateway: Gateway) -> None:
identity: Final = _identity()
response, error = _responses_frames(
gateway, openai_error_stream(identity, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
)
assert [frame["type"] for frame in data_frames(response.text)] == ["response.created", "response.failed"]
_assert_rate_limit_failure(error)
_assert_failure_logged(response.headers["x-litellm-call-id"], "429", RATE_LIMIT_MESSAGE)
def test_native_responses_stream_error_after_text_fails_with_the_provider_code(gateway: Gateway) -> None:
identity: Final = _identity()
stream: Final = (
frame(created(identity)),
frame(delta(identity, "partial")),
frame(openai_error_event("rate_limit_exceeded", RATE_LIMIT_MESSAGE)),
)
response, error = _responses_frames(gateway, stream)
kinds: Final = [frame["type"] for frame in data_frames(response.text)]
assert kinds == ["response.created", "response.output_text.delta", "response.failed"], kinds
_assert_rate_limit_failure(error)
def _failed_error(events: list[ResponseStreamEvent]) -> tuple[str, str]:
assert [event.type for event in events] == ["response.created", "response.failed"], events
failed: Final = events[-1]
assert isinstance(failed, ResponseFailedEvent) and failed.response.error is not None, failed
return failed.response.error.code, failed.response.error.message
def test_openai_sdk_responses_stream_fails_with_insufficient_quota(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "insufficient_quota", _QUOTA_MESSAGE)
with wire_server(serve(stream, OPENAI_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model="openai/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key")
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
events: Final = list(
client.responses.create(model=model, input=marker, stream=True, extra_body={"cache": {"no-cache": True}})
)
code, message = _failed_error(events)
assert code == "insufficient_quota" and _QUOTA_MESSAGE in message, events
_assert_one_attempt(wire, marker)
async def test_async_openai_sdk_responses_stream_fails_with_the_server_error_message(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "server_error", _SERVER_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
response: Final = await client.responses.create(
model=model, input=marker, stream=True, extra_body={"cache": {"no-cache": True}}
)
events: Final = [event async for event in response]
code, message = _failed_error(events)
assert code == "server_error" and _SERVER_MESSAGE in message, events
_assert_one_attempt(wire, marker)
def _assert_messages_error(error: Mapping[str, JsonValue], error_type: str, provider_message: str) -> None:
assert error["type"] == error_type, error
message: Final = string_value(error["message"])
assert provider_message in message and SENTINEL_PREFIX not in message, message
def test_messages_stream_top_level_rate_limit_is_a_rate_limit_errorframe(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
served: Final = _post(gateway, "/v1/messages", messages_body("", marker), stream)
assert served.response.status_code == 200, served.response.text
events: Final = sse_events(served.response.text)
assert events[0][0] == "message_start" and events[-1][0] == "error", events
error: Final = object_value(events[-1][1]["error"])
_assert_messages_error(error, "rate_limit_error", RATE_LIMIT_MESSAGE)
assert string_value(error["message"]).count(RATE_LIMIT_PREFIX) == 1, error
_assert_one_attempt(served.wire, marker)
def _anthropic_tools() -> list[anthropic.types.ToolParam]: # mutable-ok: the Anthropic SDK tools parameter is a list
return [
{
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
}
]
async def _drain_async_messages_stream(client: anthropic.AsyncAnthropic, model: str, marker: str) -> None:
async with client.messages.stream(
model=model,
max_tokens=64,
messages=[{"role": "user", "content": marker}],
tools=_anthropic_tools(),
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
) as events:
async for _ in events:
pass
def _drain_messages_stream(client: anthropic.Anthropic, model: str, marker: str) -> None:
with client.messages.stream(
model=model,
max_tokens=64,
messages=[{"role": "user", "content": marker}],
tools=_anthropic_tools(),
extra_body={"num_retries": 0, "cache": {"no-cache": True}},
) as events:
for _ in events:
pass
async def test_async_anthropic_sdk_stream_raises_the_top_level_rate_limit(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "rate_limit_exceeded", RATE_LIMIT_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
client: Final = anthropic.AsyncAnthropic(
base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0
)
with pytest.raises(anthropic.APIStatusError) as raised:
await _drain_async_messages_stream(client, model, marker)
_assert_messages_error(
object_value(object_value(raised.value.body)["error"]), "rate_limit_error", RATE_LIMIT_MESSAGE
)
_assert_one_attempt(wire, marker)
def test_sync_anthropic_sdk_stream_raises_the_top_level_server_error_message(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
stream: Final = openai_error_stream(_identity(), "server_error", _SERVER_MESSAGE)
with wire_server(serve(stream, AZURE_TARGET)) as wire, gateway.scenario() as scenario:
model: Final = _azure_model(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
with pytest.raises(anthropic.APIStatusError) as raised:
_drain_messages_stream(client, model, marker)
_assert_messages_error(object_value(object_value(raised.value.body)["error"]), "api_error", _SERVER_MESSAGE)
_assert_one_attempt(wire, marker)