The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""Router-wide, deployment and `user_config` defaults against the params the Router binds positionally, the
moderation merge, unknown and A2A models, and every `user_config` shape on the three LLM endpoints."""
from __future__ import annotations
import asyncio
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from typing import Final
import anthropic
import httpx
import openai
import pytest
from integration._support.client import Gateway, gateway_from_environment
from integration.compatibility._status_code_audit import (
UNSET,
CHAT,
CHAT_FRAMES,
ENDPOINT_BODIES,
ENDPOINT_PATHS,
MESSAGE,
MESSAGE_FRAMES,
MODERATION,
RESPONSE,
RESPONSES_FRAMES,
ROUTER_DEFAULTS,
USER_MESSAGES,
assembled_text,
assert_no_provider_call,
deployment,
error_body,
invalid_request,
json_response,
one_outbound,
owned_gateway,
post,
sse_response,
stream_finished,
stream_lines,
)
from pydantic import JsonValue
from tests.integration.cost_calculation.cost_tracking_case import BinaryResponse, StoredResponse
pytestmark: Final = pytest.mark.timeout(180)
@dataclass(frozen=True, slots=True)
class _Rig:
gateway: Gateway
identities: Mapping[str, str]
deployments: Mapping[str, dict[str, JsonValue]]
_SPECS: Final[tuple[tuple[str, str, StoredResponse, dict[str, JsonValue]], ...]] = (
("audit-image", "openai/gpt-image-1", json_response({"created": 1, "data": []}), {}),
("audit-image-dep", "openai/gpt-image-1", json_response({"created": 1, "data": []}), {"prompt": "deployment"}),
("audit-text", "openai/gpt-3.5-turbo-instruct", json_response({"choices": []}), {}),
("audit-text-dep", "openai/gpt-3.5-turbo-instruct", json_response({"choices": []}), {"prompt": "deployment"}),
("audit-speech", "openai/gpt-4o-mini-tts", BinaryResponse(content_type="audio/mpeg", length=16), {}),
(
"audit-speech-dep",
"openai/gpt-4o-mini-tts",
BinaryResponse(content_type="audio/mpeg", length=16),
{"input": "deployment"},
),
("audit-transcribe", "openai/whisper-1", json_response({"text": "scripted"}), {}),
("audit-moderation", "openai/omni-moderation-latest", json_response(MODERATION), {}),
("audit-chat", "openai/gpt-4o-mini", json_response(CHAT), {}),
("audit-chat-stream", "openai/gpt-4o-mini", sse_response(CHAT_FRAMES), {}),
("audit-responses", "openai/gpt-4o-mini", json_response(RESPONSE), {}),
("audit-responses-stream", "openai/gpt-4o-mini", sse_response(RESPONSES_FRAMES), {}),
("audit-message", "anthropic/claude-haiku-4-5", json_response(MESSAGE), {}),
("audit-message-stream", "anthropic/claude-haiku-4-5", sse_response(MESSAGE_FRAMES), {}),
)
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
"""A proxy whose router-wide defaults name every positional and route-required param, with client-side
`user_config` allowed, and one deployment per route (plus `-dep` copies that carry the param themselves)."""
with gateway_from_environment() as gateway, gateway.scenario() as scenario:
built: Final = tuple(
deployment(scenario, name, model, response, **extra) for name, model, response, extra in _SPECS
)
identities: Final = {name: identity for (name, *_), (identity, _) in zip(_SPECS, built, strict=True)}
deployments: Final = {name: entry for (name, *_), (_, entry) in zip(_SPECS, built, strict=True)}
config: Final[dict[str, JsonValue]] = {
"model_list": [entry for _, entry in built],
"router_settings": {"default_litellm_params": ROUTER_DEFAULTS, "num_retries": 0},
"general_settings": {"allow_client_side_credentials": True},
}
with owned_gateway(tmp_path_factory.mktemp("audit-defaults"), config) as owned:
yield _Rig(owned, identities, deployments)
_POSITIONAL: Final[dict[str, tuple[str, str, str, str, dict[str, JsonValue]]]] = {
"image": ("/v1/images/generations", "/image/generations", "audit-image", "prompt", {}),
"text-completion": ("/v1/completions", "/completions", "audit-text", "prompt", {}),
"speech": ("/v1/audio/speech", "/audio/speech", "audit-speech", "input", {"voice": "alloy"}),
}
def _user_config(rig: _Rig, name: str, defaults: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return {"model_list": [rig.deployments[name]], "default_litellm_params": dict(defaults)}
@pytest.mark.parametrize("route", tuple(_POSITIONAL))
@pytest.mark.parametrize("source", ("deployment-param", "user-config-default", "user-config-null"))
def test_positional_param_from_any_default_source_returns_400(rig: _Rig, route: str, source: str) -> None:
path, error_route, name, param, extra = _POSITIONAL[route]
user_config: Final[dict[str, JsonValue]] = (
{"user_config": _user_config(rig, name, {param: "user config default"})}
if source == "user-config-default"
else {"user_config": None}
if source == "user-config-null"
else {}
)
body: Final[dict[str, JsonValue]] = {
"model": f"{name}-dep" if source == "deployment-param" else name,
**extra,
**user_config,
}
response: Final = post(rig.gateway, path, body)
error: Final = invalid_request(response)
assert error.get("param") == param, response.text
assert error.get("message") == f"{error_route}: Missing required parameter: '{param}'.", response.text
assert_no_provider_call(rig.gateway, rig.identities[name], rig.identities[f"{name}-dep"])
@pytest.mark.parametrize("route", tuple(_POSITIONAL))
def test_positional_param_in_the_body_reaches_upstream(rig: _Rig, route: str) -> None:
pytest.skip(
"BUG: a router-wide default_litellm_params value replaces the caller's own positional param "
"(prompt/input) on its way to the provider; same on the merge base"
)
path, _error_route, name, param, extra = _POSITIONAL[route]
response: Final = post(rig.gateway, path, {"model": name, param: f"explicit {route}", **extra})
assert response.status_code == 200, response.text
assert one_outbound(rig.gateway, rig.identities[name]).get(param) == f"explicit {route}"
def test_transcription_without_file_is_rejected_by_the_form_before_routing(rig: _Rig) -> None:
response: Final = rig.gateway.client.post(
"/v1/audio/transcriptions",
data={"model": "audit-transcribe"},
files={"note": ("note.txt", b"not audio", "text/plain")},
headers={"Authorization": f"Bearer {rig.gateway.key}"},
)
assert response.status_code == 422, response.text
detail: Final = response.json()["detail"]
assert [entry["loc"] for entry in detail] == [["body", "file"]], response.text
assert_no_provider_call(rig.gateway, rig.identities["audit-transcribe"])
def test_transcription_with_file_reaches_upstream(rig: _Rig) -> None:
response: Final = rig.gateway.client.post(
"/v1/audio/transcriptions",
data={"model": "audit-transcribe"},
files={"file": ("clip.wav", b"RIFF0000WAVEfmt ", "audio/wav")},
headers={"Authorization": f"Bearer {rig.gateway.key}"},
)
assert response.status_code == 200, response.text
assert response.json()["text"] == "scripted"
outbound: Final = one_outbound(rig.gateway, rig.identities["audit-transcribe"])
assert outbound.get("model") == "whisper-1"
assert outbound.get("file") == {"filename": "clip.wav", "content_type": "audio/x-wav"}
@pytest.mark.parametrize(
("body", "expected_input"),
(({}, "router default input"), ({"input": "explicit input"}, "explicit input")),
ids=("router-default", "explicit"),
)
def test_listed_moderation_model_merges_the_router_default(
rig: _Rig, body: dict[str, JsonValue], expected_input: str
) -> None:
response: Final = post(rig.gateway, "/v1/moderations", {"model": "audit-moderation", **body})
assert response.status_code == 200, response.text
assert one_outbound(rig.gateway, rig.identities["audit-moderation"]).get("input") == expected_input
@pytest.mark.parametrize(
("path", "body", "message"),
(
pytest.param(
"/v1/images/generations",
{"model": "audit-unknown-model"},
"/image/generations: Missing required parameter: 'prompt'.",
id="image-unknown-model",
),
pytest.param(
"/v1/images/generations",
{"model": "a2a/audit-unknown-agent"},
"/image/generations: Missing required parameter: 'prompt'.",
id="image-a2a-model",
),
pytest.param(
"/v1/messages",
{"model": "a2a/audit-unknown-agent", "messages": USER_MESSAGES},
"anthropic_messages: Invalid model name passed in model=a2a/audit-unknown-agent. "
"Call `/v1/models` to view available models for your key.",
id="messages-a2a-model",
),
),
)
def test_model_outside_the_router_with_a_router_default_param(
rig: _Rig, path: str, body: dict[str, JsonValue], message: str
) -> None:
response: Final = post(rig.gateway, path, body)
assert response.status_code == 400, response.text
error: Final = error_body(response)
assert error.get("type") == "invalid_request_error", response.text
assert error.get("message") == message, response.text
@pytest.mark.parametrize(
("user_config", "message"),
(
pytest.param(None, None, id="null"),
pytest.param(UNSET, None, id="missing"),
pytest.param({}, "There are no healthy deployments for this model", id="empty-object"),
pytest.param("", "Invalid request format: 'str' object has no attribute 'items'", id="empty-string"),
pytest.param(7, "Invalid request format: 'int' object has no attribute 'items'", id="int"),
pytest.param([], "Invalid request format: 'list' object has no attribute 'items'", id="list"),
pytest.param("x" * 5000, "Invalid request format: 'str' object has no attribute 'items'", id="5kb-string"),
),
)
def test_chat_user_config_shapes(rig: _Rig, user_config: object, message: str | None) -> None:
body: Final[dict[str, JsonValue]] = {
"model": "audit-chat",
"messages": USER_MESSAGES,
**({} if user_config is UNSET else {"user_config": user_config}), # pyright: ignore[reportAssignmentType] # the sad shapes are deliberately not JSON objects
}
response: Final = post(rig.gateway, "/v1/chat/completions", body)
if message is None:
assert response.status_code == 200, response.text
assert one_outbound(rig.gateway, rig.identities["audit-chat"]).get("max_tokens") == 32
return
assert response.status_code == 400, response.text
assert message in str(error_body(response).get("message")), response.text
assert_no_provider_call(rig.gateway, rig.identities["audit-chat"])
def test_messages_empty_user_config_does_not_inherit_router_defaults(rig: _Rig) -> None:
response: Final = post(
rig.gateway, "/v1/messages", {"model": "audit-message", "messages": USER_MESSAGES, "user_config": {}}
)
assert response.status_code == 400, response.text
assert error_body(response).get("message") == "anthropic_messages: Missing required parameter: 'max_tokens'."
assert_no_provider_call(rig.gateway, rig.identities["audit-message"])
@pytest.mark.parametrize(
("endpoint", "model", "stream", "expected_max_tokens"),
(
pytest.param("chat", "audit-chat", False, 32, id="chat"),
pytest.param("chat", "audit-chat-stream", True, 32, id="chat-stream"),
pytest.param("responses", "audit-responses", False, None, id="responses"),
pytest.param("responses", "audit-responses-stream", True, None, id="responses-stream"),
pytest.param("messages", "audit-message-stream", True, 32, id="messages-stream"),
),
)
def test_null_user_config_serves_every_endpoint(
rig: _Rig, endpoint: str, model: str, stream: bool, expected_max_tokens: int | None
) -> None:
body: Final[dict[str, JsonValue]] = {
**{key: value for key, value in ENDPOINT_BODIES[endpoint].items() if key != "max_tokens"},
"model": model,
"user_config": None,
**({"stream": True} if stream else {}),
}
if stream:
status, lines = stream_lines(rig.gateway, ENDPOINT_PATHS[endpoint], body)
assert status == 200, lines
assert stream_finished(endpoint, lines), lines
assert assembled_text(lines) == "streamed response", lines
else:
response: Final = post(rig.gateway, ENDPOINT_PATHS[endpoint], body)
assert response.status_code == 200, response.text
assert "scripted" in response.text, response.text
outbound: Final = one_outbound(rig.gateway, rig.identities[model])
assert "user_config" not in outbound, outbound
assert outbound.get("max_tokens") == expected_max_tokens, outbound
def test_null_user_config_without_a_valid_key_is_401(rig: _Rig) -> None:
response: Final = post(
rig.gateway,
"/v1/chat/completions",
{"model": "audit-chat", "messages": USER_MESSAGES, "user_config": None},
key="sk-audit-not-a-key",
)
assert response.status_code == 401, response.text
assert_no_provider_call(rig.gateway, rig.identities["audit-chat"])
def _openai_chat(rig: _Rig, asynchronous: bool) -> str | None:
base_url: Final = f"{rig.gateway.client.base_url}/v1"
if not asynchronous:
with openai.OpenAI(base_url=base_url, api_key=rig.gateway.key, max_retries=0) as client:
completion: Final = client.chat.completions.create(
model="audit-chat",
messages=[{"role": "user", "content": "Hello"}],
extra_body={"user_config": None},
)
return completion.choices[0].message.content
async def call() -> str | None:
async with openai.AsyncOpenAI(base_url=base_url, api_key=rig.gateway.key, max_retries=0) as client:
completion: Final = await client.chat.completions.create(
model="audit-chat",
messages=[{"role": "user", "content": "Hello"}],
extra_body={"user_config": None},
)
return completion.choices[0].message.content
return asyncio.run(call())
def _anthropic_message(rig: _Rig, asynchronous: bool) -> str:
base_url: Final = str(rig.gateway.client.base_url)
if not asynchronous:
with anthropic.Anthropic(base_url=base_url, api_key=rig.gateway.key, max_retries=0) as client:
message: Final = client.messages.create(
model="audit-message",
max_tokens=8,
messages=[{"role": "user", "content": "Hello"}],
extra_body={"user_config": None},
)
block: Final = message.content[0]
assert isinstance(block, anthropic.types.TextBlock), message
return block.text
async def call() -> str:
async with anthropic.AsyncAnthropic(base_url=base_url, api_key=rig.gateway.key, max_retries=0) as client:
message: Final = await client.messages.create(
model="audit-message",
max_tokens=8,
messages=[{"role": "user", "content": "Hello"}],
extra_body={"user_config": None},
)
block: Final = message.content[0]
assert isinstance(block, anthropic.types.TextBlock), message
return block.text
return asyncio.run(call())
@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
def test_null_user_config_through_the_openai_sdk(rig: _Rig, asynchronous: bool) -> None:
assert _openai_chat(rig, asynchronous) == "scripted"
assert one_outbound(rig.gateway, rig.identities["audit-chat"]).get("max_tokens") == 32
@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
def test_null_user_config_through_the_anthropic_sdk(rig: _Rig, asynchronous: bool) -> None:
assert _anthropic_message(rig, asynchronous) == "scripted"
assert one_outbound(rig.gateway, rig.identities["audit-message"]).get("max_tokens") == 8
def test_null_user_config_without_the_client_side_opt_in_is_rejected(gateway: Gateway) -> None:
response: Final = gateway.client.post(
"/v1/chat/completions",
json={"model": "audit-not-routed", "messages": USER_MESSAGES, "user_config": None},
headers={"Authorization": f"Bearer {gateway.key}"},
)
assert response.status_code == 401, response.text
assert "user_config is not allowed in request body" in response.text, response.text
def test_raw_httpx_async_null_user_config(rig: _Rig) -> None:
async def call() -> httpx.Response:
async with httpx.AsyncClient(base_url=str(rig.gateway.client.base_url), trust_env=False) as client:
return await client.post(
"/v1/responses",
json={"model": "audit-responses", "input": "Hello", "store": False, "user_config": None},
headers={"Authorization": f"Bearer {rig.gateway.key}"},
timeout=30,
)
response: Final = asyncio.run(call())
assert response.status_code == 200, response.text
assert one_outbound(rig.gateway, rig.identities["audit-responses"]).get("input") == "Hello"