The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import uuid
from collections.abc import Mapping
from pathlib import Path
from typing import Final, Literal
import httpx
import pytest
import yaml
from pydantic import JsonValue, TypeAdapter
from tests.integration._support.client import Gateway, Scenario, object_value, string_value
from tests.integration._support.process import owned_proxy
from tests.integration._support.wire import Reply, Request, Wire, wire_server
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_CHAT_RESPONSE: Final = {
"object": "chat.completion",
"created": 1700000000,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "scripted response"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5},
}
def _customer(scenario: Scenario, models: list[str] | None = None) -> str:
identity: Final = f"integration-customer-{uuid.uuid4().hex}"
body: Final[dict[str, JsonValue]] = {
"user_id": identity,
**({"models": models} if models is not None else {}),
}
scenario.gateway.post("/customer/new", body)
scenario.cleanups.callback(scenario.gateway.post, "/customer/delete", {"user_ids": [identity]})
return identity
def _reply(request: Request) -> Reply:
if request.method != "POST" or not request.body:
return Reply(body=b'{"object":"list","data":[]}')
body: Final = _JSON_OBJECT.validate_json(request.body)
response: Final = {
"id": f"chatcmpl-{uuid.uuid4().hex}",
"model": string_value(body["model"]),
**_CHAT_RESPONSE,
}
return Reply(body=json.dumps(response).encode())
def _fallback_reply(request: Request) -> Reply:
if request.method == "POST" and request.body:
body: Final = _JSON_OBJECT.validate_json(request.body)
if "scripted-primary-" in string_value(body["model"]):
return Reply(status=500, body=b'{"error":{"message":"scripted primary failure","type":"server_error"}}')
return _reply(request)
def _chat(
gateway: Gateway,
key: str,
model: str,
marker: str,
*,
customer: str | None = None,
headers: Mapping[str, str] | None = None,
fallbacks: tuple[str, ...] | None = None,
) -> httpx.Response:
body: Final[dict[str, JsonValue]] = {
"model": model,
"messages": [{"role": "user", "content": marker}],
**({"user": customer} if customer is not None else {}),
**({"fallbacks": list(fallbacks)} if fallbacks is not None else {}),
}
return gateway.request("POST", "/v1/chat/completions", body, key=key, headers=headers)
def _error_type(response: httpx.Response) -> str:
error: Final = object_value(_JSON_OBJECT.validate_json(response.content)["error"])
return string_value(error["type"])
def _assert_upstream(wire: Wire, marker: str, expected: int) -> tuple[Request, ...]:
matching: Final = tuple(request for request in wire.drain() if marker.encode() in request.body)
assert len(matching) == expected, matching
return matching
@pytest.mark.parametrize(
"customer_state",
("without_models", "empty_models", "no_customer_id", "unknown_customer_id"),
)
def test_unrestricted_customers_keep_key_model_access(
gateway: Gateway,
customer_state: Literal["without_models", "empty_models", "no_customer_id", "unknown_customer_id"],
) -> None:
with wire_server(_reply) as wire, gateway.scenario() as scenario:
first: Final = scenario.model(api_base=f"{wire.url}/v1")
second: Final = scenario.model(api_base=f"{wire.url}/v1")
key: Final = scenario.key(models=[first, second])
customer: Final = (
_customer(scenario)
if customer_state == "without_models"
else _customer(scenario, [])
if customer_state == "empty_models"
else f"missing-{uuid.uuid4().hex}"
if customer_state == "unknown_customer_id"
else None
)
for model in (first, second):
marker: Final = uuid.uuid4().hex
response: Final = _chat(gateway, key, model, marker, customer=customer)
assert response.status_code == 200, response.text
assert len(_assert_upstream(wire, marker, 1)) == 1
def test_customer_and_key_model_lists_are_both_enforced(gateway: Gateway) -> None:
with wire_server(_reply) as wire, gateway.scenario() as scenario:
model_a: Final = scenario.model(api_base=f"{wire.url}/v1")
model_b: Final = scenario.model(api_base=f"{wire.url}/v1")
model_c: Final = scenario.model(api_base=f"{wire.url}/v1")
key: Final = scenario.key(models=[model_a, model_b])
headers: Final = {"x-litellm-end-user-id": _customer(scenario, [model_b, model_c])}
marker_a: Final = uuid.uuid4().hex
denied_by_customer: Final = _chat(gateway, key, model_a, marker_a, headers=headers)
assert denied_by_customer.status_code == 403, denied_by_customer.text
assert _error_type(denied_by_customer) == "customer_model_access_denied"
assert _assert_upstream(wire, marker_a, 0) == ()
marker_b: Final = uuid.uuid4().hex
allowed: Final = _chat(gateway, key, model_b, marker_b, headers=headers)
assert allowed.status_code == 200, allowed.text
assert len(_assert_upstream(wire, marker_b, 1)) == 1
marker_c: Final = uuid.uuid4().hex
denied_by_key: Final = _chat(gateway, key, model_c, marker_c, headers=headers)
assert denied_by_key.status_code == 403, denied_by_key.text
assert _error_type(denied_by_key) == "key_model_access_denied"
assert _assert_upstream(wire, marker_c, 0) == ()
def test_request_body_fallback_outside_customer_allowlist_is_denied(gateway: Gateway) -> None:
with wire_server(_fallback_reply) as wire, gateway.scenario() as scenario:
primary: Final = scenario.model(api_base=f"{wire.url}/v1")
fallback: Final = scenario.model(api_base=f"{wire.url}/v1")
key: Final = scenario.key(models=[primary, fallback])
customer: Final = _customer(scenario, [primary])
marker: Final = uuid.uuid4().hex
response: Final = _chat(
gateway,
key,
primary,
marker,
customer=customer,
fallbacks=(fallback,),
)
assert response.status_code == 403, response.text
assert _error_type(response) == "customer_model_access_denied"
assert _assert_upstream(wire, marker, 0) == ()
def _router_config(
directory: Path,
wire: Wire,
primary: str,
fallback: str,
allowed_fallback: str,
primary_provider: str,
fallback_provider: str,
allowed_provider: str,
*,
fallback_target: str,
enforce: bool,
) -> Path:
models: Final = (
(primary, primary_provider),
(fallback, fallback_provider),
(allowed_fallback, allowed_provider),
)
config: Final = {
"model_list": [
{
"model_name": name,
"litellm_params": {
"model": provider_model,
"api_key": "integration-provider-key",
"api_base": f"{wire.url}/v1",
},
}
for name, provider_model in models
],
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"store_model_in_db": False,
"enforce_fallback_model_access": enforce,
},
"litellm_settings": {"cache": False},
"router_settings": {
"num_retries": 0,
"disable_cooldowns": True,
"fallbacks": [{primary: [fallback_target]}],
},
}
path: Final = directory / f"customer-fallback-{uuid.uuid4().hex}.yaml"
path.write_text(yaml.safe_dump(config))
return path
@pytest.mark.parametrize(
("enforce", "allow_fallback", "expected_status", "expected_upstream_count"),
(
(True, False, 500, 1),
(False, False, 200, 2),
(True, True, 200, 2),
),
)
def test_router_config_fallback_customer_allowlist(
gateway: Gateway,
tmp_path: Path,
enforce: bool,
allow_fallback: bool,
expected_status: int,
expected_upstream_count: int,
) -> None:
with wire_server(_fallback_reply) as wire, gateway.scenario() as model_scenario:
primary_provider: Final = f"openai/scripted-primary-{uuid.uuid4().hex}"
fallback_provider: Final = f"openai/scripted-fallback-{uuid.uuid4().hex}"
allowed_provider: Final = f"openai/scripted-allowed-{uuid.uuid4().hex}"
primary: Final = model_scenario.model(api_base=f"{wire.url}/v1", model=primary_provider)
fallback: Final = model_scenario.model(api_base=f"{wire.url}/v1", model=fallback_provider)
allowed_fallback: Final = model_scenario.model(api_base=f"{wire.url}/v1", model=allowed_provider)
fallback_target: Final = allowed_fallback if allow_fallback else fallback
config: Final = _router_config(
tmp_path,
wire,
primary,
fallback,
allowed_fallback,
primary_provider,
fallback_provider,
allowed_provider,
fallback_target=fallback_target,
enforce=enforce,
)
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
with candidate.scenario() as scenario:
key: Final = scenario.key(models=[primary, fallback, allowed_fallback])
customer_models: Final = [primary, fallback_target] if allow_fallback else [primary]
customer: Final = _customer(scenario, customer_models)
marker: Final = uuid.uuid4().hex
response: Final = _chat(candidate, key, primary, marker, customer=customer)
assert response.status_code == expected_status, response.text
requests: Final = _assert_upstream(wire, marker, expected_upstream_count)
models: Final = tuple(
string_value(_JSON_OBJECT.validate_json(request.body)["model"]) for request in requests
)
assert "scripted-primary-" in models[0]
if expected_upstream_count == 2:
assert ("scripted-allowed-" in models[1]) is allow_fallback
assert ("scripted-fallback-" in models[1]) is not allow_fallback
def test_customer_crud_sets_and_clears_models_immediately(gateway: Gateway) -> None:
with wire_server(_reply) as wire, gateway.scenario() as scenario:
model_a: Final = scenario.model(api_base=f"{wire.url}/v1")
model_b: Final = scenario.model(api_base=f"{wire.url}/v1")
key: Final = scenario.key(models=[model_a, model_b])
customer: Final = _customer(scenario)
assert gateway.get("/customer/info", {"end_user_id": customer})["models"] == []
gateway.post("/customer/update", {"user_id": customer, "models": [model_a]})
assert gateway.get("/customer/info", {"end_user_id": customer})["models"] == [model_a]
denied_marker: Final = uuid.uuid4().hex
denied: Final = _chat(gateway, key, model_b, denied_marker, customer=customer)
assert denied.status_code == 403, denied.text
assert _error_type(denied) == "customer_model_access_denied"
assert _assert_upstream(wire, denied_marker, 0) == ()
gateway.post("/customer/update", {"user_id": customer, "models": []})
assert gateway.get("/customer/info", {"end_user_id": customer})["models"] == []
allowed_marker: Final = uuid.uuid4().hex
allowed: Final = _chat(gateway, key, model_b, allowed_marker, customer=customer)
assert allowed.status_code == 200, allowed.text
assert len(_assert_upstream(wire, allowed_marker, 1)) == 1