The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import itertools
import json
import uuid
from collections.abc import Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
import httpx
import pytest
import yaml
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from integration.providers._cache_control_marks_support import anthropic_peer, marker_of
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 300)
_FIRST: Final = "heap-bound-first"
_SECOND: Final = "heap-bound-second"
_SLOTS: Final = 8
_CYCLES: Final = 6 * _SLOTS
_FILLERS: Final = 2 * _SLOTS
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _rig_config() -> dict[str, JsonValue]:
return _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
def _config(directory: Path, api_base: str) -> Path:
config: Final = _rig_config()
deployments: Final = [
{
"model_name": name,
"litellm_params": {"model": "anthropic/claude-opus-5-5", "api_base": api_base, "api_key": "synthetic"},
}
for name in (_FIRST, _SECOND)
]
merged: Final = {
**config,
"model_list": deployments,
"general_settings": {
**object_value(config["general_settings"]),
"user_api_key_cache_ttl": 3600,
"user_api_key_cache_max_size": _SLOTS,
},
"litellm_settings": {
**object_value(config["litellm_settings"]),
"cache": False,
"enable_redis_auth_cache": False,
},
}
path: Final = directory / f"heap-bound-{uuid.uuid4().hex}.yaml"
path.write_text(yaml.safe_dump(merged))
return path
def _chat(gateway: Gateway, key: str, model: str) -> tuple[int, str]:
marker: Final = uuid.uuid4().hex
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": f"Reply. marker-{marker}"}]},
key=key,
)
return response.status_code, marker
def _cycle(gateway: Gateway, key: str, turn: int) -> tuple[tuple[int, int], str]:
allowed, denied = (_SECOND, _FIRST) if turn % 2 == 0 else (_FIRST, _SECOND)
gateway.post("/key/update", {"key": key, "models": [allowed]})
allowed_status, marker = _chat(gateway, key, allowed)
denied_status, _ = _chat(gateway, key, denied)
return (allowed_status, denied_status), marker
def _fillers(scenario: Scenario, count: int) -> Sequence[str]:
keys: Final = tuple(string_value(scenario.gateway.post("/key/generate", {})["key"]) for _ in range(count))
scenario.cleanups.callback(scenario.gateway.post, "/key/delete", {"keys": list(keys)})
return keys
def _reached(peer: Wire) -> list[str]:
return sorted(marker_of(request) for request in peer.drain())
def test_key_updates_past_twice_the_key_cache_slots_take_effect_on_the_next_request(
gateway: Gateway, tmp_path: Path
) -> None:
with wire_server(anthropic_peer) as peer:
config: Final = _config(tmp_path, peer.url)
with (
owned_proxy_process(gateway, tmp_path, {}, config=config, workers=1) as owned,
owned.gateway.scenario() as scenario,
):
anchor: Final = scenario.key(models=[_FIRST])
anchor_status, anchor_marker = _chat(owned.gateway, anchor, _FIRST)
churned: Final = scenario.key(models=[_FIRST])
cycles: Final = tuple(_cycle(owned.gateway, churned, turn) for turn in range(_CYCLES))
assert [statuses for statuses, _ in cycles] == [(200, 403)] * _CYCLES
filled: Final = tuple(
owned.gateway.request("GET", "/v1/models", key=key).status_code for key in _fillers(scenario, _FILLERS)
)
assert filled == (200,) * _FILLERS
after_anchor, after_anchor_marker = _chat(owned.gateway, anchor, _FIRST)
last_allowed: Final = _SECOND if (_CYCLES - 1) % 2 == 0 else _FIRST
after_churned, after_churned_marker = _chat(owned.gateway, churned, last_allowed)
assert (anchor_status, after_anchor, after_churned) == (200, 200, 200)
assert _reached(peer) == sorted(
(anchor_marker, *(marker for _, marker in cycles), after_anchor_marker, after_churned_marker)
)
_RESPONSE_SLOTS: Final = DEFAULT_MAX_SIZE_IN_MEMORY
_OVERFLOW: Final = 20
_BURST_PER_ENDPOINT: Final = 6
_NO_STORE: Final[Mapping[str, JsonValue]] = MappingProxyType({"no-store": True})
_FRESH_TTL: Final[Mapping[str, JsonValue]] = MappingProxyType({"ttl": 600})
_EMBEDDING: Final = json.dumps(
{
"object": "list",
"data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
"model": "text-embedding-3-large",
"usage": {"prompt_tokens": 1, "total_tokens": 1},
}
).encode()
class _PeerBody(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
model: str = ""
stream: bool = False
def _response_cache_peer(request: Request) -> Reply:
body: Final = _PeerBody.model_validate_json(request.body or b"{}")
identity: Final = f"{marker_of(request)}-{uuid.uuid4().hex[:8]}"
match request.target.split("?", 1)[0]:
case "/v1/messages":
return anthropic_peer(request)
case "/v1/chat/completions":
return chat_reply(f"chatcmpl-{identity}", body.model, "scripted", stream=body.stream)
case "/v1/responses":
return responses_reply(f"resp_{identity}", body.model, "scripted", stream=body.stream)
case "/v1/embeddings":
return Reply(body=_EMBEDDING)
case _:
return Reply(status=404, body=b'{"error": {"message": "unscripted route"}}')
_NAN_TTL: Final[Mapping[str, JsonValue]] = MappingProxyType({"ttl": "NaN"})
_UNUSABLE_TTLS: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"lowercase-nan": "nan", "word": "abc", "empty-string": "", "list": [1], "object": {"x": 1}}
)
def _prompt_fields(path: str, text: str) -> dict[str, JsonValue]:
match path:
case "/v1/chat/completions":
return {"messages": [{"role": "user", "content": text}]}
case "/v1/messages":
return {"max_tokens": 16, "messages": [{"role": "user", "content": text}]}
case _:
return {"input": text}
@dataclass(frozen=True, slots=True)
class _Endpoint:
path: str
model: str
streamed: bool
def body(self, marker: str, cache: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return {
"model": self.model,
**_prompt_fields(self.path, f"Reply. marker-{marker}"),
**({"stream": True} if self.streamed else {}),
"cache": dict(cache),
}
_CHAT: Final = _Endpoint("/v1/chat/completions", "rc-openai", streamed=False)
_ENDPOINTS: Final = MappingProxyType(
{
"chat": _CHAT,
"chat-stream": _Endpoint("/v1/chat/completions", "rc-openai", streamed=True),
"messages": _Endpoint("/v1/messages", "rc-anthropic", streamed=False),
"messages-stream": _Endpoint("/v1/messages", "rc-anthropic", streamed=True),
"responses": _Endpoint("/v1/responses", "rc-openai", streamed=False),
"responses-stream": _Endpoint("/v1/responses", "rc-openai", streamed=True),
"embeddings": _Endpoint("/v1/embeddings", "rc-embedding", streamed=False),
}
)
def _posted_markers(peer: Wire) -> frozenset[str]:
return frozenset(marker_of(request) for request in peer.drain() if request.method == "POST")
@dataclass(frozen=True, slots=True)
class _CacheRig:
gateway: Gateway
peer: Wire
def send(self, endpoint: _Endpoint, marker: str, cache: Mapping[str, JsonValue]) -> int:
return self.gateway.request("POST", endpoint.path, endpoint.body(marker, cache)).status_code
def served(self, endpoint: _Endpoint, marker: str) -> bool:
self.peer.drain()
response: Final = self.gateway.request("POST", endpoint.path, endpoint.body(marker, _NO_STORE))
assert response.status_code == 200, response.text
return marker not in _posted_markers(self.peer)
def _response_cache_config(directory: Path, peer_url: str) -> Path:
config: Final = _rig_config()
deployments: Final = [
{
"model_name": "rc-openai",
"litellm_params": {"model": "openai/gpt-5.6", "api_base": f"{peer_url}/v1", "api_key": "synthetic"},
},
{
"model_name": "rc-anthropic",
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_base": peer_url, "api_key": "synthetic"},
},
{
"model_name": "rc-embedding",
"litellm_params": {
"model": "openai/text-embedding-3-large",
"api_base": f"{peer_url}/v1",
"api_key": "synthetic",
},
},
]
merged: Final = {
**config,
"model_list": deployments,
"litellm_settings": {
**object_value(config["litellm_settings"]),
"cache": True,
"cache_params": {"type": "local"},
"enable_redis_auth_cache": False,
},
}
path: Final = directory / f"response-cache-bound-{uuid.uuid4().hex}.yaml"
path.write_text(yaml.safe_dump(merged))
return path
@pytest.fixture(scope="module")
def response_cache_rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_CacheRig]:
directory: Final = tmp_path_factory.mktemp("response-cache-bound")
with (
wire_server(answering_model_discovery(_response_cache_peer)) as peer,
owned_proxy_process(
session_gateway, directory, {}, config=_response_cache_config(directory, peer.url), workers=1
) as owned,
):
yield _CacheRig(owned.gateway, peer)
def _flood(rig: _CacheRig, endpoint: _Endpoint, cache: Mapping[str, JsonValue], count: int) -> tuple[str, ...]:
markers: Final = tuple(uuid.uuid4().hex for _ in range(count))
statuses: Final = tuple(rig.send(endpoint, marker, cache) for marker in markers)
assert statuses == (200,) * count
return markers
def _served_indexes(rig: _CacheRig, endpoint: _Endpoint, markers: Sequence[str]) -> tuple[int, ...]:
return tuple(index for index, marker in enumerate(markers) if rig.served(endpoint, marker))
def _assert_newest_suffix(served: Sequence[int], flooded: int) -> None:
summary: Final = f"{len(served)} of {flooded} served, first {served[:3]}, last {served[-3:]}"
assert served, summary
assert tuple(served) == tuple(range(served[0], flooded)), summary
def _assert_next_response_cached(rig: _CacheRig, endpoint: _Endpoint) -> None:
fresh: Final = uuid.uuid4().hex
assert rig.send(endpoint, fresh, _FRESH_TTL) == 200
assert eventually(lambda: rig.served(endpoint, fresh), lambda hit: hit, return_last_on_timeout=True), (
"a response with a usable ttl was not cached after the flood"
)
def _assert_flood_keeps_the_newest_responses(
rig: _CacheRig, endpoint: _Endpoint, cache: Mapping[str, JsonValue]
) -> None:
markers: Final = _flood(rig, endpoint, cache, _RESPONSE_SLOTS + _OVERFLOW)
eventually(lambda: rig.served(endpoint, markers[-1]), lambda hit: hit, return_last_on_timeout=True)
served: Final = _served_indexes(rig, endpoint, markers)
_assert_newest_suffix(served, len(markers))
assert _RESPONSE_SLOTS - 1 <= len(served) <= _RESPONSE_SLOTS, f"{len(served)} served"
_assert_next_response_cached(rig, endpoint)
@pytest.mark.parametrize("endpoint", tuple(_ENDPOINTS.values()), ids=tuple(_ENDPOINTS))
def test_nan_ttl_flood_past_the_response_cache_size_keeps_the_newest_responses_and_caches_the_next_one(
response_cache_rig: _CacheRig, endpoint: _Endpoint
) -> None:
_assert_flood_keeps_the_newest_responses(response_cache_rig, endpoint, _NAN_TTL)
@pytest.mark.parametrize("ttl", tuple(_UNUSABLE_TTLS.values()), ids=tuple(_UNUSABLE_TTLS))
def test_unusable_ttl_shapes_flooded_past_the_response_cache_size_keep_the_newest_responses(
response_cache_rig: _CacheRig, ttl: JsonValue
) -> None:
_assert_flood_keeps_the_newest_responses(response_cache_rig, _CHAT, {"ttl": ttl})
def _post(gateway: Gateway, endpoint: _Endpoint, marker: str) -> int:
with httpx.Client(base_url=gateway.client.base_url, timeout=60, trust_env=False) as client:
return client.post(
endpoint.path,
json=endpoint.body(marker, _NAN_TTL),
headers={"Authorization": f"Bearer {gateway.key}"},
).status_code
def _liveliness(gateway: Gateway) -> int:
with httpx.Client(base_url=gateway.client.base_url, timeout=60, trust_env=False) as client:
return client.get("/health/liveliness").status_code
def test_concurrent_burst_into_a_response_cache_full_of_nan_ttls_caches_every_burst_response(
response_cache_rig: _CacheRig,
) -> None:
rig: Final = response_cache_rig
fill: Final = _flood(rig, _CHAT, _NAN_TTL, _RESPONSE_SLOTS)
eventually(lambda: rig.served(_CHAT, fill[-1]), lambda hit: hit)
burst: Final = tuple(
(endpoint, uuid.uuid4().hex)
for endpoint, _ in itertools.product(_ENDPOINTS.values(), range(_BURST_PER_ENDPOINT))
)
with ThreadPoolExecutor(2 * len(burst)) as pool:
answers: Final = tuple(pool.submit(_post, rig.gateway, endpoint, marker) for endpoint, marker in burst)
probes: Final = tuple(pool.submit(_liveliness, rig.gateway) for _ in burst)
statuses: Final = tuple(answer.result() for answer in answers)
liveliness: Final = tuple(probe.result() for probe in probes)
assert statuses == (200,) * len(burst)
assert liveliness == (200,) * len(burst)
missing: Final = eventually(
lambda: tuple(marker for endpoint, marker in burst if not rig.served(endpoint, marker)),
lambda left: not left,
return_last_on_timeout=True,
)
assert not missing, f"{len(missing)} of {len(burst)} burst responses were never served from the cache"
served_fill: Final = _served_indexes(rig, _CHAT, fill)
_assert_newest_suffix(served_fill, len(fill))
assert _RESPONSE_SLOTS - len(burst) <= len(served_fill) + len(burst) <= _RESPONSE_SLOTS, (
f"{len(served_fill)} fill and {len(burst)} burst responses served"
)
_assert_next_response_cached(rig, _CHAT)