The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import threading
import uuid
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import ExitStack
from hashlib import sha256
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, held, object_value
from integration._support.database import read_rows
from integration._support.process import WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PROJECT: Final = "scripted-project"
_ADAPTER_MODEL: Final = "settings-adapter-openai"
_NATIVE_MODEL: Final = "settings-native-gemini"
_REPLY_TEXT: Final = "settings reply"
_PROMPT_TOKENS: Final = 13
_COMPLETION_TOKENS: Final = 4
_OPENAI_USAGE: Final = {
"prompt_tokens": _PROMPT_TOKENS,
"completion_tokens": _COMPLETION_TOKENS,
"total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS,
}
_GEMINI_USAGE: Final = {
"promptTokenCount": _PROMPT_TOKENS,
"candidatesTokenCount": _COMPLETION_TOKENS,
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
}
_ZERO_USAGE: Final = {"promptTokenCount": 0, "candidatesTokenCount": 0, "totalTokenCount": 0}
_BURST_SELECT: Final = (
'SELECT api_key, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs"'
" WHERE api_key = ANY(string_to_array(%s, ','))"
)
_CACHE_SELECT: Final = (
'SELECT cache_hit, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "startTime"'
)
_CALLER_OPTIONS: Final = {
"caller_sends_none": None,
"caller_opts_in": {"include_usage": True},
"caller_opts_out": {"include_usage": False},
"caller_sends_empty_object": {},
}
_UPSTREAM_ASKS_FOR_USAGE: Final = {
"always_off": {
"caller_sends_none": False,
"caller_opts_in": True,
"caller_opts_out": False,
"caller_sends_empty_object": False,
},
"always_on": {
"caller_sends_none": True,
"caller_opts_in": True,
"caller_opts_out": False,
"caller_sends_empty_object": True,
},
}
_SETTING_VALUES: Final = {"always_off": False, "always_on": True}
_STREAMS_PER_WORKER: Final = 6
def _openai_frame(chunk: Mapping[str, JsonValue], response_id: str) -> bytes:
body: Final = {"id": response_id, "object": "chat.completion.chunk", "created": 1700000000, **chunk}
return f"data: {json.dumps(body)}\n\n".encode()
def _openai_frames(include_usage: bool) -> tuple[bytes, ...]:
response_id: Final = f"chatcmpl-{uuid.uuid4().hex}"
usage: Final = (_openai_frame({"choices": [], "usage": _OPENAI_USAGE}, response_id),) if include_usage else ()
return (
_openai_frame({"choices": [{"index": 0, "delta": {"role": "assistant", "content": _REPLY_TEXT}}]}, response_id),
_openai_frame({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, response_id),
*usage,
b"data: [DONE]\n\n",
)
def _native_frames() -> tuple[bytes, ...]:
text: Final = {"candidates": [{"content": {"parts": [{"text": _REPLY_TEXT}], "role": "model"}, "index": 0}]}
stop: Final = {
"candidates": [{"content": {"parts": [], "role": "model"}, "finishReason": "STOP", "index": 0}],
"usageMetadata": _GEMINI_USAGE,
}
return (f"data: {json.dumps(text)}\n\n".encode(), f"data: {json.dumps(stop)}\n\n".encode())
def _asks_for_usage(body: Mapping[str, JsonValue]) -> bool:
options: Final = body.get("stream_options")
return isinstance(options, dict) and options.get("include_usage") is True
def _peer(gate: threading.Event | None = None) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
if ":streamGenerateContent" in request.target:
return Reply(content_type="text/event-stream", chunks=_native_frames())
body: Final = _JSON_OBJECT.validate_json(request.body)
return Reply(
content_type="text/event-stream",
chunks=_openai_frames(_asks_for_usage(body)),
gate_after_first=gate,
)
return respond
def _owned_config(path: Path, gateway: Gateway, wire: Wire, general: Mapping[str, JsonValue]) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
path.write_text(
yaml.safe_dump(
{
**config,
"model_list": [
{
"model_name": _ADAPTER_MODEL,
"litellm_params": {
"model": "openai/gpt-5.4-mini",
"api_base": wire.url,
"api_key": "synthetic-provider-key",
},
},
{
"model_name": _NATIVE_MODEL,
"litellm_params": {
"model": "vertex_ai/gemini-2.5-flash",
"api_base": wire.url,
"vertex_project": _PROJECT,
"vertex_location": "us-central1",
"vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
},
},
],
"general_settings": {**(config.get("general_settings") or {}), **general},
}
)
)
return path
def _stream_body(marker: str, stream_options: JsonValue) -> dict[str, JsonValue]:
base: Final = {"contents": [{"role": "user", "parts": [{"text": f"settings probe {marker}"}]}]}
return base if stream_options is None else {**base, "stream_options": stream_options}
def _stream(client: httpx.Client, model: str, key: str, body: Mapping[str, JsonValue]) -> httpx.Response:
return client.post(
f"/v1beta/models/{model}:streamGenerateContent?alt=sse",
json=dict(body),
headers={"Authorization": f"Bearer {key}"},
)
def _frames(response: httpx.Response) -> list[dict[str, JsonValue]]:
parts: Final = [part for part in response.text.split("\n\n") if part.strip()]
assert all(part.startswith("data: ") for part in parts), response.text
return [_JSON_OBJECT.validate_json(part.removeprefix("data: ")) for part in parts]
def _stop_frames(frames: Sequence[Mapping[str, JsonValue]]) -> list[Mapping[str, JsonValue]]:
return [
frame
for frame in frames
if any(object_value(candidate).get("finishReason") for candidate in frame.get("candidates") or [])
]
def _model_bodies(wire: Wire) -> list[dict[str, JsonValue]]:
return [_JSON_OBJECT.validate_json(request.body) for request in wire.drain()]
def _assert_one_stop_with(response: httpx.Response, usage: Mapping[str, int]) -> None:
assert response.status_code == 200, response.text
stops: Final = _stop_frames(_frames(response))
assert len(stops) == 1, response.text
assert stops[0]["usageMetadata"] == usage, response.text
def _assert_caller_option_reaches_upstream(
client: httpx.Client, key: str, wire: Wire, setting: str, caller: str
) -> None:
response: Final = _stream(client, _ADAPTER_MODEL, key, _stream_body(uuid.uuid4().hex, _CALLER_OPTIONS[caller]))
asks: Final = _UPSTREAM_ASKS_FOR_USAGE[setting][caller]
_assert_one_stop_with(response, _GEMINI_USAGE if asks else _ZERO_USAGE)
bodies: Final = _model_bodies(wire)
assert len(bodies) == 1, (caller, bodies)
assert bodies[0].get("stream_options") == ({"include_usage": True} if asks else None), (caller, bodies[0])
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
@pytest.mark.parametrize("setting", tuple(_SETTING_VALUES))
def test_always_include_stream_usage_decides_what_the_adapter_asks_upstream(
gateway: Gateway, tmp_path: Path, setting: str
) -> None:
with ExitStack() as stack:
wire: Final = stack.enter_context(wire_server(_peer()))
config: Final = _owned_config(
tmp_path / f"{setting}.yaml", gateway, wire, {"always_include_stream_usage": _SETTING_VALUES[setting]}
)
owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2))
scenario: Final = stack.enter_context(owned.gateway.scenario())
key: Final = scenario.key(models=[_ADAPTER_MODEL, _NATIVE_MODEL])
for caller in _CALLER_OPTIONS:
_assert_caller_option_reaches_upstream(owned.gateway.client, key, wire, setting, caller)
native: Final = _stream(owned.gateway.client, _NATIVE_MODEL, key, _stream_body(uuid.uuid4().hex, None))
_assert_one_stop_with(native, _GEMINI_USAGE)
native_bodies: Final = _model_bodies(wire)
assert len(native_bodies) == 1, native_bodies
assert "stream_options" not in native_bodies[0] and "streamOptions" not in native_bodies[0], native_bodies[0]
def _cache_rows(key: str) -> list[dict[str, JsonValue]]:
digest: Final = sha256(key.encode()).hexdigest()
eventually(lambda: read_rows(_CACHE_SELECT, (digest,)), lambda rows: len(rows) >= 2, seconds=70)
return held(lambda: read_rows(_CACHE_SELECT, (digest,)), lambda rows: len(rows) == 2, holding=3)
def _assert_cache_hit_replays_the_stop_usage(
client: httpx.Client, key: str, wire: Wire, setting: str, caller: str
) -> None:
body: Final = _stream_body(uuid.uuid4().hex, _CALLER_OPTIONS[caller])
usage: Final = _GEMINI_USAGE if _UPSTREAM_ASKS_FOR_USAGE[setting][caller] else _ZERO_USAGE
_assert_one_stop_with(_stream(client, _ADAPTER_MODEL, key, body), usage)
_assert_one_stop_with(_stream(client, _ADAPTER_MODEL, key, body), usage)
assert len(_model_bodies(wire)) == 1, caller
rows: Final = _cache_rows(key)
assert [row["cache_hit"] == "True" for row in rows] == [False, True], (caller, rows)
assert (rows[1]["prompt_tokens"], rows[1]["completion_tokens"]) == (
rows[0]["prompt_tokens"],
rows[0]["completion_tokens"],
), (caller, rows)
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
@pytest.mark.parametrize("setting", tuple(_SETTING_VALUES))
def test_always_include_stream_usage_cache_hit_replays_the_stop_usage_the_miss_sent(
gateway: Gateway, tmp_path: Path, setting: str
) -> None:
with ExitStack() as stack:
wire: Final = stack.enter_context(wire_server(_peer()))
config: Final = _owned_config(
tmp_path / f"{setting}-cache.yaml", gateway, wire, {"always_include_stream_usage": _SETTING_VALUES[setting]}
)
owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2))
scenario: Final = stack.enter_context(owned.gateway.scenario())
for caller in _CALLER_OPTIONS:
_assert_cache_hit_replays_the_stop_usage(
owned.gateway.client, scenario.key(models=[_ADAPTER_MODEL]), wire, setting, caller
)
def _burst_rows(digests: Sequence[str]) -> list[dict[str, JsonValue]]:
return read_rows(_BURST_SELECT, (",".join(digests),))
def _rows_by_key(digests: Sequence[str], expected: int) -> list[dict[str, JsonValue]]:
eventually(lambda: _burst_rows(digests), lambda rows: len(rows) >= expected, seconds=120)
return held(lambda: _burst_rows(digests), lambda rows: len(rows) == expected, holding=5)
def _stream_or_dropped(client: httpx.Client, key: str) -> httpx.Response | None:
try:
return _stream(client, _ADAPTER_MODEL, key, _stream_body(uuid.uuid4().hex, None))
except httpx.TransportError:
return None
def _held_streams(
stack: ExitStack, pool: ThreadPoolExecutor, url: str, keys: Sequence[str], wire: Wire, held: int
) -> tuple[Future[httpx.Response | None], ...]:
clients: Final = tuple(stack.enter_context(httpx.Client(base_url=url, timeout=60, trust_env=False)) for _ in keys)
futures: Final = tuple(
pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True)
)
eventually(wire.received.qsize, lambda size: size >= held, seconds=60)
return futures
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_worker_killed_mid_stream_leaves_the_sibling_billing_each_stream_once(gateway: Gateway, tmp_path: Path) -> None:
gate: Final = threading.Event()
with ExitStack() as stack:
wire: Final = stack.enter_context(wire_server(_peer(gate)))
stack.callback(gate.set)
config: Final = _owned_config(tmp_path / "worker-kill.yaml", gateway, wire, {})
owned: Final = stack.enter_context(
owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2, healthcheck=WorkerHealthcheck.LENIENT)
)
victim, survivor = sorted(eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30))
scenario: Final = stack.enter_context(owned.gateway.scenario())
victim_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
survivor_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
proxy_url: Final = str(owned.gateway.client.base_url)
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=2 * _STREAMS_PER_WORKER))
with owned.suspended(survivor):
on_victim: Final = _held_streams(stack, pool, proxy_url, victim_keys, wire, _STREAMS_PER_WORKER)
with owned.suspended(victim):
on_survivor: Final = _held_streams(stack, pool, proxy_url, survivor_keys, wire, 2 * _STREAMS_PER_WORKER)
with owned.replacing_worker(victim) as kill:
kill()
gate.set()
dropped: Final = tuple(future.result(timeout=60) for future in on_victim)
survivors: Final = tuple(
zip(survivor_keys, (future.result(timeout=60) for future in on_survivor), strict=True)
)
assert dropped == (None,) * _STREAMS_PER_WORKER, dropped
for _, result in survivors:
assert result is not None
_assert_one_stop_with(result, _GEMINI_USAGE)
second_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(6))
second: Final = tuple(_stream_or_dropped(owned.gateway.client, key) for key in second_keys)
for result in second:
assert result is not None
_assert_one_stop_with(result, _GEMINI_USAGE)
billed: Final = tuple(key for key, _ in survivors) + second_keys
digests: Final = tuple(sha256(key.encode()).hexdigest() for key in billed)
rows: Final = _rows_by_key(digests, len(billed))
assert sorted(str(row["api_key"]) for row in rows) == sorted(digests)
assert {(row["prompt_tokens"], row["completion_tokens"]) for row in rows} == {
(_PROMPT_TOKENS, _COMPLETION_TOKENS)
}, rows
assert owned.process.poll() is None