The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import json
import threading
import time
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from types import MappingProxyType
from typing import Final
import anthropic
import openai
import yaml
from integration._support.client import Gateway, JsonValue, eventually, object_value
from integration._support.wire import Reply, Request
BUCKET: Final = "integration-bucket"
PREFIX: Final = "integration-logs"
@dataclass(slots=True)
class RecordingS3Sink:
"""Records every accepted PUT body by target, tracks peak concurrency, and can reject a leading
run of PUT attempts with a chosen status before accepting. Serves stored bodies back on GET."""
fail_attempts: int = 0
fail_until: float = 0.0
fail_status: int = 503
fail_code: str = "SinkFailure"
fail_body: bytes | None = None
delay_seconds: float = 0.5
lock: threading.Lock = field(default_factory=threading.Lock)
in_flight: int = 0
peak: int = 0
attempts: int = 0
attempt_log: list[tuple[float, int]] = field(default_factory=list) # mutable-ok: appended under lock per PUT
attempt_counts: dict[str, int] = field(default_factory=dict) # mutable-ok: per-target PUT counts under lock
store: dict[str, bytes] = field(default_factory=dict) # mutable-ok: GET reads must see writes from earlier PUTs
def respond(self, request: Request) -> Reply:
if request.method == "GET":
body: Final = self.store.get(request.target)
if body is None:
return Reply(status=404)
return Reply(body=body)
assert request.method == "PUT", request.method
assert request.target.startswith(f"/{BUCKET}/{PREFIX}/"), request.target
with self.lock:
self.attempts += 1
self.attempt_counts[request.target] = self.attempt_counts.get(request.target, 0) + 1
self.in_flight += 1
self.peak = max(self.peak, self.in_flight)
self.attempt_log.append((time.time(), self.in_flight))
failing: Final = self.attempts <= self.fail_attempts or time.time() < self.fail_until
if not failing:
self.store[request.target] = request.body
time.sleep(self.delay_seconds)
with self.lock:
self.in_flight -= 1
if failing:
return Reply(
status=self.fail_status,
body=self.fail_body
if self.fail_body is not None
else f"<Error><Code>{self.fail_code}</Code></Error>".encode(),
content_type="application/xml",
)
return Reply()
def peak_between(self, start: float, end: float) -> int:
with self.lock:
samples: Final = tuple(in_flight for when, in_flight in self.attempt_log if start <= when < end)
return max(samples, default=0)
def objects(self) -> Mapping[str, bytes]:
with self.lock:
return MappingProxyType(dict(self.store))
def payloads(self) -> tuple[dict[str, JsonValue], ...]:
return tuple(object_value(json.loads(line)) for body in self.objects().values() for line in body.splitlines())
def s3_config(
path: Path, sink_url: str, extra: Mapping[str, JsonValue], settings: Mapping[str, JsonValue] | None = None
) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["litellm_settings"].update(
{
"callbacks": ["s3_v2"],
"s3_callback_params": {
"s3_bucket_name": BUCKET,
"s3_region_name": "us-east-1",
"s3_endpoint_url": sink_url,
"s3_path": PREFIX,
"s3_aws_access_key_id": "AKIAIOSFODNN7EXAMPLE",
"s3_aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
**extra,
},
**(settings or {}),
}
)
target: Final = path / "s3_v2.yaml"
target.write_text(yaml.safe_dump(config))
return target
def _chat_completion(identity: str) -> dict[str, JsonValue]:
return {
"id": identity,
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15},
}
def _chat_stream_frames(identity: str) -> tuple[bytes, ...]:
chunks: Final = (
{
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}],
},
{
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15},
},
)
return tuple(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks) + (b"data: [DONE]\n\n",)
def _messages_completion(identity: str) -> dict[str, JsonValue]:
return {
"id": identity,
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5-20250929",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 11, "output_tokens": 4},
}
def _messages_stream_frames(identity: str) -> tuple[bytes, ...]:
events: Final = (
(
"message_start",
{
"type": "message_start",
"message": {
"id": identity,
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5-20250929",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 11, "output_tokens": 1},
},
},
),
(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
),
(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}},
),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
(
"message_delta",
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}},
),
("message_stop", {"type": "message_stop"}),
)
return tuple(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events)
def _responses_completion(identity: str) -> dict[str, JsonValue]:
return {
"id": identity,
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-4o-mini",
"output": [
{
"type": "message",
"id": f"msg_{identity}",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
"usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15},
}
def _responses_stream_frames(identity: str) -> tuple[bytes, ...]:
events: Final = (
(
"response.created",
{
"type": "response.created",
"response": {**_responses_completion(identity), "status": "in_progress", "output": []},
},
),
(
"response.output_text.delta",
{
"type": "response.output_text.delta",
"item_id": f"msg_{identity}",
"output_index": 0,
"content_index": 0,
"delta": "ok",
},
),
("response.completed", {"type": "response.completed", "response": _responses_completion(identity)}),
)
return tuple(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events)
def surface_reply(request: Request) -> Reply:
"""Scripted upstream that echoes the caller's marker string back as the response id."""
if request.method != "POST" or not request.body:
return Reply(status=404)
body: Final = json.loads(request.body)
if request.target.endswith("/chat/completions"):
identity: Final = body["messages"][0]["content"]
if body.get("stream"):
return Reply(content_type="text/event-stream", chunks=_chat_stream_frames(identity))
return Reply(body=json.dumps(_chat_completion(identity)).encode())
if request.target.endswith("/messages"):
identity_messages: Final = body["messages"][0]["content"]
if body.get("stream"):
return Reply(content_type="text/event-stream", chunks=_messages_stream_frames(identity_messages))
return Reply(body=json.dumps(_messages_completion(identity_messages)).encode())
assert request.target.endswith("/responses"), request.target
identity_responses: Final = body["input"]
if body.get("stream"):
return Reply(content_type="text/event-stream", chunks=_responses_stream_frames(identity_responses))
return Reply(body=json.dumps(_responses_completion(identity_responses)).encode())
SURFACES: Final = ("chat", "chat_stream", "messages", "messages_stream", "responses", "responses_stream")
def call_surface(
candidate: Gateway,
surface: str,
openai_model: str,
anthropic_model: str,
key: str,
marker: str,
no_cache: bool = True,
) -> tuple[str, str | None]:
"""Drive one request through the given surface; return (client-visible response id, x-litellm-call-id)."""
base: Final = str(candidate.client.base_url).rstrip("/")
headers: Final = {"Authorization": f"Bearer {key}"}
if surface == "chat":
reply: Final = openai.OpenAI(base_url=f"{base}/v1", api_key=key).chat.completions.create(
model=openai_model,
messages=[{"role": "user", "content": marker}],
extra_body={"cache": {"no-cache": True}} if no_cache else {},
)
return reply.id, None
async def chat_stream() -> str:
stream = await openai.AsyncOpenAI(base_url=f"{base}/v1", api_key=key).chat.completions.create(
model=openai_model,
messages=[{"role": "user", "content": marker}],
stream=True,
extra_body={"cache": {"no-cache": True}} if no_cache else {},
)
seen = ""
async for chunk in stream:
seen = chunk.id # rebind-ok: the stream yields one chunk at a time
return seen
if surface == "chat_stream":
return asyncio.run(chat_stream()), None
if surface in ("messages", "messages_stream"):
client: Final = anthropic.Anthropic(base_url=base, api_key="anthropic-placeholder", default_headers=headers)
if surface == "messages":
reply_messages: Final = client.messages.create(
model=anthropic_model, max_tokens=16, messages=[{"role": "user", "content": marker}]
)
return reply_messages.id, None
with client.messages.stream(
model=anthropic_model, max_tokens=16, messages=[{"role": "user", "content": marker}]
) as stream:
final: Final = stream.get_final_message()
return final.id, None
if surface == "responses":
response: Final = candidate.request(
"POST",
"/v1/responses",
{"model": openai_model, "input": marker, **({"cache": {"no-cache": True}} if no_cache else {})},
key=key,
)
assert response.status_code == 200, response.text
return str(response.json()["id"]), response.headers.get("x-litellm-call-id")
assert surface == "responses_stream", surface
with candidate.client.stream(
"POST",
"/v1/responses",
json={"model": openai_model, "input": marker, "stream": True},
headers=headers,
) as response:
text: Final = response.read().decode()
assert response.status_code == 200, text
call_id: Final = response.headers.get("x-litellm-call-id")
assert marker in text, text
return marker, call_id
def collect_payloads(sink: RecordingS3Sink, count: int, seconds: float = 60) -> tuple[dict[str, JsonValue], ...]:
"""Wait until `count` stored payload lines exist, then return every stored payload object."""
def delivered() -> int:
return sum(len(body.splitlines()) for body in sink.objects().values())
eventually(delivered, lambda total: total >= count, seconds=seconds)
return sink.payloads()
def mixed_burst(
candidate: Gateway, openai_model: str, anthropic_model: str, key: str, marker: str, per_surface: int = 8
) -> tuple[tuple[str, str | None], ...]:
"""Fire `per_surface` requests on every surface; returns (response id, x-litellm-call-id) per request."""
jobs: Final = tuple(
(surface, f"{marker}-{surface}-{index}") for surface in SURFACES for index in range(per_surface)
)
def call(job: tuple[str, str]) -> tuple[str, str | None]:
surface, identity = job
return call_surface(candidate, surface, openai_model, anthropic_model, key, identity)
with ThreadPoolExecutor(max_workers=48) as pool:
return tuple(pool.map(call, jobs))
def matched_ids(
payloads: tuple[dict[str, JsonValue], ...], answered: tuple[tuple[str, str | None], ...]
) -> frozenset[str]:
"""Every payload must be accountable to an answered request by response id or litellm_call_id."""
response_ids: Final = frozenset(observed for observed, _ in answered)
call_ids: Final = frozenset(call_id for _, call_id in answered if call_id is not None)
landed: Final = []
for payload in payloads:
if payload["id"] in response_ids:
landed.append(payload["id"])
continue
uncached: Final = str(payload["id"]).rsplit("_cache_hit", 1)[0]
if uncached in response_ids:
landed.append(str(payload["id"]))
continue
assert payload["litellm_call_id"] in call_ids, f"unmatched payload {payload['id']!r}"
landed.append(str(payload["id"]))
return frozenset(landed)