The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import base64
import json
import os
import struct
import zlib
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Final
import pytest
from integration._support.upstream import INTERNAL_FIELDS
from integration._support.wire import Reply, Request, wire_server
import litellm
from tests._support.stream_chunk_size import keys_at_every_depth
TEXT: Final = "wire control"
OPENAI_RESPONSE: Final = {
"id": "chatcmpl-wire",
"object": "chat.completion",
"created": 1,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": TEXT}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14},
}
ANTHROPIC_RESPONSE: Final = {
"id": "msg_wire",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": TEXT}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 4},
}
GEMINI_RESPONSE: Final = {
"candidates": [{"content": {"role": "model", "parts": [{"text": TEXT}]}, "finishReason": "STOP", "index": 0}],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 4, "totalTokenCount": 14},
}
CONVERSE_RESPONSE: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": TEXT}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 14},
"metrics": {"latencyMs": 1},
}
OPENAI_STREAM_CHUNKS: Final = (
{
"id": "chatcmpl-wire",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": TEXT}, "finish_reason": None}],
},
{
"id": "chatcmpl-wire",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
},
)
ANTHROPIC_STREAM_EVENTS: Final = (
{
"type": "message_start",
"message": {
"id": "msg_wire",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 10, "output_tokens": 1},
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": TEXT}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}},
{"type": "message_stop"},
)
GEMINI_STREAM_CHUNKS: Final = (
{"candidates": [{"content": {"role": "model", "parts": [{"text": TEXT}]}, "index": 0}]},
{
"candidates": [{"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0}],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 4, "totalTokenCount": 14},
},
)
CONVERSE_STREAM_EVENTS: Final = (
("contentBlockDelta", {"delta": {"text": TEXT}, "contentBlockIndex": 0}),
("messageStop", {"stopReason": "end_turn"}),
("metadata", {"usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 14}, "metrics": {"latencyMs": 1}}),
)
NON_STREAM_BODIES: Final = {
"openai": OPENAI_RESPONSE,
"azure": OPENAI_RESPONSE,
"anthropic": ANTHROPIC_RESPONSE,
"gemini": GEMINI_RESPONSE,
"converse": CONVERSE_RESPONSE,
"invoke": ANTHROPIC_RESPONSE,
}
PROVIDERS: Final = ("openai", "azure", "anthropic", "gemini", "converse", "invoke")
def _aws_string_header(name: str, value: str) -> bytes:
name_bytes: Final = name.encode()
value_bytes: Final = value.encode()
return struct.pack("!B", len(name_bytes)) + name_bytes + b"\x07" + struct.pack("!H", len(value_bytes)) + value_bytes
def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes:
body: Final = json.dumps(payload, separators=(",", ":")).encode()
headers: Final = (
_aws_string_header(":event-type", event_type)
+ _aws_string_header(":content-type", "application/json")
+ _aws_string_header(":message-type", "event")
)
prelude: Final = struct.pack("!II", 12 + len(headers) + len(body) + 4, len(headers))
message: Final = prelude + struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) + headers + body
return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF)
def _sse_reply(frames: tuple[bytes, ...]) -> Reply:
return Reply(chunks=frames, content_type="text/event-stream")
def _stream_reply(provider: str) -> Reply:
match provider:
case "openai" | "azure":
return _sse_reply(
tuple(
f"data: {json.dumps(chunk, separators=(',', ':'))}\n\n".encode() for chunk in OPENAI_STREAM_CHUNKS
)
+ (b"data: [DONE]\n\n",)
)
case "anthropic":
return _sse_reply(
tuple(
f"event: {event['type']}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()
for event in ANTHROPIC_STREAM_EVENTS
)
)
case "gemini":
return _sse_reply(
tuple(
f"data: {json.dumps(chunk, separators=(',', ':'))}\r\n\r\n".encode()
for chunk in GEMINI_STREAM_CHUNKS
)
)
case "converse":
return Reply(
chunks=tuple(_aws_event_frame(event_type, payload) for event_type, payload in CONVERSE_STREAM_EVENTS),
content_type="application/vnd.amazon.eventstream",
)
case "invoke":
return Reply(
chunks=tuple(
_aws_event_frame(
"chunk", {"bytes": base64.b64encode(json.dumps(event, separators=(",", ":")).encode()).decode()}
)
for event in ANTHROPIC_STREAM_EVENTS
),
content_type="application/vnd.amazon.eventstream",
)
def _request_parameters(provider: str, wire_url: str) -> dict[str, object]:
common: Final = {"messages": [{"role": "user", "content": "synthetic chunk control"}]}
match provider:
case "openai":
return {**common, "model": "openai/gpt-4.1-mini", "api_key": "synthetic-openai-key", "api_base": wire_url}
case "azure":
return {
**common,
"model": "azure/gpt-4.1-mini",
"api_key": "synthetic-azure-key",
"api_base": wire_url,
"api_version": "2025-01-01-preview",
}
case "anthropic":
return {
**common,
"model": "anthropic/claude-sonnet-4-5",
"api_key": "synthetic-anthropic-key",
"api_base": wire_url,
}
case "gemini":
return {
**common,
"model": "gemini/gemini-2.5-flash",
"api_key": "synthetic-gemini-key",
"api_base": wire_url,
}
case "converse":
return {
**common,
"model": "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_access_key_id": "fake",
"aws_secret_access_key": "fake",
"aws_region_name": "us-east-1",
"aws_bedrock_runtime_endpoint": wire_url,
}
case "invoke":
return {
**common,
"model": "bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_access_key_id": "fake",
"aws_secret_access_key": "fake",
"aws_region_name": "us-east-1",
"aws_bedrock_runtime_endpoint": wire_url,
}
def _expected_target(provider: str, streaming: bool) -> str:
match provider:
case "openai":
return "/chat/completions"
case "azure":
return "/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2025-01-01-preview"
case "anthropic":
return "/v1/messages"
case "gemini":
return ":streamGenerateContent" if streaming else ":generateContent"
case "converse":
return "/converse-stream" if streaming else "/converse"
case "invoke":
return "/invoke-with-response-stream" if streaming else "/invoke"
def _at(value: object, *path: str) -> object:
if not path:
return value
assert isinstance(value, Mapping)
return _at(value[path[0]], *path[1:])
def _custom_key(body: Mapping[str, object], provider: str) -> object:
match provider:
case "anthropic":
return _at(body, "extra_body", "custom_provider_key")
case "converse":
return _at(body, "additionalModelRequestFields", "extra_body", "custom_provider_key")
return _at(body, "custom_provider_key")
def _peer(provider: str) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
body: Final = json.loads(request.body) if request.body else {}
streaming: Final = (
(isinstance(body, dict) and body.get("stream") is True)
or "streamGenerateContent" in request.target
or request.target.endswith(("-stream",))
)
expected: Final = _expected_target(provider, streaming)
assert expected in request.target, f"{provider}: expected {expected} in {request.target}"
return _stream_reply(provider) if streaming else Reply(body=json.dumps(NON_STREAM_BODIES[provider]).encode())
return respond
@pytest.fixture
def provider_wire_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
empty: Final = tmp_path / "empty-aws-config"
empty.write_text("")
for name in tuple(name for name in os.environ if name.startswith("AWS_")):
monkeypatch.delenv(name, raising=False)
for name, value in {
"AWS_CONFIG_FILE": str(empty),
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
"AWS_EC2_METADATA_DISABLED": "true",
"LITELLM_RUST": "false",
}.items():
monkeypatch.setenv(name, value)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
@pytest.mark.parametrize("provider", PROVIDERS)
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("stream", [False, True])
async def test_internal_params_never_reach_provider_body(
provider_wire_environment: None,
provider: str,
asynchronous: bool,
stream: bool,
) -> None:
with wire_server(_peer(provider)) as wire:
parameters: Final = {
**_request_parameters(provider, wire.url),
"stream": stream,
"stream_chunk_size": 64,
"_litellm_undeclared_sentinel": "internal",
"extra_body": {"custom_provider_key": 1},
"max_tokens": 16,
"timeout": 5,
"num_retries": 0,
}
result: Final = (
await litellm.acompletion(**parameters)
if asynchronous
else await asyncio.to_thread(litellm.completion, **parameters)
)
if stream:
chunks: Final = [chunk async for chunk in result] if asynchronous else [chunk for chunk in result]
text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices)
assert text == TEXT
else:
assert result.choices[0].message.content == TEXT
requests: Final = wire.drain()
assert len(requests) == 1
body: Final = json.loads(requests[0].body)
keys: Final = keys_at_every_depth(body)
assert "stream_chunk_size" not in keys
assert not INTERNAL_FIELDS.intersection(keys)
assert not frozenset(key for key in keys if key.startswith("_litellm_")), keys
assert _custom_key(body, provider) == 1