The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final
import httpx
from integration._support.client import JSON_OBJECT, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request
from pydantic import JsonValue
AZURE_TARGET: Final = "/openai/v1/responses?api-version="
OPENAI_TARGET: Final = "/responses"
RATE_LIMIT_MESSAGE: Final = "Your requests to gpt-6 have exceeded token rate limit."
RATE_LIMIT_PREFIX: Final = "litellm.RateLimitError: "
SENTINEL_PREFIX: Final = "litellm.MidStreamFallbackError: "
def frame(event: Mapping[str, JsonValue]) -> bytes:
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
def response_object(identity: str, status: str, **fields: JsonValue) -> dict[str, JsonValue]:
return {
"id": identity,
"object": "response",
"created_at": 1,
"status": status,
"model": "gpt-6",
"output": [],
"usage": None,
**fields,
}
def created(identity: str) -> Mapping[str, JsonValue]:
return {"type": "response.created", "sequence_number": 0, "response": response_object(identity, "in_progress")}
def error_event(error: Mapping[str, JsonValue] | None) -> Mapping[str, JsonValue]:
return {"type": "error", "sequence_number": 1, **({} if error is None else {"error": dict(error)})}
def openai_error_event(code: str, message: str) -> Mapping[str, JsonValue]:
return {"type": "error", "sequence_number": 1, "code": code, "message": message, "param": None}
def openai_error_stream(identity: str, code: str, message: str) -> tuple[bytes, ...]:
return (frame(created(identity)), frame(openai_error_event(code, message)))
def azure_rate_limit() -> Mapping[str, JsonValue]:
return {
"type": "too_many_requests",
"code": "rate_limit_exceeded",
"headers": {"x-ms-fe-error": "true"},
"message": RATE_LIMIT_MESSAGE,
"param": None,
}
def failed(identity: str, code: str, message: str) -> Mapping[str, JsonValue]:
return {
"type": "response.failed",
"sequence_number": 2,
"response": response_object(identity, "failed", error={"code": code, "message": message}),
}
def delta(identity: str, text: str) -> Mapping[str, JsonValue]:
return {
"type": "response.output_text.delta",
"item_id": f"msg_{identity}",
"output_index": 0,
"content_index": 0,
"delta": text,
}
def completed(identity: str, text: str) -> Mapping[str, JsonValue]:
message: Final = {
"type": "message",
"id": f"msg_{identity}",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": text, "annotations": []}],
}
usage: Final = {
"input_tokens": 11,
"output_tokens": 4,
"total_tokens": 15,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0},
}
return {
"type": "response.completed",
"sequence_number": 3,
"response": response_object(identity, "completed", output=[message], usage=usage),
}
def rate_limited_stream(identity: str) -> tuple[bytes, ...]:
return (
frame(created(identity)),
frame(error_event(azure_rate_limit())),
frame(failed(identity, "rate_limit_exceeded", RATE_LIMIT_MESSAGE)),
)
def healthy_stream(identity: str, text: str) -> tuple[bytes, ...]:
return (frame(created(identity)), frame(delta(identity, text)), frame(completed(identity, text)))
def serve(stream: tuple[bytes, ...], target: str) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target.startswith(target), request.target
return Reply(content_type="text/event-stream", chunks=stream)
return respond
def function_tools() -> list[JsonValue]:
return [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
]
def chat_content(frames: Sequence[Mapping[str, JsonValue]]) -> str:
def deltas() -> Iterator[str]:
for chunk in frames:
for choice in chunk.get("choices") or []:
yield string_value(object_value(object_value(choice)["delta"]).get("content") or "")
return "".join(deltas())
def chat_body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]:
return {
"model": model,
"stream": True,
"messages": [{"role": "user", "content": marker}],
"tools": function_tools(),
"num_retries": 0,
"cache": {"no-cache": True},
**extra,
}
def messages_body(model: str, marker: str) -> dict[str, JsonValue]:
return {
"model": model,
"max_tokens": 64,
"stream": True,
"messages": [{"role": "user", "content": marker}],
"tools": [
{
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
}
],
"num_retries": 0,
"cache": {"no-cache": True},
}
def error_body(response: httpx.Response) -> Mapping[str, JsonValue]:
return object_value(JSON_OBJECT.validate_json(response.content)["error"])
def data_frames(text: str) -> tuple[Mapping[str, JsonValue], ...]:
return tuple(
JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in text.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"
)
def sse_events(text: str) -> tuple[tuple[str, Mapping[str, JsonValue]], ...]:
def parse(block: str) -> tuple[str, Mapping[str, JsonValue]]:
lines: Final = block.splitlines()
event: Final = next(line.removeprefix("event: ") for line in lines if line.startswith("event: "))
data: Final = next(line.removeprefix("data: ") for line in lines if line.startswith("data: "))
return event, JSON_OBJECT.validate_json(data)
return tuple(parse(block) for block in text.strip().split("\n\n") if "event: " in block)
def spend_row(call_id: str) -> Mapping[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT litellm_call_id, status, model_group, metadata FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s',
(call_id,),
),
lambda found: len(found) >= 1,
seconds=70,
)
assert len(rows) == 1, rows
return rows[0]
def assert_provider_typed_rate_limit(
error: Mapping[str, JsonValue], provider_message: str = RATE_LIMIT_MESSAGE
) -> None:
assert error["type"] == "throttling_error", error
assert str(error["code"]) == "429", error
message: Final = string_value(error["message"])
assert message.startswith(RATE_LIMIT_PREFIX) and provider_message in message, message
assert SENTINEL_PREFIX not in message, message