The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import math
import re
import time
import uuid
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from types import MappingProxyType
from typing import Final
from urllib.parse import unquote, urlsplit
import httpx
import jwt
import yaml
from cryptography.hazmat.primitives.asymmetric import rsa
from integration._support.client import Gateway, eventually, object_value
from integration._support.wire import Reply, Request
from pydantic import JsonValue
from redis import Redis
RECORDS: Final = "max_batch_file_records"
UPLOADS: Final = "max_batch_file_uploads_per_day"
DOWNLOADS: Final = "max_file_downloads_per_minute"
DAY_SECONDS: Final = 24 * 60 * 60
MINUTE_SECONDS: Final = 60
PROVIDER_KEY: Final = "integration-provider-key"
ROUTED_MODEL: Final = "batch-file-caps-routed"
PROVIDER_REJECTS: Final = "provider-rejects-this-upload"
MISSING_FILE: Final = "file-missing-"
IN_KEY: Final = "in this key's metadata"
IN_TEAM: Final = "in this team's metadata"
IN_GENERAL_SETTINGS: Final = "in general_settings"
THIS_KEY: Final = "this key"
JWT_KEY_ID: Final = "batch-file-caps-signing-key"
COMPLETION_TEXT: Final = "batch file caps completion"
_MARKER: Final = re.compile(rb"caps[0-9a-f]{32}")
_PURPOSE: Final = re.compile(rb'name="purpose"\r\n\r\n([A-Za-z_-]+)')
_CONTENT_PATH: Final = re.compile(r"/v1/files/(.+)/content")
_FILE_PATH: Final = re.compile(r"/v1/files/(.+)")
def marker() -> str:
return "caps" + uuid.uuid4().hex
def batch_line(mark: str, index: int) -> bytes:
return json.dumps(
{
"custom_id": f"{mark}-{index}",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "batch file caps"}]},
}
).encode()
def batch_file(mark: str, records: int, separator: bytes = b"\n", ending: bytes = b"\n") -> bytes:
return separator.join(batch_line(mark, index) for index in range(records)) + ending
def file_content(file_id: str) -> bytes:
return json.dumps({"id": "batch_req_1", "custom_id": file_id, "response": {"status_code": 200}}).encode() + b"\n"
def file_object(file_id: str, purpose: str) -> dict[str, JsonValue]:
return {
"id": file_id,
"object": "file",
"bytes": 128,
"created_at": 1700000000,
"filename": "batch.jsonl",
"purpose": purpose,
"status": "processed",
}
def _provider_error(status: int, message: str) -> Reply:
return Reply(
status=status,
body=json.dumps(
{"error": {"message": message, "type": "invalid_request_error", "param": None, "code": None}}
).encode(),
)
def _uploaded(request: Request) -> Reply:
if PROVIDER_REJECTS.encode() in request.body:
return _provider_error(400, "The provider rejected this batch file.")
mark: Final = _MARKER.search(request.body)
purpose: Final = _PURPOSE.search(request.body)
return Reply(
body=json.dumps(
file_object(
f"file-{mark.group().decode()}" if mark is not None else f"file-{uuid.uuid4().hex}",
purpose.group(1).decode() if purpose is not None else "batch",
)
).encode()
)
def _downloaded(file_id: str) -> Reply:
if file_id.startswith(MISSING_FILE):
return _provider_error(404, f"No such File object: {file_id}")
return Reply(body=file_content(file_id), content_type="application/octet-stream")
def _completion() -> Reply:
return Reply(
body=json.dumps(
{
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1700000000,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": COMPLETION_TEXT},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7},
}
).encode()
)
def provider(request: Request) -> Reply:
path: Final = urlsplit(request.target).path
if request.method == "POST" and path == "/v1/files":
return _uploaded(request)
if request.method == "POST" and path == "/v1/chat/completions":
return _completion()
content: Final = _CONTENT_PATH.fullmatch(path)
if request.method == "GET" and content is not None:
return _downloaded(unquote(content.group(1)))
described: Final = _FILE_PATH.fullmatch(path)
if request.method == "GET" and described is not None:
return Reply(body=json.dumps(file_object(unquote(described.group(1)), "batch")).encode())
return _provider_error(404, f"No scripted reply for {request.method} {path}")
def seen(requests: tuple[Request, ...], mark: str) -> tuple[Request, ...]:
return tuple(request for request in requests if mark in request.target or mark.encode() in request.body)
def uploads_seen(requests: tuple[Request, ...], mark: str) -> tuple[Request, ...]:
return tuple(
request for request in seen(requests, mark) if (request.method, request.target) == ("POST", "/v1/files")
)
def downloads_seen(requests: tuple[Request, ...], file_id: str) -> tuple[Request, ...]:
return tuple(
request for request in requests if (request.method, request.target) == ("GET", f"/v1/files/{file_id}/content")
)
def assert_forwarded_upload(request: Request, content: bytes) -> None:
assert request.headers["authorization"] == f"Bearer {PROVIDER_KEY}", request.headers
assert content in request.body, request.body
assert b'name="purpose"\r\n\r\nbatch' in request.body, request.body
AUTH_CACHE_TTL_SECONDS: Final = 5
def caps_config(directory: Path, provider_url: str, general_settings: Mapping[str, JsonValue]) -> Path:
base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
path: Final = directory / f"batch_file_caps_{uuid.uuid4().hex}.yaml"
path.write_text(
yaml.safe_dump(
{
**base,
"model_list": [
{
"model_name": ROUTED_MODEL,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": PROVIDER_KEY,
"api_base": f"{provider_url}/v1",
},
}
],
"general_settings": {
**base["general_settings"],
"user_api_key_cache_ttl": AUTH_CACHE_TTL_SECONDS,
**general_settings,
},
"files_settings": [
{"custom_llm_provider": "openai", "api_base": f"{provider_url}/v1", "api_key": PROVIDER_KEY}
],
}
)
)
return path
def provider_environment(provider_url: str) -> dict[str, str]:
return {"OPENAI_BASE_URL": f"{provider_url}/v1", "OPENAI_API_KEY": PROVIDER_KEY}
def upload(
candidate: Gateway,
key: str,
content: bytes,
*,
path: str = "/v1/files",
purpose: str = "batch",
filename: str = "batch.jsonl",
fields: Mapping[str, str] = MappingProxyType({}),
) -> httpx.Response:
return candidate.request_multipart(
path, {"purpose": purpose, **fields}, {"file": (filename, content, "application/jsonl")}, key=key
)
def download(
candidate: Gateway,
key: str,
file_id: str,
*,
route: str = "/v1/files/{}/content",
params: Mapping[str, str] | None = None,
) -> httpx.Response:
return candidate.request("GET", route.format(file_id), key=key, params=params)
def window_end(window_seconds: int, room_seconds: int) -> float:
started: Final = eventually(
time.time,
lambda now: window_seconds - now % window_seconds >= room_seconds,
seconds=room_seconds + 5,
)
return (started // window_seconds + 1) * window_seconds
@dataclass(frozen=True, slots=True)
class Timed:
response: httpx.Response
before: float
after: float
def timed(send: Callable[[], httpx.Response]) -> Timed:
before: Final = time.time()
response: Final = send()
return Timed(response, before, time.time())
def _assert_rate_limited(observed: Timed, ends: float, what: str, held: str, reset: str) -> None:
response: Final = observed.response
assert response.status_code == 429, response.text
assert observed.after < ends, f"The counted sequence ran past its window: {observed.after} >= {ends}"
retry_after: Final = int(response.headers["retry-after"])
earliest: Final = max(1, math.ceil(ends - observed.after))
latest: Final = max(1, math.ceil(ends - observed.before))
assert earliest <= retry_after <= latest, (retry_after, earliest, latest)
assert response.json() == {
"error": {
"message": f"{what}: {held}. {reset.format(retry_after)}",
"type": "rate_limit_error",
"param": None,
"code": "429",
}
}, response.text
def assert_upload_limited(observed: Timed, day_ends: float, limit: int, holder: str, source: str) -> None:
_assert_rate_limited(
observed,
day_ends,
"Batch file upload limit reached, the file was not forwarded to the provider",
f"{UPLOADS} is {limit} for {holder} (set {source})",
"The count resets at 00:00 UTC, in {} seconds.",
)
def assert_download_limited(
observed: Timed, minute_ends: float, file_id: str, limit: int, holder: str, source: str
) -> None:
_assert_rate_limited(
observed,
minute_ends,
f"Download limit reached for file {file_id}",
f"{DOWNLOADS} is {limit} for {holder} (set {source})",
"Retry in {} seconds.",
)
def assert_too_many_records(response: httpx.Response, limit: int, source: str) -> None:
assert response.status_code == 413, response.text
assert response.json() == {
"error": {
"message": (
f"Batch input file has more than {limit} records, which exceeds the {RECORDS} of {limit} "
f"set {source}. The file was not forwarded to the provider."
),
"type": "invalid_request_error",
"param": "file",
"code": "413",
}
}, response.text
@dataclass(frozen=True, slots=True)
class Counter:
name: str
count: int
ttl: int
def hashed(key: str) -> str:
return sha256(key.encode()).hexdigest()
def counters(cache: Redis, setting: str, holder: str) -> tuple[Counter, ...]:
return tuple(
Counter(name.decode(), int(cache.get(name) or 0), cache.ttl(name))
for name in sorted(cache.scan_iter(match=f"*litellm:file_usage:{setting}:*{holder}*", count=1000))
)
def config_entry(gateway: Gateway, setting: str) -> Mapping[str, JsonValue]:
response: Final = gateway.request("GET", "/config/list", params={"config_type": "general_settings"})
assert response.status_code == 200, response.text
(entry,) = (object_value(field) for field in response.json() if field["field_name"] == setting)
return entry
@dataclass(frozen=True, slots=True)
class Signer:
private_key: rsa.RSAPrivateKey
jwks: bytes
def signer() -> Signer:
private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
public_jwk: Final = jwt.algorithms.RSAAlgorithm.to_jwk(private_key.public_key())
return Signer(private_key, json.dumps({"keys": [{**json.loads(public_jwk), "kid": JWT_KEY_ID}]}).encode())
def signed_token(identity: Signer, subject: str) -> str:
now: Final = int(time.time())
return jwt.encode(
{"sub": subject, "iat": now, "exp": now + 300, "jti": uuid.uuid4().hex},
identity.private_key,
algorithm="RS256",
headers={"kid": JWT_KEY_ID},
)