The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import base64
import hashlib
import hmac
import json
import threading
import time
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from types import MappingProxyType
from typing import Final
from urllib.parse import parse_qs, parse_qsl, quote, unquote, urlsplit
import yaml
from integration._support.client import JsonValue, eventually, object_value
from integration._support.wire import Reply, Request
ACCOUNT: Final = "litellmaudit"
FILE_SYSTEM: Final = "litellm-logs"
SINK_HOSTS: Final = (f"{ACCOUNT}.dfs.core.localhost", f"{ACCOUNT}.blob.core.localhost")
ACCOUNT_KEY: Final = base64.b64encode(b"synthetic-account-key-for-integration-tests").decode()
AUTHENTICATION_FAILED: Final = (
b'{"error":{"code":"AuthenticationFailed","message":"Server failed to authenticate the request. '
b'Make sure the value of Authorization header is formed correctly including the signature."}}'
)
_SIGNED_HEADERS: Final = (
"content-encoding",
"content-language",
"content-length",
"content-md5",
"content-type",
"date",
"if-modified-since",
"if-match",
"if-none-match",
"if-unmodified-since",
"byte_range",
)
def shared_key_signature(request: Request) -> str:
"""The SharedKey signature the service computes for a request: canonical headers, the account plus the
path exactly as sent on the wire, then the decoded query. The aio client signs a directory-scoped file
path with `%3D` but sends a bare `=`, so a padded name fails here the way it fails on the service."""
headers: Final = {name.lower(): value for name, value in request.headers.items() if value}
standard: Final = tuple(
"" if name == "content-length" and headers.get(name) == "0" else headers.get(name, "")
for name in _SIGNED_HEADERS
)
canonical_headers: Final = "".join(
f"{name}:{value}\n" for name, value in sorted(headers.items()) if name.startswith("x-ms-")
)
parts: Final = urlsplit(request.target)
canonical_resource: Final = f"/{ACCOUNT}{parts.path}"
canonical_query: Final = "".join(
f"\n{name.lower()}:{unquote(value)}" for name, value in sorted(parse_qsl(parts.query, keep_blank_values=True))
)
string_to_sign: Final = (
f"{request.method}\n" + "\n".join(standard) + "\n" + canonical_headers + canonical_resource + canonical_query
)
digest: Final = hmac.new(base64.b64decode(ACCOUNT_KEY), string_to_sign.encode(), hashlib.sha256).digest()
return f"SharedKey {ACCOUNT}:{base64.b64encode(digest).decode()}"
@dataclass(slots=True)
class RecordingDataLakeSink:
"""Speaks enough of the Azure Data Lake Gen2 REST surface for the SDK's account-key upload: filesystem
HEAD/PUT, blob HEAD for `exists`, PUT ?resource=directory|file, PATCH ?action=append|flush. Flushed
files are kept by path and can be failed, delayed or served slowly for the chaos cells."""
fail_status: int = 0
delay_seconds: float = 0.0
lock: threading.Lock = field(default_factory=threading.Lock)
directories: set[str] = field(default_factory=set) # mutable-ok: the sink is the durable store for the run
pending: dict[str, bytearray] = field(default_factory=dict) # mutable-ok: append lands before flush
files: dict[str, bytes] = field(default_factory=dict) # mutable-ok: flushed files must be readable later
flush_count: dict[str, int] = field(default_factory=dict) # mutable-ok: re-flush of one path means double upload
rejected: list[str] = field(default_factory=list) # mutable-ok: rejected request methods seen while failing
unauthenticated: list[str] = field(
default_factory=list
) # mutable-ok: targets whose SharedKey signature did not verify
in_flight: int = 0
peak: int = 0
attempt_count: int = 0
def respond(self, request: Request) -> Reply:
parts: Final = urlsplit(request.target)
query: Final = {name: values[-1] for name, values in parse_qs(parts.query).items()}
path: Final = unquote(parts.path)
with self.lock:
self.attempt_count += 1
if self.fail_status:
self.rejected.append(request.method)
return Reply(status=self.fail_status, body=b'{"error":{"code":"SinkFailure"}}')
presented: Final = next(
(value for name, value in request.headers.items() if name.lower() == "authorization"), ""
)
if presented != shared_key_signature(request):
self.unauthenticated.append(request.target)
return Reply(
status=403, headers={"x-ms-error-code": "AuthenticationFailed"}, body=AUTHENTICATION_FAILED
)
if path != f"/{FILE_SYSTEM}" and not path.startswith(f"/{FILE_SYSTEM}/"):
return Reply(status=400, body=b'{"error":{"code":"InvalidUri"}}')
self.in_flight += 1
self.peak = max(self.peak, self.in_flight)
try:
if self.delay_seconds:
time.sleep(self.delay_seconds)
with self.lock:
return self._apply(request, path, query)
finally:
with self.lock:
self.in_flight -= 1
def _apply(self, request: Request, path: str, query: Mapping[str, str]) -> Reply:
stamp: Final = {"etag": '"0x1"', "last-modified": "Thu, 01 Jan 2026 00:00:00 GMT", "x-ms-request-id": "sink"}
empty: Final = "text/plain"
if path == f"/{FILE_SYSTEM}":
if request.method in ("HEAD", "GET"):
return Reply(headers={**stamp, "x-ms-namespace-enabled": "true"}, body=b"{}", content_type=empty)
if request.method == "PUT" and query.get("resource") == "filesystem":
return Reply(status=201, headers=stamp, body=b"", content_type=empty)
return Reply(status=400, body=b'{"error":{"code":"InvalidUri"}}')
if request.method == "HEAD":
if path in self.directories:
return Reply(headers={**stamp, "x-ms-meta-hdi_isfolder": "true"}, body=b"", content_type=empty)
if path in self.files:
return Reply(headers=stamp, body=b"", content_type=empty)
return Reply(status=404, headers={"x-ms-error-code": "PathNotFound"}, body=b"", content_type=empty)
if request.method == "GET":
if path in self.files:
return Reply(headers=stamp, body=self.files[path])
return Reply(status=404, headers={"x-ms-error-code": "PathNotFound"}, body=b"", content_type=empty)
if request.method == "PUT":
if query.get("resource") == "directory":
self.directories.add(path)
return Reply(status=201, headers=stamp, body=b"", content_type=empty)
assert query.get("resource") == "file", request.target
self.pending[path] = bytearray()
return Reply(status=201, headers=stamp, body=b"", content_type=empty)
assert request.method == "PATCH", request.method
if query.get("action") == "append":
assert int(query["position"]) == len(self.pending[path]), request.target
self.pending[path].extend(request.body)
return Reply(status=202, headers=stamp, body=b"", content_type=empty)
assert query.get("action") == "flush", request.target
assert int(query["position"]) == len(self.pending[path]), request.target
self.files[path] = bytes(self.pending.pop(path))
self.flush_count[path] = self.flush_count.get(path, 0) + 1
return Reply(status=200, headers=stamp, body=b"", content_type=empty)
def attempts(self) -> int:
with self.lock:
return self.attempt_count
def rejected_methods(self) -> tuple[str, ...]:
with self.lock:
return tuple(self.rejected)
def unauthenticated_targets(self) -> tuple[str, ...]:
with self.lock:
return tuple(self.unauthenticated)
def duplicated(self) -> tuple[str, ...]:
with self.lock:
return tuple(path for path, count in self.flush_count.items() if count > 1)
def stored(self) -> Mapping[str, bytes]:
with self.lock:
return MappingProxyType(dict(self.files))
def payloads(self) -> Mapping[str, dict[str, JsonValue]]:
return MappingProxyType({path: object_value(json.loads(body)) for path, body in self.stored().items()})
def azure_storage_config(
path: Path, settings: Mapping[str, JsonValue] | None = None, *, callback_setting: str = "callbacks"
) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["litellm_settings"].update({callback_setting: ["azure_storage"], **(settings or {})})
target: Final = path / "azure_storage.yaml"
target.write_text(yaml.safe_dump(config))
return target
def azure_storage_environment(sink_url: str, cert_file: Path) -> Mapping[str, str]:
port: Final = urlsplit(sink_url).port
return MappingProxyType(
{
"AZURE_STORAGE_ACCOUNT_NAME": ACCOUNT,
"AZURE_STORAGE_FILE_SYSTEM": FILE_SYSTEM,
"AZURE_STORAGE_ACCOUNT_KEY": ACCOUNT_KEY,
"AZURE_STORAGE_ENDPOINT_SUFFIX": f"core.localhost:{port}",
"SSL_CERT_FILE": str(cert_file),
}
)
def collect_files(sink: RecordingDataLakeSink, count: int, seconds: float = 60) -> tuple[dict[str, JsonValue], ...]:
"""Wait until `count` flushed files exist, then return every stored payload."""
eventually(lambda: len(sink.stored()), lambda total: total >= count, seconds=seconds)
return tuple(sink.payloads().values())