The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from __future__ import annotations
import base64
import json
import os
import re
import uuid
from collections import deque
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Final
from urllib.parse import urlsplit
from integration._support import claude_code as cc
from integration._support.wire import Reply, Request
from pydantic import JsonValue, TypeAdapter
MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
THOUGHT: Final = "plan the answer"
USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}
CHAT_USAGE: Final[dict[str, JsonValue]] = {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}
CLAUDE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 20, "output_tokens": 7}
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]])
MINTED_ID: Final = re.compile(r"^rs_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$")
_INNER_ID: Final = re.compile(r"response_id:([^;]+)")
_WRAPPER_PREFIX: Final = "litellm:custom_llm_provider:"
_PROXY_WRAPPED_PREFIX: Final = "litellm_proxy:responses_api:response_id:"
def signature(marker: str) -> str:
return f"sig-{marker}"
def answer(marker: str | None) -> str:
return "ok" if marker is None else f"answer marker-{marker}"
def newest_marker(text: str) -> str | None:
found: Final = MARKER.findall(text)
return str(found[-1]) if found else None
def error(status: int, message: str, code: str) -> Reply:
body: Final = {"error": {"message": message, "type": "invalid_request_error", "param": None, "code": code}}
return Reply(status=status, body=json.dumps(body).encode())
def sse(event: Mapping[str, JsonValue]) -> bytes:
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
def chat_sse(frame: Mapping[str, JsonValue]) -> bytes:
return b"data: " + json.dumps(frame).encode() + b"\n\n"
def thinking_json(marker: str) -> str:
return json.dumps([{"type": "thinking", "thinking": THOUGHT, "signature": signature(marker)}])
def minted_item(marker: str, **extra: JsonValue) -> dict[str, JsonValue]:
return {"type": "reasoning", "id": f"rs_{uuid.uuid4()}", "encrypted_content": thinking_json(marker), **extra}
def agents_sdk_history(marker: str, *reasoning: dict[str, JsonValue]) -> list[dict[str, JsonValue]]:
return [
{"role": "user", "content": "Pick a city and look up its weather."},
*reasoning,
{
"type": "message",
"id": f"msg_{uuid.uuid4()}",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Prague", "annotations": []}],
},
{"type": "function_call", "call_id": "call_weather", "name": "weather", "arguments": '{"city": "Prague"}'},
{"type": "function_call_output", "call_id": "call_weather", "output": '{"celsius": 18}'},
{"role": "user", "content": f"Now answer marker-{marker}"},
]
def without(history: Sequence[dict[str, JsonValue]], dropped: Sequence[dict[str, JsonValue]]) -> list[JsonValue]:
return [item for item in history if all(item is not gone for gone in dropped)]
def reasoning_items(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]:
return [item for item in ITEMS.validate_python(body["input"]) if item.get("type") == "reasoning"]
def _decoded_wrapper(value: str) -> str | None:
try:
decoded: Final = base64.b64decode(value.removeprefix("resp_"), validate=True).decode()
except (ValueError, UnicodeDecodeError):
return None
return decoded if decoded.startswith(_WRAPPER_PREFIX) else None
def response_identities(value: str) -> frozenset[str]:
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with
salt: Final = os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt")
opened: Final = decrypt_if_encrypted_with(value.removeprefix("resp_"), salt)
sealed: Final = opened is not None and opened.startswith(_PROXY_WRAPPED_PREFIX)
wrapped: Final = opened.removeprefix(_PROXY_WRAPPED_PREFIX).split(";", 1)[0] if sealed and opened else value
decoded: Final = _decoded_wrapper(wrapped)
if decoded is None:
return frozenset({wrapped})
inner: Final = _INNER_ID.search(decoded)
assert inner is not None, decoded
return frozenset({wrapped, inner.group(1)})
def same_response(left: str, right: str) -> bool:
return bool(response_identities(left) & response_identities(right))
@dataclass(frozen=True, slots=True)
class ResponsesVendor:
claude_model: str = cc.OPUS
pause_between_chunks: float = 0
minted: deque[str] = field(default_factory=deque)
def respond(self, request: Request) -> Reply:
path: Final = urlsplit(request.target).path
if request.method == "GET":
return Reply(body=json.dumps({"object": "list", "data": [{"id": "gpt-5.6", "object": "model"}]}).encode())
body: Final = JSON_OBJECT.validate_json(request.body)
if path.endswith("/messages"):
return self._claude(body)
if path.endswith("/chat/completions"):
return self._chat(body)
assert path.endswith("/responses"), request.target
verdict: Final = self._verdict(body)
return verdict if verdict is not None else self._responses(body)
def _verdict(self, body: Mapping[str, JsonValue]) -> Reply | None:
received: Final = body.get("input")
if isinstance(received, str):
return None
items: Final = ITEMS.validate_python(received)
if not items and "previous_response_id" not in body:
return error(
400, 'One of "input" or "previous_response_id" must be provided.', "missing_required_parameter"
)
for index, item in enumerate(items):
if item.get("type") != "reasoning":
continue
item_id: Final = item.get("id")
if item_id is not None and not isinstance(item_id, str):
return error(400, f"Invalid type for 'input[{index}].id': expected a string.", "invalid_type")
if "summary" not in item:
return error(
400, f"Missing required parameter: 'input[{index}].summary'.", "missing_required_parameter"
)
if item_id == "":
return error(400, f"Invalid 'input[{index}].id': empty string.", "invalid_value")
if isinstance(item_id, str) and item_id not in self.minted:
return error(404, f"Item with id '{item_id}' not found.", "invalid_request_error")
return None
def _responses(self, body: Mapping[str, JsonValue]) -> Reply:
marker: Final = newest_marker(json.dumps(body))
tag: Final = uuid.uuid4().hex
self.minted.append(f"rs_{tag}")
reasoning: Final[dict[str, JsonValue]] = {
"id": f"rs_{tag}",
"type": "reasoning",
"summary": [],
"encrypted_content": f"gAAAAA-vendor-{tag}",
}
message: Final[dict[str, JsonValue]] = {
"id": f"msg_{tag}",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": answer(marker), "annotations": []}],
}
response: Final[dict[str, JsonValue]] = {
"id": f"resp_{tag}",
"object": "response",
"created_at": 1,
"status": "completed",
"model": body["model"],
"output": [reasoning, message],
"usage": USAGE,
}
if body.get("stream") is not True:
return Reply(body=json.dumps(response).encode())
events: Final[tuple[dict[str, JsonValue], ...]] = (
{
"type": "response.created",
"sequence_number": 0,
"response": {**response, "status": "in_progress", "output": []},
},
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": reasoning},
{"type": "response.output_item.done", "sequence_number": 2, "output_index": 0, "item": reasoning},
{
"type": "response.output_item.added",
"sequence_number": 3,
"output_index": 1,
"item": {**message, "content": []},
},
{
"type": "response.output_text.delta",
"sequence_number": 4,
"item_id": f"msg_{tag}",
"output_index": 1,
"content_index": 0,
"delta": answer(marker),
},
{"type": "response.output_item.done", "sequence_number": 5, "output_index": 1, "item": message},
{"type": "response.completed", "sequence_number": 6, "response": response},
)
return Reply(
content_type="text/event-stream",
chunks=tuple(sse(event) for event in events),
pause_between_chunks=self.pause_between_chunks,
)
def _chat(self, body: Mapping[str, JsonValue]) -> Reply:
marker: Final = newest_marker(json.dumps(body))
tag: Final = uuid.uuid4().hex
if body.get("stream") is not True:
return Reply(
body=json.dumps(
{
"id": f"chatcmpl-{tag}",
"object": "chat.completion",
"created": 1,
"model": body["model"],
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": answer(marker)},
"finish_reason": "stop",
}
],
"usage": CHAT_USAGE,
}
).encode()
)
chunk: Final[dict[str, JsonValue]] = {
"id": f"chatcmpl-{tag}",
"object": "chat.completion.chunk",
"created": 1,
"model": body["model"],
}
frames: Final[tuple[dict[str, JsonValue], ...]] = (
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": answer(marker)}}]},
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": CHAT_USAGE},
)
return Reply(
content_type="text/event-stream",
chunks=(*(chat_sse(frame) for frame in frames), b"data: [DONE]\n\n"),
pause_between_chunks=self.pause_between_chunks,
)
def _claude(self, body: Mapping[str, JsonValue]) -> Reply:
marker: Final = newest_marker(json.dumps(body))
content: Final = (
{"type": "thinking", "thinking": THOUGHT, "signature": signature(marker or "")},
{"type": "text", "text": answer(marker)},
)
identity: Final = f"msg_{uuid.uuid4().hex}"
if body.get("stream") is True:
return Reply(
content_type="text/event-stream",
chunks=cc.message_stream(identity, self.claude_model, content, CLAUDE_USAGE),
pause_between_chunks=self.pause_between_chunks,
)
return Reply(body=cc.message_reply(identity, self.claude_model, content, CLAUDE_USAGE))