The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import base64
import json
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from typing import Final
from integration._support.claude_code import sse_frame
from integration._support.upstream import _aws_event_frame
from integration._support.wire import Reply, Request
from pydantic import JsonValue, TypeAdapter
MODEL: Final = "claude-sonnet-5-5"
BEDROCK_MODEL: Final = "anthropic.claude-sonnet-5-5"
THINKING_PARTS: Final = ("alpha ", "beta")
THINKING: Final = "alpha beta"
SIGNATURE: Final = "scripted-signature-" + "s" * 32
NO_CACHE: Final = {"cache": {"no-cache": True}}
EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
JSON_LIST: Final = TypeAdapter(list[JsonValue])
BLOCKS: Final = TypeAdapter(list[dict[str, JsonValue]])
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
_STREAMING_TARGETS: Final = ("/invoke-with-response-stream", ":streamRawPredict")
Event = dict[str, JsonValue]
def prompt(marker: str) -> str:
return f"think it through for marker-{marker}"
def answer(marker: str) -> str:
return f"answer marker-{marker}"
def identity(marker: str) -> str:
return f"msg_{marker}"
def marker_of(request: Request) -> str:
found: Final = _MARKER.findall(request.body.decode())
assert found, request.body
return found[-1]
def _event(**fields: JsonValue) -> Event:
return dict(fields)
def thinking_events(index: int, parts: Sequence[JsonValue], signatures: Sequence[JsonValue]) -> tuple[Event, ...]:
start: Final = _event(
type="content_block_start", index=index, content_block={"type": "thinking", "thinking": "", "signature": ""}
)
thought: Final = tuple(
_event(type="content_block_delta", index=index, delta={"type": "thinking_delta", "thinking": part})
for part in parts
)
signed: Final = tuple(
_event(type="content_block_delta", index=index, delta={"type": "signature_delta", "signature": signature})
for signature in signatures
)
return (start, *thought, *signed, _event(type="content_block_stop", index=index))
def redacted_events(index: int, data: str) -> tuple[Event, ...]:
return (
_event(type="content_block_start", index=index, content_block={"type": "redacted_thinking", "data": data}),
_event(type="content_block_stop", index=index),
)
def text_events(index: int, text: str) -> tuple[Event, ...]:
return (
_event(type="content_block_start", index=index, content_block={"type": "text", "text": ""}),
_event(type="content_block_delta", index=index, delta={"type": "text_delta", "text": text}),
_event(type="content_block_stop", index=index),
)
def message_events(marker: str, blocks: Sequence[Sequence[Event]]) -> tuple[Event, ...]:
start: Final = _event(
type="message_start",
message={
"id": identity(marker),
"type": "message",
"role": "assistant",
"model": MODEL,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 1},
},
)
delta: Final = _event(
type="message_delta", delta={"stop_reason": "end_turn", "stop_sequence": None}, usage={"output_tokens": 9}
)
return (start, *chain.from_iterable(blocks), delta, _event(type="message_stop"))
def standard_events(
marker: str,
*,
parts: Sequence[JsonValue] = THINKING_PARTS,
signatures: Sequence[JsonValue] = (SIGNATURE,),
) -> tuple[Event, ...]:
return message_events(marker, (thinking_events(0, parts, signatures), text_events(1, answer(marker))))
def sse_chunks(events: Sequence[Event]) -> tuple[bytes, ...]:
return tuple(sse_frame(str(event["type"]), event) for event in events)
def aws_chunks(events: Sequence[Event]) -> tuple[bytes, ...]:
return tuple(
_aws_event_frame(
"chunk",
{"bytes": base64.b64encode(json.dumps(event, separators=(",", ":")).encode()).decode()},
"sc",
"u",
)
for event in events
)
def message_body(marker: str) -> bytes:
return json.dumps(
{
"id": identity(marker),
"type": "message",
"role": "assistant",
"model": MODEL,
"content": [
{"type": "thinking", "thinking": THINKING, "signature": SIGNATURE},
{"type": "text", "text": answer(marker)},
],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 9},
}
).encode()
def streams(request: Request) -> bool:
if request.target.endswith(_STREAMING_TARGETS):
return True
return JSON_OBJECT.validate_json(request.body).get("stream") is True
def stream_reply(request: Request, events: Sequence[Event], *, abort_after: int | None = None) -> Reply:
if request.target.endswith("/invoke-with-response-stream"):
return Reply(content_type=EVENT_STREAM, chunks=aws_chunks(events), abort_after=abort_after)
return Reply(content_type="text/event-stream", chunks=sse_chunks(events), abort_after=abort_after)
def standard_peer(request: Request) -> Reply:
marker: Final = marker_of(request)
if streams(request):
return stream_reply(request, standard_events(marker))
return Reply(body=message_body(marker))
def chunks_of(text: str) -> tuple[Event, ...]:
return tuple(
JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in text.splitlines()
if line.startswith("data: {")
)
def delta_of(chunk: Mapping[str, JsonValue]) -> Event:
choices: Final = JSON_LIST.validate_python(chunk.get("choices") or [])
if not choices:
return {}
return JSON_OBJECT.validate_python(JSON_OBJECT.validate_python(choices[0]).get("delta") or {})
def deltas_of(chunks: Sequence[Mapping[str, JsonValue]]) -> tuple[Event, ...]:
return tuple(delta_of(chunk) for chunk in chunks)
def blocks_of(delta: Mapping[str, JsonValue]) -> tuple[Event, ...]:
return tuple(BLOCKS.validate_python(delta.get("thinking_blocks") or []))
def all_blocks(deltas: Sequence[Mapping[str, JsonValue]]) -> tuple[Event, ...]:
return tuple(chain.from_iterable(blocks_of(delta) for delta in deltas))
def signed_blocks(deltas: Sequence[Mapping[str, JsonValue]]) -> tuple[Event, ...]:
return tuple(block for block in all_blocks(deltas) if block.get("signature"))
def reasoning_text(deltas: Sequence[Mapping[str, JsonValue]]) -> str:
return "".join(str(delta.get("reasoning_content") or "") for delta in deltas)
def content_text(deltas: Sequence[Mapping[str, JsonValue]]) -> str:
return "".join(str(delta.get("content") or "") for delta in deltas)
def thinking_block(thinking: str, signature: JsonValue) -> Event:
return {"type": "thinking", "thinking": thinking, "signature": signature}
def signature_only(signature: JsonValue = SIGNATURE, index: int = 0) -> Event:
return {**thinking_block("", signature), "index": index}
@dataclass(frozen=True, slots=True)
class _Accumulated:
closed: tuple[Event, ...]
text: str
def _fold(state: _Accumulated, block: Mapping[str, JsonValue]) -> _Accumulated:
if block.get("type") == "redacted_thinking":
redacted: Event = {"type": "redacted_thinking", "data": block.get("data")}
return _Accumulated((*state.closed, redacted), state.text)
text: Final = state.text + str(block.get("thinking") or "")
signature: Final = block.get("signature")
if not signature:
return _Accumulated(state.closed, text)
return _Accumulated((*state.closed, thinking_block(text, signature)), "")
def accumulate(deltas: Sequence[Mapping[str, JsonValue]]) -> tuple[Event, ...]:
return reduce(_fold, all_blocks(deltas), _Accumulated((), "")).closed
def logged_thinking(response: Mapping[str, JsonValue]) -> tuple[Event, ...]:
if "choices" in response:
choice: Final = JSON_OBJECT.validate_python(JSON_LIST.validate_python(response["choices"])[0])
message: Final = JSON_OBJECT.validate_python(choice.get("message") or {})
return tuple(BLOCKS.validate_python(message.get("thinking_blocks") or []))
content: Final = BLOCKS.validate_python(response.get("content") or [])
return tuple(block for block in content if block.get("type") in ("thinking", "redacted_thinking"))