Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

16.5 KB
Raw Blame
import json
import re
import uuid
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final

import yaml
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter

# https://platform.claude.com/docs/en/build-with-claude/prompt-caching (read 2026-09-28): at most 4 blocks with cache_control
ANTHROPIC_CACHE_CONTROL_CAP: Final = 4
# https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html (read 2026-09-28): at most 4 cache checkpoints
BEDROCK_CACHE_CHECKPOINT_CAP: Final = 4

ANTHROPIC_MODEL: Final = "claude-opus-5-5"
BEDROCK_MODEL: Final = "anthropic.claude-opus-5-5"
UNMAPPED_BEDROCK_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/scripted"
PROVIDER_KEY: Final = "synthetic-provider-key"
SYSTEM: Final = "Use the provided tool results to answer the user."
ASK: Final = "Look up the weather in London, Paris, and Tokyo."
CITIES: Final = ("London", "Paris", "Tokyo")
EPHEMERAL: Final[dict[str, JsonValue]] = {"type": "ephemeral"}
POINTS: Final[list[JsonValue]] = [
    {"location": "message", "role": "system"},
    {"location": "message", "index": -1},
]
TOOL: Final[dict[str, JsonValue]] = {
    "type": "function",
    "function": {
        "name": "lookup_weather",
        "description": "Look up the weather in a city.",
        "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
    },
}
SYSTEM_LABEL: Final = f"system:{SYSTEM}"
ASK_LABEL: Final = f"user:text:{ASK}"
_MARKER: Final = re.compile(rb"marker-([0-9a-f]{32})")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])


@dataclass(frozen=True, slots=True)
class Mark:
    label: str
    ttl: str | None


def new_marker() -> str:
    return uuid.uuid4().hex


def final_text(marker: str) -> str:
    return f"Summarize the results in one word. marker-{marker}"


def final_label(marker: str) -> str:
    return f"user:text:{final_text(marker)}"


def call_id(city: str) -> str:
    return f"call_weather_{city.lower()}"


def tool_use_label(city: str) -> str:
    return f"assistant:tool_use:{call_id(city)}"


def tool_call(city: str, **fields: JsonValue) -> dict[str, JsonValue]:
    return {
        "id": call_id(city),
        "type": "function",
        "function": {"name": "lookup_weather", "arguments": json.dumps({"city": city})},
        **fields,
    }


def marked_calls(mark: JsonValue = EPHEMERAL, cities: Sequence[str] = CITIES) -> list[JsonValue]:
    return [tool_call(city, cache_control=mark) for city in cities]


def client_marked() -> list[str]:
    return [ASK_LABEL, *(tool_use_label(city) for city in CITIES)]


def ask(*, marked: bool) -> dict[str, JsonValue]:
    block: Final[dict[str, JsonValue]] = {"type": "text", "text": ASK}
    return {"role": "user", "content": [{**block, "cache_control": EPHEMERAL} if marked else block]}


def tool_results(calls: Sequence[JsonValue]) -> list[JsonValue]:
    return [
        {"role": "tool", "tool_call_id": call["id"], "content": "sunny."}
        for call in calls
        if isinstance(call, dict) and str(call.get("id", "")).startswith("call_")
    ]


def conversation(
    marker: str,
    calls: Sequence[JsonValue],
    *,
    ask_marked: bool = True,
    assistant: dict[str, JsonValue] | None = None,
) -> list[JsonValue]:
    turn: Final[dict[str, JsonValue]] = {
        "role": "assistant",
        "content": "",
        "tool_calls": list(calls),
        **(assistant or {}),
    }
    return [
        {"role": "system", "content": SYSTEM},
        ask(marked=ask_marked),
        turn,
        *tool_results(calls),
        {"role": "user", "content": final_text(marker)},
    ]


def chat_body(model: str, messages: Sequence[JsonValue], **fields: JsonValue) -> dict[str, JsonValue]:
    return {"model": model, "messages": list(messages), "tools": [TOOL], "max_tokens": 64, **fields}


def messages_body(model: str, marker: str, *, stream: bool) -> dict[str, JsonValue]:
    return {
        "model": model,
        "max_tokens": 64,
        "stream": stream,
        "system": [{"type": "text", "text": SYSTEM}],
        "tools": [
            {
                "name": "lookup_weather",
                "description": "Look up the weather in a city.",
                "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
            }
        ],
        "messages": [
            {"role": "user", "content": [{"type": "text", "text": ASK, "cache_control": EPHEMERAL}]},
            {
                "role": "assistant",
                "content": [
                    {
                        "type": "tool_use",
                        "id": call_id(city),
                        "name": "lookup_weather",
                        "input": {"city": city},
                        "cache_control": EPHEMERAL,
                    }
                    for city in CITIES
                ],
            },
            {
                "role": "user",
                "content": [
                    *({"type": "tool_result", "tool_use_id": call_id(city), "content": "sunny."} for city in CITIES),
                    {"type": "text", "text": final_text(marker)},
                ],
            },
        ],
    }


def responses_body(model: str, marker: str) -> dict[str, JsonValue]:
    items: Final[list[JsonValue]] = [
        {"role": "user", "content": [{"type": "input_text", "text": ASK, "cache_control": EPHEMERAL}]},
        *(
            {
                "type": "function_call",
                "call_id": call_id(city),
                "name": "lookup_weather",
                "arguments": json.dumps({"city": city}),
                "cache_control": EPHEMERAL,
            }
            for city in CITIES
        ),
        *({"type": "function_call_output", "call_id": call_id(city), "output": "sunny."} for city in CITIES),
        {"role": "user", "content": final_text(marker)},
    ]
    return {
        "model": model,
        "instructions": SYSTEM,
        "max_output_tokens": 64,
        "tools": [{"type": "function", "name": "lookup_weather", "parameters": {"type": "object"}}],
        "input": items,
    }


def marker_of(request: Request) -> str:
    found: Final = _MARKER.findall(request.body)
    return found[-1].decode() if found else "unmarked"


class _AnthropicBlock(BaseModel):
    model_config = ConfigDict(extra="allow", frozen=True)
    type: str
    text: str | None = None
    id: str | None = None
    tool_use_id: str | None = None
    name: str | None = None
    cache_control: JsonValue = None


class _AnthropicMessage(BaseModel):
    model_config = ConfigDict(extra="allow", frozen=True)
    role: str
    content: str | tuple[_AnthropicBlock, ...]


class _AnthropicTool(BaseModel):
    model_config = ConfigDict(extra="allow", frozen=True)
    name: str = ""
    cache_control: JsonValue = None


class _AnthropicBody(BaseModel):
    model_config = ConfigDict(extra="allow", frozen=True)
    system: str | tuple[_AnthropicBlock, ...] = ()
    messages: tuple[_AnthropicMessage, ...] = ()
    tools: tuple[_AnthropicTool, ...] = ()
    cache_control: JsonValue = None
    stream: bool = False


def _ttl(cache_control: JsonValue) -> str | None:
    if not isinstance(cache_control, dict):
        return None
    ttl: Final = cache_control.get("ttl")
    return ttl if isinstance(ttl, str) else None


def _block_label(role: str, block: _AnthropicBlock) -> str:
    detail: Final = block.text if block.type == "text" else block.id or block.tool_use_id or ""
    return f"{role}:{block.type}:{detail}"


def _message_blocks(body: _AnthropicBody) -> Iterator[tuple[str, _AnthropicBlock]]:
    for message in body.messages:
        if isinstance(message.content, tuple):
            yield from ((message.role, block) for block in message.content)


def _anthropic_body(request: Request) -> _AnthropicBody:
    return _AnthropicBody.model_validate_json(request.body)


def anthropic_marks(request: Request) -> tuple[Mark, ...]:
    body: Final = _anthropic_body(request)
    system: Final = body.system if isinstance(body.system, tuple) else ()
    return (
        *(Mark(f"tool:{tool.name}", _ttl(tool.cache_control)) for tool in body.tools if tool.cache_control is not None),
        *(
            Mark(f"system:{block.text}", _ttl(block.cache_control))
            for block in system
            if block.cache_control is not None
        ),
        *(
            Mark(_block_label(role, block), _ttl(block.cache_control))
            for role, block in _message_blocks(body)
            if block.cache_control is not None
        ),
        *((Mark("request", _ttl(body.cache_control)),) if body.cache_control is not None else ()),
    )


def anthropic_labels(request: Request) -> list[str]:
    return [mark.label for mark in anthropic_marks(request)]


def _anthropic_error(message: str) -> Reply:
    return Reply(
        status=400,
        body=json.dumps({"type": "error", "error": {"type": "invalid_request_error", "message": message}}).encode(),
    )


def _ttl_out_of_order(marks: Sequence[Mark]) -> bool:
    first_short: Final = next((index for index, mark in enumerate(marks) if mark.ttl != "1h"), len(marks))
    return any(mark.ttl == "1h" for mark in marks[first_short:])


def _usage() -> dict[str, JsonValue]:
    return {"input_tokens": 12, "output_tokens": 1, "cache_creation_input_tokens": 12, "cache_read_input_tokens": 0}


def _anthropic_message(identity: str) -> dict[str, JsonValue]:
    return {
        "id": identity,
        "type": "message",
        "role": "assistant",
        "model": ANTHROPIC_MODEL,
        "content": [{"type": "text", "text": "sunny"}],
        "stop_reason": "end_turn",
        "stop_sequence": None,
        "usage": _usage(),
    }


def _event(name: str, data: dict[str, JsonValue]) -> bytes:
    return f"event: {name}\ndata: {json.dumps(data)}\n\n".encode()


def _anthropic_stream(identity: str) -> tuple[bytes, ...]:
    return (
        _event(
            "message_start",
            {"type": "message_start", "message": {**_anthropic_message(identity), "content": [], "stop_reason": None}},
        ),
        _event(
            "content_block_start",
            {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
        ),
        _event(
            "content_block_delta",
            {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "sunny"}},
        ),
        _event("content_block_stop", {"type": "content_block_stop", "index": 0}),
        _event(
            "message_delta",
            {
                "type": "message_delta",
                "delta": {"stop_reason": "end_turn", "stop_sequence": None},
                "usage": {"output_tokens": 1},
            },
        ),
        _event("message_stop", {"type": "message_stop"}),
    )


def anthropic_peer(request: Request) -> Reply:
    marks: Final = anthropic_marks(request)
    if len(marks) > ANTHROPIC_CACHE_CONTROL_CAP:
        return _anthropic_error(
            f"A maximum of {ANTHROPIC_CACHE_CONTROL_CAP} blocks with cache_control may be provided. Found {len(marks)}."
        )
    if _ttl_out_of_order(marks):
        return _anthropic_error("a ttl='1h' cache_control block must not come after a ttl='5m' cache_control block")
    identity: Final = f"msg_{marker_of(request)}"
    if _anthropic_body(request).stream:
        return Reply(content_type="text/event-stream", chunks=_anthropic_stream(identity))
    return Reply(body=json.dumps(_anthropic_message(identity)).encode())


def _bedrock_label(role: str, block: JsonValue) -> str:
    if not isinstance(block, dict):
        return f"{role}:start"
    if isinstance(block.get("text"), str):
        return f"{role}:text:{block['text']}"
    for kind in ("toolUse", "toolResult"):
        inner = block.get(kind)
        if isinstance(inner, dict):
            return f"{role}:{kind}:{inner.get('toolUseId')}"
    spec: Final = block.get("toolSpec")
    return f"tool:{spec.get('name')}" if isinstance(spec, dict) else f"{role}:other"


def _cache_points(role: str, blocks: JsonValue) -> Iterator[str]:
    listed: Final = blocks if isinstance(blocks, list) else []
    for previous, block in zip([None, *listed], listed):
        if isinstance(block, dict) and "cachePoint" in block:
            yield _bedrock_label(role, previous)


def _bedrock_sections(body: dict[str, JsonValue]) -> Iterator[tuple[str, JsonValue]]:
    tool_config: Final = body.get("toolConfig")
    yield ("tool", tool_config.get("tools") if isinstance(tool_config, dict) else None)
    yield ("system", body.get("system"))
    messages: Final = body.get("messages")
    for message in messages if isinstance(messages, list) else []:
        if isinstance(message, dict):
            yield (str(message.get("role")), message.get("content"))


def bedrock_labels(request: Request) -> list[str]:
    body: Final = _JSON_OBJECT.validate_json(request.body)
    return [label for role, blocks in _bedrock_sections(body) for label in _cache_points(role, blocks)]


def bedrock_peer(request: Request) -> Reply:
    found: Final = len(bedrock_labels(request))
    if found > BEDROCK_CACHE_CHECKPOINT_CAP:
        return Reply(
            status=400,
            headers={"x-amzn-errortype": "ValidationException"},
            body=json.dumps(
                {
                    "message": f"A maximum of {BEDROCK_CACHE_CHECKPOINT_CAP} cache checkpoints may be provided. Found {found}."
                }
            ).encode(),
        )
    return Reply(
        body=json.dumps(
            {
                "output": {"message": {"role": "assistant", "content": [{"text": "sunny"}]}},
                "stopReason": "end_turn",
                "usage": {
                    "inputTokens": 12,
                    "outputTokens": 1,
                    "totalTokens": 13,
                    "cacheWriteInputTokens": 12,
                    "cacheReadInputTokens": 0,
                },
                "metrics": {"latencyMs": 1},
            }
        ).encode()
    )


def _without_cache_points(value: JsonValue) -> JsonValue:
    if isinstance(value, dict):
        return {key: _without_cache_points(inner) for key, inner in value.items()}
    if isinstance(value, list):
        return [
            _without_cache_points(inner) for inner in value if not (isinstance(inner, dict) and "cachePoint" in inner)
        ]
    return value


def sent_without_cache_points(requests: Sequence[Request], marker: str) -> list[JsonValue]:
    return [
        _without_cache_points(_JSON_OBJECT.validate_json(request.body.replace(marker.encode(), b"marker")))
        for request in requests
    ]


def gateway_injected(response_id: str) -> bool:
    rows: Final = eventually(
        lambda: read_rows(
            """SELECT metadata->>'litellm_gateway_injected_cache' AS injected FROM "LiteLLM_SpendLogs" """
            "WHERE request_id=%s",
            (response_id,),
        ),
        lambda values: len(values) == 1,
        seconds=70,
    )
    return rows[0]["injected"] is not None


def post_chat(gateway: Gateway, body: dict[str, JsonValue], *, key: str | None = None) -> tuple[int, str, str]:
    response: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
    identity: Final = _JSON_OBJECT.validate_json(response.content).get("id") if response.status_code == 200 else None
    return response.status_code, str(identity), response.text


def anthropic_deployment(name: str, api_base: str, **fields: JsonValue) -> dict[str, JsonValue]:
    return {
        "model_name": name,
        "litellm_params": {
            "model": f"anthropic/{ANTHROPIC_MODEL}",
            "api_base": api_base,
            "api_key": PROVIDER_KEY,
            **fields,
        },
    }


def owned_config(
    directory: Path,
    model_list: Sequence[JsonValue],
    *,
    litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}),
    router_settings: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Path:
    config: Final = _JSON_OBJECT.validate_python(
        yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
    )
    merged: Final = {
        **config,
        "model_list": list(model_list),
        "litellm_settings": {**object_value(config["litellm_settings"]), **litellm_settings},
        "router_settings": {**object_value(config["router_settings"]), "num_retries": 0, **router_settings},
    }
    path: Final = directory / f"cache-control-marks-{uuid.uuid4().hex}.yaml"
    path.write_text(yaml.safe_dump(merged))
    return path