Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

21.0 KB
Raw Blame
from __future__ import annotations

import base64
import json
import os
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
from functools import reduce
from types import MappingProxyType
from typing import Final, Literal

import httpx
import pytest
from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, string_value
from integration._support.database import read_rows
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire
from pydantic import JsonValue

from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with

Provider = Literal["gemini", "vertex_ai"]
Endpoint = Literal["chat", "messages", "responses"]
Answer = Callable[[bool], Reply]

BACKEND: Final = "gemini-4-argon"
API_KEY: Final = "synthetic-gemini-key"
PROJECT: Final = "scripted-project"
LOCATION: Final = "us-central1"
MODEL_PATH: Final = f"/v1/projects/{PROJECT}/locations/{LOCATION}/publishers/google/models/{BACKEND}"
CONTINUED_MODEL_PATH: Final = MODEL_PATH.replace("/v1/", "/v1beta1/", 1)
SALT: Final = os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt")
CAP: Final = 1000
INPUT_COST: Final = 0.001
OUTPUT_COST: Final = 0.002
NO_CACHE: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}})
NOTHING_EXTRA: Final[Mapping[str, JsonValue]] = MappingProxyType({})
PROMPT: Final = "Write the long answer."
CHECKPOINT: Final = "checkpoint-token-never-for-clients"
HOP_TOKEN: Final = "hop-token-litellm-sends-back"
CUT_TOKEN: Final = "cut-token-for-the-caller"
REFUSAL: Final = "The scripted upstream refused this continuation request"
PROVIDERS: Final[tuple[Provider, ...]] = ("gemini", "vertex_ai")
ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses")
SPEND_COLUMNS: Final = (
    'SELECT request_id, call_type, status, cache_hit, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs"'
)


@dataclass(frozen=True, slots=True)
class Hop:
    pieces: tuple[str, str]
    finish: str
    token: str | None
    prompt_tokens: int
    output_tokens: int

    @property
    def text(self) -> str:
        return "".join(self.pieces)

    @property
    def usage(self) -> tuple[int, int]:
        return self.prompt_tokens, self.output_tokens


FIRST_HALF: Final = Hop(("The first ", "half, "), "CONTINUATION", HOP_TOKEN, 20, 7)
SECOND_HALF: Final = Hop(("then the ", "second half."), "STOP", None, 27, 5)
CUT: Final = Hop(("Cut at the ", "caller's cap"), "MAX_TOKENS", CUT_TOKEN, 20, 7)
PLAIN: Final = Hop(("A short ", "answer."), "STOP", None, 20, 7)
RESUMED: Final = Hop(("and the ", "rest of it."), "STOP", None, 27, 5)


def summed(*hops: Hop) -> tuple[int, int]:
    return sum(hop.prompt_tokens for hop in hops), sum(hop.output_tokens for hop in hops)


def frame(
    response_id: str, text: str, usage: tuple[int, int], finish: str | None = None, token: str | None = None
) -> dict[str, JsonValue]:
    prompt_tokens, output_tokens = usage
    return {
        "responseId": response_id,
        "candidates": [
            {
                "content": {"role": "model", "parts": [{"text": text}]},
                "index": 0,
                **({"finishReason": finish} if finish else {}),
                **({"continuationToken": token} if token else {}),
            }
        ],
        "usageMetadata": {
            "promptTokenCount": prompt_tokens,
            "candidatesTokenCount": output_tokens,
            "totalTokenCount": prompt_tokens + output_tokens,
        },
        "modelVersion": BACKEND,
    }


def stream_frames(response_id: str, hop: Hop) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]:
    opening, closing = hop.pieces
    return (
        frame(response_id, opening, (hop.prompt_tokens, 1), token=CHECKPOINT),
        frame(response_id, closing, hop.usage, hop.finish, hop.token),
    )


def sse(frames: Sequence[Mapping[str, JsonValue]], abort_after: int | None = None) -> Reply:
    return Reply(
        content_type="text/event-stream",
        chunks=tuple(f"data: {json.dumps(item)}\n\n".encode() for item in frames),
        abort_after=abort_after,
    )


def hop_reply(response_id: str, hop: Hop, stream: bool, abort_after: int | None = None) -> Reply:
    if stream:
        return sse(stream_frames(response_id, hop), abort_after)
    return Reply(body=json.dumps(frame(response_id, hop.text, hop.usage, hop.finish, hop.token)).encode())


def answers(response_id: str, hop: Hop) -> Answer:
    return lambda stream: hop_reply(response_id, hop, stream)


def refuses(status: int) -> Answer:
    error: Final = {"error": {"code": status, "message": REFUSAL, "status": "INVALID_ARGUMENT"}}
    return lambda _stream: Reply(status=status, body=json.dumps(error).encode())


def drops_before_answering(_stream: bool) -> Reply:
    return Reply(drop_connection=True)


def drops_mid_body(response_id: str, hop: Hop) -> Answer:
    return lambda stream: replace(hop_reply(response_id, hop, stream, abort_after=1), pause_between_chunks=0.5)


def streams(request: Request) -> bool:
    return ":streamGenerateContent" in request.target


def upstream(first: Answer, continued: Answer | None = None) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        if "continuationToken" not in JSON_OBJECT.validate_json(request.body):
            return first(streams(request))
        assert continued is not None, f"Unscripted continuation request: {request.target}"
        return continued(streams(request))

    return respond


def register_vertex(
    scenario: Scenario,
    wire: Wire,
    token_url: str,
    *,
    extra_headers: JsonValue = None,
    backend: str = BACKEND,
    settings: Mapping[str, JsonValue] = NOTHING_EXTRA,
) -> str:
    return scenario.model(
        model=f"vertex_ai/{backend}",
        api_base=f"{wire.url}{MODEL_PATH.removesuffix(BACKEND)}{backend}",
        api_key=None,
        vertex_project=PROJECT,
        vertex_location=LOCATION,
        vertex_credentials=service_account_json(PROJECT, token_url),
        input_cost_per_token=INPUT_COST,
        output_cost_per_token=OUTPUT_COST,
        extra_headers=extra_headers,
        model_info=None,
        **settings,
    )


def register(
    gateway: Gateway,
    scenario: Scenario,
    provider: Provider,
    wire: Wire,
    *,
    backend: str = BACKEND,
    settings: Mapping[str, JsonValue] = NOTHING_EXTRA,
) -> str:
    if provider == "gemini":
        return scenario.model(
            model=f"gemini/{backend}",
            api_base=wire.url,
            api_key=API_KEY,
            input_cost_per_token=INPUT_COST,
            output_cost_per_token=OUTPUT_COST,
            model_info=None,
            **settings,
        )
    return register_vertex(scenario, wire, gateway.upstream_url.rstrip("/"), backend=backend, settings=settings)


def target(provider: Provider, stream: bool, continued: bool = False) -> str:
    vertex_path: Final = CONTINUED_MODEL_PATH if continued else MODEL_PATH
    prefix: Final = f"/models/{BACKEND}" if provider == "gemini" else vertex_path
    return f"{prefix}:streamGenerateContent?alt=sse" if stream else f"{prefix}:generateContent"


def expected_auth(provider: Provider) -> tuple[str, str]:
    return ("x-goog-api-key", API_KEY) if provider == "gemini" else ("authorization", "Bearer scripted-token")


def received(
    wire: Wire, provider: Provider, stream: bool, continued: tuple[bool, ...]
) -> tuple[dict[str, JsonValue], ...]:
    requests: Final = wire.drain()
    assert [(request.method, request.target) for request in requests] == [
        ("POST", target(provider, stream, resumed)) for resumed in continued
    ], requests
    header, value = expected_auth(provider)
    assert [request.headers[header] for request in requests] == [value] * len(requests), requests
    return tuple(JSON_OBJECT.validate_json(request.body) for request in requests)


def path_of(endpoint: Endpoint) -> str:
    match endpoint:
        case "chat":
            return "/v1/chat/completions"
        case "messages":
            return "/v1/messages"
        case "responses":
            return "/v1/responses"


def request_body(
    endpoint: Endpoint,
    model: str,
    stream: bool,
    extra: Mapping[str, JsonValue] = NO_CACHE,
    *,
    prompt: str = PROMPT,
    cap: int | None = CAP,
) -> dict[str, JsonValue]:
    match endpoint:
        case "chat":
            return {
                "model": model,
                "messages": [{"role": "user", "content": prompt}],
                "stream": stream,
                **({"max_tokens": cap} if cap is not None else {}),
                **({"stream_options": {"include_usage": True}} if stream else {}),
                **extra,
            }
        case "messages":
            return {
                "model": model,
                "messages": [{"role": "user", "content": prompt}],
                "stream": stream,
                **({"max_tokens": cap} if cap is not None else {}),
                **extra,
            }
        case "responses":
            return {
                "model": model,
                "input": prompt,
                "stream": stream,
                **({"max_output_tokens": cap} if cap is not None else {}),
                **extra,
            }


def post(
    gateway: Gateway, endpoint: Endpoint, body: Mapping[str, JsonValue], *, key: str | None = None
) -> httpx.Response:
    return gateway.request("POST", path_of(endpoint), body, key=key, headers={"anthropic-version": "2023-06-01"})


def _step(value: JsonValue, key: str | int) -> JsonValue:
    if isinstance(key, str):
        return value.get(key) if isinstance(value, dict) else None
    return value[key] if isinstance(value, list) and len(value) > key else None


def at(value: JsonValue, *keys: str | int) -> JsonValue:
    return reduce(_step, keys, value)


def text_of(value: JsonValue) -> str:
    return value if isinstance(value, str) else ""


def items_of(value: JsonValue) -> tuple[JsonValue, ...]:
    return tuple(value) if isinstance(value, list) else ()


def count_of(value: JsonValue) -> int | None:
    return value if isinstance(value, int) and not isinstance(value, bool) else None


def sse_payloads(text: str) -> tuple[dict[str, JsonValue], ...]:
    lines: Final = tuple(line for line in text.splitlines() if line.startswith("data: ") and line != "data: [DONE]")
    return tuple(JSON_OBJECT.validate_json(line.removeprefix("data: ").encode()) for line in lines)


def payloads(text: str, stream: bool) -> tuple[dict[str, JsonValue], ...]:
    return sse_payloads(text) if stream else (JSON_OBJECT.validate_json(text.encode()),)


@dataclass(frozen=True, slots=True)
class Seen:
    ids: frozenset[str]
    answer: str
    streamed: str
    finishes: tuple[str, ...]
    usage: tuple[int | None, int | None] | None
    initial_prompt_tokens: JsonValue
    tokens: tuple[str, ...]
    token_carriers: tuple[int, ...]
    content_carriers: tuple[int, ...]
    incomplete_reason: JsonValue
    errors: tuple[dict[str, JsonValue], ...]
    raw: str


def _named(values: Sequence[JsonValue]) -> tuple[str, ...]:
    return tuple(value for value in values if isinstance(value, str))


def _errors(frames: Sequence[dict[str, JsonValue]]) -> tuple[dict[str, JsonValue], ...]:
    return tuple(
        item
        for item in frames
        if item.get("error") is not None or item.get("type") in ("error", "response.failed", "response.error")
    )


def _chat_unary(payload: dict[str, JsonValue], raw: str) -> Seen:
    choices: Final = items_of(payload.get("choices"))
    answer: Final = text_of(at(payload, "choices", 0, "message", "content"))
    return Seen(
        ids=frozenset(_named((payload.get("id"),))),
        answer=answer,
        streamed=answer,
        finishes=_named(tuple(at(choice, "finish_reason") for choice in choices)),
        usage=(count_of(at(payload, "usage", "prompt_tokens")), count_of(at(payload, "usage", "completion_tokens"))),
        initial_prompt_tokens=at(payload, "usage", "prompt_tokens_details", "initial_prompt_tokens"),
        tokens=_named(
            tuple(at(choice, "message", "provider_specific_fields", "continuation_token") for choice in choices)
        ),
        token_carriers=(),
        content_carriers=(),
        incomplete_reason=None,
        errors=_errors((payload,)),
        raw=raw,
    )


def _chunk_token(chunk: dict[str, JsonValue]) -> JsonValue:
    return at(chunk, "choices", 0, "delta", "provider_specific_fields", "continuation_token")


def _chat_stream(chunks: tuple[dict[str, JsonValue], ...], raw: str) -> Seen:
    contents: Final = tuple(text_of(at(chunk, "choices", 0, "delta", "content")) for chunk in chunks)
    metered: Final = tuple(chunk for chunk in chunks if isinstance(chunk.get("usage"), dict))
    answer: Final = "".join(contents)
    return Seen(
        ids=frozenset(_named(tuple(chunk.get("id") for chunk in chunks))),
        answer=answer,
        streamed=answer,
        finishes=_named(tuple(at(chunk, "choices", 0, "finish_reason") for chunk in chunks)),
        usage=(
            (
                count_of(at(metered[-1], "usage", "prompt_tokens")),
                count_of(at(metered[-1], "usage", "completion_tokens")),
            )
            if metered
            else None
        ),
        initial_prompt_tokens=(
            at(metered[-1], "usage", "prompt_tokens_details", "initial_prompt_tokens") if metered else None
        ),
        tokens=_named(tuple(_chunk_token(chunk) for chunk in chunks)),
        token_carriers=tuple(index for index, chunk in enumerate(chunks) if _chunk_token(chunk) is not None),
        content_carriers=tuple(index for index, content in enumerate(contents) if content),
        incomplete_reason=None,
        errors=_errors(chunks),
        raw=raw,
    )


def _messages_unary(payload: dict[str, JsonValue], raw: str) -> Seen:
    blocks: Final = items_of(payload.get("content"))
    answer: Final = "".join(text_of(at(block, "text")) for block in blocks if at(block, "type") == "text")
    return Seen(
        ids=frozenset(_named((payload.get("id"),))),
        answer=answer,
        streamed=answer,
        finishes=_named((payload.get("stop_reason"),)),
        usage=(count_of(at(payload, "usage", "input_tokens")), count_of(at(payload, "usage", "output_tokens"))),
        initial_prompt_tokens=None,
        tokens=(),
        token_carriers=(),
        content_carriers=(),
        incomplete_reason=None,
        errors=_errors((payload,)),
        raw=raw,
    )


def _messages_stream(events: tuple[dict[str, JsonValue], ...], raw: str) -> Seen:
    closings: Final = tuple(event for event in events if event.get("type") == "message_delta")
    answer: Final = "".join(
        text_of(at(event, "delta", "text")) for event in events if at(event, "delta", "type") == "text_delta"
    )
    return Seen(
        ids=frozenset(
            _named(tuple(at(event, "message", "id") for event in events if event.get("type") == "message_start"))
        ),
        answer=answer,
        streamed=answer,
        finishes=_named(tuple(at(event, "delta", "stop_reason") for event in closings)),
        usage=(
            (
                count_of(at(closings[-1], "usage", "input_tokens")),
                count_of(at(closings[-1], "usage", "output_tokens")),
            )
            if closings
            else None
        ),
        initial_prompt_tokens=None,
        tokens=(),
        token_carriers=(),
        content_carriers=(),
        incomplete_reason=None,
        errors=_errors(events),
        raw=raw,
    )


def _part_texts(item: JsonValue) -> tuple[str, ...]:
    return tuple(
        text_of(at(part, "text")) for part in items_of(at(item, "content")) if at(part, "type") == "output_text"
    )


def output_text(response: JsonValue) -> str:
    messages: Final = tuple(item for item in items_of(at(response, "output")) if at(item, "type") == "message")
    return "".join("".join(_part_texts(item)) for item in messages)


def _responses(response: JsonValue, streamed: str | None, errors: tuple[dict[str, JsonValue], ...], raw: str) -> Seen:
    answer: Final = output_text(response)
    return Seen(
        ids=frozenset(_named((at(response, "id"),))),
        answer=answer,
        streamed=answer if streamed is None else streamed,
        finishes=_named((at(response, "status"),)),
        usage=(
            (count_of(at(response, "usage", "input_tokens")), count_of(at(response, "usage", "output_tokens")))
            if response is not None
            else None
        ),
        initial_prompt_tokens=None,
        tokens=_named((at(response, "provider_specific_fields", "continuation_token"),)),
        token_carriers=(),
        content_carriers=(),
        incomplete_reason=at(response, "incomplete_details", "reason"),
        errors=errors,
        raw=raw,
    )


def _responses_stream(events: tuple[dict[str, JsonValue], ...], raw: str) -> Seen:
    settled: Final = tuple(
        event.get("response") for event in events if event.get("type") in ("response.completed", "response.incomplete")
    )
    deltas: Final = "".join(
        text_of(event.get("delta")) for event in events if event.get("type") == "response.output_text.delta"
    )
    return _responses(settled[-1] if settled else None, deltas, _errors(events), raw)


def seen(endpoint: Endpoint, stream: bool, frames: tuple[dict[str, JsonValue], ...], raw: str) -> Seen:
    match (endpoint, stream):
        case ("chat", False):
            return _chat_unary(frames[0], raw)
        case ("chat", True):
            return _chat_stream(frames, raw)
        case ("messages", False):
            return _messages_unary(frames[0], raw)
        case ("messages", True):
            return _messages_stream(frames, raw)
        case ("responses", False):
            return _responses(frames[0], None, _errors(frames), raw)
        case _:
            return _responses_stream(frames, raw)


def answered(endpoint: Endpoint, stream: bool, response: httpx.Response) -> Seen:
    assert response.status_code == 200, response.text
    return seen(endpoint, stream, payloads(response.text, stream), response.text)


def ends_cleanly(endpoint: Endpoint, raw: str) -> bool:
    if endpoint == "chat":
        return raw.rstrip().endswith("data: [DONE]")
    closing: Final = sse_payloads(raw)[-1].get("type")
    if endpoint == "messages":
        return closing == "message_stop"
    return closing in ("response.completed", "response.incomplete")


def complete(endpoint: Endpoint) -> str:
    match endpoint:
        case "chat":
            return "stop"
        case "messages":
            return "end_turn"
        case "responses":
            return "completed"


def cut_off(endpoint: Endpoint) -> str:
    match endpoint:
        case "chat":
            return "length"
        case "messages":
            return "max_tokens"
        case "responses":
            return "incomplete"


def carries_token(endpoint: Endpoint, stream: bool) -> bool:
    return endpoint == "chat" or (endpoint == "responses" and stream)


def single(values: frozenset[str]) -> str:
    assert len(values) == 1, values
    return next(iter(values))


def upstream_response_id(endpoint: Endpoint, caller_id: str) -> str:
    if endpoint != "responses":
        return caller_id
    decrypted: Final = decrypt_if_encrypted_with(caller_id.removeprefix("resp_"), SALT)
    assert decrypted is not None, caller_id
    issued: Final = decrypted.split(";")[0].split("response_id:")[-1]
    return base64.b64decode(issued.removeprefix("resp_")).decode().split(";")[-1].removeprefix("response_id:")


def assert_one_id(endpoint: Endpoint, stream: bool, observed: Seen, response_id: str) -> None:
    caller_id: Final = single(observed.ids)
    if endpoint == "messages" and stream:
        assert caller_id.startswith("msg_"), caller_id
        return
    assert upstream_response_id(endpoint, caller_id) == response_id, (caller_id, response_id)


def call_type(endpoint: Endpoint) -> str:
    match endpoint:
        case "chat":
            return "acompletion"
        case "messages":
            return "anthropic_messages"
        case "responses":
            return "aresponses"


def number(value: JsonValue) -> float:
    assert isinstance(value, (int, float)) and not isinstance(value, bool), value
    return float(value)


def cost_of(usage: tuple[int, int]) -> float:
    prompt_tokens, completion_tokens = usage
    return prompt_tokens * INPUT_COST + completion_tokens * OUTPUT_COST


def spend_rows(model: str, expected: int = 1) -> Sequence[dict[str, JsonValue]]:
    rows: Final = eventually(
        lambda: read_rows(f"{SPEND_COLUMNS} WHERE model_group=%s ORDER BY request_id", (model,)),
        lambda found: len(found) >= expected,
        seconds=70,
    )
    assert len(rows) == expected, rows
    return rows


def assert_billed(model: str, endpoint: Endpoint, usage: tuple[int, int], identities: frozenset[str]) -> None:
    (row,) = spend_rows(model)
    assert (row["call_type"], row["status"], row["prompt_tokens"], row["completion_tokens"]) == (
        call_type(endpoint),
        "success",
        *usage,
    ), row
    assert number(row["spend"]) == pytest.approx(cost_of(usage)), row
    assert string_value(row["request_id"]) in identities, (row, identities)