Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

24.3 KB
Raw Blame
import json
import uuid
from collections.abc import Mapping
from pathlib import Path
from typing import Final, Literal

import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.upstream import JsonResponse, SseResponse, delete_scenario, register_scenario
from pydantic import BaseModel, JsonValue, TypeAdapter

pytestmark: Final = pytest.mark.shares_worker_state

_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])


class _AnthropicTextBlock(BaseModel):
    type: Literal["text"]
    text: str


class _AnthropicMessageUsage(BaseModel):
    input_tokens: int
    output_tokens: int


class _AnthropicMessageResponse(BaseModel):
    id: str
    type: Literal["message"]
    role: Literal["assistant"]
    model: str
    content: list[_AnthropicTextBlock]
    stop_reason: str | None
    stop_sequence: str | None
    usage: _AnthropicMessageUsage


@pytest.mark.covers("spend.end_user.charged_when_key_has_no_user_id_and_auth_cache_is_redis")
def test_end_user_spend_lands_for_key_without_user_id_when_auth_cache_is_redis(gateway: Gateway) -> None:
    with gateway.scenario() as scenario:
        model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
        key: Final = scenario.key(models=[model])
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        response: Final = gateway.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": f"end user spend {end_user}"}], "user": end_user},
            key=key,
        )
        assert response.status_code == 200, response.text
        assert response.json()["usage"]["total_tokens"] == 40, response.text
        charged: Final = eventually(
            lambda: read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (end_user,)),
            lambda values: len(values) == 1 and float(values[0]["spend"]) >= 0.06,
            seconds=70,
        )
        assert float(charged[0]["spend"]) == pytest.approx(20 * 0.001 + 20 * 0.002)


def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]:
    response: Final = upstream.get("/__observations")
    response.raise_for_status()
    requests: Final = response.json()["requests"]
    assert isinstance(requests, list)
    return requests


def _observation(path: str, body: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    return {
        "path": path,
        "authorization": "Bearer integration-provider-key",
        "body": dict(body),
        "method": "POST",
        "api_key": "",
    }


def _new_customer(gateway: Gateway, user_id: str, **fields: JsonValue) -> None:
    response: Final = gateway.request("POST", "/customer/new", {"user_id": user_id, **fields})
    assert response.status_code == 200, response.text


def _charged_spend(user_id: str) -> float:
    charged: Final = eventually(
        lambda: read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (user_id,)),
        lambda values: len(values) == 1 and float(str(values[0]["spend"])) >= 0.05,
        seconds=70,
    )
    return float(str(charged[0]["spend"]))


def _assert_daily_row(user_id: str, model: str, upstream_model: str = "openai/gpt-4o-mini") -> None:
    daily: Final = eventually(
        lambda: read_rows(
            'SELECT model, model_group, spend, api_requests FROM "LiteLLM_DailyEndUserSpend" WHERE end_user_id=%s',
            (user_id,),
        ),
        lambda rows: len(rows) == 1,
        seconds=70,
    )
    assert daily[0]["model"] == upstream_model
    assert daily[0]["model_group"] == model
    assert float(str(daily[0]["spend"])) == pytest.approx(0.06)
    assert int(str(daily[0]["api_requests"])) == 1


def _assert_budget_refusal(response: httpx.Response, end_user: str) -> None:
    assert response.status_code == 422, response.text
    error: Final = object_value(response.json()["error"])
    assert error["type"] == "budget_exceeded"
    assert error["param"] is None
    assert error["code"] == "422"
    assert str(error["message"]).startswith(f"ExceededBudget: End User={end_user} over budget. Spend="), error[
        "message"
    ]


def _json_object(raw: bytes) -> dict[str, JsonValue]:
    return _JSON_OBJECT.validate_json(raw)


_CHAT_SPELLINGS: Final = (
    pytest.param("x-litellm-end-user-id", id="header-x-litellm-end-user-id"),
    pytest.param("x-litellm-customer-id", id="header-x-litellm-customer-id"),
    pytest.param("litellm_metadata.user", id="body-litellm_metadata-user"),
    pytest.param("metadata.user_id", id="body-metadata-user_id"),
    pytest.param("litellm_metadata.user:json-string", id="body-litellm_metadata-json-string-user"),
    pytest.param("metadata.user_id:json-string", id="body-metadata-json-string-user_id"),
)


def _chat_request(
    gateway: Gateway,
    model: str,
    key: str,
    spelling: str,
    end_user: str,
    text: str,
    stream: bool = False,
) -> httpx.Response:
    headers: Final[dict[str, str]] = {}
    body: Final[dict[str, JsonValue]] = {
        "model": model,
        "messages": [{"role": "user", "content": text}],
    }
    if spelling.startswith("x-"):
        headers[spelling] = end_user
    elif spelling == "litellm_metadata.user":
        body["litellm_metadata"] = {"user": end_user}
    elif spelling == "litellm_metadata.user:json-string":
        body["litellm_metadata"] = json.dumps({"user": end_user})
    elif spelling == "metadata.user_id:json-string":
        body["metadata"] = json.dumps({"user_id": end_user})
    else:
        body["metadata"] = {"user_id": end_user}
    if stream:
        body["stream"] = True
    return gateway.request("POST", "/v1/chat/completions", body, key=key, headers=headers)


@pytest.mark.parametrize(
    "spelling",
    (
        pytest.param("x-litellm-end-user-id", id="header"),
        pytest.param("metadata.user_id", id="metadata"),
    ),
)
def test_streamed_end_user_spelling_is_charged_and_budget_enforced(gateway: Gateway, spelling: str) -> None:
    with (
        gateway.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        _new_customer(gateway, end_user, max_budget=0.05)
        scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
        key: Final = scenario.key(models=[model])

        text: Final = f"streamed end user spelling {uuid.uuid4().hex}"
        served: Final = _chat_request(gateway, model, key, spelling, end_user, text, stream=True)
        assert served.status_code == 200, served.text
        assert served.headers["content-type"].startswith("text/event-stream"), served.headers
        chunks: Final = tuple(
            _json_object(line.removeprefix("data: ").strip().encode())
            for line in served.text.splitlines()
            if line.startswith("data: ") and line.removeprefix("data: ").strip() != "[DONE]"
        )
        assert chunks
        assert object_value(chunks[-1]["choices"][0])["finish_reason"] == "stop"
        assert "data: [DONE]" in served.text
        observed: Final = _observed(upstream)
        assert observed == [
            _observation(
                "/v1/chat/completions",
                {
                    "messages": [{"role": "user", "content": text}],
                    "model": "gpt-4o-mini",
                    "stream": True,
                    "stream_options": {"include_usage": True},
                },
            )
        ]
        spend: Final = _charged_spend(end_user)
        assert spend == pytest.approx(20 * 0.001 + 20 * 0.002)
        _assert_daily_row(end_user, model)

        upstream.delete("/__observations").raise_for_status()
        denied: Final = _chat_request(
            gateway,
            model,
            key,
            spelling,
            end_user,
            f"again {uuid.uuid4().hex}",
            stream=True,
        )
        assert denied.headers["content-type"].startswith("application/json"), denied.headers
        assert "data:" not in denied.text
        _assert_budget_refusal(denied, end_user)
        assert _observed(upstream) == []


@pytest.mark.parametrize("stream", (False, True), ids=("non-streaming", "streaming"))
def test_anthropic_messages_metadata_user_id_is_charged_and_budget_enforced(gateway: Gateway, stream: bool) -> None:
    with (
        gateway.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        _new_customer(gateway, end_user, max_budget=0.05)
        scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        scenario_id: Final = f"sc-{uuid.uuid4().hex}"
        response: Final = (
            SseResponse(
                content_type="text/event-stream",
                frames=(
                    "event: message_start\ndata: "
                    '{"type":"message_start","message":{"id":"msg_$REQUEST_ID","type":"message",'
                    '"role":"assistant","model":"claude-sonnet-5-5","content":[],"stop_reason":null,'
                    '"stop_sequence":null,"usage":{"input_tokens":20,"output_tokens":0}}}',
                    "event: content_block_start\ndata: "
                    '{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
                    "event: content_block_delta\ndata: "
                    '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted answer"}}',
                    'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
                    "event: message_delta\ndata: "
                    '{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},'
                    '"usage":{"output_tokens":20}}',
                    'event: message_stop\ndata: {"type":"message_stop"}',
                ),
            )
            if stream
            else JsonResponse(
                content_type="application/json",
                body={
                    "id": "msg_$REQUEST_ID",
                    "type": "message",
                    "role": "assistant",
                    "model": "claude-sonnet-5-5",
                    "content": [{"type": "text", "text": "scripted answer"}],
                    "stop_reason": "end_turn",
                    "stop_sequence": None,
                    "usage": {"input_tokens": 20, "output_tokens": 20},
                },
            )
        )
        handle: Final = register_scenario(scenario_id, response)
        scenario.cleanups.callback(delete_scenario, handle)
        model: Final = scenario.model(
            model="anthropic/claude-sonnet-5-5",
            api_base=handle.api_base(),
            input_cost_per_token=0.001,
            output_cost_per_token=0.002,
        )
        key: Final = scenario.key(models=[model])
        text: Final = f"anthropic end user {uuid.uuid4().hex}"
        body: Final[dict[str, JsonValue]] = {
            "model": model,
            "max_tokens": 64,
            "messages": [{"role": "user", "content": text}],
            "metadata": {"user_id": end_user},
            "stream": stream,
        }
        served: Final = gateway.client.request(
            "POST",
            "/v1/messages",
            json=body,
            headers={"x-api-key": key, "anthropic-version": "2023-06-01"},
        )
        assert served.status_code == 200, served.text
        if stream:
            assert served.headers["content-type"].startswith("text/event-stream"), served.headers
            message_events: Final = tuple(
                _json_object(line.removeprefix("data: ").strip().encode())
                for line in served.text.splitlines()
                if line.startswith("data: ")
            )
            assert tuple(str(event["type"]) for event in message_events) == (
                "message_start",
                "content_block_start",
                "content_block_delta",
                "content_block_stop",
                "message_delta",
                "message_stop",
            )
        else:
            message: Final = _AnthropicMessageResponse.model_validate_json(served.content)
            assert message.type == "message"
            assert message.role == "assistant"
            assert message.content == [_AnthropicTextBlock(type="text", text="scripted answer")]
            assert message.stop_reason == "end_turn"
            assert message.stop_sequence is None
            assert message.usage == _AnthropicMessageUsage(input_tokens=20, output_tokens=20)
        observed: Final = _observed(upstream)
        assert observed == [
            {
                "path": f"/{scenario_id}/v1/messages",
                "authorization": "",
                "body": {
                    "model": "claude-sonnet-5-5",
                    "max_tokens": 64,
                    "messages": [{"role": "user", "content": text}],
                    "metadata": {"user_id": end_user},
                    "stream": stream,
                },
                "method": "POST",
                "api_key": "",
            }
        ]
        spend: Final = _charged_spend(end_user)
        assert spend == pytest.approx(20 * 0.001 + 20 * 0.002)
        _assert_daily_row(end_user, model, "anthropic/claude-sonnet-5-5")

        upstream.delete("/__observations").raise_for_status()
        denied: Final = gateway.client.request(
            "POST",
            "/v1/messages",
            json={**body, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}]},
            headers={"x-api-key": key, "anthropic-version": "2023-06-01"},
        )
        _assert_budget_refusal(denied, end_user)
        assert _observed(upstream) == []


@pytest.mark.parametrize("spelling", _CHAT_SPELLINGS)
def test_end_user_id_spelling_is_charged_and_budget_enforced(gateway: Gateway, spelling: str) -> None:
    with (
        gateway.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        _new_customer(gateway, end_user, max_budget=0.05)
        scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
        key: Final = scenario.key(models=[model])

        text: Final = f"end user spelling {uuid.uuid4().hex}"
        served: Final = _chat_request(gateway, model, key, spelling, end_user, text)
        assert served.status_code == 200, served.text
        assert _observed(upstream) == [
            _observation(
                "/v1/chat/completions",
                {"messages": [{"role": "user", "content": text}], "model": "gpt-4o-mini"},
            )
        ]
        spend: Final = _charged_spend(end_user)
        assert spend == pytest.approx(20 * 0.001 + 20 * 0.002)
        _assert_daily_row(end_user, model)

        upstream.delete("/__observations").raise_for_status()
        denied: Final = _chat_request(gateway, model, key, spelling, end_user, f"again {uuid.uuid4().hex}")
        _assert_budget_refusal(denied, end_user)
        assert _observed(upstream) == []


def test_safety_identifier_on_responses_is_charged_and_budget_enforced(gateway: Gateway) -> None:
    with (
        gateway.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        scenario_id: Final = f"sc-{uuid.uuid4().hex}"
        handle: Final = register_scenario(
            scenario_id,
            JsonResponse(
                content_type="application/json",
                body={
                    "id": f"resp_{scenario_id}",
                    "object": "response",
                    "created_at": 1759700000,
                    "status": "completed",
                    "model": "gpt-4o-mini",
                    "output": [
                        {
                            "type": "message",
                            "id": "msg_1",
                            "status": "completed",
                            "role": "assistant",
                            "content": [{"type": "output_text", "text": "scripted answer", "annotations": []}],
                        }
                    ],
                    "usage": {
                        "input_tokens": 20,
                        "output_tokens": 20,
                        "total_tokens": 40,
                        "input_tokens_details": {"cached_tokens": 0},
                        "output_tokens_details": {"reasoning_tokens": 0},
                    },
                },
            ),
        )
        scenario.cleanups.callback(
            lambda: httpx.delete(f"{handle.control_url}/__scenarios/{handle.scenario_id}", trust_env=False)
        )
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        _new_customer(gateway, end_user, max_budget=0.05)
        scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        model: Final = scenario.model(
            api_base=f"{gateway.upstream_url}/{handle.scenario_id}/v1",
            input_cost_per_token=0.001,
            output_cost_per_token=0.002,
        )
        key: Final = scenario.key(models=[model])

        input_message: Final[dict[str, JsonValue]] = {
            "role": "user",
            "content": [{"type": "input_text", "text": f"responses spelling {uuid.uuid4().hex}"}],
        }

        def call() -> httpx.Response:
            return gateway.request(
                "POST",
                "/v1/responses",
                {"model": model, "input": [input_message], "safety_identifier": end_user},
                key=key,
            )

        served: Final = call()
        assert served.status_code == 200, served.text
        assert _observed(upstream) == [
            _observation(
                f"/{handle.scenario_id}/v1/responses",
                {"model": "gpt-4o-mini", "input": [input_message], "safety_identifier": end_user},
            )
        ]
        spend: Final = _charged_spend(end_user)
        assert spend == pytest.approx(0.06)
        _assert_daily_row(end_user, model)

        upstream.delete("/__observations").raise_for_status()
        denied: Final = call()
        _assert_budget_refusal(denied, end_user)
        assert _observed(upstream) == []


def test_end_user_header_wins_over_body_user(gateway: Gateway) -> None:
    with (
        gateway.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        header_user: Final = f"integration-end-user-a-{uuid.uuid4().hex}"
        body_user: Final = f"integration-end-user-b-{uuid.uuid4().hex}"
        _new_customer(gateway, header_user, max_budget=0.05)
        _new_customer(gateway, body_user, max_budget=1.0)
        for end_user in (header_user, body_user):
            scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
        key: Final = scenario.key(models=[model])

        text: Final = f"header precedence {uuid.uuid4().hex}"
        served: Final = gateway.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": text}], "user": body_user},
            key=key,
            headers={"x-litellm-end-user-id": header_user},
        )
        assert served.status_code == 200, served.text
        assert _observed(upstream) == [
            _observation(
                "/v1/chat/completions",
                {
                    "messages": [{"role": "user", "content": text}],
                    "model": "gpt-4o-mini",
                    "user": body_user,
                },
            )
        ]
        assert _charged_spend(header_user) == pytest.approx(0.06)
        _assert_daily_row(header_user, model)
        body_rows: Final = read_rows('SELECT spend FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (body_user,))
        assert all(float(str(row["spend"])) == 0.0 for row in body_rows), body_rows
        assert (
            read_rows(
                'SELECT id FROM "LiteLLM_DailyEndUserSpend" WHERE end_user_id=%s',
                (body_user,),
            )
            == []
        )

        upstream.delete("/__observations").raise_for_status()
        denied: Final = gateway.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}], "user": body_user},
            key=key,
            headers={"x-litellm-end-user-id": header_user},
        )
        _assert_budget_refusal(denied, header_user)
        assert _observed(upstream) == []


def _config_with_general_settings(tmp_path: Path, general_settings: Mapping[str, JsonValue]) -> Path:
    config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
    config["general_settings"] = {**object_value(config["general_settings"]), **dict(general_settings)}
    path: Final = tmp_path / "config.yaml"
    path.write_text(yaml.safe_dump(config))
    return path


@pytest.mark.parametrize(
    ("general_settings", "injects_user_body_field"),
    (
        pytest.param(
            {"user_header_mappings": [{"header_name": "x-integration-end-user", "litellm_user_role": "customer"}]},
            False,
            id="config-user_header_mappings",
        ),
        pytest.param({"user_header_name": "x-integration-end-user"}, True, id="config-user_header_name"),
    ),
)
def test_configured_end_user_header_is_charged_and_budget_enforced(
    gateway: Gateway, tmp_path: Path, general_settings: Mapping[str, JsonValue], injects_user_body_field: bool
) -> None:
    config: Final = _config_with_general_settings(tmp_path, general_settings)
    with (
        owned_proxy(gateway, tmp_path, {}, config=config) as candidate,
        candidate.scenario() as scenario,
        httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
    ):
        upstream.delete("/__observations").raise_for_status()
        end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
        _new_customer(candidate, end_user, max_budget=0.05)
        scenario.cleanups.callback(candidate.request, "POST", "/customer/delete", {"user_ids": [end_user]})
        model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
        key: Final = scenario.key(models=[model])
        headers: Final = {"x-integration-end-user": end_user}

        text: Final = f"configured header {uuid.uuid4().hex}"
        served: Final = candidate.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": text}]},
            key=key,
            headers=headers,
        )
        assert served.status_code == 200, served.text
        expected_body: Final[dict[str, JsonValue]] = {
            "messages": [{"role": "user", "content": text}],
            "model": "gpt-4o-mini",
        }
        if injects_user_body_field:
            expected_body["user"] = end_user
        assert _observed(upstream) == [_observation("/v1/chat/completions", expected_body)]
        assert _charged_spend(end_user) == pytest.approx(0.06)
        _assert_daily_row(end_user, model)

        upstream.delete("/__observations").raise_for_status()
        denied: Final = candidate.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}]},
            key=key,
            headers=headers,
        )
        _assert_budget_refusal(denied, end_user)
        assert _observed(upstream) == []