Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

21.5 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
import threading
import uuid
from collections import Counter
from collections.abc import Callable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from typing import Final

import anthropic
import httpx
import openai
import pytest
import yaml
from pydantic import JsonValue

from tests.integration._support.client import Gateway, eventually, gateway_from_environment
from tests.integration._support.process import owned_proxy
from tests.integration._support.prometheus_series import Sample, scrape
from tests.integration._support.wire import Reply, Request, Wire, wire_server

_FAILURES: Final = "litellm_deployment_failure_responses_total"
_TOTALS: Final = "litellm_deployment_total_requests_total"
_SERVED: Final = "limited"
_UNREACHABLE: Final = "unreachable"
_STALLED: Final = "stalled"
_NEVER_RELEASED: Final = threading.Event()
_IN_HOUSE_HANDLER: Final = """
from litellm import CustomLLM


class InHouse(CustomLLM):
    pass


handler = InHouse()
"""


@dataclass(frozen=True, slots=True)
class _Deployment:
    group: str
    model: str
    provider: str
    extra: Mapping[str, str | int]
    model_id: str = ""


@dataclass(frozen=True, slots=True)
class _Rig:
    gateway: Gateway
    upstream: Wire


def _deployment(group: str, model: str, provider: str, **extra: str | int) -> _Deployment:
    return _Deployment(group, model, provider, extra, f"{group}-{uuid.uuid4().hex[:8]}")


_CHAT: Final = _deployment("deployment-failure-chat", "openai/refused-chat", "openai")
_EMBEDDING: Final = _deployment("deployment-failure-embedding", "openai/refused-embedding", "openai")
_RESPONSES: Final = _deployment("deployment-failure-responses", "openai/refused-responses", "openai")
_MESSAGES: Final = _deployment("deployment-failure-messages", "anthropic/claude-haiku-4-5", "anthropic")
_UNSUPPORTED: Final = _deployment("deployment-failure-unsupported", "anthropic/claude-haiku-4-5", "anthropic")
_LIMITED: Final = _deployment("deployment-failure-limited", f"openai/{_SERVED}-openai", "openai", rpm=1)
_LIMITED_AZURE: Final = _deployment("gpt-4o-mini", f"azure/{_SERVED}-azure", "azure", rpm=1, api_version="2024-10-21")
_STREAMED_CHAT: Final = _deployment("deployment-failure-streamed-chat", "openai/refused-streamed-chat", "openai")
_STREAMED_RESPONSES: Final = _deployment(
    "deployment-failure-streamed-responses", "openai/refused-streamed-responses", "openai"
)
_STREAMED_MESSAGES: Final = _deployment(
    "deployment-failure-streamed-messages", "anthropic/claude-haiku-4-5", "anthropic"
)
_UNREACHABLE_EMBEDDING: Final = _deployment(
    "deployment-failure-unreachable", f"openai/{_UNREACHABLE}-embedding", "openai"
)
_STALLED_EMBEDDING: Final = _deployment(
    "deployment-failure-stalled", f"openai/{_STALLED}-embedding", "openai", timeout=1
)
_JSON_PROVIDER: Final = _deployment("deployment-failure-json-provider", "abliteration/blocked", "abliteration", rpm=0)
_CUSTOM_PROVIDER: Final = _deployment("deployment-failure-custom-provider", "in-house/blocked", "in-house", rpm=0)
_PROMPT_MANAGEMENT: Final = _deployment(
    "deployment-failure-prompt-management", "litellm_agent/anthropic/claude-haiku-4-5", "anthropic"
)
_UNPREFIXED: Final = _deployment("deployment-failure-unprefixed", "gpt-4o-mini", "", rpm=0)
_FORGED_EMBEDDING: Final = _deployment("deployment-failure-forged-embedding", "openai/refused-forged", "openai")
_FORGED_BLOCKED: Final = _deployment("deployment-failure-forged-blocked", "openai/blocked-forged", "openai", rpm=0)
_UNAUTHENTICATED: Final = _deployment("deployment-failure-unauthenticated", "openai/refused-unauthenticated", "openai")
_SDK: Final = {
    client: _deployment(f"deployment-failure-{client}", model, provider)
    for client, model, provider in (
        ("openai-sync", "openai/refused-sdk-sync", "openai"),
        ("openai-async", "openai/refused-sdk-async", "openai"),
        ("anthropic-sync", "anthropic/claude-haiku-4-5", "anthropic"),
        ("anthropic-async", "anthropic/claude-haiku-4-5", "anthropic"),
    )
}
_BURST_EMBEDDING: Final = _deployment("burst-embedding", "openai/refused-burst-embedding", "openai")
_BURST_RESPONSES: Final = _deployment("burst-responses", "openai/refused-burst-responses", "openai")
_BURST_MESSAGES: Final = _deployment("burst-messages", "anthropic/claude-haiku-4-5", "anthropic")
_BURST_UNSUPPORTED: Final = _deployment("burst-unsupported", "anthropic/claude-haiku-4-5", "anthropic")
_BURST_BLOCKED: Final = _deployment("burst-blocked", "gemini/blocked-burst", "gemini", rpm=0)
_BURST: Final = (_BURST_EMBEDDING, _BURST_RESPONSES, _BURST_MESSAGES, _BURST_UNSUPPORTED, _BURST_BLOCKED)
_DEPLOYMENTS: Final = (
    _CHAT,
    _EMBEDDING,
    _RESPONSES,
    _MESSAGES,
    _UNSUPPORTED,
    _LIMITED,
    _LIMITED_AZURE,
    _STREAMED_CHAT,
    _STREAMED_RESPONSES,
    _STREAMED_MESSAGES,
    _UNREACHABLE_EMBEDDING,
    _STALLED_EMBEDDING,
    _JSON_PROVIDER,
    _CUSTOM_PROVIDER,
    _PROMPT_MANAGEMENT,
    _UNPREFIXED,
    _FORGED_EMBEDDING,
    _FORGED_BLOCKED,
    _UNAUTHENTICATED,
    *_SDK.values(),
    *_BURST,
)


def _respond(request: Request) -> Reply:
    model: Final = json.loads(request.body).get("model", "") if request.body else ""
    if model.startswith(_UNREACHABLE):
        return Reply(drop_connection=True)
    if model.startswith(_STALLED):
        return Reply(chunks=(b"{}",), gate_before_first=_NEVER_RELEASED)
    if _SERVED in request.target or model.startswith(_SERVED):
        return Reply(
            body=json.dumps(
                {
                    "id": f"chatcmpl-{uuid.uuid4().hex}",
                    "object": "chat.completion",
                    "created": 1,
                    "model": "gpt-4o-mini",
                    "choices": [
                        {"index": 0, "message": {"role": "assistant", "content": "served"}, "finish_reason": "stop"}
                    ],
                    "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
                }
            ).encode()
        )
    if request.target.endswith("/v1/messages"):
        return Reply(
            status=400,
            body=json.dumps(
                {"type": "error", "error": {"type": "invalid_request_error", "message": "refused upstream"}}
            ).encode(),
        )
    return Reply(
        status=400,
        body=json.dumps(
            {"error": {"message": "refused upstream", "type": "invalid_request_error", "code": "refused"}}
        ).encode(),
    )


def _config(directory: Path, upstream: Wire) -> Path:
    configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
    configuration["model_list"] = [
        *configuration["model_list"],
        *(
            {
                "model_name": deployment.group,
                "litellm_params": {
                    "model": deployment.model,
                    "api_key": "sk-fixture",
                    "api_base": upstream.url if deployment.provider != "openai" else f"{upstream.url}/v1",
                    **deployment.extra,
                },
                "model_info": {"id": deployment.model_id},
            }
            for deployment in _DEPLOYMENTS
        ),
    ]
    (directory / "in_house_handler.py").write_text(_IN_HOUSE_HANDLER)
    configuration["litellm_settings"]["callbacks"] = ["prometheus"]
    configuration["litellm_settings"]["custom_provider_map"] = [
        {"provider": "in-house", "custom_handler": "in_house_handler.handler"}
    ]
    router_settings: Final = configuration.setdefault("router_settings", {})
    router_settings["num_retries"] = 0
    router_settings["optional_pre_call_checks"] = ["enforce_model_rate_limits"]
    path: Final = directory / "prometheus-deployment-failure-api-provider.yaml"
    path.write_text(yaml.safe_dump(configuration))
    return path


@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
    directory: Final = tmp_path_factory.mktemp("prometheus-deployment-failure-api-provider")
    samples: Final = directory / "prometheus-multiproc"
    samples.mkdir()
    with (
        wire_server(_respond) as upstream,
        gateway_from_environment() as shared,
        owned_proxy(
            shared,
            directory,
            {"PROMETHEUS_MULTIPROC_DIR": str(samples)},
            config=_config(directory, upstream),
            workers=2,
        ) as owned,
    ):
        yield _Rig(owned, upstream)


def _series(samples: Sequence[Sample], name: str, model_id: str) -> tuple[Sample, ...]:
    return tuple(sample for sample in samples if sample.name == name and sample.labels.get("model_id") == model_id)


def _providers(rig: _Rig, name: str, model_id: str) -> frozenset[str]:
    found: Final = eventually(lambda: _series(scrape(rig.gateway), name, model_id), bool, seconds=30)
    return frozenset(sample.labels["api_provider"] for sample in found)


def _assert_labeled(rig: _Rig, deployment: _Deployment) -> None:
    assert _providers(rig, _FAILURES, deployment.model_id) == {deployment.provider}
    assert _providers(rig, _TOTALS, deployment.model_id) == {deployment.provider}


def _counted(samples: Sequence[Sample], name: str, model_id: str) -> Mapping[str, float]:
    counts: Final[Counter[str]] = Counter()
    for sample in _series(samples, name, model_id):
        counts[sample.labels["api_provider"]] += sample.value
    return counts


def _counts(rig: _Rig, name: str, deployment: _Deployment, expected: int) -> Mapping[str, float]:
    return eventually(
        lambda: _counted(scrape(rig.gateway), name, deployment.model_id),
        lambda counts: sum(counts.values()) >= expected,
        seconds=60,
    )


def _post(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> httpx.Response:
    return rig.gateway.request("POST", path, body)


def _prompt() -> str:
    return f"deployment failure {uuid.uuid4().hex}"


def _chat(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]:
    return {"model": deployment.group, "messages": [{"role": "user", "content": _prompt()}], **extra}


def _messages(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]:
    return {"model": deployment.group, "max_tokens": 16, "messages": [{"role": "user", "content": _prompt()}], **extra}


def test_a_refused_chat_completion_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None:
    response: Final = _post(rig, "/v1/chat/completions", _chat(_CHAT))
    assert response.status_code == 400, response.text
    _assert_labeled(rig, _CHAT)


def test_a_refused_embedding_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None:
    response: Final = _post(rig, "/v1/embeddings", {"model": _EMBEDDING.group, "input": _prompt()})
    assert response.status_code == 400, response.text
    assert any(request.target.endswith("/v1/embeddings") for request in rig.upstream.drain())
    _assert_labeled(rig, _EMBEDDING)


def test_a_refused_responses_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None:
    response: Final = _post(rig, "/v1/responses", {"model": _RESPONSES.group, "input": _prompt()})
    assert response.status_code == 400, response.text
    assert any(request.target.endswith("/v1/responses") for request in rig.upstream.drain())
    _assert_labeled(rig, _RESPONSES)


def test_a_refused_anthropic_messages_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None:
    response: Final = _post(rig, "/v1/messages", _messages(_MESSAGES))
    assert response.status_code == 400, response.text
    assert any(request.target.endswith("/v1/messages") for request in rig.upstream.drain())
    _assert_labeled(rig, _MESSAGES)


def test_a_param_the_deployment_does_not_support_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None:
    rig.upstream.drain()
    response: Final = _post(rig, "/v1/chat/completions", _chat(_UNSUPPORTED, logit_bias={"50256": -100}))
    assert response.status_code == 400, response.text
    assert "logit_bias" in response.text
    assert rig.upstream.drain() == ()
    _assert_labeled(rig, _UNSUPPORTED)


def _statuses(rig: _Rig, deployment: _Deployment) -> Iterator[int]:
    for _ in range(8):
        yield _post(rig, "/v1/chat/completions", _chat(deployment)).status_code


def _until_rate_limited(rig: _Rig, deployment: _Deployment) -> None:
    first_refusal: Final = next((status for status in _statuses(rig, deployment) if status != 200), None)
    assert first_refusal == 429, f"expected the rpm limit to refuse a request, first non-200 was {first_refusal}"


@pytest.mark.parametrize(
    "deployment", [_LIMITED, _LIMITED_AZURE], ids=["alias-group", "group-named-like-another-provider"]
)
def test_a_deployment_over_its_rpm_limit_labels_the_failure_with_the_deployment_provider(
    rig: _Rig, deployment: _Deployment
) -> None:
    _until_rate_limited(rig, deployment)
    _assert_labeled(rig, deployment)


@pytest.mark.parametrize(
    ("path", "body"),
    [
        ("/v1/chat/completions", _chat(_STREAMED_CHAT, stream=True)),
        ("/v1/responses", {"model": _STREAMED_RESPONSES.group, "input": _prompt(), "stream": True}),
        ("/v1/messages", _messages(_STREAMED_MESSAGES, stream=True)),
    ],
    ids=["chat", "responses", "messages"],
)
def test_a_refused_streaming_call_labels_the_deployment_failure_with_its_provider(
    rig: _Rig, path: str, body: Mapping[str, JsonValue]
) -> None:
    deployment: Final = next(deployment for deployment in _DEPLOYMENTS if deployment.group == body["model"])
    response: Final = _post(rig, path, body)
    assert response.status_code == 400, response.text
    _assert_labeled(rig, deployment)


@pytest.mark.parametrize(
    ("deployment", "status"),
    [(_UNREACHABLE_EMBEDDING, 500), (_STALLED_EMBEDDING, 408)],
    ids=["connection-dropped", "read-timeout"],
)
def test_an_upstream_that_never_answers_labels_the_deployment_failure_with_its_provider(
    rig: _Rig, deployment: _Deployment, status: int
) -> None:
    response: Final = _post(rig, "/v1/embeddings", {"model": deployment.group, "input": _prompt()})
    assert response.status_code == status, response.text
    _assert_labeled(rig, deployment)


@pytest.mark.parametrize(
    "deployment",
    [_JSON_PROVIDER, _CUSTOM_PROVIDER, _UNPREFIXED],
    ids=["json-configured-provider", "custom-provider-map", "unprefixed-model"],
)
def test_a_call_blocked_before_it_leaves_labels_the_failure_by_the_deployment_model_prefix(
    rig: _Rig, deployment: _Deployment
) -> None:
    rig.upstream.drain()
    response: Final = _post(rig, "/v1/chat/completions", _chat(deployment))
    assert response.status_code == 429, response.text
    assert rig.upstream.drain() == ()
    _assert_labeled(rig, deployment)


def test_a_prompt_management_deployment_labels_the_failure_with_the_provider_it_resolved_to(rig: _Rig) -> None:
    response: Final = _post(rig, "/v1/chat/completions", _chat(_PROMPT_MANAGEMENT, logit_bias={"50256": -100}))
    assert response.status_code == 400, response.text
    found: Final = eventually(
        lambda: tuple(
            sample
            for sample in scrape(rig.gateway)
            if sample.name == _FAILURES and sample.labels.get("model_group") == _PROMPT_MANAGEMENT.group
        ),
        bool,
        seconds=30,
    )
    assert {sample.labels["api_provider"] for sample in found} == {_PROMPT_MANAGEMENT.provider}


@pytest.mark.parametrize(
    "forged",
    ["bedrock/forged", 5, ["bedrock/forged"], "", f"bedrock/{'x' * 5000}"],
    ids=["provider-string", "int", "list", "empty", "five-kb"],
)
@pytest.mark.parametrize(
    ("path", "deployment", "status"),
    [("/v1/embeddings", _FORGED_EMBEDDING, 400), ("/v1/chat/completions", _FORGED_BLOCKED, 429)],
    ids=["refused-upstream", "blocked-before-the-call"],
)
def test_a_caller_cannot_pick_the_label_through_request_metadata(
    rig: _Rig, path: str, deployment: _Deployment, status: int, forged: JsonValue
) -> None:
    bucket: Final[dict[str, JsonValue]] = {"deployment": forged, "model_info": {"id": deployment.model_id}}
    body: Final = (
        _chat(deployment, metadata=bucket, litellm_metadata=bucket)
        if path == "/v1/chat/completions"
        else {"model": deployment.group, "input": _prompt(), "metadata": bucket, "litellm_metadata": bucket}
    )
    response: Final = _post(rig, path, body)
    assert response.status_code == status, response.text
    _assert_labeled(rig, deployment)
    every_provider: Final = {
        sample.labels.get("api_provider") for sample in scrape(rig.gateway) if sample.name in (_FAILURES, _TOTALS)
    }
    assert not any(str(provider).startswith("bedrock") for provider in every_provider), every_provider


def test_an_unauthenticated_request_leaves_the_deployment_series_alone(rig: _Rig) -> None:
    body: Final[dict[str, JsonValue]] = {"model": _UNAUTHENTICATED.group, "input": _prompt()}
    rejected: Final = rig.gateway.request("POST", "/v1/embeddings", body, key="sk-not-a-litellm-key")
    assert rejected.status_code == 401, rejected.text
    refused: Final = _post(rig, "/v1/embeddings", body)
    assert refused.status_code == 400, refused.text
    assert _counts(rig, _FAILURES, _UNAUTHENTICATED, 1) == {"openai": 1}
    assert _counts(rig, _TOTALS, _UNAUTHENTICATED, 1) == {"openai": 1}


def _sdk_v1(gateway: Gateway) -> str:
    return f"{str(gateway.client.base_url).rstrip('/')}/v1"


def _sdk_origin(gateway: Gateway) -> str:
    return str(gateway.client.base_url).rstrip("/")


def _openai_sync(gateway: Gateway, deployment: _Deployment) -> int:
    with (
        httpx.Client(timeout=15, trust_env=False) as transport,
        openai.OpenAI(api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport) as client,
    ):
        with pytest.raises(openai.APIStatusError) as refusal:
            client.embeddings.create(model=deployment.group, input=_prompt())
        return refusal.value.status_code


def _openai_async(gateway: Gateway, deployment: _Deployment) -> int:
    async def run() -> int:
        async with (
            httpx.AsyncClient(timeout=15, trust_env=False) as transport,
            openai.AsyncOpenAI(
                api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport
            ) as client,
        ):
            with pytest.raises(openai.APIStatusError) as refusal:
                await client.embeddings.create(model=deployment.group, input=_prompt())
            return refusal.value.status_code

    return asyncio.run(run())


def _anthropic_sync(gateway: Gateway, deployment: _Deployment) -> int:
    with (
        httpx.Client(timeout=15, trust_env=False) as transport,
        anthropic.Anthropic(
            api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport
        ) as client,
    ):
        with pytest.raises(anthropic.APIStatusError) as refusal:
            client.messages.create(
                model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}]
            )
        return refusal.value.status_code


def _anthropic_async(gateway: Gateway, deployment: _Deployment) -> int:
    async def run() -> int:
        async with (
            httpx.AsyncClient(timeout=15, trust_env=False) as transport,
            anthropic.AsyncAnthropic(
                api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport
            ) as client,
        ):
            with pytest.raises(anthropic.APIStatusError) as refusal:
                await client.messages.create(
                    model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}]
                )
            return refusal.value.status_code

    return asyncio.run(run())


_SDK_CALLS: Final[Mapping[str, Callable[[Gateway, _Deployment], int]]] = {
    "openai-sync": _openai_sync,
    "openai-async": _openai_async,
    "anthropic-sync": _anthropic_sync,
    "anthropic-async": _anthropic_async,
}


@pytest.mark.parametrize("client", list(_SDK_CALLS))
def test_a_refusal_through_the_provider_sdk_labels_the_deployment_failure_with_its_provider(
    rig: _Rig, client: str
) -> None:
    assert _SDK_CALLS[client](rig.gateway, _SDK[client]) == 400
    _assert_labeled(rig, _SDK[client])


_BURST_SIZE: Final = 10
_BURST_CALLS: Final[Mapping[str, tuple[str, Callable[[], Mapping[str, JsonValue]], int]]] = {
    _BURST_EMBEDDING.model_id: ("/v1/embeddings", lambda: {"model": _BURST_EMBEDDING.group, "input": _prompt()}, 400),
    _BURST_RESPONSES.model_id: ("/v1/responses", lambda: {"model": _BURST_RESPONSES.group, "input": _prompt()}, 400),
    _BURST_MESSAGES.model_id: ("/v1/messages", lambda: _messages(_BURST_MESSAGES), 400),
    _BURST_UNSUPPORTED.model_id: (
        "/v1/chat/completions",
        lambda: _chat(_BURST_UNSUPPORTED, logit_bias={"50256": -100}),
        400,
    ),
    _BURST_BLOCKED.model_id: ("/v1/chat/completions", lambda: _chat(_BURST_BLOCKED), 429),
}


def test_a_concurrent_burst_of_failures_counts_each_one_once_under_its_deployment_provider(rig: _Rig) -> None:
    calls: Final = [deployment.model_id for deployment in _BURST for _ in range(_BURST_SIZE)]
    with ThreadPoolExecutor(max_workers=len(calls)) as pool:
        statuses: Final = Counter(
            pool.map(
                lambda model_id: (
                    model_id,
                    _post(rig, _BURST_CALLS[model_id][0], _BURST_CALLS[model_id][1]()).status_code,
                ),
                calls,
            )
        )
    assert statuses == Counter({(model_id, status): _BURST_SIZE for model_id, (_, _, status) in _BURST_CALLS.items()})
    for deployment in _BURST:
        assert _counts(rig, _FAILURES, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE}
        assert _counts(rig, _TOTALS, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE}