Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

8.6 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
from collections.abc import Callable, Mapping
from typing import Final

import pytest
from integration._support.wire import Reply, Request, Wire, wire_server

import litellm
from litellm import Router

_MODEL: Final = "gpt-5-wire-unmapped"
_API_KEY: Final = "synthetic-unsupported-params-key"
_PROMPT: Final = "which deployment answers when the requested model rejects the request?"
_CLIENTS: Final = ("sync", "async")
_PRIMARY_TO_BACKUP: Final = [{"primary": ["backup"]}]
_UNSUPPORTED_BY_PRIMARY: Final = {"reasoning_effort": "xhigh"}


def _answers(text: str) -> Reply:
    body: Final = {
        "id": "chatcmpl-bad-request-wire",
        "object": "chat.completion",
        "created": 1700000000,
        "model": _MODEL,
        "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
        "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
    }
    return Reply(body=json.dumps(body).encode())


_UPSTREAM_500: Final = Reply(status=500, body=json.dumps({"error": {"message": "overloaded"}}).encode())
_UPSTREAM_400: Final = Reply(
    status=400,
    body=json.dumps({"error": {"message": "Unsupported value: temperature", "type": "invalid_request_error"}}).encode(),
)
_UPSTREAM_CONTEXT_WINDOW: Final = Reply(
    status=400,
    body=json.dumps(
        {
            "error": {
                "message": "This model's maximum context length is 8192 tokens",
                "type": "invalid_request_error",
                "code": "context_length_exceeded",
            }
        }
    ).encode(),
)
_UPSTREAM_CONTENT_POLICY: Final = Reply(
    status=400,
    body=json.dumps(
        {
            "error": {
                "message": "Your request was rejected",
                "type": "invalid_request_error",
                "code": "content_policy_violation",
            }
        }
    ).encode(),
)
_BACKUP_ANSWERS: Final = _answers("answered by the backup")


def _peer(replies: Mapping[str, Reply]) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        assert request.method == "POST", request.method
        deployment, _, route = request.target.lstrip("/").partition("/")
        assert route == "chat/completions", request.target
        return replies[deployment]

    return respond


def _deployments_hit(wire: Wire) -> tuple[str, ...]:
    return tuple(request.target.lstrip("/").partition("/")[0] for request in wire.drain())


_ALL_DAY_FOR_TEAM_A: Final = (
    {"start": "00:00", "end": "12:00", "timezone": "UTC", "team_ids": ["team-a"]},
    {"start": "12:00", "end": "00:00", "timezone": "UTC", "team_ids": ["team-a"]},
)


def _router(wire: Wire, primary_model_info: Mapping[str, object] | None = None, **settings: object) -> Router:
    return Router(
        model_list=[
            {
                "model_name": "primary",
                "litellm_params": {"model": f"openai/{_MODEL}", "api_base": f"{wire.url}/primary", "api_key": _API_KEY},
                "model_info": dict(primary_model_info or {}),
            },
            {
                "model_name": "backup",
                "litellm_params": {
                    "model": f"openai/{_MODEL}",
                    "api_base": f"{wire.url}/backup",
                    "api_key": _API_KEY,
                    "drop_params": True,
                },
            },
        ],
        num_retries=0,
        disable_cooldowns=True,
        **{"fallbacks": _PRIMARY_TO_BACKUP, **settings},
    )


_MESSAGES: Final = [{"role": "user", "content": _PROMPT}]


def _content(response: litellm.ModelResponse) -> str:
    choice: Final = response.choices[0]
    assert isinstance(choice, litellm.Choices), choice
    return str(choice.message.content)


async def _acomplete(router: Router, **request: object) -> str:
    response: Final = await router.acompletion(model="primary", messages=_MESSAGES, **request)
    assert isinstance(response, litellm.ModelResponse), response
    return _content(response)


def _complete(client: str, router: Router, **request: object) -> str:
    if client == "async":
        return asyncio.run(_acomplete(router, **request))
    response: Final = router.completion(model="primary", messages=_MESSAGES, **request)
    assert isinstance(response, litellm.ModelResponse), response
    return _content(response)


@pytest.mark.parametrize("client", _CLIENTS)
def test_client_side_param_rejection_reaches_the_caller_without_trying_fallbacks(client: str) -> None:
    with wire_server(_peer({"backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire, fallback_on_unsupported_params=False)
        with pytest.raises(litellm.UnsupportedParamsError, match="reasoning_effort"):
            _complete(client, router, **_UNSUPPORTED_BY_PRIMARY)
        assert _deployments_hit(wire) == ()


@pytest.mark.parametrize("client", _CLIENTS)
def test_client_side_param_rejection_still_falls_back_by_default(client: str) -> None:
    with wire_server(_peer({"backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire)
        assert _complete(client, router, **_UNSUPPORTED_BY_PRIMARY) == "answered by the backup"
        assert _deployments_hit(wire) == ("backup",)


@pytest.mark.parametrize("client", _CLIENTS)
def test_upstream_500_still_falls_back_when_bad_requests_raise(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_500, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire, fallback_on_unsupported_params=False)
        assert _complete(client, router) == "answered by the backup"
        assert _deployments_hit(wire) == ("primary", "backup")


@pytest.mark.parametrize("client", _CLIENTS)
def test_router_raised_400_still_falls_back_when_bad_requests_raise(client: str) -> None:
    with wire_server(_peer({"backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(
            wire, primary_model_info={"access_windows": _ALL_DAY_FOR_TEAM_A}, fallback_on_unsupported_params=False
        )
        answer: Final = _complete(client, router, metadata={"user_api_key_team_id": "team-b"})
        assert answer == "answered by the backup"
        assert _deployments_hit(wire) == ("backup",)


@pytest.mark.parametrize("client", _CLIENTS)
def test_context_window_error_still_uses_its_own_fallbacks_when_bad_requests_raise(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_CONTEXT_WINDOW, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(
            wire, fallback_on_unsupported_params=False, fallbacks=None, context_window_fallbacks=_PRIMARY_TO_BACKUP
        )
        assert _complete(client, router) == "answered by the backup"
        assert _deployments_hit(wire) == ("primary", "backup")


@pytest.mark.parametrize("client", _CLIENTS)
def test_content_policy_error_still_uses_its_own_fallbacks_when_bad_requests_raise(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_CONTENT_POLICY, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(
            wire, fallback_on_unsupported_params=False, fallbacks=None, content_policy_fallbacks=_PRIMARY_TO_BACKUP
        )
        assert _complete(client, router) == "answered by the backup"
        assert _deployments_hit(wire) == ("primary", "backup")


@pytest.mark.parametrize("client", _CLIENTS)
def test_upstream_400_reaches_the_caller_without_trying_fallbacks(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_400, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire, fallback_on_unsupported_params=False)
        with pytest.raises(litellm.BadRequestError, match="Unsupported value: temperature"):
            _complete(client, router)
        assert _deployments_hit(wire) == ("primary",)


@pytest.mark.parametrize("client", _CLIENTS)
def test_flag_applied_through_update_settings_stops_fallbacks_on_upstream_400(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_400, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire)
        router.update_settings(fallback_on_unsupported_params=False)
        with pytest.raises(litellm.BadRequestError, match="Unsupported value: temperature"):
            _complete(client, router)
        assert _deployments_hit(wire) == ("primary",)


@pytest.mark.parametrize("client", _CLIENTS)
def test_upstream_400_still_falls_back_by_default(client: str) -> None:
    with wire_server(_peer({"primary": _UPSTREAM_400, "backup": _BACKUP_ANSWERS})) as wire:
        router: Final = _router(wire)
        assert _complete(client, router) == "answered by the backup"
        assert _deployments_hit(wire) == ("primary", "backup")