Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

21.3 KB
Raw Blame
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Final

import pytest
from pydantic import JsonValue

from litellm import Router
from litellm.caching.dual_cache import DualCache
from litellm.llms.anthropic.prompt_cache_prediction import TokenCounter, cache_scope, parse_prompt
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.cache_aware_routing import (
    CacheAwareChoice,
    choose_cached_model,
    eligible_models,
    select_cached_model,
)
from litellm.proxy.hooks.prompt_cache_prediction import CacheObservation, _cache_key
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.types.router import PreRoutingHookResponse

_CALLER: Final = "test-cache-aware-caller"
_PROVIDER_KEY: Final = "test-cache-aware-provider"
_NOW: Final = 1000.0


@dataclass(frozen=True, slots=True)
class _Counts:
    total: int | None = 51000
    prefix: int | None = 50000

    async def __call__(self, model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
        return self.total if "max_tokens" in body else self.prefix


def _counter_for_model(model: str) -> TokenCounter:
    return _Counts()


def _forbidden_counter(model: str) -> TokenCounter:
    raise AssertionError("No provider counts should run without a warm eligible alternative")


def _body(text: str = "Stable cached context") -> dict[str, JsonValue]:
    return {
        "max_tokens": 20000,
        "messages": [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}},
                    {"type": "text", "text": "What is 2 + 2?"},
                ],
            }
        ],
    }


def _router(
    strong_output_rate: float = 0.000015, *, free: bool = False, cheap_limit: int = 30000, strong_limit: int = 30000
) -> Router:
    return Router(
        model_list=[
            {
                "model_name": "cheap",
                "litellm_params": {
                    "model": "anthropic/claude-haiku-4-5",
                    "api_key": _PROVIDER_KEY,
                    "input_cost_per_token": 0 if free else 0.000001,
                    "output_cost_per_token": 0 if free else 0.000004,
                    "cache_read_input_token_cost": 0 if free else 0.0000001,
                    "cache_creation_input_token_cost": 0 if free else 0.00000125,
                },
                "model_info": {"id": "test-cache-cheap", "max_input_tokens": 100000, "max_output_tokens": cheap_limit},
            },
            {
                "model_name": "strong",
                "litellm_params": {
                    "model": "anthropic/claude-sonnet-5",
                    "api_key": _PROVIDER_KEY,
                    "input_cost_per_token": 0 if free else 0.000003,
                    "output_cost_per_token": 0 if free else strong_output_rate,
                    "cache_read_input_token_cost": 0 if free else 0.0000003,
                    "cache_creation_input_token_cost": 0 if free else 0.00000375,
                },
                "model_info": {
                    "id": "test-cache-strong",
                    "max_input_tokens": 100000,
                    "max_output_tokens": strong_limit,
                },
            },
        ]
    )


def _config(**overrides: object) -> ComplexityRouterConfig:
    return ComplexityRouterConfig.model_validate(
        {
            "tiers": {"SIMPLE": "cheap", "COMPLEX": "strong"},
            "cache_aware_routing": True,
            **overrides,
        }
    )


def _response(tier: str = "SIMPLE", model: str = "cheap") -> PreRoutingHookResponse:
    return PreRoutingHookResponse(
        model=model,
        messages=None,
        routing_decision={
            "router_model_name": "smart",
            "router_type": "complexity",
            "routed_model": model,
            "tier": tier,
            "cause": "heuristic_scorer",
        },
    )


async def _observed(cache: DualCache, *, caller: str = _CALLER, expires_at: float = 1290.0) -> None:
    prefix: Final = parse_prompt(_body())
    assert prefix is not None
    scope: Final = cache_scope(caller, "test-cache-strong", _PROVIDER_KEY, "claude-sonnet-5")
    observation: Final = CacheObservation(
        fingerprint=prefix.fingerprint,
        cached_tokens=50000,
        observed_at=990.0,
        expires_at=expires_at,
    )
    await cache.async_set_cache(_cache_key(scope, prefix.fingerprint), observation.model_dump_json(), ttl=3600)


async def _select(
    *,
    router: Router,
    config: ComplexityRouterConfig,
    response: PreRoutingHookResponse,
    body: Mapping[str, JsonValue],
    request_kwargs: Mapping[str, object],
    messages: Sequence[Mapping[str, object]] | None,
    caller: UserAPIKeyAuth,
    cache: DualCache,
    counter_for_model: Callable[[str], TokenCounter],
    now: float,
) -> CacheAwareChoice | None:
    complexity: Final = ComplexityRouter("smart", router, config.model_dump())
    return await select_cached_model(
        router=router,
        config=config,
        params_for_model=complexity._litellm_params_for_model,
        response=response,
        body=body,
        request_kwargs=request_kwargs,
        messages=messages,
        caller=caller,
        cache=cache,
        counter_for_model=counter_for_model,
        now=now,
    )


@pytest.mark.asyncio
async def test_warm_stronger_model_wins_after_counting_input_and_output_cost() -> None:
    cache: Final = DualCache()
    await _observed(cache)
    choice: Final = await _select(
        router=_router(),
        config=_config(),
        response=_response(),
        body=_body(),
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
        cache=cache,
        counter_for_model=_counter_for_model,
        now=_NOW,
    )
    assert choice is not None
    assert (choice.model, choice.tier, choice.deployment_id) == ("strong", "COMPLEX", "test-cache-strong")
    assert choice.original_cost == pytest.approx(50000 * 0.00000125 + 1000 * 0.000001 + 1024 * 0.000004)
    assert choice.estimated_cost == pytest.approx(50000 * 0.0000003 + 1000 * 0.000003 + 1024 * 0.000015)


@pytest.mark.asyncio
async def test_output_price_can_outweigh_the_cache_saving() -> None:
    cache: Final = DualCache()
    await _observed(cache)
    choice: Final = await _select(
        router=_router(strong_output_rate=0.001),
        config=_config(),
        response=_response(),
        body=_body(),
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
        cache=cache,
        counter_for_model=_counter_for_model,
        now=_NOW,
    )
    assert choice is None


@pytest.mark.asyncio
@pytest.mark.parametrize("case", ["missing", "expired", "different_caller", "changed_prefix", "unauthorized"])
async def test_no_cache_discount_without_fresh_authorized_matching_evidence(case: str) -> None:
    cache: Final = DualCache()
    if case != "missing":
        await _observed(
            cache,
            caller="someone-else" if case == "different_caller" else _CALLER,
            expires_at=999.0 if case == "expired" else 1290.0,
        )
    choice: Final = await _select(
        router=_router(),
        config=_config(),
        response=_response(),
        body=_body("Changed context" if case == "changed_prefix" else "Stable cached context"),
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap"] if case == "unauthorized" else ["cheap", "strong"]),
        cache=cache,
        counter_for_model=_forbidden_counter,
        now=_NOW,
    )
    assert choice is None


@pytest.mark.asyncio
async def test_disabled_setting_does_not_access_prediction_services() -> None:
    config: Final = ComplexityRouterConfig(tiers={"SIMPLE": "cheap"})
    assert config.cache_aware_routing is False
    assert (
        await choose_cached_model(
            router=_router(),
            config=config,
            params_for_model=ComplexityRouter("smart", _router(), config.model_dump())._litellm_params_for_model,
            response=_response(),
            request_kwargs={},
            messages=None,
        )
        is None
    )


def test_cache_prices_cannot_add_a_model_below_the_classified_tier() -> None:
    response: Final = _response("COMPLEX", "strong")
    assert response.routing_decision is not None
    assert eligible_models(_config(), response.routing_decision) == (("COMPLEX", "strong"),)


@pytest.mark.parametrize(
    "overrides", [{"adaptive": True}, {"session_affinity": True}, {"classification_mode": "user_turn"}]
)
def test_existing_pinned_or_adaptive_policies_are_preserved(overrides: Mapping[str, object]) -> None:
    response: Final = _response()
    assert response.routing_decision is not None
    assert eligible_models(_config(**overrides), response.routing_decision) == ()


@pytest.mark.parametrize("total,prefix", [(None, 50000), (51000, None), (1000, 50000)])
@pytest.mark.asyncio
async def test_unavailable_or_inconsistent_counts_keep_the_classified_model(
    total: int | None, prefix: int | None
) -> None:
    cache: Final = DualCache()
    await _observed(cache)
    assert (
        await _select(
            router=_router(),
            config=_config(),
            response=_response(),
            body=_body(),
            request_kwargs={},
            messages=None,
            caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
            cache=cache,
            counter_for_model=lambda _: _Counts(total, prefix),
            now=_NOW,
        )
        is None
    )


@pytest.mark.asyncio
async def test_output_estimate_is_capped_by_the_requested_limit() -> None:
    cache: Final = DualCache()
    await _observed(cache)
    choice: Final = await _select(
        router=_router(),
        config=_config(cache_aware_routing_output_tokens=100000, max_tokens_from_tier_model=False),
        response=_response(),
        body={**_body(), "max_tokens": 1},
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
        cache=cache,
        counter_for_model=_counter_for_model,
        now=_NOW,
    )
    assert choice is not None
    assert choice.estimated_cost == pytest.approx(50000 * 0.0000003 + 1000 * 0.000003 + 0.000015)


@pytest.mark.asyncio
async def test_warm_model_that_cannot_fit_the_request_is_not_selected() -> None:
    cache: Final = DualCache()
    await _observed(cache)
    assert (
        await _select(
            router=_router(),
            config=_config(max_tokens_from_tier_model=False),
            response=_response(),
            body={**_body(), "max_tokens": 100000000},
            request_kwargs={},
            messages=None,
            caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
            cache=cache,
            counter_for_model=_counter_for_model,
            now=_NOW,
        )
        is None
    )


def test_repeated_model_in_multiple_tiers_is_only_considered_once() -> None:
    decision: Final = _response().routing_decision
    assert decision is not None
    assert eligible_models(_config(tiers={"SIMPLE": "cheap", "MEDIUM": "strong", "COMPLEX": "strong"}), decision) == (
        ("SIMPLE", "cheap"),
        ("MEDIUM", "strong"),
    )


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "enabled,behavior,expected",
    [
        (False, "success", "cheap"),
        (True, "success", "strong"),
        (True, "tier_cost", "cheap"),
        (True, "tier_context", "cheap"),
        (True, "error", "cheap"),
        (True, "deadline", "cheap"),
        (True, "cancel", None),
        (True, "transformed", "cheap"),
        (True, "unsupported_shape", "cheap"),
        (True, "custom_endpoint", "cheap"),
        (True, "compaction", "cheap"),
        (True, "guardrail", "cheap"),
    ],
)
async def test_router_applies_opt_in_and_preserves_failure_semantics(
    monkeypatch: pytest.MonkeyPatch, enabled: bool, behavior: str, expected: str | None
) -> None:
    import asyncio
    import json

    import httpx

    import litellm
    from litellm.caching.llm_caching_handler import LLMClientCache
    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
    from litellm.proxy import proxy_server
    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
    from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
    from litellm.proxy.utils import ProxyLogging
    from litellm.router_strategy.complexity_router.context_compaction import initialize_compaction_state

    config: Final = _config(
        cache_aware_routing=enabled, cache_aware_routing_timeout_ms=1 if behavior == "deadline" else 2000
    )
    models: Final = _router(
        strong_output_rate=0.000048 if behavior == "tier_cost" else 0.000015,
        cheap_limit=50 if behavior == "tier_cost" else 30000,
        strong_limit=60000 if behavior == "tier_context" else 30000,
    ).get_model_list()
    assert models is not None
    router: Final = Router(
        model_list=[
            *models,
            {
                "model_name": "smart",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": config.model_dump(),
                    **({"temperature": 0.1} if behavior == "transformed" else {}),
                },
            },
        ]
    )
    logging: Final = ProxyLogging(UserApiKeyCache())
    logging.proxy_hook_mapping["parallel_request_limiter"] = PROXY_MaxParallelRequestsHandler_v3(
        logging.internal_usage_cache
    )
    await _observed(logging.internal_usage_cache.dual_cache, expires_at=1e100)
    monkeypatch.setattr(proxy_server, "llm_router", router)
    monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging)
    monkeypatch.setattr(litellm, "callbacks", [])
    monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())

    requests: Final = asyncio.Queue[httpx.Request]()

    async def count(request: httpx.Request) -> httpx.Response:
        requests.put_nowait(request)
        assert enabled and behavior not in ("transformed", "unsupported_shape", "custom_endpoint")
        if behavior == "error":
            return httpx.Response(503, json={"error": "Provider unavailable"})
        if behavior == "cancel":
            raise asyncio.CancelledError()
        if behavior == "deadline":
            await asyncio.Future()
        payload: Final = json.loads(request.content)
        assert request.url == "https://api.anthropic.com/v1/messages/count_tokens"
        assert request.headers["x-api-key"] == _PROVIDER_KEY
        return httpx.Response(200, json={"input_tokens": 51000 if "What is 2 + 2?" in str(payload) else 50000})

    async with httpx.AsyncClient(transport=httpx.MockTransport(count)) as client:
        handler: Final = AsyncHTTPHandler()
        await handler.client.aclose()
        handler.client = client
        litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientanthropic", handler)
        body: Final = {
            **_body(),
            **({"max_tokens": 1000} if behavior in ("tier_cost", "tier_context") else {}),
            **({"thinking": {"type": "enabled", "budget_tokens": 10000}} if behavior == "unsupported_shape" else {}),
        }
        kwargs: Final = {
            "litellm_metadata": {
                "user_api_key_auth": UserAPIKeyAuth(api_key=_CALLER, models=["smart", "cheap", "strong"])
            },
            "proxy_server_request": {"url": "http://localhost/v1/messages", "body": body, "headers": {}},
            **({"api_base": "https://custom.example"} if behavior == "custom_endpoint" else {}),
            **(
                {"_context_compaction_state": initialize_compaction_state({}, "messages")}
                if behavior == "compaction"
                else {}
            ),
            **({"guardrails": ["test-guardrail"]} if behavior == "guardrail" else {}),
        }
        if expected is None:
            with pytest.raises(asyncio.CancelledError):
                await router.async_pre_routing_hook(model="smart", request_kwargs=kwargs, messages=body["messages"])
            return
        response: Final = await router.async_pre_routing_hook(
            model="smart", request_kwargs=kwargs, messages=body["messages"]
        )
        assert response is not None
        assert response.model == expected
        if not enabled or behavior in (
            "transformed",
            "unsupported_shape",
            "custom_endpoint",
            "compaction",
            "guardrail",
        ):
            assert requests.qsize() == 0
        if enabled and behavior == "success":
            assert requests.qsize() == 4
        assert response.routing_decision is not None
        assert response.routing_decision["cause"] == (
            "prompt_cache_cost" if expected == "strong" else "heuristic_scorer"
        )


@pytest.mark.asyncio
async def test_equal_costs_keep_the_classified_model() -> None:
    cache: Final = DualCache()
    await _observed(cache)
    assert (
        await _select(
            router=_router(free=True),
            config=_config(),
            response=_response(),
            body=_body(),
            request_kwargs={},
            messages=None,
            caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
            cache=cache,
            counter_for_model=_counter_for_model,
            now=_NOW,
        )
        is None
    )


@pytest.mark.parametrize(
    "cause", ["llm_v2_classifier", "capability_classifier", "heuristic_first_short_circuit", "hybrid_short_circuit"]
)
def test_successful_classifiers_can_consider_cache_costs(cause: str) -> None:
    response: Final = PreRoutingHookResponse.model_validate(
        {
            "model": "cheap",
            "messages": None,
            "routing_decision": {"tier": "SIMPLE", "cause": cause},
        }
    )
    assert response.routing_decision is not None
    assert eligible_models(_config(), response.routing_decision) == (("SIMPLE", "cheap"), ("COMPLEX", "strong"))


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "cheap_limit,strong_limit,requested,from_tier,output_rate,expected_limits",
    [
        (50, 1000, 1000, True, 0.000048, None),
        (30000, 30000, 1, True, 0.000049, None),
        (30000, 60000, 1, True, 0.000015, None),
        (50, 100, 20000, True, 0.000048, (50, 100)),
        (30000, 30000, 1, False, 0.000048, (1, 1)),
    ],
)
async def test_each_candidate_uses_its_effective_routed_output_limit(
    cheap_limit: int,
    strong_limit: int,
    requested: int,
    from_tier: bool,
    output_rate: float,
    expected_limits: tuple[int, int] | None,
) -> None:
    cache: Final = DualCache()
    await _observed(cache)
    choice: Final = await _select(
        router=_router(strong_output_rate=output_rate, cheap_limit=cheap_limit, strong_limit=strong_limit),
        config=_config(max_tokens_from_tier_model=from_tier),
        response=_response(),
        body={**_body(), "max_tokens": requested},
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong"]),
        cache=cache,
        counter_for_model=_counter_for_model,
        now=_NOW,
    )
    if expected_limits is None:
        assert choice is None
        return
    assert choice is not None
    assert choice.original_cost == pytest.approx(50000 * 0.00000125 + 1000 * 0.000001 + expected_limits[0] * 0.000004)
    assert choice.estimated_cost == pytest.approx(
        50000 * 0.0000003 + 1000 * 0.000003 + expected_limits[1] * output_rate
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("warm,authorized", [(False, True), (True, True), (True, False)])
async def test_authorization_only_runs_for_original_and_warm_alternatives_before_provider_counts(
    monkeypatch: pytest.MonkeyPatch, warm: bool, authorized: bool
) -> None:
    from unittest.mock import AsyncMock

    from litellm.proxy.common_utils import cache_aware_routing

    cache: Final = DualCache()
    if warm:
        await _observed(cache)
    models: Final = _router().get_model_list()
    assert models is not None
    router: Final = Router(
        model_list=[
            *models,
            {**models[0], "model_name": "cold", "model_info": {"id": "test-cache-cold"}},
        ]
    )
    authorization: Final = AsyncMock(wraps=cache_aware_routing.can_key_call_resolved_model)
    monkeypatch.setattr(cache_aware_routing, "can_key_call_resolved_model", authorization)

    def counter_for_model(model: str) -> TokenCounter:
        assert warm and authorized
        assert authorization.await_count == 2
        return _Counts()

    choice: Final = await _select(
        router=router,
        config=_config(tiers={"SIMPLE": "cheap", "MEDIUM": "cold", "COMPLEX": "strong"}),
        response=_response(),
        body=_body(),
        request_kwargs={},
        messages=None,
        caller=UserAPIKeyAuth(api_key=_CALLER, models=["cheap", "strong", "cold"] if authorized else ["cheap", "cold"]),
        cache=cache,
        counter_for_model=counter_for_model,
        now=_NOW,
    )
    assert (choice is not None) == (warm and authorized)
    assert tuple(call.kwargs["model"] for call in authorization.await_args_list) == (
        ("cheap", "strong") if warm else ()
    )