Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

11.2 KB
Raw Blame
import asyncio
import time
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final

import pytest

import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import Litellm_EntityType
from litellm.proxy.hooks.model_max_budget_limiter import (
    PROXY_VirtualKeyModelMaxBudgetLimiter,
    model_budget_spend_cache_key,
    team_member_budget_entity_id,
)
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import LiteLLMBatch, Usage

KEY_HASH: Final = "key-hash-batch"
USER_ID: Final = "user-batch"
TEAM_ID: Final = "team-batch"
MODEL_GROUP: Final = "batch-qa-primary"
BATCH_COST: Final = 2.925e-05
CHAT_COST: Final = 0.001
KEY_SPEND_KEY: Final = f"virtual_key_spend:{KEY_HASH}:{MODEL_GROUP}:1d"
USER_SPEND_KEY: Final = f"user_model_spend:{USER_ID}:{MODEL_GROUP}:1d"


def _batch(batch_id: str, status: str) -> LiteLLMBatch:
    return LiteLLMBatch(
        id=batch_id,
        completion_window="24h",
        created_at=1,
        endpoint="/v1/chat/completions",
        input_file_id="file-batch",
        object="batch",
        status=status,
        usage=Usage(prompt_tokens=20, completion_tokens=18, total_tokens=38),
    )


def _event(
    call_type: str,
    response_cost: float,
    team_member_model_max_budget: Mapping[str, object] | None = None,
    user_id: str = USER_ID,
    team_id: str = TEAM_ID,
) -> dict[str, object]:
    return {
        "call_type": call_type,
        "standard_logging_object": {
            "call_type": call_type,
            "response_cost": response_cost,
            "model": "openai/gpt-5.4-mini",
            "model_group": MODEL_GROUP,
            "metadata": {
                "user_api_key_hash": KEY_HASH,
                "user_api_key_user_id": user_id,
                "user_api_key_team_id": team_id,
            },
        },
        "litellm_params": {
            "metadata": {
                "user_api_key_model_max_budget": {MODEL_GROUP: {"budget_limit": 0.0001, "time_period": "1d"}},
                "user_api_key_user_model_max_budget": {MODEL_GROUP: {"budget_limit": 0.0001, "time_period": "1d"}},
                **(
                    {"user_api_key_team_member_model_max_budget": team_member_model_max_budget}
                    if team_member_model_max_budget is not None
                    else {}
                ),
            }
        },
    }


async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None:
    await limiter.async_log_success_event(
        _event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None
    )


async def _chat(
    limiter: PROXY_VirtualKeyModelMaxBudgetLimiter,
    team_member_model_max_budget: Mapping[str, object] | None = None,
    user_id: str = USER_ID,
    team_id: str = TEAM_ID,
) -> None:
    await limiter.async_log_success_event(
        _event(
            "acompletion",
            CHAT_COST,
            team_member_model_max_budget=team_member_model_max_budget,
            user_id=user_id,
            team_id=team_id,
        ),
        response_obj=None,
        start_time=None,
        end_time=None,
    )


async def _spend(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, spend_key: str) -> float:
    return await limiter.dual_cache.async_get_cache(key=spend_key) or 0.0


def _local_spend(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, spend_key: str) -> float:
    return limiter.dual_cache.in_memory_cache.get_cache(key=spend_key) or 0.0


@pytest.mark.asyncio
async def test_team_member_budget_is_charged_and_enforced_after_success():
    budget: Final = {MODEL_GROUP: {"max_budget": 0.0005, "budget_duration": "1d"}}
    limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())

    await _chat(limiter, team_member_model_max_budget=budget)

    entity_id: Final = team_member_budget_entity_id(user_id=USER_ID, team_id=TEAM_ID)
    spend_key: Final = model_budget_spend_cache_key(
        entity_type=Litellm_EntityType.TEAM_MEMBER,
        entity_id=entity_id,
        budget_model=MODEL_GROUP,
        budget_duration="1d",
    )
    assert await _spend(limiter, spend_key) == CHAT_COST

    with pytest.raises(litellm.BudgetExceededError, match="LiteLLM Team Member"):
        await limiter.is_team_member_within_model_budget(
            user_id=USER_ID,
            team_id=TEAM_ID,
            team_member_model_max_budget=budget,
            model=MODEL_GROUP,
        )


def test_team_member_budget_entity_id_is_unambiguous():
    first_entity_id: Final = team_member_budget_entity_id(user_id="a:b", team_id="c")
    second_entity_id: Final = team_member_budget_entity_id(user_id="a", team_id="b:c")

    assert first_entity_id == "3:a:b:c"
    assert second_entity_id == "1:a:b:c"

    first_spend_key: Final = model_budget_spend_cache_key(
        entity_type=Litellm_EntityType.TEAM_MEMBER,
        entity_id=first_entity_id,
        budget_model=MODEL_GROUP,
        budget_duration="1d",
    )
    second_spend_key: Final = model_budget_spend_cache_key(
        entity_type=Litellm_EntityType.TEAM_MEMBER,
        entity_id=second_entity_id,
        budget_model=MODEL_GROUP,
        budget_duration="1d",
    )
    assert first_spend_key != second_spend_key


@pytest.mark.asyncio
async def test_team_member_budget_spend_isolated_by_user_and_team():
    budget: Final = {MODEL_GROUP: {"max_budget": CHAT_COST * 2, "budget_duration": "1d"}}
    limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())

    await _chat(limiter, team_member_model_max_budget=budget)

    assert await limiter.is_team_member_within_model_budget(
        user_id="different-user",
        team_id=TEAM_ID,
        team_member_model_max_budget=budget,
        model=MODEL_GROUP,
    )
    assert await limiter.is_team_member_within_model_budget(
        user_id=USER_ID,
        team_id="different-team",
        team_member_model_max_budget=budget,
        model=MODEL_GROUP,
    )


@pytest.mark.asyncio
async def test_zero_team_member_model_budget_rejects_first_request():
    limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())

    with pytest.raises(litellm.BudgetExceededError, match="LiteLLM Team Member"):
        await limiter.is_team_member_within_model_budget(
            user_id=USER_ID,
            team_id=TEAM_ID,
            team_member_model_max_budget={MODEL_GROUP: {"max_budget": 0, "budget_duration": "1d"}},
            model=MODEL_GROUP,
        )


class _Clock:
    def __init__(self) -> None:
        self.seconds = 0.0

    def now(self) -> float:
        return self.seconds

    def advance(self, seconds: float) -> None:
        self.seconds = self.seconds + seconds


class _SharedRedisDouble:
    def __init__(self, now: Callable[[], float] = time.time) -> None:
        self.now = now
        self.entries: Mapping[str, tuple[float, float | None]] = MappingProxyType({})

    def _live(self, key: str) -> tuple[float, float | None] | None:
        entry: Final = self.entries.get(key)
        if entry is None:
            return None
        expires_at: Final = entry[1]
        if expires_at is not None and expires_at <= self.now():
            return None
        return entry

    def _store(self, key: str, value: float, expires_at: float | None) -> None:
        self.entries = MappingProxyType({**self.entries, key: (value, expires_at)})

    async def async_get_cache(self, key: str, **kwargs: object) -> float | None:
        await asyncio.sleep(0)
        entry: Final = self._live(key)
        return None if entry is None else entry[0]

    async def async_set_cache(self, key: str, value: float, ttl: int | None = None, **kwargs: object) -> None:
        await asyncio.sleep(0)
        self._store(key, value, None if ttl is None else self.now() + ttl)

    async def async_increment(
        self,
        key: str,
        value: float,
        ttl: int | None = None,
        parent_otel_span: object = None,
        refresh_ttl: bool = False,
    ) -> float:
        await asyncio.sleep(0)
        live: Final = self._live(key)
        total: Final = value if live is None else live[0] + value
        kept_expiry: Final = None if live is None else live[1]
        expires_at: Final = (
            kept_expiry if ttl is None or (kept_expiry is not None and not refresh_ttl) else self.now() + ttl
        )
        self._store(key, total, expires_at)
        return total

    async def async_increment_pipeline(self, increment_list: list[RedisPipelineIncrementOperation]) -> list[float]:
        return [await self.async_increment(op["key"], op["increment_value"], ttl=op["ttl"]) for op in increment_list]


def _worker(redis: _SharedRedisDouble) -> PROXY_VirtualKeyModelMaxBudgetLimiter:
    return PROXY_VirtualKeyModelMaxBudgetLimiter(
        dual_cache=DualCache(redis_cache=redis)  # pyright: ignore[reportArgumentType]  # duck-typed Redis double
    )


async def _drain_redis_pushes() -> None:
    await asyncio.gather(*(task for task in asyncio.all_tasks() if task is not asyncio.current_task()))


@pytest.mark.asyncio
async def test_polls_of_a_finished_batch_charge_each_per_model_budget_once():
    limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
    first: Final = _batch("batch_first", "completed")

    await _poll(limiter, _batch("batch_first", "in_progress"), response_cost=0)
    for _ in range(3):
        await _poll(limiter, first, response_cost=BATCH_COST)

    assert await _spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
    assert await _spend(limiter, USER_SPEND_KEY) == pytest.approx(BATCH_COST)


@pytest.mark.asyncio
async def test_a_second_batch_and_chat_requests_still_charge_the_budget():
    limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())

    await _poll(limiter, _batch("batch_first", "completed"), response_cost=BATCH_COST)
    await _poll(limiter, _batch("batch_first", "completed"), response_cost=BATCH_COST)
    await _poll(limiter, _batch("batch_second", "completed"), response_cost=BATCH_COST)
    await _chat(limiter)
    await _chat(limiter)

    assert await _spend(limiter, KEY_SPEND_KEY) == pytest.approx(2 * BATCH_COST + 2 * CHAT_COST)


@pytest.mark.asyncio
async def test_two_workers_polling_the_same_finished_batch_at_once_charge_it_once():
    redis: Final = _SharedRedisDouble()
    worker_a: Final = _worker(redis)
    worker_b: Final = _worker(redis)
    finished: Final = _batch("batch_first", "completed")

    await asyncio.gather(_poll(worker_a, finished, BATCH_COST), _poll(worker_b, finished, BATCH_COST))
    await _drain_redis_pushes()

    assert _local_spend(worker_a, KEY_SPEND_KEY) + _local_spend(worker_b, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
    assert await redis.async_get_cache(KEY_SPEND_KEY) == pytest.approx(BATCH_COST)


@pytest.mark.asyncio
async def test_a_batch_polled_within_every_budget_window_is_never_charged_again():
    clock: Final = _Clock()
    limiter: Final = _worker(_SharedRedisDouble(now=clock.now))
    finished: Final = _batch("batch_first", "completed")

    await _poll(limiter, finished, BATCH_COST)
    clock.advance(12 * 3600)
    await _poll(limiter, finished, BATCH_COST)
    clock.advance(18 * 3600)
    await _poll(limiter, finished, BATCH_COST)

    assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)