Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

31.9 KB
Raw Blame
"""Exact Redis round-trip counts for the spend counters admission reads within one auth scope."""

from __future__ import annotations

import asyncio
from collections.abc import Mapping, Sequence
from unittest.mock import AsyncMock, MagicMock

import pytest

import litellm
import litellm.proxy.proxy_server as ps
from litellm.caching.redis_cache import RedisCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
from litellm.proxy.spend_tracking.spend_counter_batch import (
    SpendCounterBatch,
    active_spend_counter_batch,
    admission_counter_keys,
    bind_admission_counter_keys,
    post_call_counter_keys,
    release_spend_counter_batch,
    spend_counter_batch_scope,
)

TOKEN = UserAPIKeyAuth(token="hashed", team_id="team", user_id="user", org_id="org")
TOKEN_KEYS = frozenset(
    {
        "spend:key:hashed",
        "spend:team:team",
        "spend:team_member:user:team",
        "spend:user:user",
        "spend:end_user:eu",
        "spend:org:org",
    }
)


class CountingRedis(RedisCache):
    def __init__(self, store: dict[str, object] | None = None, fail: bool = False) -> None:
        self.store: dict[str, object] = dict(store or {})
        self.fail = fail
        self.commands: list[str] = []

    async def async_get_cache(self, key: str, **kwargs: object) -> object:
        if self.fail:
            raise ConnectionError("redis down")
        self.commands.append(f"GET {key}")
        return self.store.get(key)

    async def async_batch_get_cache(self, key_list: Sequence[str], **kwargs: object) -> dict[str, object]:
        if self.fail:
            raise ConnectionError("redis down")
        self.commands.append(f"MGET {' '.join(key_list)}")
        return {key: self.store.get(key) for key in key_list}

    def get_ttl(self, **kwargs: object) -> int | None:
        return None

    async def async_increment(self, key: str, value: float, **kwargs: object) -> float:
        self.commands.append(f"INCRBYFLOAT {key} {value}")
        return self._incr(key, value)

    async def async_increment_pipeline(
        self, increment_list: Sequence[Mapping[str, object]], **kwargs: object
    ) -> list[float]:
        self.commands.append(f"PIPELINE {' '.join(str(op['key']) for op in increment_list)}")
        return [self._incr(str(op["key"]), float(str(op["increment_value"]))) for op in increment_list]

    def _incr(self, key: str, value: float) -> float:
        total = float(str(self.store.get(key, 0.0))) + value
        self.store[key] = total
        return total


def _spend_counter_cache(redis: RedisCache | None, in_memory: dict[str, float] | None = None) -> MagicMock:
    cache = MagicMock()
    cache.redis_cache = redis
    cache.in_memory_cache.get_cache = MagicMock(side_effect=lambda key: (in_memory or {}).get(key))
    return cache


def test_admission_counter_keys_cover_every_entity_the_checks_read():
    assert admission_counter_keys(TOKEN, end_user_id="eu") == TOKEN_KEYS
    assert admission_counter_keys(UserAPIKeyAuth(token="hashed"), end_user_id=None) == {"spend:key:hashed"}
    assert "spend:team_member:user:team" not in admission_counter_keys(
        UserAPIKeyAuth(token="hashed", user_id="user"), end_user_id=None
    )


def test_post_call_counter_keys_skip_ids_that_are_not_strings():
    """A synthetic logging payload (batch cost polling, tests) can carry placeholders where the ids belong; those
    have no counter, and deriving the key set must never raise inside the cost callback."""
    placeholder = object()
    assert post_call_counter_keys(
        token=placeholder,  # pyright: ignore[reportArgumentType]  # synthetic payload placeholder, not an id
        team_id="team",
        user_id=None,
        org_id=placeholder,  # pyright: ignore[reportArgumentType]  # synthetic payload placeholder, not an id
        end_user_id="eu",
        tags=[placeholder, "t1"],
        model_access_groups=None,
    ) == {"spend:team:team", "spend:end_user:eu", "spend:tag:t1"}


@pytest.mark.asyncio
async def test_bound_counters_share_one_mget_and_a_clean_miss_is_authoritative():
    redis = CountingRedis({"spend:key:hashed": 1.5, "spend:team:team": 2.5})
    batch = SpendCounterBatch(redis)
    batch.bind(TOKEN_KEYS)

    reads = await asyncio.gather(*(batch.read(key) for key in sorted(TOKEN_KEYS)))

    assert len(redis.commands) == 1
    assert set(redis.commands[0].split()[1:]) == TOKEN_KEYS
    assert dict(zip(sorted(TOKEN_KEYS), reads)) == {
        "spend:end_user:eu": (None, True),
        "spend:key:hashed": (1.5, True),
        "spend:org:org": (None, True),
        "spend:team:team": (2.5, True),
        "spend:team_member:user:team": (None, True),
        "spend:user:user": (None, True),
    }


@pytest.mark.asyncio
async def test_unbound_counter_and_closed_batch_leave_the_read_to_the_caller():
    redis = CountingRedis({"spend:key:hashed": 1.0})
    batch = SpendCounterBatch(redis)
    batch.bind(frozenset({"spend:key:hashed"}))

    assert await batch.read("spend:tag:prod") is None
    assert redis.commands == []
    assert await batch.read("spend:key:hashed") == (1.0, True)

    batch.close()
    batch.bind(frozenset({"spend:org:org"}))
    assert await batch.read("spend:key:hashed") is None
    assert await batch.read("spend:org:org") is None
    assert len(redis.commands) == 1


@pytest.mark.asyncio
async def test_keys_bound_after_the_first_read_join_one_more_mget_for_only_the_new_keys():
    redis = CountingRedis({"spend:key:hashed": 1.0, "spend:org:org": 9.0})
    batch = SpendCounterBatch(redis)
    batch.bind(frozenset({"spend:key:hashed"}))
    assert await batch.read("spend:key:hashed") == (1.0, True)

    batch.bind(frozenset({"spend:org:org", "spend:key:hashed"}))
    assert await batch.read("spend:org:org") == (9.0, True)
    assert await batch.read("spend:key:hashed") == (1.0, True)

    assert redis.commands == ["MGET spend:key:hashed", "MGET spend:org:org"]


@pytest.mark.asyncio
async def test_a_recorded_write_result_answers_later_reads_without_another_redis_read():
    redis = CountingRedis({"spend:key:hashed": 1.0})
    batch = SpendCounterBatch(redis)
    batch.bind(frozenset({"spend:key:hashed"}))
    assert await batch.read("spend:key:hashed") == (1.0, True)

    batch.record("spend:key:hashed", 3.5)
    batch.record("spend:org:org", 7.0)

    assert await batch.read("spend:key:hashed") == (3.5, True)
    assert await batch.read("spend:org:org") == (7.0, True)
    assert redis.commands == ["MGET spend:key:hashed"]


@pytest.mark.asyncio
async def test_a_forgotten_counter_is_read_fresh_from_redis_when_it_is_bound_again():
    redis = CountingRedis({"spend:key:hashed": 1.0})
    batch = SpendCounterBatch(redis)
    batch.bind(frozenset({"spend:key:hashed"}))
    batch.record("spend:key:hashed", 3.5)

    batch.forget("spend:key:hashed")
    assert await batch.read("spend:key:hashed") is None

    redis.store["spend:key:hashed"] = 9.0
    batch.bind(frozenset({"spend:key:hashed"}))
    assert await batch.read("spend:key:hashed") == (9.0, True)
    assert redis.commands == ["MGET spend:key:hashed"]


@pytest.mark.asyncio
async def test_failed_mget_hands_every_counter_back_to_the_caller():
    batch = SpendCounterBatch(CountingRedis(fail=True))
    batch.bind(TOKEN_KEYS)

    assert await batch.read("spend:key:hashed") is None


@pytest.mark.asyncio
async def test_non_numeric_counter_payload_hands_the_batch_back_to_the_caller():
    redis = CountingRedis()
    redis.store["spend:key:hashed"] = "garbage"
    batch = SpendCounterBatch(redis)
    batch.bind(frozenset({"spend:key:hashed"}))

    assert await batch.read("spend:key:hashed") is None


def test_scope_installs_a_batch_only_when_redis_exists_and_release_closes_without_clearing():
    assert active_spend_counter_batch() is None
    with spend_counter_batch_scope(None):
        assert active_spend_counter_batch() is None
        bind_admission_counter_keys(TOKEN, end_user_id=None)

    with spend_counter_batch_scope(CountingRedis()):
        batch = active_spend_counter_batch()
        assert batch is not None
        bind_admission_counter_keys(TOKEN, end_user_id="eu")
        assert batch.counter_keys == TOKEN_KEYS
        release_spend_counter_batch()
        assert active_spend_counter_batch() is batch
        batch.bind(frozenset({"spend:tag:x"}))
        assert batch.counter_keys == TOKEN_KEYS
    assert active_spend_counter_batch() is None


@pytest.mark.asyncio
async def test_get_current_spend_inside_the_scope_costs_one_mget_for_all_admission_counters(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 3.0, "spend:team:team": 4.0, "spend:org:org": 5.0})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id="eu")
        key_spend = await ps.get_current_spend(counter_key="spend:key:hashed", fallback_spend=0.0)
        team_spend = await ps.get_current_spend(counter_key="spend:team:team", fallback_spend=0.0)
        org_spend = await ps.get_current_spend(counter_key="spend:org:org", fallback_spend=0.0)
        user_spend = await ps.get_current_spend(counter_key="spend:user:user", fallback_spend=7.0)

    assert (key_spend, team_spend, org_spend, user_spend) == (3.0, 4.0, 5.0, 7.0)
    assert [c for c in redis.commands if c.startswith("GET ")] == [], "the cold reseed reuses the MGET miss"
    assert [c for c in redis.commands if c.startswith("MGET ")] == [f"MGET {' '.join(sorted(TOKEN_KEYS))}"]


@pytest.mark.asyncio
async def test_get_current_spend_outside_the_scope_still_reads_redis_per_counter(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 3.0})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))

    assert await ps.get_current_spend(counter_key="spend:key:hashed", fallback_spend=0.0) == 3.0
    assert redis.commands == ["GET spend:key:hashed"]


@pytest.mark.asyncio
async def test_after_release_a_read_goes_to_redis_directly_so_read_then_write_sees_fresh_values(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 3.0})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        assert await ps.read_spend_counter_cache_value("spend:key:hashed") == (3.0, True)
        redis.store["spend:key:hashed"] = 8.0
        assert await ps.read_spend_counter_cache_value("spend:key:hashed") == (3.0, True)
        release_spend_counter_batch()
        assert await ps.read_spend_counter_cache_value("spend:key:hashed") == (8.0, True)

    assert [c.split()[0] for c in redis.commands] == ["MGET", "GET"]


@pytest.mark.asyncio
async def test_batched_clean_miss_does_not_fall_back_to_the_per_pod_in_memory_copy(monkeypatch):
    redis = CountingRedis()
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis, in_memory={"spend:key:hashed": 99.0}))

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        assert await ps.read_spend_counter_cache_value("spend:key:hashed") == (None, True)


@pytest.mark.asyncio
async def test_batched_redis_failure_falls_back_to_the_per_pod_in_memory_copy(monkeypatch):
    redis = CountingRedis(fail=True)
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis, in_memory={"spend:key:hashed": 99.0}))

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        assert await ps.read_spend_counter_cache_value("spend:key:hashed") == (99.0, False)


@pytest.mark.asyncio
async def test_scope_is_per_task_so_concurrent_requests_do_not_share_a_batch():
    redis = CountingRedis({"spend:key:a": 1.0, "spend:key:b": 2.0})

    async def request(token: str) -> tuple[float | None, bool] | None:
        with spend_counter_batch_scope(redis):
            bind_admission_counter_keys(UserAPIKeyAuth(token=token), end_user_id=None)
            batch = active_spend_counter_batch()
            assert batch is not None
            await asyncio.sleep(0)
            return await batch.read(f"spend:key:{token}")

    assert await asyncio.gather(request("a"), request("b")) == [(1.0, True), (2.0, True)]
    assert sorted(redis.commands) == ["MGET spend:key:a", "MGET spend:key:b"]


@pytest.mark.asyncio
async def test_batch_reads_never_touch_a_prisma_client_when_redis_answers(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 3.0})
    prisma = MagicMock()
    prisma.db.litellm_verificationtoken.find_unique = AsyncMock()
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", prisma)

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        assert await ps.get_current_spend(counter_key="spend:key:hashed", fallback_spend=0.0) == 3.0

    assert prisma.db.mock_calls == []


def _reseed_prisma(spend: float) -> MagicMock:
    prisma = MagicMock()
    prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=MagicMock(spend=spend))
    return prisma


@pytest.mark.asyncio
async def test_reseed_reuses_the_admission_mget_instead_of_its_own_get():
    redis = CountingRedis({"spend:key:hashed": 7.5})
    prisma = _reseed_prisma(spend=1.0)
    cache = _spend_counter_cache(redis)

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        value = await SpendCounterReseed.coalesced(prisma, cache, counter_key="spend:key:hashed")

    assert value == 7.5
    assert redis.commands == [
        "MGET spend:key:hashed spend:org:org spend:team:team spend:team_member:user:team spend:user:user"
    ]
    assert prisma.db.mock_calls == []


@pytest.mark.asyncio
async def test_reseed_treats_a_batched_clean_miss_as_authoritative_and_seeds_from_the_db():
    redis = CountingRedis()
    redis.async_set_cache = AsyncMock(return_value=True)
    prisma = _reseed_prisma(spend=2.25)
    cache = _spend_counter_cache(redis, in_memory={"spend:key:hashed": 99.0})

    with spend_counter_batch_scope(redis):
        bind_admission_counter_keys(TOKEN, end_user_id=None)
        value = await SpendCounterReseed.coalesced(prisma, cache, counter_key="spend:key:hashed")

    assert value == 2.25
    assert [c for c in redis.commands if c.startswith("GET")] == []
    redis.async_set_cache.assert_awaited_once_with(key="spend:key:hashed", value=2.25, nx=True)


@pytest.mark.asyncio
async def test_reseed_outside_the_scope_still_re_checks_redis_itself():
    redis = CountingRedis({"spend:key:hashed": 4.0})

    value = await SpendCounterReseed.coalesced(
        _reseed_prisma(spend=1.0), _spend_counter_cache(redis), "spend:key:hashed"
    )

    assert value == 4.0
    assert redis.commands == ["GET spend:key:hashed"]


POST_CALL_KEYS = TOKEN_KEYS | {"spend:tag:prod", "spend:model_access_group:premium"}


@pytest.mark.asyncio
async def test_post_call_increment_for_every_entity_costs_one_mget_and_one_pipeline(monkeypatch):
    redis = CountingRedis({key: 1.0 for key in POST_CALL_KEYS})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)

    await ps.increment_spend_counters(
        token="hashed",
        team_id="team",
        user_id="user",
        org_id="org",
        end_user_id="eu",
        tags=["prod"],
        model_access_groups=["premium"],
        response_cost=0.5,
    )

    assert [c.split()[0] for c in redis.commands] == ["MGET", "PIPELINE"], redis.commands
    assert set(redis.commands[0].split()[1:]) == POST_CALL_KEYS
    assert set(redis.commands[1].split()[1:]) == POST_CALL_KEYS
    assert {key: redis.store[key] for key in POST_CALL_KEYS} == {key: 1.5 for key in POST_CALL_KEYS}


@pytest.mark.asyncio
async def test_post_call_cold_counters_seed_from_the_mget_miss_without_a_second_read(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 1.0})
    redis.async_set_cache = AsyncMock(return_value=True)
    prisma = MagicMock()
    prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=MagicMock(spend=4.0))
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", prisma)

    await ps.increment_spend_counters(token="hashed", team_id="team", user_id=None, response_cost=0.5)

    assert [c.split()[0] for c in redis.commands] == ["MGET", "PIPELINE"], redis.commands
    redis.async_set_cache.assert_awaited_once_with(key="spend:team:team", value=4.0, nx=True)
    assert redis.store["spend:key:hashed"] == 1.5


RESERVED_KEYS = frozenset(
    {"spend:key:hashed", "spend:team:team", "spend:team_member:user:team", "spend:end_user:eu", "spend:org:org"}
)


def _reservation(reserved_cost: float, counter_keys: frozenset[str] = RESERVED_KEYS) -> dict[str, object]:
    return {
        "reserved_cost": reserved_cost,
        "entries": [
            {"counter_key": key, "entity_type": "Key", "entity_id": key, "reserved_cost": reserved_cost}
            for key in sorted(counter_keys)
        ],
    }


@pytest.mark.asyncio
async def test_post_call_with_a_reservation_costs_one_mget_and_one_pipeline_for_reconcile_and_increments(monkeypatch):
    redis = CountingRedis({key: 1.0 for key in POST_CALL_KEYS})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    reservation = _reservation(reserved_cost=0.4)

    await ps.increment_spend_counters(
        token="hashed",
        team_id="team",
        user_id="user",
        org_id="org",
        end_user_id="eu",
        tags=["prod"],
        model_access_groups=["premium"],
        response_cost=0.5,
        budget_reservation=reservation,
    )

    assert [c.split()[0] for c in redis.commands] == ["MGET", "PIPELINE"], redis.commands
    assert set(redis.commands[0].split()[1:]) == POST_CALL_KEYS, "reconcile and warm checks share the MGET"
    assert set(redis.commands[1].split()[1:]) == POST_CALL_KEYS, "reconcile adjustments ride the increment pipeline"
    assert {key: round(redis.store[key], 6) for key in POST_CALL_KEYS} == {
        key: (1.1 if key in RESERVED_KEYS else 1.5) for key in POST_CALL_KEYS
    }
    assert [round(entry["applied_adjustment"], 6) for entry in reservation["entries"]] == [0.1] * len(RESERVED_KEYS)
    assert reservation["finalized"] is True


@pytest.mark.asyncio
async def test_a_stale_counter_repair_updates_the_open_batch_instead_of_forcing_a_second_mget(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 1.0, "spend:team:team": 1.0})

    async def set_max(key: str, value: float, **kwargs: object) -> float:
        redis.commands.append(f"SETMAX {key} {value}")
        redis.store[key] = max(float(str(redis.store.get(key, 0.0))), value)
        return float(str(redis.store[key]))

    redis.async_set_max = set_max
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))

    with spend_counter_batch_scope(redis, counter_keys=frozenset({"spend:key:hashed", "spend:team:team"})):
        assert await ps.read_spend_counter_cache_value(counter_key="spend:team:team") == (1.0, True)
        await ps._repair_stale_spend_counter(counter_key="spend:team:team", db_spend=4.0)
        assert await ps.read_spend_counter_cache_value(counter_key="spend:team:team") == (4.0, True)
        assert await ps.read_spend_counter_cache_value(counter_key="spend:key:hashed") == (1.0, True)

    assert [c.split()[0] for c in redis.commands] == ["MGET", "SETMAX"], redis.commands


@pytest.mark.asyncio
async def test_reconcile_settles_a_flushed_counter_on_its_own_after_the_shared_pipeline(monkeypatch):
    from litellm.proxy.spend_tracking.budget_reservation import reconcile_budget_reservation

    redis = CountingRedis({key: 1.0 for key in RESERVED_KEYS - {"spend:team:team"}})
    redis.async_set_max = AsyncMock(return_value=4.0)
    prisma = MagicMock()
    prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=MagicMock(spend=4.0))
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", prisma)
    reservation = _reservation(reserved_cost=0.4)

    await reconcile_budget_reservation(budget_reservation=reservation, actual_cost=0.5)

    assert [c.split()[0] for c in redis.commands] == ["MGET", "PIPELINE", "INCRBYFLOAT"], redis.commands
    assert set(redis.commands[1].split()[1:]) == RESERVED_KEYS - {"spend:team:team"}
    assert redis.commands[2] == "INCRBYFLOAT spend:team:team 0.5"
    redis.async_set_max.assert_awaited_once()
    assert redis.async_set_max.await_args.kwargs["key"] == "spend:team:team"
    assert all(round(entry["applied_adjustment"], 6) == 0.1 for entry in reservation["entries"])


@pytest.mark.asyncio
async def test_pre_call_resize_against_an_inconsistent_counter_writes_nothing_and_denies(monkeypatch):
    from litellm.proxy.spend_tracking.budget_reservation import _resize_applied_reservation

    redis = CountingRedis({"spend:key:hashed": 1.0})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    entries = _reservation(reserved_cost=0.4, counter_keys=frozenset({"spend:key:hashed", "spend:team:team"}))[
        "entries"
    ]

    with pytest.raises(RuntimeError, match="spend:team:team"):
        await _resize_applied_reservation(entries=entries, current_reserved_cost=0.4, new_reserved_cost=0.9)

    assert [c.split()[0] for c in redis.commands] == ["MGET"], redis.commands
    assert redis.store["spend:key:hashed"] == 1.0
    assert all("applied_adjustment" not in entry for entry in entries)


@pytest.mark.asyncio
async def test_a_failed_post_call_pipeline_invalidates_every_counter_it_carried_and_stamps_nothing(monkeypatch):
    redis = CountingRedis({key: 1.0 for key in POST_CALL_KEYS})
    redis.async_delete_cache = AsyncMock()

    async def _pipeline(increment_list: Sequence[Mapping[str, object]], **kwargs: object) -> list[float]:
        raise ConnectionError("redis down")

    redis.async_increment_pipeline = _pipeline  # pyright: ignore[reportAttributeAccessIssue]  # instance override
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    reservation = _reservation(reserved_cost=0.4)

    with pytest.raises(ConnectionError):
        await ps.increment_spend_counters(
            token="hashed",
            team_id="team",
            user_id="user",
            org_id="org",
            end_user_id="eu",
            response_cost=0.5,
            budget_reservation=reservation,
        )

    assert [c.split()[0] for c in redis.commands] == ["MGET"], redis.commands
    assert {call.kwargs["key"] for call in redis.async_delete_cache.await_args_list} == RESERVED_KEYS | {
        "spend:user:user"
    }
    assert all("applied_adjustment" not in entry for entry in reservation["entries"])
    assert {key: redis.store[key] for key in POST_CALL_KEYS} == {key: 1.0 for key in POST_CALL_KEYS}


def test_a_scope_opened_inside_an_open_scope_joins_its_batch_and_a_closed_one_gets_its_own():
    redis = CountingRedis()
    with spend_counter_batch_scope(redis, counter_keys=frozenset({"spend:key:a"})):
        outer = active_spend_counter_batch()
        assert outer is not None
        with spend_counter_batch_scope(redis, counter_keys=frozenset({"spend:key:b"})):
            assert active_spend_counter_batch() is outer
        assert outer.counter_keys == {"spend:key:a", "spend:key:b"}
        release_spend_counter_batch()
        with spend_counter_batch_scope(redis, counter_keys=frozenset({"spend:key:c"})):
            inner = active_spend_counter_batch()
            assert inner is not outer
            assert inner is not None and inner.counter_keys == {"spend:key:c"}
        assert active_spend_counter_batch() is outer


@pytest.mark.asyncio
async def test_reservation_inside_the_admission_scope_reuses_its_mget_and_reserves_in_one_pipeline(monkeypatch):
    from litellm.caching.dual_cache import DualCache
    from litellm.proxy._types import LiteLLM_TeamTableCachedObj
    from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request

    redis = CountingRedis({"spend:key:hashed": 1.0, "spend:team:team": 2.0})
    redis.default_ttl = 3600
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    monkeypatch.setattr("litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", lambda **_: 0.5)
    token = UserAPIKeyAuth(token="hashed", team_id="team", max_budget=10.0)

    with spend_counter_batch_scope(redis, counter_keys=admission_counter_keys(token, end_user_id=None)):
        reservation = await reserve_budget_for_request(
            request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]},
            route="/chat/completions",
            llm_router=None,
            valid_token=token,
            team_object=LiteLLM_TeamTableCachedObj(team_id="team", max_budget=20.0),
            user_object=None,
            prisma_client=None,
            user_api_key_cache=DualCache(),
            proxy_logging_obj=MagicMock(),
        )

    assert reservation is not None
    assert [c.split()[0] for c in redis.commands] == ["MGET", "PIPELINE"], redis.commands
    assert set(redis.commands[0].split()[1:]) == {"spend:key:hashed", "spend:team:team"}
    assert redis.commands[1] == "PIPELINE spend:key:hashed spend:team:team"
    assert redis.store == {"spend:key:hashed": 1.5, "spend:team:team": 2.5}
    assert [entry["counter_key"] for entry in reservation["entries"]] == ["spend:key:hashed", "spend:team:team"]


@pytest.mark.asyncio
async def test_a_failed_reservation_pipeline_drops_every_counter_and_reserves_nothing(monkeypatch):
    from litellm.caching.dual_cache import DualCache
    from litellm.proxy._types import LiteLLM_TeamTableCachedObj
    from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request

    redis = CountingRedis({"spend:key:hashed": 1.0, "spend:team:team": 2.0})
    redis.default_ttl = 3600
    redis.async_delete_cache = AsyncMock()

    async def _pipeline(increment_list: Sequence[Mapping[str, object]], **kwargs: object) -> list[float]:
        raise ConnectionError("redis down")

    redis.async_increment_pipeline = _pipeline  # pyright: ignore[reportAttributeAccessIssue]  # instance override
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    monkeypatch.setattr("litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", lambda **_: 0.5)
    token = UserAPIKeyAuth(token="hashed", team_id="team", max_budget=10.0)

    reservation = await reserve_budget_for_request(
        request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]},
        route="/chat/completions",
        llm_router=None,
        valid_token=token,
        team_object=LiteLLM_TeamTableCachedObj(team_id="team", max_budget=20.0),
        user_object=None,
        prisma_client=None,
        user_api_key_cache=DualCache(),
        proxy_logging_obj=MagicMock(),
    )

    assert reservation is None
    assert {call.kwargs["key"] for call in redis.async_delete_cache.await_args_list} == {
        "spend:key:hashed",
        "spend:team:team",
    }
    assert redis.store == {"spend:key:hashed": 1.0, "spend:team:team": 2.0}


@pytest.mark.asyncio
async def test_post_call_lifecycle_reads_the_counters_after_the_db_update_and_writes_one_pipeline(monkeypatch):
    from litellm.proxy.hooks.proxy_track_cost_callback import _update_database_and_spend_counters

    redis = CountingRedis({key: 1.0 for key in POST_CALL_KEYS})
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    proxy_logging_obj = MagicMock()

    async def _update_database(**kwargs: object) -> bool:
        redis.commands.append("DB")
        return True

    proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(side_effect=_update_database)
    reservation = _reservation(reserved_cost=0.4)

    charged = await _update_database_and_spend_counters(
        proxy_logging_obj=proxy_logging_obj,
        increment_spend_counters=ps.increment_spend_counters,
        user_api_key="hashed",
        user_id="user",
        end_user_id="eu",
        team_id="team",
        org_id="org",
        kwargs={},
        completion_response=None,
        start_time=None,
        end_time=None,
        response_cost=0.5,
        budget_reservation=reservation,
        request_tags=["prod"],
        model_access_groups=["premium"],
    )

    assert charged is True
    proxy_logging_obj.db_spend_update_writer.update_database.assert_awaited_once()
    assert [c.split()[0] for c in redis.commands] == ["MGET", "DB", "MGET", "PIPELINE"], redis.commands
    assert set(redis.commands[0].split()[1:]) == RESERVED_KEYS
    assert set(redis.commands[2].split()[1:]) == POST_CALL_KEYS
    assert set(redis.commands[3].split()[1:]) == POST_CALL_KEYS
    assert {key: round(redis.store[key], 6) for key in POST_CALL_KEYS} == {
        key: (1.1 if key in RESERVED_KEYS else 1.5) for key in POST_CALL_KEYS
    }
    assert [round(entry["applied_adjustment"], 6) for entry in reservation["entries"]] == [0.1] * len(RESERVED_KEYS)
    assert reservation["finalized"] is True
    assert active_spend_counter_batch() is None


def _reservation_fixture(monkeypatch, redis: CountingRedis) -> None:
    redis.default_ttl = 3600
    monkeypatch.setattr(ps, "spend_counter_cache", _spend_counter_cache(redis))
    monkeypatch.setattr(ps, "prisma_client", None)
    monkeypatch.setattr("litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", lambda **_: 0.5)


async def _reserve(redis: CountingRedis, token: UserAPIKeyAuth, team_max_budget: float) -> dict | None:
    from litellm.caching.dual_cache import DualCache
    from litellm.proxy._types import LiteLLM_TeamTableCachedObj
    from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request

    with spend_counter_batch_scope(redis, counter_keys=admission_counter_keys(token, end_user_id=None)):
        return await reserve_budget_for_request(
            request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]},
            route="/chat/completions",
            llm_router=None,
            valid_token=token,
            team_object=LiteLLM_TeamTableCachedObj(team_id="team", max_budget=team_max_budget),
            user_object=None,
            prisma_client=None,
            user_api_key_cache=DualCache(),
            proxy_logging_obj=MagicMock(),
        )


@pytest.mark.asyncio
async def test_a_rejected_counter_is_charged_alone_so_the_counters_after_it_are_never_touched(monkeypatch):
    """Only counters the admission MGET says still fit the estimate share the reservation pipeline; a counter that
    does not is charged on its own first, so its rejection never inflates a sibling counter, not even briefly."""
    redis = CountingRedis({"spend:key:hashed": 10.0, "spend:team:team": 2.0})
    _reservation_fixture(monkeypatch, redis)
    token = UserAPIKeyAuth(token="hashed", team_id="team", max_budget=10.0)

    with pytest.raises(litellm.BudgetExceededError):
        await _reserve(redis, token, team_max_budget=20.0)

    writes = [c for c in redis.commands if not c.startswith("MGET")]
    assert writes and all("spend:team:team" not in c for c in writes), redis.commands
    assert redis.store == {"spend:key:hashed": 10.0, "spend:team:team": 2.0}


@pytest.mark.asyncio
async def test_a_resized_reservation_is_carried_at_its_resized_cost_to_the_counters_charged_after_it(monkeypatch):
    redis = CountingRedis({"spend:key:hashed": 9.8, "spend:team:team": 2.0})
    _reservation_fixture(monkeypatch, redis)
    token = UserAPIKeyAuth(token="hashed", team_id="team", max_budget=10.0)

    reservation = await _reserve(redis, token, team_max_budget=2.1)

    assert reservation is not None
    assert reservation["reserved_cost"] == pytest.approx(0.1)
    assert redis.store["spend:key:hashed"] == pytest.approx(9.9)
    assert redis.store["spend:team:team"] == pytest.approx(2.1)