The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""Spend accumulation for model access group budgets."""
import asyncio
from collections.abc import Mapping, Sequence
import pytest
from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
from litellm.proxy._types import DBSpendUpdateTransactions, Litellm_EntityType, SpendUpdateQueueItem
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter, debitable_model_access_groups
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue
from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer
from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups
class _FakeRouter:
"""Deployment lookup returning the access groups each deployment declares."""
def __init__(self, deployments: Mapping[str, Sequence[str] | None]) -> None:
self._deployments = deployments
def get_model_info(self, id: str) -> dict | None:
if id not in self._deployments:
return None
declared = self._deployments[id]
model_info: dict = {"id": id}
if declared is not None:
model_info["access_groups"] = list(declared)
return {"model_name": "some-model", "model_info": model_info}
class _FakeBatchTable:
def __init__(self) -> None:
self.calls: list[tuple[dict, dict]] = []
def update_many(self, where: dict, data: dict) -> None:
self.calls.append((where, data))
class _FakeBatcher:
def __init__(self) -> None:
self.tables: dict[str, _FakeBatchTable] = {}
def __getattr__(self, name: str) -> _FakeBatchTable:
return self.tables.setdefault(name, _FakeBatchTable())
class _FakeBatchManager:
def __init__(self, batcher: _FakeBatcher) -> None:
self._batcher = batcher
async def __aenter__(self) -> _FakeBatcher:
return self._batcher
async def __aexit__(self, *exc_info: object) -> bool:
return False
class _FakeTransaction:
def __init__(self, batcher: _FakeBatcher) -> None:
self._batcher = batcher
def batch_(self) -> _FakeBatchManager:
return _FakeBatchManager(self._batcher)
async def execute_raw(self, query: str, *args: object) -> int:
return 0
async def __aenter__(self) -> "_FakeTransaction":
return self
async def __aexit__(self, *exc_info: object) -> bool:
return False
class _FakeDb:
def __init__(self, batcher: _FakeBatcher) -> None:
self._batcher = batcher
def tx(self, timeout: object = None) -> _FakeTransaction:
return _FakeTransaction(self._batcher)
class _FakePrismaClient:
def __init__(self) -> None:
self.batcher = _FakeBatcher()
self.db = _FakeDb(self.batcher)
def _empty_transactions(**overrides: dict[str, float]) -> DBSpendUpdateTransactions:
return DBSpendUpdateTransactions(
user_list_transactions=overrides.get("user_list_transactions", {}),
end_user_list_transactions=overrides.get("end_user_list_transactions", {}),
key_list_transactions=overrides.get("key_list_transactions", {}),
team_list_transactions=overrides.get("team_list_transactions", {}),
team_member_list_transactions=overrides.get("team_member_list_transactions", {}),
org_list_transactions=overrides.get("org_list_transactions", {}),
tag_list_transactions=overrides.get("tag_list_transactions", {}),
agent_list_transactions=overrides.get("agent_list_transactions", {}),
model_access_group_list_transactions=overrides.get("model_access_group_list_transactions", {}),
)
async def _drain(queue: SpendUpdateQueue) -> list[SpendUpdateQueueItem]:
return await queue.flush_all_updates_from_in_memory_queue()
# --- enqueue ---------------------------------------------------------------
@pytest.mark.asyncio
async def test_single_matched_group_enqueues_one_item_with_full_cost():
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=0.42,
request_model_access_groups=["premium-pool"],
served_model_id="deployment-1",
prisma_client=object(),
router=_FakeRouter({"deployment-1": ["premium-pool"]}),
)
updates = await _drain(writer.spend_update_queue)
assert updates == [
SpendUpdateQueueItem(
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP,
entity_id="premium-pool",
response_cost=0.42,
)
]
@pytest.mark.asyncio
async def test_every_matched_group_is_charged_the_full_cost_not_a_split():
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=0.30,
request_model_access_groups=["pool-a", "pool-b", "pool-c"],
served_model_id="deployment-1",
prisma_client=object(),
router=_FakeRouter({"deployment-1": ["pool-a", "pool-b", "pool-c"]}),
)
updates = await _drain(writer.spend_update_queue)
assert [update["entity_id"] for update in updates] == ["pool-a", "pool-b", "pool-c"]
assert [update["response_cost"] for update in updates] == [0.30, 0.30, 0.30]
assert {update["entity_type"] for update in updates} == {Litellm_EntityType.MODEL_ACCESS_GROUP}
@pytest.mark.parametrize("attributed", [None, [], ()])
@pytest.mark.asyncio
async def test_no_attributed_groups_enqueues_nothing(attributed):
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=1.0,
request_model_access_groups=attributed,
served_model_id="deployment-1",
prisma_client=object(),
router=_FakeRouter({"deployment-1": ["premium-pool"]}),
)
assert await _drain(writer.spend_update_queue) == []
@pytest.mark.asyncio
async def test_no_prisma_client_enqueues_nothing():
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=1.0,
request_model_access_groups=["premium-pool"],
served_model_id="deployment-1",
prisma_client=None,
router=_FakeRouter({"deployment-1": ["premium-pool"]}),
)
assert await _drain(writer.spend_update_queue) == []
@pytest.mark.asyncio
async def test_group_outside_the_attributed_set_is_never_debited():
"""The served deployment also sits in a pool auth never attributed; that pool stays untouched."""
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=0.10,
request_model_access_groups=["premium-pool"],
served_model_id="deployment-1",
prisma_client=object(),
router=_FakeRouter({"deployment-1": ["premium-pool", "unattributed-pool"]}),
)
updates = await _drain(writer.spend_update_queue)
assert [update["entity_id"] for update in updates] == ["premium-pool"]
# --- fallback guard --------------------------------------------------------
def test_fallback_to_a_model_in_another_pool_debits_nothing():
assert (
debitable_model_access_groups(
attributed=["premium-pool"],
served_model_id="fallback-deployment",
router=_FakeRouter({"fallback-deployment": ["cheap-pool"]}),
)
== ()
)
def test_fallback_to_a_model_in_no_pool_debits_nothing():
assert (
debitable_model_access_groups(
attributed=["premium-pool"],
served_model_id="fallback-deployment",
router=_FakeRouter({"fallback-deployment": None}),
)
== ()
)
def test_attributed_set_stands_when_the_served_deployment_is_unknown():
assert debitable_model_access_groups(
attributed=["premium-pool"],
served_model_id="not-in-router",
router=_FakeRouter({"deployment-1": ["premium-pool"]}),
) == ("premium-pool",)
def test_attributed_set_stands_without_a_router():
assert debitable_model_access_groups(
attributed=["premium-pool", "premium-pool"],
served_model_id="deployment-1",
router=None,
) == ("premium-pool",)
def test_partial_overlap_keeps_only_the_intersection():
assert debitable_model_access_groups(
attributed=["pool-a", "pool-b"],
served_model_id="deployment-1",
router=_FakeRouter({"deployment-1": ["pool-b", "pool-c"]}),
) == ("pool-b",)
def test_only_real_group_names_ever_become_entity_ids():
"""Whatever shape the attributed set arrives in, an empty or non-string name never reaches the queue."""
assert debitable_model_access_groups(
attributed=["pool-a", "", "pool-a", None, 7],
served_model_id=None,
router=None,
) == ("pool-a",)
# --- metadata extraction ---------------------------------------------------
def test_access_groups_read_from_request_metadata():
kwargs = {"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["pool-a", "pool-b", "pool-a"]}}}
assert get_request_model_access_groups(kwargs) == ("pool-a", "pool-b")
def test_access_groups_read_from_litellm_metadata():
kwargs = {"litellm_params": {"litellm_metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["pool-a"]}}}
assert get_request_model_access_groups(kwargs) == ("pool-a",)
def test_standard_logging_payload_wins_over_metadata():
kwargs = {
"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["from-metadata"]}},
"standard_logging_object": {"request_model_access_groups": ["from-payload"]},
}
assert get_request_model_access_groups(kwargs) == ("from-payload",)
def test_metadata_is_used_when_the_logging_payload_carries_no_groups():
kwargs = {
"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["from-metadata"]}},
"standard_logging_object": {"request_model_access_groups": []},
}
assert get_request_model_access_groups(kwargs) == ("from-metadata",)
@pytest.mark.parametrize("stamped", ["pool-a", 7, {"pool-a": 1}])
def test_non_list_access_group_metadata_is_ignored(stamped):
kwargs = {"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: stamped}}}
assert get_request_model_access_groups(kwargs) == ()
def test_key_absent_from_metadata_yields_no_groups():
"""The chat path only stamps the key when something matched, so absent must mean nothing to debit."""
kwargs = {"litellm_params": {"metadata": {"user_api_key_user_id": "u-1"}}}
assert get_request_model_access_groups(kwargs) == ()
def test_explicit_none_yields_no_groups():
"""The pass-through path stamps the key unconditionally, so it can be present and None."""
kwargs = {"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: None}}}
assert get_request_model_access_groups(kwargs) == ()
@pytest.mark.parametrize(
"metadata",
[
{"user_api_key_user_id": "u-1"},
{MODEL_ACCESS_GROUP_METADATA_KEY: None},
],
ids=["key-absent", "key-present-but-none"],
)
@pytest.mark.asyncio
async def test_neither_absent_nor_none_metadata_debits_anything(metadata):
writer = DBSpendUpdateWriter()
await writer._update_model_access_group_db(
response_cost=0.5,
request_model_access_groups=get_request_model_access_groups({"litellm_params": {"metadata": metadata}}),
served_model_id="deployment-1",
prisma_client=object(),
router=_FakeRouter({"deployment-1": ["premium-pool"]}),
)
assert await _drain(writer.spend_update_queue) == []
def test_detached_sub_call_falls_back_to_the_auth_object():
"""Sub-calls inherit only the identity keys, so the groups come off user_api_key_auth there."""
class _Auth:
matched_model_access_groups = ["premium-pool"]
kwargs = {"litellm_params": {"metadata": {"user_api_key_auth": _Auth()}}}
assert get_request_model_access_groups(kwargs) == ("premium-pool",)
def test_stamped_metadata_wins_over_the_auth_object():
class _Auth:
matched_model_access_groups = ["stale-pool"]
kwargs = {
"litellm_params": {
"metadata": {
MODEL_ACCESS_GROUP_METADATA_KEY: ["fresh-pool"],
"user_api_key_auth": _Auth(),
}
}
}
assert get_request_model_access_groups(kwargs) == ("fresh-pool",)
def test_auth_object_without_matched_groups_yields_no_groups():
class _Auth:
matched_model_access_groups = None
kwargs = {"litellm_params": {"metadata": {"user_api_key_auth": _Auth()}}}
assert get_request_model_access_groups(kwargs) == ()
def test_non_string_entries_are_dropped():
kwargs = {"litellm_params": {"metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["pool-a", None, "", 3]}}}
assert get_request_model_access_groups(kwargs) == ("pool-a",)
def test_missing_metadata_yields_no_groups():
assert get_request_model_access_groups(None) == ()
assert get_request_model_access_groups({}) == ()
assert get_request_model_access_groups({"litellm_params": {}}) == ()
# --- queue bucketing and redis round trip ----------------------------------
def test_access_group_updates_aggregate_into_their_own_bucket():
queue = SpendUpdateQueue()
transactions = queue.get_aggregated_db_spend_update_transactions(
[
SpendUpdateQueueItem(
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP, entity_id="pool-a", response_cost=0.1
),
SpendUpdateQueueItem(
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP, entity_id="pool-a", response_cost=0.2
),
SpendUpdateQueueItem(
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP, entity_id="pool-b", response_cost=0.5
),
SpendUpdateQueueItem(entity_type=Litellm_EntityType.TAG, entity_id="pool-a", response_cost=9.0),
]
)
assert transactions["model_access_group_list_transactions"] == {"pool-a": pytest.approx(0.3), "pool-b": 0.5}
assert transactions["tag_list_transactions"] == {"pool-a": 9.0}
def test_access_group_transactions_survive_the_redis_buffer_merge():
merged = RedisUpdateBuffer._combine_list_of_transactions(
[
_empty_transactions(model_access_group_list_transactions={"pool-a": 0.25}),
_empty_transactions(model_access_group_list_transactions={"pool-a": 0.25, "pool-b": 1.0}),
]
)
assert merged["model_access_group_list_transactions"] == {"pool-a": 0.5, "pool-b": 1.0}
@pytest.mark.asyncio
async def test_redis_buffer_requeues_access_group_transactions_as_queue_items():
queue = SpendUpdateQueue()
daily_queue = DailySpendUpdateQueue()
await RedisUpdateBuffer._restore_spend_updates_to_in_memory_queues(
db_spend_update_transactions=_empty_transactions(model_access_group_list_transactions={"pool-a": 0.75}),
daily_spend_update_transactions=None,
daily_team_spend_update_transactions=None,
daily_org_spend_update_transactions=None,
daily_end_user_spend_update_transactions=None,
daily_agent_spend_update_transactions=None,
window_spend_update_transactions=None,
spend_update_queue=queue,
daily_spend_update_queue=daily_queue,
daily_team_spend_update_queue=daily_queue,
daily_org_spend_update_queue=daily_queue,
daily_end_user_spend_update_queue=daily_queue,
daily_agent_spend_update_queue=daily_queue,
window_spend_update_queue=None,
)
updates = await _drain(queue)
assert updates == [
SpendUpdateQueueItem(
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP,
entity_id="pool-a",
response_cost=0.75,
)
]
# --- flush to postgres -----------------------------------------------------
@pytest.mark.asyncio
async def test_commit_increments_spend_on_the_model_access_group_budget_table():
prisma_client = _FakePrismaClient()
await DBSpendUpdateWriter()._commit_spend_updates_to_db(
prisma_client=prisma_client,
n_retry_times=0,
proxy_logging_obj=None,
db_spend_update_transactions=_empty_transactions(
model_access_group_list_transactions={"pool-b": 0.5, "pool-a": 0.25}
),
)
assert prisma_client.batcher.tables["litellm_modelaccessgroupbudgettable"].calls == [
({"access_group_name": "pool-a"}, {"spend": {"increment": 0.25}}),
({"access_group_name": "pool-b"}, {"spend": {"increment": 0.5}}),
]
assert "litellm_tagtable" not in prisma_client.batcher.tables
# --- end-to-end through the batched fan-out --------------------------------
@pytest.mark.asyncio
async def test_batch_database_updates_enqueues_access_group_spend():
writer = DBSpendUpdateWriter()
await writer._batch_database_updates(
response_cost=0.15,
user_id=None,
hashed_token=None,
team_id=None,
org_id=None,
end_user_id=None,
prisma_client=object(),
litellm_proxy_budget_name=None,
payload={"model_id": "deployment-1", "spend": 0.15},
request_model_access_groups=("pool-a", "pool-b"),
)
await asyncio.sleep(0)
access_group_updates = [
update
for update in await _drain(writer.spend_update_queue)
if update["entity_type"] is Litellm_EntityType.MODEL_ACCESS_GROUP
]
assert [(update["entity_id"], update["response_cost"]) for update in access_group_updates] == [
("pool-a", 0.15),
("pool-b", 0.15),
]
@pytest.mark.asyncio
async def test_batch_database_updates_enqueues_nothing_without_access_groups():
writer = DBSpendUpdateWriter()
await writer._batch_database_updates(
response_cost=0.15,
user_id=None,
hashed_token=None,
team_id=None,
org_id=None,
end_user_id=None,
prisma_client=object(),
litellm_proxy_budget_name=None,
payload={"model_id": "deployment-1", "spend": 0.15},
)
await asyncio.sleep(0)
updates = await _drain(writer.spend_update_queue)
assert [update for update in updates if update["entity_type"] is Litellm_EntityType.MODEL_ACCESS_GROUP] == []