The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""Pin module-level spend functions.
Symbols pinned here:
- ``update_spend``
- ``update_daily_tag_spend``
- ``update_spend_logs_job``
- ``_monitor_spend_logs_queue``
- ``_raise_failed_update_spend_exception``
"""
from __future__ import annotations
import asyncio
import json
from collections.abc import Callable
from contextlib import suppress
from datetime import datetime, timezone
from typing import Any, Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.constants import REDIS_SPEND_LOGS_BUFFER_KEY
from litellm.proxy.utils import (
MAX_SPEND_LOG_DRAIN_ITERATIONS,
monitor_spend_logs_queue,
raise_failed_update_spend_exception,
drain_spend_logs_queue,
recover_parked_spend_logs,
update_daily_tag_spend,
update_spend,
update_spend_logs_job,
)
@pytest.mark.asyncio
async def test_update_spend_invokes_writer_and_skips_empty_queue(
mock_prisma_client: Any,
) -> None:
proxy_logging = MagicMock()
proxy_logging.db_spend_update_writer = MagicMock()
proxy_logging.db_spend_update_writer.db_update_spend_transaction_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = []
await update_spend(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
handler = proxy_logging.db_spend_update_writer.db_update_spend_transaction_handler
pinned = {
"handler_called": handler.await_count,
"handler_kwargs": handler.await_args.kwargs,
"queue_empty": mock_prisma_client.spend_log_transactions,
}
assert pinned == {
"handler_called": 1,
"handler_kwargs": {
"prisma_client": mock_prisma_client,
"n_retry_times": 3,
"proxy_logging_obj": proxy_logging,
},
"queue_empty": [],
}
@pytest.mark.asyncio
async def test_update_spend_processes_logs_when_queue_nonempty(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
proxy_logging = MagicMock()
proxy_logging.db_spend_update_writer = MagicMock()
proxy_logging.db_spend_update_writer.db_update_spend_transaction_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
import litellm.proxy.utils as utils_mod
job_mock = AsyncMock()
monkeypatch.setattr(utils_mod, "update_spend_logs_job", job_mock)
await update_spend(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert job_mock.await_count == 1
@pytest.mark.asyncio
async def test_update_spend_handler_failure_propagates(
mock_prisma_client: Any,
) -> None:
proxy_logging = MagicMock()
proxy_logging.db_spend_update_writer = MagicMock()
proxy_logging.db_spend_update_writer.db_update_spend_transaction_handler = AsyncMock(
side_effect=RuntimeError("handler down")
)
with pytest.raises(RuntimeError, match="handler down"):
await update_spend(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
@pytest.mark.asyncio
async def test_update_daily_tag_spend_redis_path_when_buffered(
mock_prisma_client: Any,
) -> None:
proxy_logging = MagicMock()
writer = MagicMock()
proxy_logging.db_spend_update_writer = writer
writer.redis_update_buffer = MagicMock()
writer.redis_update_buffer.should_commit_spend_updates_to_redis = MagicMock(return_value=True)
writer.commit_daily_tag_spend_to_db_with_redis = AsyncMock()
writer.commit_daily_tag_spend_to_db = AsyncMock()
await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
redis_kwargs = writer.commit_daily_tag_spend_to_db_with_redis.await_args.kwargs
pinned = {
"redis_calls": writer.commit_daily_tag_spend_to_db_with_redis.await_count,
"direct_calls": writer.commit_daily_tag_spend_to_db.await_count,
"redis_kwargs_keys": sorted(redis_kwargs.keys()),
"redis_n_retries": redis_kwargs["n_retry_times"],
}
assert pinned == {
"redis_calls": 1,
"direct_calls": 0,
"redis_kwargs_keys": sorted(["prisma_client", "n_retry_times", "proxy_logging_obj"]),
"redis_n_retries": 3,
}
@pytest.mark.asyncio
async def test_update_daily_tag_spend_direct_path_when_no_redis(
mock_prisma_client: Any,
) -> None:
proxy_logging = MagicMock()
writer = MagicMock()
proxy_logging.db_spend_update_writer = writer
writer.redis_update_buffer = MagicMock()
writer.redis_update_buffer.should_commit_spend_updates_to_redis = MagicMock(return_value=False)
writer.commit_daily_tag_spend_to_db_with_redis = AsyncMock()
writer.commit_daily_tag_spend_to_db = AsyncMock()
await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
assert writer.commit_daily_tag_spend_to_db.await_count == 1
assert writer.commit_daily_tag_spend_to_db_with_redis.await_count == 0
@pytest.mark.asyncio
async def test_update_daily_tag_spend_logs_and_swallows_errors(
mock_prisma_client: Any,
) -> None:
"""A failure in the commit path is logged but not re-raised; this matches
the historical behavior of this site (see plain ``logger.error`` rather
than ``spend_log_error``).
"""
proxy_logging = MagicMock()
proxy_logging.db_spend_update_writer = MagicMock()
proxy_logging.db_spend_update_writer.redis_update_buffer = MagicMock()
proxy_logging.db_spend_update_writer.redis_update_buffer.should_commit_spend_updates_to_redis = MagicMock(
return_value=False
)
proxy_logging.db_spend_update_writer.commit_daily_tag_spend_to_db = AsyncMock(
side_effect=RuntimeError("commit boom")
)
await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
@pytest.mark.asyncio
async def test_update_spend_logs_job_skips_when_queue_empty(
mock_prisma_client: Any,
) -> None:
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 0
@pytest.mark.asyncio
async def test_update_spend_logs_job_drains_tool_queue_when_spend_queue_empty(
mock_prisma_client: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
# Regression: a spend-log write failure aborts a run before the tool drain,
# so tool transactions can outlive the spend queue; the job must still run
# for them instead of early-returning on the empty spend queue.
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.tool_usage_transactions = [MagicMock()]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
flush_stub = AsyncMock()
monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", flush_stub, raising=False)
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert len(flush_stub.await_args.kwargs["transactions"]) == 1
assert mock_prisma_client.tool_usage_transactions == []
@pytest.mark.asyncio
async def test_update_spend_logs_job_drains_the_whole_model_usage_queue_in_one_run(
mock_prisma_client: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.model_usage_rollup as model_usage_mod
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
queued = [MagicMock() for _ in range(25_000)]
mock_prisma_client.model_usage_transactions = list(queued)
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False)
flush_stub = AsyncMock()
monkeypatch.setattr(model_usage_mod, "flush_model_usage_transactions", flush_stub, raising=False)
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert flush_stub.await_args.kwargs["transactions"] == queued
assert mock_prisma_client.model_usage_transactions == []
@pytest.mark.asyncio
async def test_update_spend_logs_job_processes_and_clears_queue(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
# Stub auxiliary imports so the test focuses on the spend-logs write path.
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False)
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
pinned = {
"create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count,
"queue_after": mock_prisma_client.spend_log_transactions,
"first_data_request_id": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["data"][0][
"request_id"
],
"skip_duplicates_set": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["skip_duplicates"],
}
assert pinned == {
"create_many_calls": 1,
"queue_after": [],
"first_data_request_id": "r1",
"skip_duplicates_set": True,
}
@pytest.mark.asyncio
async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled(
mock_prisma_client: Any, make_spend_log_row: Any
) -> None:
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
]
row_arriving_mid_flush = make_spend_log_row(request_id="r3")
async def _cancel_mid_write(*args: Any, **kwargs: Any) -> None:
mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush)
raise asyncio.CancelledError()
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_cancel_mid_write)
with pytest.raises(asyncio.CancelledError):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r1", "r2", "r3"]
@pytest.mark.asyncio
async def test_update_spend_logs_job_does_not_requeue_when_cancelled_after_write(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Rows are already committed once guardrail tracking runs, so replaying
them would double-count the non-idempotent daily guardrail increments.
"""
import litellm.proxy.guardrails.usage_tracking as guard_mod
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
monkeypatch.setattr(
guard_mod,
"process_spend_logs_guardrail_usage",
AsyncMock(side_effect=asyncio.CancelledError()),
raising=False,
)
with pytest.raises(asyncio.CancelledError):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
written: list[str] = []
async def _write(*args: Any, **kwargs: Any) -> None:
written.extend(row["request_id"] for row in kwargs["data"])
if len(written) == 1:
mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r2"))
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert written == ["r1", "r2"]
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
write_started = asyncio.Event()
written: list[str] = []
write_calls = {"n": 0}
async def _write(*args: Any, **kwargs: Any) -> None:
write_calls["n"] += 1
if write_calls["n"] == 1:
write_started.set()
await asyncio.Event().wait()
written.extend(row["request_id"] for row in kwargs["data"])
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
async def _monitor() -> None:
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
mock_prisma_client.spend_logs_queue_monitor_task = asyncio.create_task(_monitor())
await write_started.wait()
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert written == ["r1"]
assert mock_prisma_client.spend_log_transactions == []
assert mock_prisma_client.spend_logs_queue_monitor_task is None
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_gives_up_after_max_passes(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == MAX_SPEND_LOG_DRAIN_ITERATIONS
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False)
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_SIZE_THRESHOLD", 1, raising=False)
proxy_logging = MagicMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
cancel_after = {"n": 0}
async def _fake_job(*args: Any, **kwargs: Any) -> None:
cancel_after["n"] += 1
if cancel_after["n"] >= 1:
raise asyncio.CancelledError()
monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
with pytest.raises(asyncio.CancelledError):
await monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert cancel_after["n"] == 1
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_swallows_errors_and_backs_off(
mock_prisma_client: Any,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""An exception inside the loop is logged with backoff and the loop
continues running rather than crashing the monitor task.
"""
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False)
sleep_count = {"n": 0}
async def _short_sleep(_: float, *args: Any, **kwargs: Any) -> None:
sleep_count["n"] += 1
if sleep_count["n"] >= 3:
raise asyncio.CancelledError()
monkeypatch.setattr(utils_mod.asyncio, "sleep", _short_sleep)
proxy_logging = MagicMock()
bad_lock = MagicMock()
bad_lock.__aenter__ = AsyncMock(side_effect=RuntimeError("lock broken"))
bad_lock.__aexit__ = AsyncMock(return_value=False)
mock_prisma_client._spend_log_transactions_lock = bad_lock
with pytest.raises(asyncio.CancelledError):
await monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert sleep_count["n"] == 3
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_flushes_as_soon_as_one_is_requested(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A requested flush wakes the monitor mid-poll, so a Responses row reaches the DB
before the client can chain a `previous_response_id` off it.
"""
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
from litellm.proxy.utils import request_spend_log_flush
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.tool_usage_transactions = []
flushed: Final = asyncio.Event()
async def _fake_job(*args: Any, **kwargs: Any) -> None:
flushed.set()
monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
monitor: Final = asyncio.create_task(
monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
)
try:
await asyncio.sleep(0.05)
assert not flushed.is_set()
assert isinstance(mock_prisma_client.spend_log_flush_requested, asyncio.Event)
mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r1"))
request_spend_log_flush(mock_prisma_client)
await asyncio.wait_for(flushed.wait(), timeout=5.0)
finally:
monitor.cancel()
with suppress(asyncio.CancelledError):
await monitor
def test_monitor_spend_logs_queue_flush_survives_an_earlier_event_loop(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A second monitor, started in a fresh event loop, is still woken by a flush request,
so a worker whose first loop is gone keeps flushing Responses rows instead of stalling.
"""
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
from litellm.proxy.utils import request_spend_log_flush
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
mock_prisma_client.tool_usage_transactions = []
async def _flush_once_under_a_monitor() -> None:
flushed: Final = asyncio.Event()
async def _fake_job(*args: Any, **kwargs: Any) -> None:
flushed.set()
monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
mock_prisma_client.spend_log_transactions = []
monitor: Final = asyncio.create_task(
monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
)
try:
await asyncio.sleep(0.05)
assert not flushed.is_set()
mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r1"))
request_spend_log_flush(mock_prisma_client)
await asyncio.wait_for(flushed.wait(), timeout=5.0)
finally:
monitor.cancel()
with suppress(asyncio.CancelledError):
await monitor
asyncio.run(_flush_once_under_a_monitor())
asyncio.run(_flush_once_under_a_monitor())
@pytest.mark.asyncio
async def test_flush_requested_before_the_monitor_starts_costs_the_row_nothing(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A Responses row enqueued before the monitor exists still reaches the DB on its first
pass, so dropping that early request delays nothing.
"""
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
from litellm.proxy.utils import request_spend_log_flush
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
mock_prisma_client.spend_log_flush_requested = None
mock_prisma_client.tool_usage_transactions = []
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
flushed: Final = asyncio.Event()
async def _fake_job(*args: Any, **kwargs: Any) -> None:
flushed.set()
monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
request_spend_log_flush(mock_prisma_client)
assert mock_prisma_client.spend_log_flush_requested is None
monitor: Final = asyncio.create_task(
monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
)
try:
await asyncio.wait_for(flushed.wait(), timeout=5.0)
finally:
monitor.cancel()
with suppress(asyncio.CancelledError):
await monitor
def test_raise_failed_update_spend_exception_emits_failure_handler() -> None:
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
async def _runner() -> Any:
try:
raise_failed_update_spend_exception(
e=RuntimeError("boom"),
start_time=0.0,
proxy_logging_obj=proxy_logging,
)
except RuntimeError as e:
return e
return None
err = asyncio.run(_runner())
pinned = {
"raised": str(err),
"failure_handler_called": proxy_logging.failure_handler.call_count,
"call_type": (
proxy_logging.failure_handler.call_args.kwargs.get("call_type")
if proxy_logging.failure_handler.call_args
else None
),
"non_blocking_in_traceback": (
"Non-Blocking" in proxy_logging.failure_handler.call_args.kwargs["traceback_str"]
if proxy_logging.failure_handler.call_args
else False
),
}
assert pinned == {
"raised": "boom",
"failure_handler_called": 1,
"call_type": "update_spend",
"non_blocking_in_traceback": True,
}
def test_raise_failed_update_spend_exception_raises_original_error() -> None:
"""Error path: the function always re-raises the original exception so
the caller can observe the failure.
"""
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
async def _runner() -> None:
raise_failed_update_spend_exception(
e=ValueError("specific"),
start_time=0.0,
proxy_logging_obj=proxy_logging,
)
with pytest.raises(ValueError, match="specific"):
asyncio.run(_runner())
def _table_gone_error() -> Exception:
from prisma.errors import TableNotFoundError
return TableNotFoundError(
{"user_facing_error": {"error_code": "P2021", "message": "The table does not exist", "meta": {}}}
)
def _parked_request_ids(fake_redis: Any) -> list[str]:
return [json.loads(row)["request_id"] for row in fake_redis.items.get(REDIS_SPEND_LOGS_BUFFER_KEY, [])]
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_parks_unwritable_rows_in_redis_on_shutdown(
mock_prisma_client: Any, make_spend_log_row: Any, proxy_logging_with_redis: MagicMock, fake_redis: Any
) -> None:
from prisma.errors import TableNotFoundError
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_table_gone_error())
with pytest.raises(TableNotFoundError):
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
assert mock_prisma_client.spend_log_transactions == []
assert sorted(_parked_request_ids(fake_redis)) == ["r1", "r2"]
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_waits_for_an_in_flight_write_before_parking(
mock_prisma_client: Any, make_spend_log_row: Any, proxy_logging_with_redis: MagicMock, fake_redis: Any
) -> None:
db_outage_seen: Final = asyncio.Event()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="in-flight")]
async def _fail_once_shutdown_starts(*args: Any, **kwargs: Any) -> None:
await db_outage_seen.wait()
raise _table_gone_error()
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_fail_once_shutdown_starts)
scheduler_write: Final = asyncio.ensure_future(
update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
)
await asyncio.sleep(0)
assert mock_prisma_client.spend_log_transactions == []
async def _release_after_shutdown_started() -> None:
await asyncio.sleep(0.05)
db_outage_seen.set()
release: Final = asyncio.ensure_future(_release_after_shutdown_started())
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
assert _parked_request_ids(fake_redis) == ["in-flight"]
assert mock_prisma_client.spend_log_transactions == []
await release
with suppress(Exception):
await scheduler_write
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_parks_rows_left_after_max_passes(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
proxy_logging_with_redis: MagicMock,
fake_redis: Any,
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False)
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r0")]
async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="late"))
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
assert mock_prisma_client.spend_log_transactions == []
assert _parked_request_ids(fake_redis) == ["late"]
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_keeps_rows_in_memory_when_redis_is_down(
mock_prisma_client: Any, make_spend_log_row: Any, proxy_logging_with_redis: MagicMock, fake_redis: Any
) -> None:
from prisma.errors import TableNotFoundError
fake_redis.down = True
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_table_gone_error())
with pytest.raises(TableNotFoundError):
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r1"]
assert fake_redis.items == {}
@pytest.mark.asyncio
async def test_update_spend_writes_rows_parked_in_redis_by_a_previous_pod(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
proxy_logging_with_redis: MagicMock,
fake_redis: Any,
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False)
buffer = proxy_logging_with_redis.db_spend_update_writer.redis_update_buffer
assert await buffer.store_spend_logs_in_redis([make_spend_log_row(request_id="parked")]) is True
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
await update_spend(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
written = mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["data"]
assert [row["request_id"] for row in written] == ["parked"]
assert _parked_request_ids(fake_redis) == []
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_recover_parked_spend_logs_re_parks_rows_when_the_enqueue_is_cancelled(
mock_prisma_client: Any, make_spend_log_row: Any, proxy_logging_with_redis: MagicMock, fake_redis: Any
) -> None:
buffer = proxy_logging_with_redis.db_spend_update_writer.redis_update_buffer
assert await buffer.store_spend_logs_in_redis([make_spend_log_row(request_id="parked")]) is True
mock_prisma_client.spend_log_transactions = []
await mock_prisma_client._spend_log_transactions_lock.acquire()
recovery: Final = asyncio.ensure_future(
recover_parked_spend_logs(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging_with_redis)
)
await asyncio.sleep(0.01)
assert _parked_request_ids(fake_redis) == []
recovery.cancel()
with pytest.raises(asyncio.CancelledError):
await recovery
mock_prisma_client._spend_log_transactions_lock.release()
assert _parked_request_ids(fake_redis) == ["parked"]
assert mock_prisma_client.spend_log_transactions == []
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_pulls_parked_rows_before_each_flush(
mock_prisma_client: Any,
make_spend_log_row: Any,
monkeypatch: pytest.MonkeyPatch,
proxy_logging_with_redis: MagicMock,
) -> None:
import litellm.constants as constants_mod
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False)
buffer = proxy_logging_with_redis.db_spend_update_writer.redis_update_buffer
assert await buffer.store_spend_logs_in_redis([make_spend_log_row(request_id="parked")]) is True
mock_prisma_client.spend_log_transactions = []
seen: list[list[str]] = []
polls = {"n": 0}
async def _fake_job(*args: Any, **kwargs: Any) -> None:
seen.append([row["request_id"] for row in mock_prisma_client.spend_log_transactions])
raise asyncio.CancelledError()
async def _poll(*args: Any, **kwargs: Any) -> bool:
polls["n"] += 1
if polls["n"] >= 3:
raise asyncio.CancelledError()
return False
monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
monkeypatch.setattr(utils_mod, "_wait_for_spend_log_flush_request", _poll)
with pytest.raises(asyncio.CancelledError):
await monitor_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging_with_redis,
)
assert seen == [["parked"]]
def _postgres_out_of_connections() -> Exception:
"""prisma's shape for Postgres SQLSTATE 53300: a base ``DataError`` whose only
hint is the connector message."""
from prisma.errors import DataError
return DataError(
data={
"user_facing_error": {
"is_panic": False,
"message": "Error in connector: Error querying the database: FATAL: sorry, too many clients already",
"backtrace": None,
}
}
)
@pytest.mark.asyncio
async def test_update_spend_logs_job_requeues_whole_batch_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
make_spend_log_row: Callable[..., dict[str, object]],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""53300 is not a poison row: bisecting it would issue one failing statement
per row (each a fresh connection attempt against a full server) and drop
every row. The batch goes back to the queue head untouched, in one attempt,
without the in-job retry loop hammering the server."""
sleeps: Final[list[float]] = []
async def _no_sleep(seconds: float, *_: object, **__: object) -> None:
sleeps.append(seconds)
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
proxy_logging: Final = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
make_spend_log_row(request_id="r3"),
]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
with pytest.raises(Exception, match="too many clients already"):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert {
"create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count,
"queue_after": [row["request_id"] for row in mock_prisma_client.spend_log_transactions],
"backoff_sleeps": sleeps,
} == {"create_many_calls": 1, "queue_after": ["r1", "r2", "r3"], "backoff_sleeps": []}
def _tool_usage_transaction(request_id: str) -> object:
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
return ToolUsageTransaction(
request_id=request_id,
date="2026-10-02",
start_time=datetime(2026, 10, 2, tzinfo=timezone.utc),
tool_names=("get_weather",),
spend=0.01,
total_tokens=12,
)
@pytest.mark.asyncio
async def test_tool_usage_flush_requeues_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
) -> None:
"""A tool-usage batch Postgres had no connection for was never sent, so it is
safe to keep; dropping it loses the rollup increments for good. The job stops
there, like the spend-log write, so a drain loop does not re-hit the full server."""
from prisma.errors import DataError
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
mock_prisma_client.spend_log_transactions = []
first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2")
mock_prisma_client.tool_usage_transactions = [first, second]
with pytest.raises(DataError, match="too many clients already"):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert {
"index_writes": mock_prisma_client.db.litellm_spendlogtoolindex.create_many.await_count,
"queue_after": mock_prisma_client.tool_usage_transactions,
} == {"index_writes": 1, "queue_after": [first, second]}
@pytest.mark.asyncio
async def test_tool_usage_flush_still_drops_ambiguous_failures(
mock_prisma_client: MagicMock,
) -> None:
"""Anything other than a connection refusal may have reached the server, and
the rollup increments are not idempotent, so the batch is not replayed."""
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(
side_effect=RuntimeError("engine returned a malformed payload")
)
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.tool_usage_transactions = [_tool_usage_transaction("r1")]
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert mock_prisma_client.tool_usage_transactions == []
def _postgres_lock_timeout() -> Exception:
"""prisma's shape for Postgres SQLSTATE 55P03: the rollup upsert waited past
``lock_timeout`` for a row another pod held and was cancelled before it took the lock."""
from prisma.errors import DataError
return DataError(
data={
"user_facing_error": {
"is_panic": False,
"message": "Error querying the database: canceling statement due to lock timeout",
"meta": {"code": "55P03", "message": "canceling statement due to lock timeout"},
}
}
)
@pytest.mark.asyncio
async def test_tool_usage_flush_requeues_when_the_rollup_row_lock_times_out(
mock_prisma_client: MagicMock,
) -> None:
"""A rollup transaction cancelled by ``lock_timeout`` never applied: the waiter
gives its connection back instead of pinning it behind the holder, and its
increments go back to the queue head for the next flush rather than being dropped.
Unlike a full server, one contended row says nothing about the other writers, so the
job carries on: the model usage rollup still opens its own transaction and the
auto-router drain still runs."""
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock()
rollup_batch = MagicMock()
rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock())
rollup_batch.__aexit__ = AsyncMock(side_effect=_postgres_lock_timeout())
mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch)
mock_prisma_client.spend_log_transactions = []
first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2")
mock_prisma_client.tool_usage_transactions = [first, second]
mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")]
mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert {
"rollup_transactions": mock_prisma_client.db.batch_.call_count,
"queue_after": mock_prisma_client.tool_usage_transactions,
"autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions,
} == {"rollup_transactions": 2, "queue_after": [first, second], "autorouter_queue_after": []}
def _model_usage_transaction(model: str) -> object:
from litellm.proxy.db.model_usage_rollup import ModelUsageKey, ModelUsageTransaction
return ModelUsageTransaction(
key=ModelUsageKey(
date="2026-10-03", model_group="gpt", model=model, custom_llm_provider="openai", task_type="chat"
),
spend=0.001,
prompt_tokens=1,
completion_tokens=1,
successful=True,
)
def _model_usage_batch_failing_with(error: Exception, mock_prisma_client: MagicMock) -> MagicMock:
"""Point the model usage rollup's ``batch_()`` at a transaction that fails on commit, with
no tool usage queued so the tool rollup never opens one."""
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock()
mock_prisma_client.tool_usage_transactions = []
mock_prisma_client.spend_log_transactions = []
rollup_batch = MagicMock()
rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock())
rollup_batch.__aexit__ = AsyncMock(side_effect=error)
mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch)
return rollup_batch
@pytest.mark.asyncio
async def test_model_usage_flush_requeues_and_carries_on_when_its_row_lock_times_out(
mock_prisma_client: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The DailyModelUsage rollup shares its rows across every pod exactly like the tool rollup.
A batch cancelled under ``lock_timeout`` never applied, so its transactions go back to the
queue head for the next flush instead of being dropped, and because only that rollup's row
was contended the job still drains the auto-router queue behind it."""
monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None))
_model_usage_batch_failing_with(_postgres_lock_timeout(), mock_prisma_client)
first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini")
mock_prisma_client.model_usage_transactions = [first, second]
mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert {
"queue_after": mock_prisma_client.model_usage_transactions,
"autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions,
} == {"queue_after": [first, second], "autorouter_queue_after": []}
@pytest.mark.asyncio
async def test_model_usage_flush_requeues_and_stops_the_job_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A batch refused for want of a connection (53300) was never sent either, so it is kept, but
a full server will refuse the writers behind it too: the job stops there like the tool
rollup does, so the drain loop backs off instead of re-hitting the server."""
monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None))
_model_usage_batch_failing_with(_postgres_out_of_connections(), mock_prisma_client)
first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini")
mock_prisma_client.model_usage_transactions = [first, second]
mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
with pytest.raises(Exception, match="too many clients already"):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert {
"queue_after": mock_prisma_client.model_usage_transactions,
"autorouter_queue_after": len(mock_prisma_client.autorouter_turn_transactions),
} == {"queue_after": [first, second], "autorouter_queue_after": 1}
@pytest.mark.asyncio
async def test_model_usage_flush_still_drops_a_batch_that_may_have_applied(
mock_prisma_client: MagicMock,
) -> None:
"""Any other failure is ambiguous about whether the increments landed, so replaying it could
double-count: the batch is logged and dropped, and the job carries on."""
_model_usage_batch_failing_with(RuntimeError("engine lost mid-commit"), mock_prisma_client)
mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")]
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert mock_prisma_client.model_usage_transactions == []