Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

fix(proxy): flush tool discovery queue when use_redis_transaction_buffer is enabled (#47653)

mrinal-berridba4fa0e944e
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index e7909cbe600..bb6ffe675c0 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -1651,6 +1651,8 @@ class DBSpendUpdateWriter:
window_spend_update_queue=self.window_spend_update_queue,
)
+ await self._flush_tool_discovery_queue(prisma_client=prisma_client)
+
# Only commit from redis to db if this pod is the leader
if await self.pod_lock_manager.acquire_lock(
cronjob_id=DB_SPEND_UPDATE_JOB_NAME,
diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py
index d440b99a632..7437042e4bd 100644
--- a/tests/unit/proxy/db/test_db_spend_update_writer.py
+++ b/tests/unit/proxy/db/test_db_spend_update_writer.py
@@ -19,7 +19,7 @@ from redis.exceptions import DataError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._service_logger import ServiceTypes
-from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
+from litellm.constants import DB_SPEND_UPDATE_JOB_NAME, REDIS_UPDATE_BUFFER_KEY, SPEND_ROLLUP_LOCK_TIMEOUT_MS
from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem
from litellm.proxy.db.db_spend_update_writer import (
_TEAM_ADVISORY_LOCK_SQL,
@@ -29,12 +29,14 @@ from litellm.proxy.db.db_spend_update_writer import (
_SpendTableName,
)
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue
+from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer
from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
build_window_spend_transaction,
)
from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
+from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@@ -2365,6 +2367,58 @@ async def test_commit_spend_updates_uses_pipeline():
mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called()
+class _InMemoryRedis:
+ def __init__(self) -> None:
+ self.store: dict[str, str] = {}
+ self.lists: dict[str, list[object]] = {}
+
+ async def async_set_cache(self, key: str, value: str, nx: bool = False, **kwargs: object) -> bool | None:
+ if nx and key in self.store:
+ return None
+ self.store[key] = json.dumps(value)
+ return True
+
+ async def async_get_cache(self, key: str, **kwargs: object) -> str | None:
+ raw = self.store.get(key)
+ return json.loads(raw) if raw is not None else None
+
+ async def async_delete_cache(self, key: str) -> int:
+ return 1 if self.store.pop(key, None) is not None else 0
+
+ async def async_rpush_pipeline(self, rpush_list: list[RedisPipelineRpushOperation]) -> list[int]:
+ for op in rpush_list:
+ self.lists.setdefault(op["key"], []).extend(op["values"])
+ return [len(self.lists[op["key"]]) for op in rpush_list]
+
+ async def async_lpop_pipeline(self, lpop_list: list[RedisPipelineLpopOperation]) -> list[list[object] | None]:
+ return [self.lists.pop(op["key"], None) for op in lpop_list]
+
+
[email protected]("lock_held_by_another_pod", [False, True])
+async def test_commit_with_redis_writes_discovered_tools_on_leader_and_non_leader(lock_held_by_another_pod: bool):
+ redis = _InMemoryRedis()
+ if lock_held_by_another_pod:
+ assert await PodLockManager(redis_cache=redis).acquire_lock(cronjob_id=DB_SPEND_UPDATE_JOB_NAME)
+ db_writer = DBSpendUpdateWriter(redis_cache=redis)
+ db_writer.pod_lock_manager.redis_cache = redis
+ db_writer.tool_discovery_queue.add_update({"tool_name": "get_weather", "origin": "user_defined"})
+ prisma_client = MagicMock()
+ prisma_client.db.litellm_tooltable.upsert = AsyncMock()
+
+ await db_writer._commit_spend_updates_to_db_with_redis(
+ prisma_client=prisma_client,
+ n_retry_times=0,
+ proxy_logging_obj=MagicMock(),
+ )
+
+ upserted = [c.kwargs["where"] for c in prisma_client.db.litellm_tooltable.upsert.await_args_list]
+ assert upserted == [{"tool_name": "get_weather"}]
+ assert (REDIS_UPDATE_BUFFER_KEY in redis.lists) is lock_held_by_another_pod, (
+ f"spend buffer left in redis={list(redis.lists)}, only the lock holder may drain it"
+ )
+
+
@pytest.mark.asyncio
async def test_commit_with_redis_requeues_all_on_db_failure():
"""