The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(proxy): flush tool discovery queue when use_redis_transaction_buffer is enabled (#47653)
mrinal-berridba4fa0e944e
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.pyindex e7909cbe600..bb6ffe675c0 100644--- a/litellm/proxy/db/db_spend_update_writer.py+++ b/litellm/proxy/db/db_spend_update_writer.py@@ -1651,6 +1651,8 @@ class DBSpendUpdateWriter:window_spend_update_queue=self.window_spend_update_queue,)+ await self._flush_tool_discovery_queue(prisma_client=prisma_client)+# Only commit from redis to db if this pod is the leaderif await self.pod_lock_manager.acquire_lock(cronjob_id=DB_SPEND_UPDATE_JOB_NAME,diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.pyindex d440b99a632..7437042e4bd 100644--- a/tests/unit/proxy/db/test_db_spend_update_writer.py+++ b/tests/unit/proxy/db/test_db_spend_update_writer.py@@ -19,7 +19,7 @@ from redis.exceptions import DataErrorimport litellmfrom litellm._logging import verbose_proxy_loggerfrom litellm._service_logger import ServiceTypes-from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS+from litellm.constants import DB_SPEND_UPDATE_JOB_NAME, REDIS_UPDATE_BUFFER_KEY, SPEND_ROLLUP_LOCK_TIMEOUT_MSfrom litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItemfrom litellm.proxy.db.db_spend_update_writer import (_TEAM_ADVISORY_LOCK_SQL,@@ -29,12 +29,14 @@ from litellm.proxy.db.db_spend_update_writer import (_SpendTableName,)from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue+from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManagerfrom litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBufferfrom litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueuefrom litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (build_window_spend_transaction,)from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL+from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperationfrom tests.unit.proxy.db.fake_prisma_engine import engine_call@@ -2365,6 +2367,58 @@ async def test_commit_spend_updates_uses_pipeline():mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called()+class _InMemoryRedis:+ def __init__(self) -> None:+ self.store: dict[str, str] = {}+ self.lists: dict[str, list[object]] = {}++ async def async_set_cache(self, key: str, value: str, nx: bool = False, **kwargs: object) -> bool | None:+ if nx and key in self.store:+ return None+ self.store[key] = json.dumps(value)+ return True++ async def async_get_cache(self, key: str, **kwargs: object) -> str | None:+ raw = self.store.get(key)+ return json.loads(raw) if raw is not None else None++ async def async_delete_cache(self, key: str) -> int:+ return 1 if self.store.pop(key, None) is not None else 0++ async def async_rpush_pipeline(self, rpush_list: list[RedisPipelineRpushOperation]) -> list[int]:+ for op in rpush_list:+ self.lists.setdefault(op["key"], []).extend(op["values"])+ return [len(self.lists[op["key"]]) for op in rpush_list]++ async def async_lpop_pipeline(self, lpop_list: list[RedisPipelineLpopOperation]) -> list[list[object] | None]:+ return [self.lists.pop(op["key"], None) for op in lpop_list]++[email protected]("lock_held_by_another_pod", [False, True])+async def test_commit_with_redis_writes_discovered_tools_on_leader_and_non_leader(lock_held_by_another_pod: bool):+ redis = _InMemoryRedis()+ if lock_held_by_another_pod:+ assert await PodLockManager(redis_cache=redis).acquire_lock(cronjob_id=DB_SPEND_UPDATE_JOB_NAME)+ db_writer = DBSpendUpdateWriter(redis_cache=redis)+ db_writer.pod_lock_manager.redis_cache = redis+ db_writer.tool_discovery_queue.add_update({"tool_name": "get_weather", "origin": "user_defined"})+ prisma_client = MagicMock()+ prisma_client.db.litellm_tooltable.upsert = AsyncMock()++ await db_writer._commit_spend_updates_to_db_with_redis(+ prisma_client=prisma_client,+ n_retry_times=0,+ proxy_logging_obj=MagicMock(),+ )++ upserted = [c.kwargs["where"] for c in prisma_client.db.litellm_tooltable.upsert.await_args_list]+ assert upserted == [{"tool_name": "get_weather"}]+ assert (REDIS_UPDATE_BUFFER_KEY in redis.lists) is lock_held_by_another_pod, (+ f"spend buffer left in redis={list(redis.lists)}, only the lock holder may drain it"+ )++@pytest.mark.asyncioasync def test_commit_with_redis_requeues_all_on_db_failure():"""