The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""One Redis pipeline per backend for the post-call writes of a request: spend counters, rate-limit token
scripts and slot releases and deployment TPM all ride the post-call batch, which goes out once the success/failure
callbacks have run (or at the deadline when no callback phase closes it). The response-cache SET stays direct so the
next identical request can hit it while the callbacks are still running."""
from __future__ import annotations
import asyncio
import datetime
import hashlib
import json
from collections.abc import Awaitable, Callable, Mapping, Sequence
import pytest
import litellm
from litellm.caching.caching import Cache
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_batch import (
active_post_call_redis_batch,
drain_post_call_redis_batches,
flush_post_call_redis_batches,
request_redis_batch_scope,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PARALLEL_RELEASE_SCRIPT,
TOKEN_INCREMENT_SCRIPT,
ParallelSlotAcquisition,
PROXY_MaxParallelRequestsHandler_v3,
RequestRateLimiterStash,
)
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement
from litellm.proxy.utils import InternalUsageCache
from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import ModelResponse
from .test_redis_batch import FakeClient, FakeRedisCache
async def _script_outside_the_pipeline(keys: Sequence[str], args: Sequence[object]) -> object:
raise AssertionError("post-call scripts must ride the post-call pipeline")
class PostCallFakeRedisCache(FakeRedisCache):
"""Records the direct (non-pipelined) writes an owner falls back to."""
def async_register_script(self, script: str) -> Callable[..., Awaitable[object]]:
return _script_outside_the_pipeline
async def async_increment_pipeline(
self, increment_list: list[RedisPipelineIncrementOperation], **kwargs: object
) -> list[float]:
return [await self.async_increment(op["key"], op["increment_value"]) for op in increment_list]
async def async_delete_cache(self, key: str, **kwargs: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # the fake drops RedisCache's unused kwargs
self.alone.append(("DEL", key))
self.store.pop(key, None)
async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
self.alone.append(("SET", key, dict(kwargs)))
self.store[key] = value
def sha_of(script: str) -> str:
return hashlib.sha1(script.encode()).hexdigest() # noqa: S324
def _ok_replies(command: tuple[object, ...]) -> object:
match command[0]:
case "INCRBYFLOAT":
return b"7.5"
case "EXPIRE":
return 1
case "SET":
return True
case "EVALSHA":
return [3, 0]
case "MGET":
return [json.dumps({"spend": 1.0}) for _ in command[1:]]
raise AssertionError(command)
async def _run_ready_callbacks(client: FakeClient) -> None:
for _ in range(20):
if client.pipelines:
return
await asyncio.sleep(0)
def _names(client: FakeClient, index: int = 0) -> list[str]:
return [command[0] for command in client.pipelines[index].commands]
def _limiter(redis_cache: FakeRedisCache) -> PROXY_MaxParallelRequestsHandler_v3:
dual_cache = DualCache()
dual_cache.attach_redis_cache(redis_cache)
return PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(dual_cache=dual_cache))
def _slot_stash(slot_id: str, *counter_keys: str) -> RequestRateLimiterStash:
return RequestRateLimiterStash(
parallel_slot=ParallelSlotAcquisition(slot_id=slot_id, counter_keys=list(counter_keys))
)
def _token_ops(*keys: str) -> list[RedisPipelineIncrementOperation]:
return [RedisPipelineIncrementOperation(key=key, increment_value=10, ttl=60) for key in keys]
def _response_cache(redis_cache: FakeRedisCache) -> Cache:
cache = Cache(type="local")
cache.type = "redis" # pyright: ignore[reportAttributeAccessIssue] # the fake stands in for the Redis backend
cache.cache = redis_cache
return cache
def _tpm_router(redis_cache: FakeRedisCache) -> tuple[LowestTPMLoggingHandler_v2, DualCache]:
router_cache = DualCache()
router_cache.attach_redis_cache(redis_cache)
return LowestTPMLoggingHandler_v2(router_cache=router_cache, routing_args={"ttl": 60}), router_cache
def _tpm_kwargs() -> Mapping[str, object]:
return {
"standard_logging_object": {
"model_group": "gpt",
"model_id": "dep-a",
"hidden_params": {"litellm_model_name": "openai/gpt-4o-mini"},
"total_tokens": 42,
},
"litellm_params": {"metadata": {}},
}
@pytest.mark.asyncio
async def test_every_post_call_owner_rides_one_pipeline_that_goes_out_when_the_callbacks_are_done():
client = FakeClient(_ok_replies)
redis_cache = PostCallFakeRedisCache(client)
limiter = _limiter(redis_cache)
tpm, router_cache = _tpm_router(redis_cache)
with request_redis_batch_scope():
await tpm.async_log_success_event(_tpm_kwargs(), None, None, None)
await limiter.async_increment_tokens_with_ttl_preservation(_token_ops("{api_key:k1}:tokens"))
await limiter._release_stashed_parallel_slot(
_slot_stash("slot-1", "{api_key:k1}:parallel"), None, in_logging_callback=True
)
assert client.pipelines == [] # nothing goes out while the callbacks are still declaring
await flush_post_call_redis_batches()
assert len(client.pipelines) == 1
assert _names(client) == ["INCRBYFLOAT", "EXPIRE", "EVALSHA", "EVALSHA"]
evalshas = [c for c in client.pipelines[0].commands if c[0] == "EVALSHA"]
assert [c[1] for c in evalshas] == [sha_of(TOKEN_INCREMENT_SCRIPT), sha_of(PARALLEL_RELEASE_SCRIPT)]
assert redis_cache.alone == []
assert (
await router_cache.in_memory_cache.async_get_cache(
next(k for k in router_cache.in_memory_cache.cache_dict if ":tpm:" in k)
)
== 42
)
@pytest.mark.asyncio
async def test_the_response_cache_set_reaches_redis_before_the_post_call_pipeline_goes_out():
client = FakeClient(_ok_replies)
redis_cache = PostCallFakeRedisCache(client)
response_cache = _response_cache(redis_cache)
kwargs = {"messages": [{"role": "user", "content": "hi"}], "model": "gpt", "ttl": 120}
cache_key = response_cache.get_cache_key(**kwargs)
with request_redis_batch_scope():
await response_cache.async_add_cache({"id": "resp"}, **kwargs)
assert redis_cache.store[cache_key]["response"] == {"id": "resp"}
await flush_post_call_redis_batches()
assert client.pipelines == []
(direct_set,) = redis_cache.alone
assert (direct_set[0], direct_set[1], direct_set[2]["ttl"]) == ("SET", cache_key, 120)
@pytest.mark.asyncio
async def test_a_chat_response_written_through_the_handler_dual_cache_is_in_memory_and_redis_at_once():
client = FakeClient(_ok_replies)
redis_cache = PostCallFakeRedisCache(client)
response_cache = _response_cache(redis_cache)
handler_cache = DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache())
kwargs = {"messages": [{"role": "user", "content": "hi"}], "model": "gpt", "ttl": 120}
cache_key = response_cache.get_cache_key(**kwargs)
with request_redis_batch_scope():
await response_cache.async_add_cache('{"id": "resp"}', dynamic_cache_object=handler_cache, **kwargs)
in_memory = await handler_cache.in_memory_cache.async_get_cache(cache_key)
assert in_memory["response"] == '{"id": "resp"}'
assert redis_cache.store[cache_key]["response"] == '{"id": "resp"}'
await flush_post_call_redis_batches()
assert client.pipelines == []
(direct_set,) = redis_cache.alone
assert (direct_set[0], direct_set[1], direct_set[2]["ttl"]) == ("SET", cache_key, 120)
@pytest.mark.asyncio
async def test_a_failed_operation_fails_only_its_owner_and_the_owner_applies_its_own_fallback():
def replies(command: tuple[object, ...]) -> object:
if command[0] == "EVALSHA" and command[3] == "{api_key:k1}:tokens":
return Exception("ERR Lua")
return _ok_replies(command)
client = FakeClient(replies)
redis_cache = PostCallFakeRedisCache(client)
limiter = _limiter(redis_cache)
with request_redis_batch_scope():
await limiter.async_increment_tokens_with_ttl_preservation(_token_ops("{api_key:k1}:tokens"))
await limiter.async_increment_tokens_with_ttl_preservation(_token_ops("{team:t1}:tokens"))
await flush_post_call_redis_batches()
assert len(client.pipelines) == 1
# the failed group falls back to the plain increment (memory + Redis), the healthy group does not
assert redis_cache.alone == [("INCRBYFLOAT", "{api_key:k1}:tokens", 10)]
assert await limiter.internal_usage_cache.dual_cache.in_memory_cache.async_get_cache("{api_key:k1}:tokens") == 10
assert await limiter.internal_usage_cache.dual_cache.in_memory_cache.async_get_cache("{team:t1}:tokens") is None
@pytest.mark.asyncio
async def test_a_failed_slot_release_script_releases_the_slot_in_memory():
def replies(command: tuple[object, ...]) -> object:
if command[0] == "EVALSHA":
return Exception("ERR Lua")
return _ok_replies(command)
redis_cache = PostCallFakeRedisCache(FakeClient(replies))
limiter = _limiter(redis_cache)
memory = limiter.internal_usage_cache.dual_cache.in_memory_cache
await memory.async_set_cache("{api_key:k1}:parallel", {"slot-1": 1.0, "slot-2": 1.0})
with request_redis_batch_scope():
await limiter._release_stashed_parallel_slot(
_slot_stash("slot-1", "{api_key:k1}:parallel"), None, in_logging_callback=True
)
await flush_post_call_redis_batches()
assert await memory.async_get_cache("{api_key:k1}:parallel") == {"slot-2": 1.0}
class DirectScriptFakeRedisCache(PostCallFakeRedisCache):
"""Records the release script a pre-response caller runs outside the pipeline."""
def async_register_script(self, script: str) -> Callable[..., Awaitable[object]]:
async def run(keys: Sequence[str], args: Sequence[object]) -> object:
self.alone.append(("EVALSHA", tuple(keys), tuple(args)))
return [0 for _ in keys]
return run
@pytest.mark.asyncio
async def test_a_slot_released_before_the_response_reaches_redis_at_once_not_on_the_pipeline():
client = FakeClient(_ok_replies)
redis_cache = DirectScriptFakeRedisCache(client)
limiter = _limiter(redis_cache)
memory = limiter.internal_usage_cache.dual_cache.in_memory_cache
await memory.async_set_cache("{api_key:k1}:parallel", {"slot-1": 1.0})
with request_redis_batch_scope():
await limiter._release_stashed_parallel_slot(_slot_stash("slot-1", "{api_key:k1}:parallel"), None)
assert redis_cache.alone == [("EVALSHA", ("{api_key:k1}:parallel",), ("slot-1",))]
assert await memory.async_get_cache("{api_key:k1}:parallel") == 0
await flush_post_call_redis_batches()
assert client.pipelines == []
@pytest.mark.asyncio
async def test_a_released_slot_is_free_locally_at_once_and_the_older_redis_count_does_not_overwrite_the_gauge():
def replies(command: tuple[object, ...]) -> object:
if command[0] == "EVALSHA":
return [2]
return _ok_replies(command)
redis_cache = PostCallFakeRedisCache(FakeClient(replies))
limiter = _limiter(redis_cache)
memory = limiter.internal_usage_cache.dual_cache.in_memory_cache
await memory.async_set_cache("{api_key:k1}:parallel", {"slot-1": 1.0, "slot-2": 1.0, "slot-3": 1.0})
with request_redis_batch_scope():
await limiter._release_stashed_parallel_slot(
_slot_stash("slot-1", "{api_key:k1}:parallel"), None, in_logging_callback=True
)
assert await memory.async_get_cache("{api_key:k1}:parallel") == {"slot-2": 1.0, "slot-3": 1.0}
await memory.async_set_cache("{api_key:k1}:parallel", {"slot-2": 1.0, "slot-3": 1.0, "slot-4": 1.0})
await flush_post_call_redis_batches()
assert await memory.async_get_cache("{api_key:k1}:parallel") == {"slot-2": 1.0, "slot-3": 1.0, "slot-4": 1.0}
@pytest.mark.asyncio
async def test_failure_refunds_ride_the_post_call_pipeline_and_count_in_memory_at_once():
client = FakeClient(_ok_replies)
dual_cache = DualCache()
dual_cache.attach_redis_cache(PostCallFakeRedisCache(client))
refund = [RedisPipelineIncrementOperation(key="{api_key:k1}:tokens", increment_value=-500, ttl=60)]
with request_redis_batch_scope():
await dual_cache.async_increment_cache_pipeline_post_call(refund)
assert await dual_cache.in_memory_cache.async_get_cache("{api_key:k1}:tokens") == -500
assert client.pipelines == []
await flush_post_call_redis_batches()
assert client.pipelines[0].commands[0] == ("INCRBYFLOAT", "{api_key:k1}:tokens", -500)
@pytest.mark.asyncio
async def test_outside_a_request_scope_owners_write_directly_as_before():
client = FakeClient(_ok_replies)
redis_cache = PostCallFakeRedisCache(client)
dual_cache = DualCache()
dual_cache.attach_redis_cache(redis_cache)
response_cache = _response_cache(redis_cache)
await dual_cache.async_increment_cache_post_call("dep:tpm", 42, ttl=60)
await response_cache.async_add_cache({"id": "resp"}, messages=[{"role": "user", "content": "hi"}], model="gpt")
assert client.pipelines == []
assert redis_cache.alone[0] == ("INCRBYFLOAT", "dep:tpm", 42)
assert active_post_call_redis_batch(redis_cache) is None
@pytest.mark.asyncio
async def test_a_set_with_options_keeps_the_direct_path():
client = FakeClient(_ok_replies)
redis_cache = PostCallFakeRedisCache(client)
response_cache = _response_cache(redis_cache)
with request_redis_batch_scope():
await response_cache.async_add_cache(
{"id": "r"}, messages=[{"role": "user", "content": "hi"}], model="gpt", nx=True
)
await flush_post_call_redis_batches()
assert client.pipelines == []
(direct_set,) = redis_cache.alone
assert direct_set[0] == "SET" and direct_set[2]["nx"] is True
@pytest.mark.asyncio
async def test_two_backends_get_one_post_call_pipeline_each():
a_client, b_client = FakeClient(_ok_replies), FakeClient(_ok_replies)
a, b = DualCache(), DualCache()
a.attach_redis_cache(PostCallFakeRedisCache(a_client))
b.attach_redis_cache(PostCallFakeRedisCache(b_client))
with request_redis_batch_scope():
await a.async_increment_cache_post_call("x", 1, ttl=None)
await b.async_increment_cache_post_call("y", 1, ttl=None)
await a.async_increment_cache_post_call("z", 1, ttl=None)
await flush_post_call_redis_batches()
assert len(a_client.pipelines) == 1 and len(b_client.pipelines) == 1
assert [c[1] for c in a_client.pipelines[0].commands if c[0] == "INCRBYFLOAT"] == ["x", "z"]
@pytest.mark.asyncio
async def test_post_call_writes_still_waiting_on_their_callbacks_are_drained_at_shutdown():
client = FakeClient(_ok_replies)
dual_cache = DualCache()
dual_cache.attach_redis_cache(PostCallFakeRedisCache(client))
with request_redis_batch_scope(post_call_deadline=60) as request:
await dual_cache.async_increment_cache_post_call("x", 1, ttl=None)
await request.flush_all()
assert client.pipelines == []
await drain_post_call_redis_batches()
assert len(client.pipelines) == 1 and _names(client) == ["INCRBYFLOAT"]
await drain_post_call_redis_batches()
assert len(client.pipelines) == 1
@pytest.mark.asyncio
async def test_a_post_call_batch_nobody_closes_goes_out_at_the_deadline(monkeypatch: pytest.MonkeyPatch):
client = FakeClient(_ok_replies)
dual_cache = DualCache()
dual_cache.attach_redis_cache(PostCallFakeRedisCache(client))
loop = asyncio.get_running_loop()
armed_at = loop.time()
with request_redis_batch_scope(post_call_deadline=60) as request:
await dual_cache.async_increment_cache_post_call("x", 1, ttl=None)
await request.flush_all()
await _run_ready_callbacks(client)
assert client.pipelines == [], "the request boundary drains the immediate batch, not the post-call one"
monkeypatch.setattr(loop, "time", lambda: armed_at + 61)
await _run_ready_callbacks(client)
assert len(client.pipelines) == 1 and _names(client) == ["INCRBYFLOAT"]
@pytest.mark.asyncio
async def test_the_success_handler_closes_the_post_call_batch_after_the_last_callback(monkeypatch):
client = FakeClient(_ok_replies)
dual_cache = DualCache()
dual_cache.attach_redis_cache(PostCallFakeRedisCache(client))
pipelines_seen_by_callbacks: list[int] = []
class Counter(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
await dual_cache.async_increment_cache_post_call("counted", 1, ttl=None)
pipelines_seen_by_callbacks.append(len(client.pipelines))
monkeypatch.setattr(litellm, "_async_success_callback", [])
logging_obj = LitellmLogging(
model="test-model",
messages=[],
stream=False,
call_type="completion",
start_time=datetime.datetime.now(),
litellm_call_id="post-call",
function_id="post-call",
dynamic_async_success_callbacks=[Counter(), Counter()],
)
logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={})
payload = {
"id": "post-call",
"call_type": "completion",
"metadata": {},
"model_group": "test-model",
"model_parameters": {},
}
with request_redis_batch_scope():
await logging_obj.async_success_handler(result=ModelResponse(), standard_logging_object=payload)
assert pipelines_seen_by_callbacks == [0, 0]
assert len(client.pipelines) == 1 and _names(client) == ["INCRBYFLOAT", "INCRBYFLOAT"]
@pytest.mark.asyncio
async def test_spend_counter_increments_ride_the_pipeline_and_settle_into_memory(monkeypatch):
from litellm.proxy import proxy_server
client = FakeClient(_ok_replies)
spend_cache = DualCache()
spend_cache.attach_redis_cache(PostCallFakeRedisCache(client))
monkeypatch.setattr(proxy_server, "spend_counter_cache", spend_cache)
pending = [PendingSpendIncrement("spend:key:k1", 0.5), PendingSpendIncrement("spend:team:t1", 0.5)]
with request_redis_batch_scope():
await proxy_server._apply_spend_counter_increments(pending)
assert client.pipelines == []
await flush_post_call_redis_batches()
assert [c for c in client.pipelines[0].commands if c[0] == "INCRBYFLOAT"] == [
("INCRBYFLOAT", "spend:key:k1", 0.5),
("INCRBYFLOAT", "spend:team:t1", 0.5),
]
assert spend_cache.in_memory_cache.get_cache("spend:key:k1") == 7.5
@pytest.mark.asyncio
async def test_a_spend_counter_whose_increment_failed_is_invalidated_not_trusted(monkeypatch):
from litellm.proxy import proxy_server
def replies(command: tuple[object, ...]) -> object:
if command[0] == "INCRBYFLOAT" and command[1] == "spend:key:k1":
return Exception("OOM")
return _ok_replies(command)
redis_cache = PostCallFakeRedisCache(FakeClient(replies))
spend_cache = DualCache()
spend_cache.attach_redis_cache(redis_cache)
spend_cache.in_memory_cache.set_cache("spend:key:k1", 3.0)
spend_cache.in_memory_cache.set_cache("spend:team:t1", 3.0)
monkeypatch.setattr(proxy_server, "spend_counter_cache", spend_cache)
with request_redis_batch_scope():
await proxy_server._apply_spend_counter_increments(
[PendingSpendIncrement("spend:key:k1", 0.5), PendingSpendIncrement("spend:team:t1", 0.5)]
)
await flush_post_call_redis_batches()
assert spend_cache.in_memory_cache.get_cache("spend:key:k1") is None
assert redis_cache.alone == [("DEL", "spend:key:k1")]
assert spend_cache.in_memory_cache.get_cache("spend:team:t1") == 7.5
@pytest.mark.asyncio
async def test_a_cancelled_post_call_flush_keeps_the_shared_spend_counter_and_counts_the_spend_locally(monkeypatch):
from litellm.proxy import proxy_server
redis_cache = PostCallFakeRedisCache(
FakeClient(_ok_replies, fail=asyncio.CancelledError()) # pyright: ignore[reportArgumentType] # a cancel raised mid-pipeline
)
spend_cache = DualCache()
spend_cache.attach_redis_cache(redis_cache)
spend_cache.in_memory_cache.set_cache("spend:key:k1", 3.0)
monkeypatch.setattr(proxy_server, "spend_counter_cache", spend_cache)
with request_redis_batch_scope():
await proxy_server._apply_spend_counter_increments(
[PendingSpendIncrement("spend:key:k1", 0.5), PendingSpendIncrement("spend:team:t1", 0.5)]
)
with pytest.raises(asyncio.CancelledError):
await flush_post_call_redis_batches()
assert redis_cache.alone == [], "a cancel says nothing about the shared counter, so Redis keeps it"
assert spend_cache.in_memory_cache.get_cache("spend:key:k1") == 3.5, "the local copy counts the cancelled spend"
assert spend_cache.in_memory_cache.get_cache("spend:team:t1") is None, "an absent local copy is not seeded"