The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import os
from typing import Final
from uuid import uuid4
import pytest
import litellm
from litellm import Cache, acompletion
from litellm.caching.caching import LiteLLMCacheType
from litellm.caching.caching_handler import _PENDING_CACHE_WRITES
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm._service_logger import ServiceLogging
async def _drain_cache_writes() -> None:
await GLOBAL_LOGGING_WORKER.flush()
await asyncio.gather(*_PENDING_CACHE_WRITES)
@pytest.fixture
def redis_response_cache(monkeypatch: pytest.MonkeyPatch) -> Cache:
cache: Final = Cache(
type=LiteLLMCacheType.REDIS,
host=os.environ["REDIS_HOST"],
port=os.environ["REDIS_PORT"],
)
monkeypatch.setattr(litellm, "cache", cache)
return cache
@pytest.mark.asyncio
async def test_redis_cache_completion_stream(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "callbacks", [CustomLogger()])
messages: Final = [{"role": "user", "content": f"cache stream {uuid4().hex}"}]
first_response: Final = await acompletion(
model="gpt-4o-mini",
messages=messages,
caching=True,
stream=True,
mock_response="The same response is replayed.",
)
first_chunks: Final = tuple([chunk async for chunk in first_response])
for _ in range(10):
await asyncio.sleep(0)
await _drain_cache_writes()
second_response: Final = await acompletion(
model="gpt-4o-mini",
messages=messages,
caching=True,
stream=True,
mock_response="A cache miss would return this.",
)
second_chunks: Final = tuple([chunk async for chunk in second_response])
assert first_chunks[-1].id == second_chunks[-1].id
assert "".join(chunk.choices[0].delta.content or "" for chunk in first_chunks) == "The same response is replayed."
assert "".join(chunk.choices[0].delta.content or "" for chunk in second_chunks) == "The same response is replayed."
@pytest.mark.asyncio
async def test_completion_with_caching(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "service_callback", ["prometheus_system"])
service_logger: Final = ServiceLogging(mock_testing=True)
service_logger.prometheusServicesLogger.mock_testing = True
redis_response_cache.cache.service_logger_obj = service_logger
messages: Final = [{"role": "user", "content": f"prometheus cache {uuid4().hex}"}]
await acompletion(
model="gpt-4o-mini",
messages=messages,
caching=True,
mock_response="cached response",
)
await _drain_cache_writes()
await acompletion(
model="gpt-4o-mini",
messages=messages,
caching=True,
mock_response="unused cache miss response",
)
await _drain_cache_writes()
assert service_logger.mock_testing_async_success_hook == 2
assert service_logger.prometheusServicesLogger.mock_testing_success_calls == 2
assert service_logger.mock_testing_sync_failure_hook == 0
assert service_logger.mock_testing_async_failure_hook == 0