Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

3.2 KB
Raw Blame
import asyncio
import os
from typing import Final
from uuid import uuid4

import pytest

import litellm
from litellm import Cache, acompletion
from litellm.caching.caching import LiteLLMCacheType
from litellm.caching.caching_handler import _PENDING_CACHE_WRITES
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm._service_logger import ServiceLogging


async def _drain_cache_writes() -> None:
    await GLOBAL_LOGGING_WORKER.flush()
    await asyncio.gather(*_PENDING_CACHE_WRITES)


@pytest.fixture
def redis_response_cache(monkeypatch: pytest.MonkeyPatch) -> Cache:
    cache: Final = Cache(
        type=LiteLLMCacheType.REDIS,
        host=os.environ["REDIS_HOST"],
        port=os.environ["REDIS_PORT"],
    )
    monkeypatch.setattr(litellm, "cache", cache)
    return cache


@pytest.mark.asyncio
async def test_redis_cache_completion_stream(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(litellm, "callbacks", [CustomLogger()])
    messages: Final = [{"role": "user", "content": f"cache stream {uuid4().hex}"}]
    first_response: Final = await acompletion(
        model="gpt-4o-mini",
        messages=messages,
        caching=True,
        stream=True,
        mock_response="The same response is replayed.",
    )
    first_chunks: Final = tuple([chunk async for chunk in first_response])
    for _ in range(10):
        await asyncio.sleep(0)
        await _drain_cache_writes()
    second_response: Final = await acompletion(
        model="gpt-4o-mini",
        messages=messages,
        caching=True,
        stream=True,
        mock_response="A cache miss would return this.",
    )
    second_chunks: Final = tuple([chunk async for chunk in second_response])

    assert first_chunks[-1].id == second_chunks[-1].id
    assert "".join(chunk.choices[0].delta.content or "" for chunk in first_chunks) == "The same response is replayed."
    assert "".join(chunk.choices[0].delta.content or "" for chunk in second_chunks) == "The same response is replayed."


@pytest.mark.asyncio
async def test_completion_with_caching(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(litellm, "service_callback", ["prometheus_system"])
    service_logger: Final = ServiceLogging(mock_testing=True)
    service_logger.prometheusServicesLogger.mock_testing = True
    redis_response_cache.cache.service_logger_obj = service_logger
    messages: Final = [{"role": "user", "content": f"prometheus cache {uuid4().hex}"}]

    await acompletion(
        model="gpt-4o-mini",
        messages=messages,
        caching=True,
        mock_response="cached response",
    )
    await _drain_cache_writes()
    await acompletion(
        model="gpt-4o-mini",
        messages=messages,
        caching=True,
        mock_response="unused cache miss response",
    )
    await _drain_cache_writes()

    assert service_logger.mock_testing_async_success_hook == 2
    assert service_logger.prometheusServicesLogger.mock_testing_success_calls == 2
    assert service_logger.mock_testing_sync_failure_hook == 0
    assert service_logger.mock_testing_async_failure_hook == 0