Skip to content

A programming framework for agentic AI

Redis Doesn't Support Streaming (#6954)

Ben Constable2f38334c0ef0
diff --git a/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py b/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py
index 873dabd87..02c3af6b0 100644
--- a/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py
+++ b/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py
@@ -90,6 +90,7 @@ class RedisStore(CacheStore[T], Component[RedisStoreConfig]):
This method handles both primitive values and complex objects:
- Pydantic models are automatically serialized to JSON
+ - Lists containing Pydantic models are serialized to JSON
- Primitive values (strings, numbers, etc.) are stored as-is
Args:
@@ -101,10 +102,21 @@ class RedisStore(CacheStore[T], Component[RedisStoreConfig]):
# Serialize Pydantic models to JSON
serialized_value = value.model_dump_json().encode("utf-8")
self.cache.set(key, serialized_value)
+ elif isinstance(value, list):
+ # Serialize lists (which may contain Pydantic models) to JSON
+ serializable_list: list[Any] = []
+ item: Any
+ for item in value:
+ if isinstance(item, BaseModel):
+ serializable_list.append(item.model_dump())
+ else:
+ serializable_list.append(item)
+ serialized_value = json.dumps(serializable_list).encode("utf-8")
+ self.cache.set(key, serialized_value)
else:
# Backward compatibility for primitives
self.cache.set(key, cast(Any, value))
- except (redis.RedisError, ConnectionError, UnicodeEncodeError):
+ except (redis.RedisError, ConnectionError, UnicodeEncodeError, TypeError):
# Log the error but don't re-raise to maintain robustness
pass
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py b/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py
index 279cffb81..4bd324872 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py
@@ -77,6 +77,81 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheCon
asyncio.run(main())
+ For Redis caching:
+
+ .. code-block:: python
+
+ import asyncio
+
+ from autogen_core.models import UserMessage
+ from autogen_ext.models.openai import OpenAIChatCompletionClient
+ from autogen_ext.models.cache import ChatCompletionCache, CHAT_CACHE_VALUE_TYPE
+ from autogen_ext.cache_store.redis import RedisStore
+ import redis
+
+
+ async def main():
+ # Initialize the original client
+ openai_model_client = OpenAIChatCompletionClient(model="gpt-4o")
+
+ # Initialize Redis cache store
+ redis_instance = redis.Redis()
+ cache_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)
+ cache_client = ChatCompletionCache(openai_model_client, cache_store)
+
+ response = await cache_client.create([UserMessage(content="Hello, how are you?", source="user")])
+ print(response) # Should print response from OpenAI
+ response = await cache_client.create([UserMessage(content="Hello, how are you?", source="user")])
+ print(response) # Should print cached response
+
+
+ asyncio.run(main())
+
+ For streaming with Redis caching:
+
+ .. code-block:: python
+
+ import asyncio
+
+ from autogen_core.models import UserMessage, CreateResult
+ from autogen_ext.models.openai import OpenAIChatCompletionClient
+ from autogen_ext.models.cache import ChatCompletionCache, CHAT_CACHE_VALUE_TYPE
+ from autogen_ext.cache_store.redis import RedisStore
+ import redis
+
+
+ async def main():
+ # Initialize the original client
+ openai_model_client = OpenAIChatCompletionClient(model="gpt-4o")
+
+ # Initialize Redis cache store
+ redis_instance = redis.Redis()
+ cache_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)
+ cache_client = ChatCompletionCache(openai_model_client, cache_store)
+
+ # First streaming call
+ async for chunk in cache_client.create_stream(
+ [UserMessage(content="List all countries in Africa", source="user")]
+ ):
+ if isinstance(chunk, CreateResult):
+ print("\\n")
+ print("Cached: ", chunk.cached) # Should print False
+ else:
+ print(chunk, end="")
+
+ # Second streaming call (cached)
+ async for chunk in cache_client.create_stream(
+ [UserMessage(content="List all countries in Africa", source="user")]
+ ):
+ if isinstance(chunk, CreateResult):
+ print("\\n")
+ print("Cached: ", chunk.cached) # Should print True
+ else:
+ print(chunk, end="")
+
+
+ asyncio.run(main())
+
You can now use the `cached_client` as you would the original client, but with caching enabled.
Args:
@@ -171,10 +246,18 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheCon
NOTE: cancellation_token is ignored for cached results.
"""
cached_result, cache_key = self._check_cache(messages, tools, json_output, extra_create_args)
- if cached_result:
- assert isinstance(cached_result, CreateResult)
- cached_result.cached = True
- return cached_result
+ if cached_result is not None:
+ if isinstance(cached_result, CreateResult):
+ # Cache hit from previous non-streaming call
+ cached_result.cached = True
+ return cached_result
+ elif isinstance(cached_result, list):
+ # Cache hit from previous streaming call - extract the final CreateResult
+ for item in reversed(cached_result):
+ if isinstance(item, CreateResult):
+ item.cached = True
+ return item
+ # If no CreateResult found in list, fall through to make actual call
result = await self.client.create(
messages,
@@ -212,13 +295,24 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheCon
json_output,
extra_create_args,
)
- if cached_result:
- assert isinstance(cached_result, list)
- for result in cached_result:
- if isinstance(result, CreateResult):
- result.cached = True
- yield result
- return
+ if cached_result is not None:
+ if isinstance(cached_result, list):
+ # Cache hit from previous streaming call
+ for result in cached_result:
+ if isinstance(result, CreateResult):
+ result.cached = True
+ yield result
+ return
+ elif isinstance(cached_result, CreateResult):
+ # Cache hit from previous non-streaming call - convert to streaming format
+ cached_result.cached = True
+
+ # If content is a non-empty string, yield it as a streaming chunk first
+ if isinstance(cached_result.content, str) and cached_result.content:
+ yield cached_result.content
+
+ yield cached_result
+ return
result_stream = self.client.create_stream(
messages,
@@ -230,12 +324,14 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheCon
)
output_results: List[Union[str, CreateResult]] = []
- self.store.set(cache_key, output_results)
async for result in result_stream:
output_results.append(result)
yield result
+ # Store the complete results only after streaming is finished
+ self.store.set(cache_key, output_results)
+
return _generator()
async def close(self) -> None:
diff --git a/python/packages/autogen-ext/tests/cache_store/test_redis_store.py b/python/packages/autogen-ext/tests/cache_store/test_redis_store.py
index 305fe7abe..675b48cc0 100644
--- a/python/packages/autogen-ext/tests/cache_store/test_redis_store.py
+++ b/python/packages/autogen-ext/tests/cache_store/test_redis_store.py
@@ -1,8 +1,9 @@
import json
-from typing import cast
+from typing import Dict, List, Union, cast
from unittest.mock import MagicMock
import pytest
+from autogen_core.models import CreateResult, RequestUsage
from pydantic import BaseModel
redis = pytest.importorskip("redis")
@@ -180,3 +181,220 @@ def test_redis_store_nested_model_serialization() -> None:
assert retrieved_model["nested"]["id"] == 1 # type: ignore
assert retrieved_model["nested"]["data"] == "nested_data" # type: ignore
assert retrieved_model["tags"] == ["tag1", "tag2", "tag3"] # type: ignore
+
+
+def test_redis_store_list_with_strings_only() -> None:
+ """Test serialization of lists containing only strings (streaming scenario)."""
+ from autogen_ext.cache_store.redis import RedisStore
+
+ redis_instance = MagicMock()
+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)
+ test_key = "test_string_list_key"
+
+ # Create a list with only strings (partial streaming result)
+ string_list: List[Union[str, CreateResult]] = ["Hello", " world", "!", " How", " are", " you", "?"]
+
+ # Test setting the list
+ store.set(test_key, string_list)
+
+ # Verify Redis was called with JSON-serialized data
+ args, _ = redis_instance.set.call_args
+ assert args[0] == test_key
+ assert isinstance(args[1], bytes)
+
+ # Verify the serialized data is correct
+ serialized_json = args[1].decode("utf-8")
+ deserialized_data = json.loads(serialized_json)
+ assert deserialized_data == string_list
+
+ # Test retrieving the list
+ redis_instance.get.return_value = args[1] # Return the serialized data
+ retrieved_list = store.get(test_key)
+
+ assert retrieved_list is not None
+ assert isinstance(retrieved_list, list)
+ assert retrieved_list == string_list
+
+
+def test_redis_store_list_with_create_results_only() -> None:
+ """Test serialization of lists containing only CreateResult objects."""
+ from autogen_ext.cache_store.redis import RedisStore
+
+ redis_instance = MagicMock()
+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)
+ test_key = "test_create_result_list_key"
+
+ # Create a list with only CreateResult objects
+ usage = RequestUsage(prompt_tokens=10, completion_tokens=20)
+ create_result_list: List[Union[str, CreateResult]] = [
+ CreateResult(content="First response", usage=usage, finish_reason="stop", cached=False),
+ CreateResult(content="Second response", usage=usage, finish_reason="stop", cached=False),
+ ]
+
+ # Test setting the list
+ store.set(test_key, create_result_list)
+
+ # Verify Redis was called with JSON-serialized data
+ args, _ = redis_instance.set.call_args
+ assert args[0] == test_key
+ assert isinstance(args[1], bytes)
+
+ # Verify the serialized data structure
+ serialized_json = args[1].decode("utf-8")
+ deserialized_data = json.loads(serialized_json)
+
+ assert isinstance(deserialized_data, list)
+ # Type narrowing: after isinstance check, deserialized_data is known to be a list
+ deserialized_list: List[Dict[str, Union[str, int]]] = deserialized_data # Now properly typed as list
+ assert len(deserialized_list) == 2
+ assert deserialized_list[0]["content"] == "First response"
+ assert deserialized_list[1]["content"] == "Second response"
+ assert deserialized_list[0]["finish_reason"] == "stop"
+
+ # Test retrieving the list
+ redis_instance.get.return_value = args[1] # Return the serialized data
+ retrieved_list = store.get(test_key)
+
+ assert retrieved_list is not None
+ assert isinstance(retrieved_list, list)
+ assert len(retrieved_list) == 2
+
+ # The retrieved items should be dicts (as Redis returns JSON-parsed objects)
+ assert isinstance(retrieved_list[0], dict)
+ assert isinstance(retrieved_list[1], dict)
+ assert retrieved_list[0]["content"] == "First response" # type: ignore
+ assert retrieved_list[1]["content"] == "Second response" # type: ignore
+
+
+def test_redis_store_mixed_list_streaming_scenario() -> None:
+ """Test serialization of mixed lists (strings + CreateResult) for streaming cache scenario."""
+ from autogen_ext.cache_store.redis import RedisStore
+
+ redis_instance = MagicMock()
+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)
+ test_key = "test_mixed_streaming_list_key"
+
+ # Create a mixed list simulating a streaming response
+ usage = RequestUsage(prompt_tokens=15, completion_tokens=30)
+ mixed_list: List[Union[str, CreateResult]] = [
+ "The",
+ " capital",
+ " of",
+ " France",
+ " is",
+ " Paris",
+ ".",
+ CreateResult(content="The capital of France is Paris.", usage=usage, finish_reason="stop", cached=False),
+ ]
+
+ # Test setting the mixed list
+ store.set(test_key, mixed_list)
+
+ # Verify Redis was called with JSON-serialized data
+ args, _ = redis_instance.set.call_args
+ assert args[0] == test_key
+ assert isinstance(args[1], bytes)
+
+ # Verify the serialized data structure
+ serialized_json = args[1].decode("utf-8")
+ deserialized_data = json.loads(serialized_json)
+
+ assert isinstance(deserialized_data, list)
+ # Type narrowing: after isinstance check, deserialized_data is known to be a list
+ deserialized_list: List[Union[str, Dict[str, Union[str, int]]]] = deserialized_data # Now properly typed as list
+ assert len(deserialized_list) == 8 # 7 strings + 1 CreateResult
+
+ # First 7 items should be strings
+ for i in range(7):
+ assert isinstance(deserialized_list[i], str)
+
+ # Last item should be the serialized CreateResult (as dict)
+ assert isinstance(deserialized_list[7], dict)
+ assert deserialized_list[7]["content"] == "The capital of France is Paris."
+ assert deserialized_list[7]["finish_reason"] == "stop"
+ assert deserialized_data[7]["usage"]["prompt_tokens"] == 15
+ assert deserialized_data[7]["usage"]["completion_tokens"] == 30
+
+ # Test retrieving the mixed list
+ redis_instance.get.return_value = args[1] # Return the serialized data
+ retrieved_list = store.get(test_key)
+
+ assert retrieved_list is not None
+ assert isinstance(retrieved_list, list)
+ assert len(retrieved_list) == 8
+
+ # First 7 items should still be strings
+ for i in range(7):
+ assert isinstance(retrieved_list[i], str)
+ assert retrieved_list[i] == mixed_list[i]
+
+ # Last item should be a dict (CreateResult deserialized from JSON)
+ assert isinstance(retrieved_list[7], dict)
+ assert retrieved_list[7]["content"] == "The capital of France is Paris." # type: ignore
+ assert retrieved_list[7]["cached"] is False # type: ignore
+
+
+def test_redis_store_empty_list() -> None:
+ """Test serialization of empty lists."""
+ from autogen_ext.cache_store.redis import RedisStore
+
+ redis_instance = MagicMock()
+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)
+ test_key = "test_empty_list_key"
+
+ # Test setting an empty list
+ empty_list: List[Union[str, CreateResult]] = []
+ store.set(test_key, empty_list)
+
+ # Verify Redis was called with JSON-serialized data
+ args, _ = redis_instance.set.call_args
+ assert args[0] == test_key
+ assert isinstance(args[1], bytes)
+
+ # Verify the serialized data is an empty JSON array
+ serialized_json = args[1].decode("utf-8")
+ deserialized_data = json.loads(serialized_json)
+ assert deserialized_data == []
+
+ # Test retrieving the empty list
+ redis_instance.get.return_value = args[1]
+ retrieved_list = store.get(test_key)
+
+ assert retrieved_list is not None
+ assert isinstance(retrieved_list, list)
+ assert len(retrieved_list) == 0
+
+
+def test_redis_store_list_serialization_error_handling() -> None:
+ """Test error handling during list serialization."""
+ from autogen_ext.cache_store.redis import RedisStore
+
+ redis_instance = MagicMock()
+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)
+
+ # Test Redis error during set
+ redis_instance.set.side_effect = redis.RedisError("Redis connection failed")
+
+ mixed_list: List[Union[str, CreateResult]] = [
+ "test",
+ CreateResult(
+ content="test content",
+ usage=RequestUsage(prompt_tokens=1, completion_tokens=1),
+ finish_reason="stop",
+ cached=False,
+ ),
+ ]
+
+ # This should not raise an exception due to our try/except block
+ try:
+ store.set("error_key", mixed_list)
+ except Exception:
+ pytest.fail("set() method didn't handle the Redis exception properly")
+
+ # Test get with corrupted JSON data for lists
+ redis_instance.get.side_effect = None # Reset side effect
+ redis_instance.get.return_value = b'[{"invalid": json}]' # Invalid JSON
+
+ retrieved_value = store.get("corrupted_key", default=[])
+ # Should return the decoded string when JSON parsing fails (backward compatibility)
+ assert retrieved_value == '[{"invalid": json}]' # type: ignore[comparison-overlap]
diff --git a/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py b/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py
index 3a125307a..c0730904e 100644
--- a/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py
+++ b/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py
@@ -2,7 +2,7 @@ import copy
from typing import Any, Dict, List, Optional, Tuple, Union, cast
import pytest
-from autogen_core import CacheStore
+from autogen_core import CacheStore, FunctionCall
from autogen_core.models import (
ChatCompletionClient,
CreateResult,
@@ -480,3 +480,356 @@ def test_check_cache_already_correct_type() -> None:
assert isinstance(cached_result, CreateResult)
assert cached_result.content == "already correct type"
assert cache_key is not None
+
+
+async def test_redis_streaming_cache_integration() -> None:
+ """Integration test for Redis streaming cache scenario.
+ This test covers the original streaming cache issues:
+ 1. Cache is stored after streaming completes (not before)
+ 2. Redis cache properly handles lists containing CreateResult objects
+ 3. ChatCompletionCache properly reconstructs CreateResult from Redis dicts
+ """
+ from unittest.mock import MagicMock
+
+ # Skip this test if redis is not available
+ pytest.importorskip("redis")
+
+ from autogen_ext.cache_store.redis import RedisStore
+
+ # Use standardized test data
+ _, prompts, system_prompt, replay_client, _ = get_test_data()
+
+ # Mock Redis instance to control what gets stored/retrieved
+ redis_instance = MagicMock()
+ redis_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)
+
+ # Create the cached client with Redis store
+ cached_client = ChatCompletionCache(replay_client, redis_store)
+
+ # Simulate first streaming call (should cache after completion)
+ first_stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ first_stream_results.append(copy.copy(chunk))
+
+ # Verify Redis set was called with the complete streaming results
+ redis_instance.set.assert_called_once()
+ call_args = redis_instance.set.call_args
+ serialized_data = call_args[0][1]
+
+ # Verify the serialized data represents the complete stream
+ assert isinstance(serialized_data, bytes)
+ import json
+
+ deserialized = json.loads(serialized_data.decode("utf-8"))
+ assert isinstance(deserialized, list)
+ # Type narrowing: after isinstance check, deserialized is known to be a list
+ deserialized_list: List[Union[str, Dict[str, Union[str, int]]]] = deserialized # Now properly typed as list
+ # Should contain both string chunks and final CreateResult (as dict)
+ assert len(deserialized_list) > 0
+
+ # Reset the mock for the second call
+ redis_instance.reset_mock()
+
+ # Configure Redis to return the serialized data (simulating cache hit)
+ redis_instance.get.return_value = serialized_data
+
+ # Second streaming call should hit the cache
+ second_stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ second_stream_results.append(copy.copy(chunk))
+
+ # Verify Redis get was called but set was not (cache hit)
+ redis_instance.get.assert_called_once()
+ redis_instance.set.assert_not_called()
+
+ # Verify both streams have the same content
+ assert len(first_stream_results) == len(second_stream_results)
+
+ # Verify cached results are marked as cached
+ for first, second in zip(first_stream_results, second_stream_results, strict=True):
+ if isinstance(first, CreateResult) and isinstance(second, CreateResult):
+ assert not first.cached # First call should not be cached
+ assert second.cached # Second call should be cached
+ assert first.content == second.content
+ elif isinstance(first, str) and isinstance(second, str):
+ assert first == second
+ else:
+ pytest.fail(f"Unexpected chunk types: {type(first)}, {type(second)}")
+
+
+async def test_cache_cross_compatibility_create_to_stream() -> None:
+ """Test that create() cache can be used by create_stream() call.
+ This tests the scenario where:
+ 1. User calls create() - stores CreateResult
+ 2. User calls create_stream() with same inputs - should get cache hit and yield the CreateResult
+ """
+ responses, prompts, system_prompt, _, cached_client = get_test_data()
+
+ # First call: create() - should cache a CreateResult
+ create_result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])
+ assert isinstance(create_result, CreateResult)
+ assert not create_result.cached
+ assert create_result.content == responses[0]
+
+ # Second call: create_stream() with same inputs - should hit the cache
+ stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream_results.append(copy.copy(chunk))
+
+ # Should yield exactly two items: the string content + the cached CreateResult
+ assert len(stream_results) == 2
+
+ # First item should be the string content
+ assert isinstance(stream_results[0], str)
+ assert stream_results[0] == responses[0]
+
+ # Second item should be the cached CreateResult
+ assert isinstance(stream_results[1], CreateResult)
+ assert stream_results[1].cached # Should be marked as cached
+ assert stream_results[1].content == responses[0]
+
+ # Verify no additional API calls were made (cache hit)
+ initial_usage = cached_client.total_usage()
+
+ # Third call: create_stream() again - should still hit cache
+ stream_results_2: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream_results_2.append(copy.copy(chunk))
+
+ # Usage should be the same (no new API calls)
+ assert cached_client.total_usage().prompt_tokens == initial_usage.prompt_tokens
+ assert cached_client.total_usage().completion_tokens == initial_usage.completion_tokens
+
+
+async def test_cache_cross_compatibility_stream_to_create() -> None:
+ """Test that create_stream() cache can be used by create() call.
+ This tests the scenario where:
+ 1. User calls create_stream() - stores List[Union[str, CreateResult]]
+ 2. User calls create() with same inputs - should get cache hit and return the final CreateResult
+ """
+ _, prompts, system_prompt, _, cached_client = get_test_data()
+
+ # First call: create_stream() - should cache a List[Union[str, CreateResult]]
+ first_stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ first_stream_results.append(copy.copy(chunk))
+
+ # Verify we got streaming results
+ assert len(first_stream_results) > 0
+ final_create_result = None
+ for item in first_stream_results:
+ if isinstance(item, CreateResult):
+ final_create_result = item
+ break
+
+ assert final_create_result is not None
+ assert not final_create_result.cached # First call should not be cached
+
+ # Second call: create() with same inputs - should hit the streaming cache
+ create_result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])
+
+ assert isinstance(create_result, CreateResult)
+ assert create_result.cached # Should be marked as cached
+ assert create_result.content == final_create_result.content
+
+ # Verify no additional API calls were made (cache hit)
+ initial_usage = cached_client.total_usage()
+
+ # Third call: create() again - should still hit cache
+ create_result_2 = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])
+
+ # Usage should be the same (no new API calls)
+ assert cached_client.total_usage().prompt_tokens == initial_usage.prompt_tokens
+ assert cached_client.total_usage().completion_tokens == initial_usage.completion_tokens
+ assert create_result_2.cached
+
+
+async def test_cache_cross_compatibility_mixed_sequence() -> None:
+ """Test mixed sequence of create() and create_stream() calls with caching.
+ This tests a realistic scenario with multiple interleaved calls:
+ create() → create_stream() → create() → create_stream()
+ """
+ responses, prompts, system_prompt, _, cached_client = get_test_data(num_messages=4)
+
+ # Call 1: create() with prompt[0] - should make API call
+ result1 = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])
+ assert not result1.cached
+ assert result1.content == responses[0]
+ usage_after_1 = copy.copy(cached_client.total_usage())
+
+ # Call 2: create_stream() with prompt[0] - should hit cache from call 1
+ stream1_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream1_results.append(chunk)
+
+ assert len(stream1_results) == 2 # Should yield string content + cached CreateResult
+ assert isinstance(stream1_results[0], str) # First item: string content
+ assert stream1_results[0] == responses[0]
+ assert isinstance(stream1_results[1], CreateResult) # Second item: cached CreateResult
+ assert stream1_results[1].cached
+ usage_after_2 = copy.copy(cached_client.total_usage())
+ # No new API call should have been made
+ assert usage_after_2.prompt_tokens == usage_after_1.prompt_tokens
+
+ # Call 3: create_stream() with prompt[1] - should make new API call
+ stream2_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[1], source="user")]):
+ stream2_results.append(copy.copy(chunk))
+
+ # Should have made a new API call
+ usage_after_3 = copy.copy(cached_client.total_usage())
+ assert usage_after_3.prompt_tokens > usage_after_2.prompt_tokens
+
+ # Find the final CreateResult
+ final_result = None
+ for item in stream2_results:
+ if isinstance(item, CreateResult):
+ final_result = item
+ break
+ assert final_result is not None
+ assert not final_result.cached
+
+ # Call 4: create() with prompt[1] - should hit cache from call 3
+ result4 = await cached_client.create([system_prompt, UserMessage(content=prompts[1], source="user")])
+ assert result4.cached
+ assert result4.content == final_result.content
+ usage_after_4 = copy.copy(cached_client.total_usage())
+ # No new API call should have been made
+ assert usage_after_4.prompt_tokens == usage_after_3.prompt_tokens
+
+
+async def test_cache_streaming_list_without_create_result() -> None:
+ """Test edge case where streaming cache contains only strings (no CreateResult).
+ This could happen if streaming was interrupted or in unusual scenarios.
+ The create() method should handle this gracefully by falling through to make a real API call.
+ """
+ responses, prompts, system_prompt, replay_client, _ = get_test_data()
+
+ # Create a mock cache store that returns a list with only strings (no CreateResult)
+ string_only_list: List[Union[str, CreateResult]] = ["Hello", " world", "!"]
+ mock_store = MockCacheStore(return_value=string_only_list)
+ cached_client = ChatCompletionCache(replay_client, mock_store)
+
+ # Call create() - should fall through and make API call since no CreateResult in cached list
+ result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])
+
+ assert isinstance(result, CreateResult)
+ assert not result.cached # Should be from real API call, not cache
+ assert result.content == responses[0]
+
+
+async def test_create_stream_with_cached_non_streaming_result_string_content() -> None:
+ """
+ Test that when create_stream() finds a cached non-streaming result with string content,
+ it yields both the content string as a streaming chunk and then the CreateResult.
+ """
+ responses, prompts, system_prompt, replay_client, _ = get_test_data()
+
+ # Create a CreateResult with string content (simulating a cached non-streaming result)
+ cached_create_result = CreateResult(
+ content=responses[0], # This is a string
+ finish_reason="stop",
+ usage=RequestUsage(prompt_tokens=10, completion_tokens=20),
+ cached=False, # Will be set to True when retrieved from cache
+ )
+
+ # Mock cache store that returns the non-streaming CreateResult
+ mock_store = MockCacheStore(return_value=cached_create_result)
+ cached_client = ChatCompletionCache(replay_client, mock_store)
+
+ # Call create_stream() - should yield string content first, then CreateResult
+ stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream_results.append(copy.copy(chunk))
+
+ # Should have exactly 2 items: the string content, then the CreateResult
+ assert len(stream_results) == 2
+
+ # First item should be the string content
+ assert isinstance(stream_results[0], str)
+ assert stream_results[0] == responses[0]
+
+ # Second item should be the CreateResult
+ assert isinstance(stream_results[1], CreateResult)
+ assert stream_results[1].content == responses[0]
+ assert stream_results[1].finish_reason == "stop"
+ assert stream_results[1].cached is True # Should be marked as cached
+
+
+async def test_create_stream_with_cached_non_streaming_result_empty_content() -> None:
+ """
+ Test that when create_stream() finds a cached non-streaming result with empty string content,
+ it only yields the CreateResult (no separate string chunk).
+ """
+ _, prompts, system_prompt, replay_client, _ = get_test_data()
+
+ # Create a CreateResult with empty string content
+ cached_create_result = CreateResult(
+ content="", # Empty string
+ finish_reason="stop",
+ usage=RequestUsage(prompt_tokens=10, completion_tokens=0),
+ cached=False,
+ )
+
+ # Mock cache store that returns the non-streaming CreateResult
+ mock_store = MockCacheStore(return_value=cached_create_result)
+ cached_client = ChatCompletionCache(replay_client, mock_store)
+
+ # Call create_stream() - should yield only the CreateResult (no string chunk)
+ stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream_results.append(copy.copy(chunk))
+
+ # Should have exactly 1 item: just the CreateResult
+ assert len(stream_results) == 1
+
+ # Only item should be the CreateResult
+ assert isinstance(stream_results[0], CreateResult)
+ assert stream_results[0].content == ""
+ assert stream_results[0].finish_reason == "stop"
+ assert stream_results[0].cached is True
+
+
+async def test_create_stream_with_cached_non_streaming_result_non_string_content() -> None:
+ """
+ Test that when create_stream() finds a cached non-streaming result with non-string content,
+ it only yields the CreateResult (no separate string chunk).
+ """
+ _, prompts, system_prompt, replay_client, _ = get_test_data()
+
+ # Create a CreateResult with non-string content (e.g., list of function calls)
+ cached_create_result = CreateResult(
+ content=[
+ FunctionCall(id="call_123", name="test_func", arguments='{"param": "value"}')
+ ], # List of FunctionCall objects
+ finish_reason="function_calls", # Valid finish reason for function calls
+ usage=RequestUsage(prompt_tokens=10, completion_tokens=15),
+ cached=False,
+ )
+
+ # Mock cache store that returns the non-streaming CreateResult
+ mock_store = MockCacheStore(return_value=cached_create_result)
+ cached_client = ChatCompletionCache(replay_client, mock_store)
+
+ # Call create_stream() - should yield only the CreateResult (no string chunk)
+ stream_results: List[Union[str, CreateResult]] = []
+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):
+ stream_results.append(copy.copy(chunk))
+
+ # Should have exactly 1 item: just the CreateResult
+ assert len(stream_results) == 1
+
+ # Only item should be the CreateResult
+ assert isinstance(stream_results[0], CreateResult)
+ expected_function_call = FunctionCall(id="call_123", name="test_func", arguments='{"param": "value"}')
+ assert stream_results[0].content == [expected_function_call]
+ assert stream_results[0].finish_reason == "function_calls"
+ assert stream_results[0].cached is True