A programming framework for agentic AI
Redis Doesn't Support Streaming (#6954)
Ben Constable2f38334c0ef0
diff --git a/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py b/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.pyindex 873dabd87..02c3af6b0 100644--- a/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py+++ b/python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py@@ -90,6 +90,7 @@ class RedisStore(CacheStore[T], Component[RedisStoreConfig]):This method handles both primitive values and complex objects:- Pydantic models are automatically serialized to JSON+ - Lists containing Pydantic models are serialized to JSON- Primitive values (strings, numbers, etc.) are stored as-isArgs:@@ -101,10 +102,21 @@ class RedisStore(CacheStore[T], Component[RedisStoreConfig]):# Serialize Pydantic models to JSONserialized_value = value.model_dump_json().encode("utf-8")self.cache.set(key, serialized_value)+ elif isinstance(value, list):+ # Serialize lists (which may contain Pydantic models) to JSON+ serializable_list: list[Any] = []+ item: Any+ for item in value:+ if isinstance(item, BaseModel):+ serializable_list.append(item.model_dump())+ else:+ serializable_list.append(item)+ serialized_value = json.dumps(serializable_list).encode("utf-8")+ self.cache.set(key, serialized_value)else:# Backward compatibility for primitivesself.cache.set(key, cast(Any, value))- except (redis.RedisError, ConnectionError, UnicodeEncodeError):+ except (redis.RedisError, ConnectionError, UnicodeEncodeError, TypeError):# Log the error but don't re-raise to maintain robustnesspassdiff --git a/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py b/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.pyindex 279cffb81..4bd324872 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/cache/_chat_completion_cache.py@@ -77,6 +77,81 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheConasyncio.run(main())+ For Redis caching:++ .. code-block:: python++ import asyncio++ from autogen_core.models import UserMessage+ from autogen_ext.models.openai import OpenAIChatCompletionClient+ from autogen_ext.models.cache import ChatCompletionCache, CHAT_CACHE_VALUE_TYPE+ from autogen_ext.cache_store.redis import RedisStore+ import redis+++ async def main():+ # Initialize the original client+ openai_model_client = OpenAIChatCompletionClient(model="gpt-4o")++ # Initialize Redis cache store+ redis_instance = redis.Redis()+ cache_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)+ cache_client = ChatCompletionCache(openai_model_client, cache_store)++ response = await cache_client.create([UserMessage(content="Hello, how are you?", source="user")])+ print(response) # Should print response from OpenAI+ response = await cache_client.create([UserMessage(content="Hello, how are you?", source="user")])+ print(response) # Should print cached response+++ asyncio.run(main())++ For streaming with Redis caching:++ .. code-block:: python++ import asyncio++ from autogen_core.models import UserMessage, CreateResult+ from autogen_ext.models.openai import OpenAIChatCompletionClient+ from autogen_ext.models.cache import ChatCompletionCache, CHAT_CACHE_VALUE_TYPE+ from autogen_ext.cache_store.redis import RedisStore+ import redis+++ async def main():+ # Initialize the original client+ openai_model_client = OpenAIChatCompletionClient(model="gpt-4o")++ # Initialize Redis cache store+ redis_instance = redis.Redis()+ cache_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)+ cache_client = ChatCompletionCache(openai_model_client, cache_store)++ # First streaming call+ async for chunk in cache_client.create_stream(+ [UserMessage(content="List all countries in Africa", source="user")]+ ):+ if isinstance(chunk, CreateResult):+ print("\\n")+ print("Cached: ", chunk.cached) # Should print False+ else:+ print(chunk, end="")++ # Second streaming call (cached)+ async for chunk in cache_client.create_stream(+ [UserMessage(content="List all countries in Africa", source="user")]+ ):+ if isinstance(chunk, CreateResult):+ print("\\n")+ print("Cached: ", chunk.cached) # Should print True+ else:+ print(chunk, end="")+++ asyncio.run(main())+You can now use the `cached_client` as you would the original client, but with caching enabled.Args:@@ -171,10 +246,18 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheConNOTE: cancellation_token is ignored for cached results."""cached_result, cache_key = self._check_cache(messages, tools, json_output, extra_create_args)- if cached_result:- assert isinstance(cached_result, CreateResult)- cached_result.cached = True- return cached_result+ if cached_result is not None:+ if isinstance(cached_result, CreateResult):+ # Cache hit from previous non-streaming call+ cached_result.cached = True+ return cached_result+ elif isinstance(cached_result, list):+ # Cache hit from previous streaming call - extract the final CreateResult+ for item in reversed(cached_result):+ if isinstance(item, CreateResult):+ item.cached = True+ return item+ # If no CreateResult found in list, fall through to make actual callresult = await self.client.create(messages,@@ -212,13 +295,24 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheConjson_output,extra_create_args,)- if cached_result:- assert isinstance(cached_result, list)- for result in cached_result:- if isinstance(result, CreateResult):- result.cached = True- yield result- return+ if cached_result is not None:+ if isinstance(cached_result, list):+ # Cache hit from previous streaming call+ for result in cached_result:+ if isinstance(result, CreateResult):+ result.cached = True+ yield result+ return+ elif isinstance(cached_result, CreateResult):+ # Cache hit from previous non-streaming call - convert to streaming format+ cached_result.cached = True++ # If content is a non-empty string, yield it as a streaming chunk first+ if isinstance(cached_result.content, str) and cached_result.content:+ yield cached_result.content++ yield cached_result+ returnresult_stream = self.client.create_stream(messages,@@ -230,12 +324,14 @@ class ChatCompletionCache(ChatCompletionClient, Component[ChatCompletionCacheCon)output_results: List[Union[str, CreateResult]] = []- self.store.set(cache_key, output_results)async for result in result_stream:output_results.append(result)yield result+ # Store the complete results only after streaming is finished+ self.store.set(cache_key, output_results)+return _generator()async def close(self) -> None:diff --git a/python/packages/autogen-ext/tests/cache_store/test_redis_store.py b/python/packages/autogen-ext/tests/cache_store/test_redis_store.pyindex 305fe7abe..675b48cc0 100644--- a/python/packages/autogen-ext/tests/cache_store/test_redis_store.py+++ b/python/packages/autogen-ext/tests/cache_store/test_redis_store.py@@ -1,8 +1,9 @@import json-from typing import cast+from typing import Dict, List, Union, castfrom unittest.mock import MagicMockimport pytest+from autogen_core.models import CreateResult, RequestUsagefrom pydantic import BaseModelredis = pytest.importorskip("redis")@@ -180,3 +181,220 @@ def test_redis_store_nested_model_serialization() -> None:assert retrieved_model["nested"]["id"] == 1 # type: ignoreassert retrieved_model["nested"]["data"] == "nested_data" # type: ignoreassert retrieved_model["tags"] == ["tag1", "tag2", "tag3"] # type: ignore+++def test_redis_store_list_with_strings_only() -> None:+ """Test serialization of lists containing only strings (streaming scenario)."""+ from autogen_ext.cache_store.redis import RedisStore++ redis_instance = MagicMock()+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)+ test_key = "test_string_list_key"++ # Create a list with only strings (partial streaming result)+ string_list: List[Union[str, CreateResult]] = ["Hello", " world", "!", " How", " are", " you", "?"]++ # Test setting the list+ store.set(test_key, string_list)++ # Verify Redis was called with JSON-serialized data+ args, _ = redis_instance.set.call_args+ assert args[0] == test_key+ assert isinstance(args[1], bytes)++ # Verify the serialized data is correct+ serialized_json = args[1].decode("utf-8")+ deserialized_data = json.loads(serialized_json)+ assert deserialized_data == string_list++ # Test retrieving the list+ redis_instance.get.return_value = args[1] # Return the serialized data+ retrieved_list = store.get(test_key)++ assert retrieved_list is not None+ assert isinstance(retrieved_list, list)+ assert retrieved_list == string_list+++def test_redis_store_list_with_create_results_only() -> None:+ """Test serialization of lists containing only CreateResult objects."""+ from autogen_ext.cache_store.redis import RedisStore++ redis_instance = MagicMock()+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)+ test_key = "test_create_result_list_key"++ # Create a list with only CreateResult objects+ usage = RequestUsage(prompt_tokens=10, completion_tokens=20)+ create_result_list: List[Union[str, CreateResult]] = [+ CreateResult(content="First response", usage=usage, finish_reason="stop", cached=False),+ CreateResult(content="Second response", usage=usage, finish_reason="stop", cached=False),+ ]++ # Test setting the list+ store.set(test_key, create_result_list)++ # Verify Redis was called with JSON-serialized data+ args, _ = redis_instance.set.call_args+ assert args[0] == test_key+ assert isinstance(args[1], bytes)++ # Verify the serialized data structure+ serialized_json = args[1].decode("utf-8")+ deserialized_data = json.loads(serialized_json)++ assert isinstance(deserialized_data, list)+ # Type narrowing: after isinstance check, deserialized_data is known to be a list+ deserialized_list: List[Dict[str, Union[str, int]]] = deserialized_data # Now properly typed as list+ assert len(deserialized_list) == 2+ assert deserialized_list[0]["content"] == "First response"+ assert deserialized_list[1]["content"] == "Second response"+ assert deserialized_list[0]["finish_reason"] == "stop"++ # Test retrieving the list+ redis_instance.get.return_value = args[1] # Return the serialized data+ retrieved_list = store.get(test_key)++ assert retrieved_list is not None+ assert isinstance(retrieved_list, list)+ assert len(retrieved_list) == 2++ # The retrieved items should be dicts (as Redis returns JSON-parsed objects)+ assert isinstance(retrieved_list[0], dict)+ assert isinstance(retrieved_list[1], dict)+ assert retrieved_list[0]["content"] == "First response" # type: ignore+ assert retrieved_list[1]["content"] == "Second response" # type: ignore+++def test_redis_store_mixed_list_streaming_scenario() -> None:+ """Test serialization of mixed lists (strings + CreateResult) for streaming cache scenario."""+ from autogen_ext.cache_store.redis import RedisStore++ redis_instance = MagicMock()+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)+ test_key = "test_mixed_streaming_list_key"++ # Create a mixed list simulating a streaming response+ usage = RequestUsage(prompt_tokens=15, completion_tokens=30)+ mixed_list: List[Union[str, CreateResult]] = [+ "The",+ " capital",+ " of",+ " France",+ " is",+ " Paris",+ ".",+ CreateResult(content="The capital of France is Paris.", usage=usage, finish_reason="stop", cached=False),+ ]++ # Test setting the mixed list+ store.set(test_key, mixed_list)++ # Verify Redis was called with JSON-serialized data+ args, _ = redis_instance.set.call_args+ assert args[0] == test_key+ assert isinstance(args[1], bytes)++ # Verify the serialized data structure+ serialized_json = args[1].decode("utf-8")+ deserialized_data = json.loads(serialized_json)++ assert isinstance(deserialized_data, list)+ # Type narrowing: after isinstance check, deserialized_data is known to be a list+ deserialized_list: List[Union[str, Dict[str, Union[str, int]]]] = deserialized_data # Now properly typed as list+ assert len(deserialized_list) == 8 # 7 strings + 1 CreateResult++ # First 7 items should be strings+ for i in range(7):+ assert isinstance(deserialized_list[i], str)++ # Last item should be the serialized CreateResult (as dict)+ assert isinstance(deserialized_list[7], dict)+ assert deserialized_list[7]["content"] == "The capital of France is Paris."+ assert deserialized_list[7]["finish_reason"] == "stop"+ assert deserialized_data[7]["usage"]["prompt_tokens"] == 15+ assert deserialized_data[7]["usage"]["completion_tokens"] == 30++ # Test retrieving the mixed list+ redis_instance.get.return_value = args[1] # Return the serialized data+ retrieved_list = store.get(test_key)++ assert retrieved_list is not None+ assert isinstance(retrieved_list, list)+ assert len(retrieved_list) == 8++ # First 7 items should still be strings+ for i in range(7):+ assert isinstance(retrieved_list[i], str)+ assert retrieved_list[i] == mixed_list[i]++ # Last item should be a dict (CreateResult deserialized from JSON)+ assert isinstance(retrieved_list[7], dict)+ assert retrieved_list[7]["content"] == "The capital of France is Paris." # type: ignore+ assert retrieved_list[7]["cached"] is False # type: ignore+++def test_redis_store_empty_list() -> None:+ """Test serialization of empty lists."""+ from autogen_ext.cache_store.redis import RedisStore++ redis_instance = MagicMock()+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)+ test_key = "test_empty_list_key"++ # Test setting an empty list+ empty_list: List[Union[str, CreateResult]] = []+ store.set(test_key, empty_list)++ # Verify Redis was called with JSON-serialized data+ args, _ = redis_instance.set.call_args+ assert args[0] == test_key+ assert isinstance(args[1], bytes)++ # Verify the serialized data is an empty JSON array+ serialized_json = args[1].decode("utf-8")+ deserialized_data = json.loads(serialized_json)+ assert deserialized_data == []++ # Test retrieving the empty list+ redis_instance.get.return_value = args[1]+ retrieved_list = store.get(test_key)++ assert retrieved_list is not None+ assert isinstance(retrieved_list, list)+ assert len(retrieved_list) == 0+++def test_redis_store_list_serialization_error_handling() -> None:+ """Test error handling during list serialization."""+ from autogen_ext.cache_store.redis import RedisStore++ redis_instance = MagicMock()+ store = RedisStore[List[Union[str, CreateResult]]](redis_instance)++ # Test Redis error during set+ redis_instance.set.side_effect = redis.RedisError("Redis connection failed")++ mixed_list: List[Union[str, CreateResult]] = [+ "test",+ CreateResult(+ content="test content",+ usage=RequestUsage(prompt_tokens=1, completion_tokens=1),+ finish_reason="stop",+ cached=False,+ ),+ ]++ # This should not raise an exception due to our try/except block+ try:+ store.set("error_key", mixed_list)+ except Exception:+ pytest.fail("set() method didn't handle the Redis exception properly")++ # Test get with corrupted JSON data for lists+ redis_instance.get.side_effect = None # Reset side effect+ redis_instance.get.return_value = b'[{"invalid": json}]' # Invalid JSON++ retrieved_value = store.get("corrupted_key", default=[])+ # Should return the decoded string when JSON parsing fails (backward compatibility)+ assert retrieved_value == '[{"invalid": json}]' # type: ignore[comparison-overlap]diff --git a/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py b/python/packages/autogen-ext/tests/models/test_chat_completion_cache.pyindex 3a125307a..c0730904e 100644--- a/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py+++ b/python/packages/autogen-ext/tests/models/test_chat_completion_cache.py@@ -2,7 +2,7 @@ import copyfrom typing import Any, Dict, List, Optional, Tuple, Union, castimport pytest-from autogen_core import CacheStore+from autogen_core import CacheStore, FunctionCallfrom autogen_core.models import (ChatCompletionClient,CreateResult,@@ -480,3 +480,356 @@ def test_check_cache_already_correct_type() -> None:assert isinstance(cached_result, CreateResult)assert cached_result.content == "already correct type"assert cache_key is not None+++async def test_redis_streaming_cache_integration() -> None:+ """Integration test for Redis streaming cache scenario.+ This test covers the original streaming cache issues:+ 1. Cache is stored after streaming completes (not before)+ 2. Redis cache properly handles lists containing CreateResult objects+ 3. ChatCompletionCache properly reconstructs CreateResult from Redis dicts+ """+ from unittest.mock import MagicMock++ # Skip this test if redis is not available+ pytest.importorskip("redis")++ from autogen_ext.cache_store.redis import RedisStore++ # Use standardized test data+ _, prompts, system_prompt, replay_client, _ = get_test_data()++ # Mock Redis instance to control what gets stored/retrieved+ redis_instance = MagicMock()+ redis_store = RedisStore[CHAT_CACHE_VALUE_TYPE](redis_instance)++ # Create the cached client with Redis store+ cached_client = ChatCompletionCache(replay_client, redis_store)++ # Simulate first streaming call (should cache after completion)+ first_stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ first_stream_results.append(copy.copy(chunk))++ # Verify Redis set was called with the complete streaming results+ redis_instance.set.assert_called_once()+ call_args = redis_instance.set.call_args+ serialized_data = call_args[0][1]++ # Verify the serialized data represents the complete stream+ assert isinstance(serialized_data, bytes)+ import json++ deserialized = json.loads(serialized_data.decode("utf-8"))+ assert isinstance(deserialized, list)+ # Type narrowing: after isinstance check, deserialized is known to be a list+ deserialized_list: List[Union[str, Dict[str, Union[str, int]]]] = deserialized # Now properly typed as list+ # Should contain both string chunks and final CreateResult (as dict)+ assert len(deserialized_list) > 0++ # Reset the mock for the second call+ redis_instance.reset_mock()++ # Configure Redis to return the serialized data (simulating cache hit)+ redis_instance.get.return_value = serialized_data++ # Second streaming call should hit the cache+ second_stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ second_stream_results.append(copy.copy(chunk))++ # Verify Redis get was called but set was not (cache hit)+ redis_instance.get.assert_called_once()+ redis_instance.set.assert_not_called()++ # Verify both streams have the same content+ assert len(first_stream_results) == len(second_stream_results)++ # Verify cached results are marked as cached+ for first, second in zip(first_stream_results, second_stream_results, strict=True):+ if isinstance(first, CreateResult) and isinstance(second, CreateResult):+ assert not first.cached # First call should not be cached+ assert second.cached # Second call should be cached+ assert first.content == second.content+ elif isinstance(first, str) and isinstance(second, str):+ assert first == second+ else:+ pytest.fail(f"Unexpected chunk types: {type(first)}, {type(second)}")+++async def test_cache_cross_compatibility_create_to_stream() -> None:+ """Test that create() cache can be used by create_stream() call.+ This tests the scenario where:+ 1. User calls create() - stores CreateResult+ 2. User calls create_stream() with same inputs - should get cache hit and yield the CreateResult+ """+ responses, prompts, system_prompt, _, cached_client = get_test_data()++ # First call: create() - should cache a CreateResult+ create_result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])+ assert isinstance(create_result, CreateResult)+ assert not create_result.cached+ assert create_result.content == responses[0]++ # Second call: create_stream() with same inputs - should hit the cache+ stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream_results.append(copy.copy(chunk))++ # Should yield exactly two items: the string content + the cached CreateResult+ assert len(stream_results) == 2++ # First item should be the string content+ assert isinstance(stream_results[0], str)+ assert stream_results[0] == responses[0]++ # Second item should be the cached CreateResult+ assert isinstance(stream_results[1], CreateResult)+ assert stream_results[1].cached # Should be marked as cached+ assert stream_results[1].content == responses[0]++ # Verify no additional API calls were made (cache hit)+ initial_usage = cached_client.total_usage()++ # Third call: create_stream() again - should still hit cache+ stream_results_2: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream_results_2.append(copy.copy(chunk))++ # Usage should be the same (no new API calls)+ assert cached_client.total_usage().prompt_tokens == initial_usage.prompt_tokens+ assert cached_client.total_usage().completion_tokens == initial_usage.completion_tokens+++async def test_cache_cross_compatibility_stream_to_create() -> None:+ """Test that create_stream() cache can be used by create() call.+ This tests the scenario where:+ 1. User calls create_stream() - stores List[Union[str, CreateResult]]+ 2. User calls create() with same inputs - should get cache hit and return the final CreateResult+ """+ _, prompts, system_prompt, _, cached_client = get_test_data()++ # First call: create_stream() - should cache a List[Union[str, CreateResult]]+ first_stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ first_stream_results.append(copy.copy(chunk))++ # Verify we got streaming results+ assert len(first_stream_results) > 0+ final_create_result = None+ for item in first_stream_results:+ if isinstance(item, CreateResult):+ final_create_result = item+ break++ assert final_create_result is not None+ assert not final_create_result.cached # First call should not be cached++ # Second call: create() with same inputs - should hit the streaming cache+ create_result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])++ assert isinstance(create_result, CreateResult)+ assert create_result.cached # Should be marked as cached+ assert create_result.content == final_create_result.content++ # Verify no additional API calls were made (cache hit)+ initial_usage = cached_client.total_usage()++ # Third call: create() again - should still hit cache+ create_result_2 = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])++ # Usage should be the same (no new API calls)+ assert cached_client.total_usage().prompt_tokens == initial_usage.prompt_tokens+ assert cached_client.total_usage().completion_tokens == initial_usage.completion_tokens+ assert create_result_2.cached+++async def test_cache_cross_compatibility_mixed_sequence() -> None:+ """Test mixed sequence of create() and create_stream() calls with caching.+ This tests a realistic scenario with multiple interleaved calls:+ create() → create_stream() → create() → create_stream()+ """+ responses, prompts, system_prompt, _, cached_client = get_test_data(num_messages=4)++ # Call 1: create() with prompt[0] - should make API call+ result1 = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])+ assert not result1.cached+ assert result1.content == responses[0]+ usage_after_1 = copy.copy(cached_client.total_usage())++ # Call 2: create_stream() with prompt[0] - should hit cache from call 1+ stream1_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream1_results.append(chunk)++ assert len(stream1_results) == 2 # Should yield string content + cached CreateResult+ assert isinstance(stream1_results[0], str) # First item: string content+ assert stream1_results[0] == responses[0]+ assert isinstance(stream1_results[1], CreateResult) # Second item: cached CreateResult+ assert stream1_results[1].cached+ usage_after_2 = copy.copy(cached_client.total_usage())+ # No new API call should have been made+ assert usage_after_2.prompt_tokens == usage_after_1.prompt_tokens++ # Call 3: create_stream() with prompt[1] - should make new API call+ stream2_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[1], source="user")]):+ stream2_results.append(copy.copy(chunk))++ # Should have made a new API call+ usage_after_3 = copy.copy(cached_client.total_usage())+ assert usage_after_3.prompt_tokens > usage_after_2.prompt_tokens++ # Find the final CreateResult+ final_result = None+ for item in stream2_results:+ if isinstance(item, CreateResult):+ final_result = item+ break+ assert final_result is not None+ assert not final_result.cached++ # Call 4: create() with prompt[1] - should hit cache from call 3+ result4 = await cached_client.create([system_prompt, UserMessage(content=prompts[1], source="user")])+ assert result4.cached+ assert result4.content == final_result.content+ usage_after_4 = copy.copy(cached_client.total_usage())+ # No new API call should have been made+ assert usage_after_4.prompt_tokens == usage_after_3.prompt_tokens+++async def test_cache_streaming_list_without_create_result() -> None:+ """Test edge case where streaming cache contains only strings (no CreateResult).+ This could happen if streaming was interrupted or in unusual scenarios.+ The create() method should handle this gracefully by falling through to make a real API call.+ """+ responses, prompts, system_prompt, replay_client, _ = get_test_data()++ # Create a mock cache store that returns a list with only strings (no CreateResult)+ string_only_list: List[Union[str, CreateResult]] = ["Hello", " world", "!"]+ mock_store = MockCacheStore(return_value=string_only_list)+ cached_client = ChatCompletionCache(replay_client, mock_store)++ # Call create() - should fall through and make API call since no CreateResult in cached list+ result = await cached_client.create([system_prompt, UserMessage(content=prompts[0], source="user")])++ assert isinstance(result, CreateResult)+ assert not result.cached # Should be from real API call, not cache+ assert result.content == responses[0]+++async def test_create_stream_with_cached_non_streaming_result_string_content() -> None:+ """+ Test that when create_stream() finds a cached non-streaming result with string content,+ it yields both the content string as a streaming chunk and then the CreateResult.+ """+ responses, prompts, system_prompt, replay_client, _ = get_test_data()++ # Create a CreateResult with string content (simulating a cached non-streaming result)+ cached_create_result = CreateResult(+ content=responses[0], # This is a string+ finish_reason="stop",+ usage=RequestUsage(prompt_tokens=10, completion_tokens=20),+ cached=False, # Will be set to True when retrieved from cache+ )++ # Mock cache store that returns the non-streaming CreateResult+ mock_store = MockCacheStore(return_value=cached_create_result)+ cached_client = ChatCompletionCache(replay_client, mock_store)++ # Call create_stream() - should yield string content first, then CreateResult+ stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream_results.append(copy.copy(chunk))++ # Should have exactly 2 items: the string content, then the CreateResult+ assert len(stream_results) == 2++ # First item should be the string content+ assert isinstance(stream_results[0], str)+ assert stream_results[0] == responses[0]++ # Second item should be the CreateResult+ assert isinstance(stream_results[1], CreateResult)+ assert stream_results[1].content == responses[0]+ assert stream_results[1].finish_reason == "stop"+ assert stream_results[1].cached is True # Should be marked as cached+++async def test_create_stream_with_cached_non_streaming_result_empty_content() -> None:+ """+ Test that when create_stream() finds a cached non-streaming result with empty string content,+ it only yields the CreateResult (no separate string chunk).+ """+ _, prompts, system_prompt, replay_client, _ = get_test_data()++ # Create a CreateResult with empty string content+ cached_create_result = CreateResult(+ content="", # Empty string+ finish_reason="stop",+ usage=RequestUsage(prompt_tokens=10, completion_tokens=0),+ cached=False,+ )++ # Mock cache store that returns the non-streaming CreateResult+ mock_store = MockCacheStore(return_value=cached_create_result)+ cached_client = ChatCompletionCache(replay_client, mock_store)++ # Call create_stream() - should yield only the CreateResult (no string chunk)+ stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream_results.append(copy.copy(chunk))++ # Should have exactly 1 item: just the CreateResult+ assert len(stream_results) == 1++ # Only item should be the CreateResult+ assert isinstance(stream_results[0], CreateResult)+ assert stream_results[0].content == ""+ assert stream_results[0].finish_reason == "stop"+ assert stream_results[0].cached is True+++async def test_create_stream_with_cached_non_streaming_result_non_string_content() -> None:+ """+ Test that when create_stream() finds a cached non-streaming result with non-string content,+ it only yields the CreateResult (no separate string chunk).+ """+ _, prompts, system_prompt, replay_client, _ = get_test_data()++ # Create a CreateResult with non-string content (e.g., list of function calls)+ cached_create_result = CreateResult(+ content=[+ FunctionCall(id="call_123", name="test_func", arguments='{"param": "value"}')+ ], # List of FunctionCall objects+ finish_reason="function_calls", # Valid finish reason for function calls+ usage=RequestUsage(prompt_tokens=10, completion_tokens=15),+ cached=False,+ )++ # Mock cache store that returns the non-streaming CreateResult+ mock_store = MockCacheStore(return_value=cached_create_result)+ cached_client = ChatCompletionCache(replay_client, mock_store)++ # Call create_stream() - should yield only the CreateResult (no string chunk)+ stream_results: List[Union[str, CreateResult]] = []+ async for chunk in cached_client.create_stream([system_prompt, UserMessage(content=prompts[0], source="user")]):+ stream_results.append(copy.copy(chunk))++ # Should have exactly 1 item: just the CreateResult+ assert len(stream_results) == 1++ # Only item should be the CreateResult+ assert isinstance(stream_results[0], CreateResult)+ expected_function_call = FunctionCall(id="call_123", name="test_func", arguments='{"param": "value"}')+ assert stream_results[0].content == [expected_function_call]+ assert stream_results[0].finish_reason == "function_calls"+ assert stream_results[0].cached is True