Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: Fix chat completions streaming (#1078)
Matthew Zhou94899b2be131
diff --git a/letta/server/rest_api/chat_completions_interface.py b/letta/server/rest_api/chat_completions_interface.pyindex 88fde2032..9c03464ed 100644--- a/letta/server/rest_api/chat_completions_interface.py+++ b/letta/server/rest_api/chat_completions_interface.py@@ -153,7 +153,9 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):"""No-op retained for interface compatibility."""return- def process_chunk(self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime) -> None:+ def process_chunk(+ self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False+ ) -> None:"""Called externally with a ChatCompletionChunkResponse. Transformsit if necessary, then enqueues partial messages for streaming back.diff --git a/letta/streaming_interface.py b/letta/streaming_interface.pyindex 2949b94e4..3d007cad0 100644--- a/letta/streaming_interface.py+++ b/letta/streaming_interface.py@@ -48,7 +48,9 @@ class AgentChunkStreamingInterface(ABC):raise NotImplementedError@abstractmethod- def process_chunk(self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime):+ def process_chunk(+ self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False+ ):"""Process a streaming chunk from an OpenAI-compatible server"""raise NotImplementedError@@ -92,7 +94,9 @@ class StreamingCLIInterface(AgentChunkStreamingInterface):def _flush(self):pass- def process_chunk(self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime):+ def process_chunk(+ self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False+ ):assert len(chunk.choices) == 1, chunkmessage_delta = chunk.choices[0].deltadiff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.pyindex ef19bdb68..767456cae 100644--- a/tests/integration_test_chat_completions.py+++ b/tests/integration_test_chat_completions.py@@ -120,9 +120,16 @@ def test_chat_completions_streaming(mock_e2b_api_key_none, client, agent, messagf"{client.base_url}/openai/{client.api_prefix}/chat/completions", request.model_dump(exclude_none=True), client.headers)- chunks = list(response)- for idx, chunk in enumerate(chunks):- _assert_valid_chunk(chunk, idx, chunks)+ try:+ chunks = list(response)+ assert len(chunks) > 1, "Streaming response did not return enough chunks (may have failed silently)."++ for idx, chunk in enumerate(chunks):+ assert chunk, f"Empty chunk received at index {idx}."+ print(chunk)+ _assert_valid_chunk(chunk, idx, chunks)+ except Exception as e:+ pytest.fail(f"Streaming failed with exception: {e}")@pytest.mark.asyncio@@ -134,10 +141,15 @@ async def test_chat_completions_streaming_async(client, agent, message):async_client = AsyncOpenAI(base_url=f"{client.base_url}/openai/{client.api_prefix}", max_retries=0)stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))- async with stream:- async for chunk in stream:- if isinstance(chunk, ChatCompletionChunk):+ received_chunks = 0+ try:+ async with stream:+ async for chunk in stream:+ assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."assert chunk.choices[0].delta.content, f"Chunk at index 0 has no content: {chunk.model_dump_json(indent=4)}"- else:- pytest.fail(f"Unexpected chunk type: {chunk}")+ received_chunks += 1+ except Exception as e:+ pytest.fail(f"Streaming failed with exception: {e}")++ assert received_chunks > 1, "No valid streaming chunks were received."