Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: use message idx instead of chunk idx for streaming (#1587)
cthomas42bfe08ce1d5
diff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.pyindex 8d4127aeb..7e1499d9e 100644--- a/letta/llm_api/openai.py+++ b/letta/llm_api/openai.py@@ -252,6 +252,8 @@ def openai_chat_completions_process_stream(n_chunks = 0 # approx == n_tokenschunk_idx = 0+ prev_message_type = None+ message_idx = 0try:for chat_completion_chunk in openai_chat_completions_request_stream(url=url, api_key=api_key, chat_completion_request=chat_completion_request@@ -268,14 +270,17 @@ def openai_chat_completions_process_stream(if stream_interface:if isinstance(stream_interface, AgentChunkStreamingInterface):- stream_interface.process_chunk(+ message_type = stream_interface.process_chunk(chat_completion_chunk,message_id=chat_completion_response.id if create_message_id else chat_completion_chunk.id,message_date=chat_completion_response.created if create_message_datetime else chat_completion_chunk.created,expect_reasoning_content=expect_reasoning_content,name=name,- chunk_index=chunk_idx,+ message_index=message_idx,)+ if message_type != prev_message_type and message_type is not None:+ message_idx += 1+ prev_message_type = message_typeelif isinstance(stream_interface, AgentRefreshStreamingInterface):stream_interface.process_refresh(chat_completion_response)else:diff --git a/letta/server/rest_api/chat_completions_interface.py b/letta/server/rest_api/chat_completions_interface.pyindex 4b5730b38..ae58327fd 100644--- a/letta/server/rest_api/chat_completions_interface.py+++ b/letta/server/rest_api/chat_completions_interface.py@@ -155,7 +155,12 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):returndef process_chunk(- self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False+ self,+ chunk: ChatCompletionChunkResponse,+ message_id: str,+ message_date: datetime,+ expect_reasoning_content: bool = False,+ message_index: int = 0,) -> None:"""Called externally with a ChatCompletionChunkResponse. Transformsdiff --git a/letta/server/rest_api/interface.py b/letta/server/rest_api/interface.pyindex a48bc57c4..3ef301887 100644--- a/letta/server/rest_api/interface.py+++ b/letta/server/rest_api/interface.py@@ -468,7 +468,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):# and `content` needs to be handled outside the interfaceexpect_reasoning_content: bool = False,name: Optional[str] = None,- chunk_index: int = 0,+ message_index: int = 0,) -> Optional[Union[ReasoningMessage, ToolCallMessage, AssistantMessage]]:"""Example data from non-streaming response looks like:@@ -481,7 +481,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):"""choice = chunk.choices[0]message_delta = choice.delta- otid = Message.generate_otid_from_id(message_id, chunk_index)+ otid = Message.generate_otid_from_id(message_id, message_index)if (message_delta.content is None@@ -1065,7 +1065,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):message_date: datetime,expect_reasoning_content: bool = False,name: Optional[str] = None,- chunk_index: int = 0,+ message_index: int = 0,):"""Process a streaming chunk from an OpenAI-compatible server.@@ -1092,14 +1092,15 @@ class StreamingServerInterface(AgentChunkStreamingInterface):message_date=message_date,expect_reasoning_content=expect_reasoning_content,name=name,- chunk_index=chunk_index,+ message_index=message_index,)-if processed_chunk is None:returnself._push_to_buffer(processed_chunk)+ return processed_chunk.message_type+def user_message(self, msg: str, msg_obj: Optional[Message] = None):"""Letta receives a user message"""returndiff --git a/letta/streaming_interface.py b/letta/streaming_interface.pyindex 005b104e5..6fe972860 100644--- a/letta/streaming_interface.py+++ b/letta/streaming_interface.py@@ -54,7 +54,7 @@ class AgentChunkStreamingInterface(ABC):message_id: str,message_date: datetime,expect_reasoning_content: bool = False,- chunk_index: int = 0,+ message_index: int = 0,):"""Process a streaming chunk from an OpenAI-compatible server"""raise NotImplementedError@@ -100,7 +100,12 @@ class StreamingCLIInterface(AgentChunkStreamingInterface):passdef process_chunk(- self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False+ self,+ chunk: ChatCompletionChunkResponse,+ message_id: str,+ message_date: datetime,+ expect_reasoning_content: bool = False,+ message_index: int = 0,):assert len(chunk.choices) == 1, chunk