Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: use message idx instead of chunk idx for streaming (#1587)

cthomas42bfe08ce1d5
diff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.py
index 8d4127aeb..7e1499d9e 100644
--- a/letta/llm_api/openai.py
+++ b/letta/llm_api/openai.py
@@ -252,6 +252,8 @@ def openai_chat_completions_process_stream(
n_chunks = 0 # approx == n_tokens
chunk_idx = 0
+ prev_message_type = None
+ message_idx = 0
try:
for chat_completion_chunk in openai_chat_completions_request_stream(
url=url, api_key=api_key, chat_completion_request=chat_completion_request
@@ -268,14 +270,17 @@ def openai_chat_completions_process_stream(
if stream_interface:
if isinstance(stream_interface, AgentChunkStreamingInterface):
- stream_interface.process_chunk(
+ message_type = stream_interface.process_chunk(
chat_completion_chunk,
message_id=chat_completion_response.id if create_message_id else chat_completion_chunk.id,
message_date=chat_completion_response.created if create_message_datetime else chat_completion_chunk.created,
expect_reasoning_content=expect_reasoning_content,
name=name,
- chunk_index=chunk_idx,
+ message_index=message_idx,
)
+ if message_type != prev_message_type and message_type is not None:
+ message_idx += 1
+ prev_message_type = message_type
elif isinstance(stream_interface, AgentRefreshStreamingInterface):
stream_interface.process_refresh(chat_completion_response)
else:
diff --git a/letta/server/rest_api/chat_completions_interface.py b/letta/server/rest_api/chat_completions_interface.py
index 4b5730b38..ae58327fd 100644
--- a/letta/server/rest_api/chat_completions_interface.py
+++ b/letta/server/rest_api/chat_completions_interface.py
@@ -155,7 +155,12 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):
return
def process_chunk(
- self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False
+ self,
+ chunk: ChatCompletionChunkResponse,
+ message_id: str,
+ message_date: datetime,
+ expect_reasoning_content: bool = False,
+ message_index: int = 0,
) -> None:
"""
Called externally with a ChatCompletionChunkResponse. Transforms
diff --git a/letta/server/rest_api/interface.py b/letta/server/rest_api/interface.py
index a48bc57c4..3ef301887 100644
--- a/letta/server/rest_api/interface.py
+++ b/letta/server/rest_api/interface.py
@@ -468,7 +468,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
# and `content` needs to be handled outside the interface
expect_reasoning_content: bool = False,
name: Optional[str] = None,
- chunk_index: int = 0,
+ message_index: int = 0,
) -> Optional[Union[ReasoningMessage, ToolCallMessage, AssistantMessage]]:
"""
Example data from non-streaming response looks like:
@@ -481,7 +481,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
"""
choice = chunk.choices[0]
message_delta = choice.delta
- otid = Message.generate_otid_from_id(message_id, chunk_index)
+ otid = Message.generate_otid_from_id(message_id, message_index)
if (
message_delta.content is None
@@ -1065,7 +1065,7 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
message_date: datetime,
expect_reasoning_content: bool = False,
name: Optional[str] = None,
- chunk_index: int = 0,
+ message_index: int = 0,
):
"""Process a streaming chunk from an OpenAI-compatible server.
@@ -1092,14 +1092,15 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
message_date=message_date,
expect_reasoning_content=expect_reasoning_content,
name=name,
- chunk_index=chunk_index,
+ message_index=message_index,
)
-
if processed_chunk is None:
return
self._push_to_buffer(processed_chunk)
+ return processed_chunk.message_type
+
def user_message(self, msg: str, msg_obj: Optional[Message] = None):
"""Letta receives a user message"""
return
diff --git a/letta/streaming_interface.py b/letta/streaming_interface.py
index 005b104e5..6fe972860 100644
--- a/letta/streaming_interface.py
+++ b/letta/streaming_interface.py
@@ -54,7 +54,7 @@ class AgentChunkStreamingInterface(ABC):
message_id: str,
message_date: datetime,
expect_reasoning_content: bool = False,
- chunk_index: int = 0,
+ message_index: int = 0,
):
"""Process a streaming chunk from an OpenAI-compatible server"""
raise NotImplementedError
@@ -100,7 +100,12 @@ class StreamingCLIInterface(AgentChunkStreamingInterface):
pass
def process_chunk(
- self, chunk: ChatCompletionChunkResponse, message_id: str, message_date: datetime, expect_reasoning_content: bool = False
+ self,
+ chunk: ChatCompletionChunkResponse,
+ message_id: str,
+ message_date: datetime,
+ expect_reasoning_content: bool = False,
+ message_index: int = 0,
):
assert len(chunk.choices) == 1, chunk