Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: consolidate logic for finish tokens (#2779)
cthomasccb203b2646a
diff --git a/letta/agents/base_agent.py b/letta/agents/base_agent.pyindex dcc0f2ec6..d4e4e38f7 100644--- a/letta/agents/base_agent.py+++ b/letta/agents/base_agent.py@@ -13,6 +13,7 @@ from letta.schemas.letta_message import LegacyLettaMessage, LettaMessagefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.letta_response import LettaResponsefrom letta.schemas.message import Message, MessageCreate, MessageUpdate+from letta.schemas.usage import LettaUsageStatisticsfrom letta.schemas.user import Userfrom letta.services.agent_manager import AgentManagerfrom letta.services.helpers.agent_manager_helper import compile_system_message@@ -136,3 +137,9 @@ class BaseAgent(ABC):except:logger.exception(f"Failed to rebuild memory for agent id={agent_state.id} and actor=({self.actor.id}, {self.actor.name})")raise++ def get_finish_chunks_for_stream(self, usage: LettaUsageStatistics):+ return [+ usage.model_dump_json(),+ MessageStreamStatus.done.value,+ ]diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex 6d40704ff..0620ccce1 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -25,7 +25,7 @@ from letta.otel.context import get_ctx_attributesfrom letta.otel.metric_registry import MetricRegistryfrom letta.otel.tracing import log_event, trace_method, tracerfrom letta.schemas.agent import AgentState-from letta.schemas.enums import MessageRole, MessageStreamStatus+from letta.schemas.enums import MessageRolefrom letta.schemas.letta_message import MessageTypefrom letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContentfrom letta.schemas.letta_response import LettaResponse@@ -262,9 +262,7 @@ class LettaAgent(BaseAgent):)for message in letta_messages:- if not include_return_message_types:- yield f"data: {message.model_dump_json()}\n\n"- elif include_return_message_types and message.message_type in include_return_message_types:+ if include_return_message_types is None or message.message_type in include_return_message_types:yield f"data: {message.model_dump_json()}\n\n"if not should_continue:@@ -288,8 +286,8 @@ class LettaAgent(BaseAgent):request_span.end()# Return back usage- yield f"data: {usage.model_dump_json()}\n\n"- yield f"data: {MessageStreamStatus.done.model_dump_json()}\n\n"+ for finish_chunk in self.get_finish_chunks_for_stream(usage):+ yield f"data: {finish_chunk}\n\n"async def _step(self,@@ -506,10 +504,7 @@ class LettaAgent(BaseAgent):request_span.add_event(name="time_to_first_token_ms", attributes={"ttft_ms": ns_to_ms(ttft_ns)})first_chunk = False- if include_return_message_types is None:- # return all data- yield f"data: {chunk.model_dump_json()}\n\n"- elif include_return_message_types and chunk.message_type in include_return_message_types:+ if include_return_message_types is None or chunk.message_type in include_return_message_types:# filter down returned datayield f"data: {chunk.model_dump_json()}\n\n"@@ -610,9 +605,8 @@ class LettaAgent(BaseAgent):request_span.add_event(name="letta_request_ms", attributes={"duration_ms": ns_to_ms(request_ns)})request_span.end()- # TODO: Also yield out a letta usage stats SSE- yield f"data: {usage.model_dump_json()}\n\n"- yield f"data: {MessageStreamStatus.done.model_dump_json()}\n\n"+ for finish_chunk in self.get_finish_chunks_for_stream(usage):+ yield f"data: {finish_chunk}\n\n"async def _build_and_request_from_llm(self,diff --git a/letta/groups/sleeptime_multi_agent_v2.py b/letta/groups/sleeptime_multi_agent_v2.pyindex f08759416..97c9216b4 100644--- a/letta/groups/sleeptime_multi_agent_v2.py+++ b/letta/groups/sleeptime_multi_agent_v2.py@@ -144,7 +144,8 @@ class SleeptimeMultiAgentV2(BaseAgent):for message in response.messages:yield f"data: {message.model_dump_json()}\n\n"- yield f"data: {response.usage.model_dump_json()}\n\n"+ for finish_chunk in self.get_finish_chunks_for_stream(response.usage):+ yield f"data: {finish_chunk}\n\n"@trace_methodasync def step_stream(diff --git a/letta/interfaces/anthropic_streaming_interface.py b/letta/interfaces/anthropic_streaming_interface.pyindex 78bac4143..e4f331334 100644--- a/letta/interfaces/anthropic_streaming_interface.py+++ b/letta/interfaces/anthropic_streaming_interface.py@@ -90,6 +90,8 @@ class AnthropicStreamingInterface:def get_tool_call_object(self) -> ToolCall:"""Useful for agent loop"""+ if not self.tool_call_name:+ raise ValueError("No tool call returned")# hack for tool rulestry:tool_input = json.loads(self.accumulated_tool_call_args)diff --git a/letta/interfaces/openai_streaming_interface.py b/letta/interfaces/openai_streaming_interface.pyindex 7417be50b..3020e7a43 100644--- a/letta/interfaces/openai_streaming_interface.py+++ b/letta/interfaces/openai_streaming_interface.py@@ -6,6 +6,7 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_ms+from letta.log import get_loggerfrom letta.schemas.letta_message import AssistantMessage, LettaMessage, ReasoningMessage, ToolCallDelta, ToolCallMessagefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.message import Message@@ -13,6 +14,8 @@ from letta.schemas.openai.chat_completion_response import FunctionCall, ToolCallfrom letta.server.rest_api.json_parser import OptimisticJSONParserfrom letta.streaming_utils import JSONInnerThoughtsExtractor+logger = get_logger(__name__)+class OpenAIStreamingInterface:"""@@ -60,6 +63,8 @@ class OpenAIStreamingInterface:def get_tool_call_object(self) -> ToolCall:"""Useful for agent loop"""function_name = self.last_flushed_function_name if self.last_flushed_function_name else self.function_name_buffer+ if not function_name:+ raise ValueError("No tool call ID available")tool_call_id = self.last_flushed_function_id if self.last_flushed_function_id else self.function_id_bufferif not tool_call_id:raise ValueError("No tool call ID available")@@ -79,254 +84,259 @@ class OpenAIStreamingInterface:It also collects tokens and detects if a tool call is triggered."""first_chunk = True+ try:+ async with stream:+ prev_message_type = None+ message_index = 0+ async for chunk in stream:+ if first_chunk and ttft_span is not None and provider_request_start_timestamp_ns is not None:+ now = get_utc_timestamp_ns()+ ttft_ns = now - provider_request_start_timestamp_ns+ ttft_span.add_event(+ name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ns_to_ms(ttft_ns)}+ )+ first_chunk = False++ if not self.model or not self.message_id:+ self.model = chunk.model+ self.message_id = chunk.id++ # track usage+ if chunk.usage:+ self.input_tokens += chunk.usage.prompt_tokens+ self.output_tokens += chunk.usage.completion_tokens++ if chunk.choices:+ choice = chunk.choices[0]+ message_delta = choice.delta++ if message_delta.tool_calls is not None and len(message_delta.tool_calls) > 0:+ tool_call = message_delta.tool_calls[0]++ if tool_call.function.name:+ # If we're waiting for the first key, then we should hold back the name+ # ie add it to a buffer instead of returning it as a chunk+ if self.function_name_buffer is None:+ self.function_name_buffer = tool_call.function.name+ else:+ self.function_name_buffer += tool_call.function.name- async with stream:- prev_message_type = None- message_index = 0- async for chunk in stream:- if first_chunk and ttft_span is not None and provider_request_start_timestamp_ns is not None:- now = get_utc_timestamp_ns()- ttft_ns = now - provider_request_start_timestamp_ns- ttft_span.add_event(- name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ns_to_ms(ttft_ns)}- )- first_chunk = False-- if not self.model or not self.message_id:- self.model = chunk.model- self.message_id = chunk.id-- # track usage- if chunk.usage:- self.input_tokens += chunk.usage.prompt_tokens- self.output_tokens += chunk.usage.completion_tokens-- if chunk.choices:- choice = chunk.choices[0]- message_delta = choice.delta-- if message_delta.tool_calls is not None and len(message_delta.tool_calls) > 0:- tool_call = message_delta.tool_calls[0]-- if tool_call.function.name:- # If we're waiting for the first key, then we should hold back the name- # ie add it to a buffer instead of returning it as a chunk- if self.function_name_buffer is None:- self.function_name_buffer = tool_call.function.name- else:- self.function_name_buffer += tool_call.function.name-- if tool_call.id:- # Buffer until next time- if self.function_id_buffer is None:- self.function_id_buffer = tool_call.id- else:- self.function_id_buffer += tool_call.id-- if tool_call.function.arguments:- # updates_main_json, updates_inner_thoughts = self.function_args_reader.process_fragment(tool_call.function.arguments)- self.current_function_arguments += tool_call.function.arguments- updates_main_json, updates_inner_thoughts = self.function_args_reader.process_fragment(- tool_call.function.arguments- )-- # If we have inner thoughts, we should output them as a chunk- if updates_inner_thoughts:- if prev_message_type and prev_message_type != "reasoning_message":- message_index += 1- self.reasoning_messages.append(updates_inner_thoughts)- reasoning_message = ReasoningMessage(- id=self.letta_message_id,- date=datetime.now(timezone.utc),- reasoning=updates_inner_thoughts,- # name=name,- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),- )- prev_message_type = reasoning_message.message_type- yield reasoning_message-- # Additionally inner thoughts may stream back with a chunk of main JSON- # In that case, since we can only return a chunk at a time, we should buffer it- if updates_main_json:- if self.function_args_buffer is None:- self.function_args_buffer = updates_main_json- else:- self.function_args_buffer += updates_main_json+ if tool_call.id:+ # Buffer until next time+ if self.function_id_buffer is None:+ self.function_id_buffer = tool_call.id+ else:+ self.function_id_buffer += tool_call.id- # If we have main_json, we should output a ToolCallMessage- elif updates_main_json:+ if tool_call.function.arguments:+ # updates_main_json, updates_inner_thoughts = self.function_args_reader.process_fragment(tool_call.function.arguments)+ self.current_function_arguments += tool_call.function.arguments+ updates_main_json, updates_inner_thoughts = self.function_args_reader.process_fragment(+ tool_call.function.arguments+ )- # If there's something in the function_name buffer, we should release it first- # NOTE: we could output it as part of a chunk that has both name and args,- # however the frontend may expect name first, then args, so to be- # safe we'll output name first in a separate chunk- if self.function_name_buffer:+ # If we have inner thoughts, we should output them as a chunk+ if updates_inner_thoughts:+ if prev_message_type and prev_message_type != "reasoning_message":+ message_index += 1+ self.reasoning_messages.append(updates_inner_thoughts)+ reasoning_message = ReasoningMessage(+ id=self.letta_message_id,+ date=datetime.now(timezone.utc),+ reasoning=updates_inner_thoughts,+ # name=name,+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),+ )+ prev_message_type = reasoning_message.message_type+ yield reasoning_message++ # Additionally inner thoughts may stream back with a chunk of main JSON+ # In that case, since we can only return a chunk at a time, we should buffer it+ if updates_main_json:+ if self.function_args_buffer is None:+ self.function_args_buffer = updates_main_json+ else:+ self.function_args_buffer += updates_main_json- # use_assisitant_message means that we should also not release main_json raw, and instead should only release the contents of "message": "..."- if self.use_assistant_message and self.function_name_buffer == self.assistant_message_tool_name:+ # If we have main_json, we should output a ToolCallMessage+ elif updates_main_json:- # Store the ID of the tool call so allow skipping the corresponding response- if self.function_id_buffer:- self.prev_assistant_message_id = self.function_id_buffer+ # If there's something in the function_name buffer, we should release it first+ # NOTE: we could output it as part of a chunk that has both name and args,+ # however the frontend may expect name first, then args, so to be+ # safe we'll output name first in a separate chunk+ if self.function_name_buffer:- else:- if prev_message_type and prev_message_type != "tool_call_message":- message_index += 1- self.tool_call_name = str(self.function_name_buffer)- tool_call_msg = ToolCallMessage(- id=self.letta_message_id,- date=datetime.now(timezone.utc),- tool_call=ToolCallDelta(- name=self.function_name_buffer,- arguments=None,- tool_call_id=self.function_id_buffer,- ),- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),- )- prev_message_type = tool_call_msg.message_type- yield tool_call_msg-- # Record what the last function name we flushed was- self.last_flushed_function_name = self.function_name_buffer- if self.last_flushed_function_id is None:- self.last_flushed_function_id = self.function_id_buffer- # Clear the buffer- self.function_name_buffer = None- self.function_id_buffer = None- # Since we're clearing the name buffer, we should store- # any updates to the arguments inside a separate buffer-- # Add any main_json updates to the arguments buffer- if self.function_args_buffer is None:- self.function_args_buffer = updates_main_json- else:- self.function_args_buffer += updates_main_json+ # use_assisitant_message means that we should also not release main_json raw, and instead should only release the contents of "message": "..."+ if self.use_assistant_message and self.function_name_buffer == self.assistant_message_tool_name:- # If there was nothing in the name buffer, we can proceed to- # output the arguments chunk as a ToolCallMessage- else:-- # use_assisitant_message means that we should also not release main_json raw, and instead should only release the contents of "message": "..."- if self.use_assistant_message and (- self.last_flushed_function_name is not None- and self.last_flushed_function_name == self.assistant_message_tool_name- ):- # do an additional parse on the updates_main_json- if self.function_args_buffer:- updates_main_json = self.function_args_buffer + updates_main_json- self.function_args_buffer = None-- # Pretty gross hardcoding that assumes that if we're toggling into the keywords, we have the full prefix- match_str = '{"' + self.assistant_message_tool_kwarg + '":"'- if updates_main_json == match_str:- updates_main_json = None-- else:- # Some hardcoding to strip off the trailing "}"- if updates_main_json in ["}", '"}']:- updates_main_json = None- if updates_main_json and len(updates_main_json) > 0 and updates_main_json[-1:] == '"':- updates_main_json = updates_main_json[:-1]-- if not updates_main_json:- # early exit to turn into content mode- continue-- # There may be a buffer from a previous chunk, for example- # if the previous chunk had arguments but we needed to flush name- if self.function_args_buffer:- # In this case, we should release the buffer + new data at once- combined_chunk = self.function_args_buffer + updates_main_json-- if prev_message_type and prev_message_type != "assistant_message":- message_index += 1- assistant_message = AssistantMessage(- id=self.letta_message_id,- date=datetime.now(timezone.utc),- content=combined_chunk,- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),- )- prev_message_type = assistant_message.message_type- yield assistant_message# Store the ID of the tool call so allow skipping the corresponding responseif self.function_id_buffer:self.prev_assistant_message_id = self.function_id_buffer- # clear buffer- self.function_args_buffer = None- self.function_id_buffer = Noneelse:- # If there's no buffer to clear, just output a new chunk with new data- # TODO: THIS IS HORRIBLE- # TODO: WE USE THE OLD JSON PARSER EARLIER (WHICH DOES NOTHING) AND NOW THE NEW JSON PARSER- # TODO: THIS IS TOTALLY WRONG AND BAD, BUT SAVING FOR A LARGER REWRITE IN THE NEAR FUTURE- parsed_args = self.optimistic_json_parser.parse(self.current_function_arguments)-- if parsed_args.get(self.assistant_message_tool_kwarg) and parsed_args.get(- self.assistant_message_tool_kwarg- ) != self.current_json_parse_result.get(self.assistant_message_tool_kwarg):- new_content = parsed_args.get(self.assistant_message_tool_kwarg)- prev_content = self.current_json_parse_result.get(self.assistant_message_tool_kwarg, "")- # TODO: Assumes consistent state and that prev_content is subset of new_content- diff = new_content.replace(prev_content, "", 1)- self.current_json_parse_result = parsed_args- if prev_message_type and prev_message_type != "assistant_message":- message_index += 1- assistant_message = AssistantMessage(- id=self.letta_message_id,- date=datetime.now(timezone.utc),- content=diff,- # name=name,- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),- )- prev_message_type = assistant_message.message_type- yield assistant_message-- # Store the ID of the tool call so allow skipping the corresponding response- if self.function_id_buffer:- self.prev_assistant_message_id = self.function_id_buffer- # clear buffers- self.function_id_buffer = None- else:-- # There may be a buffer from a previous chunk, for example- # if the previous chunk had arguments but we needed to flush name- if self.function_args_buffer:- # In this case, we should release the buffer + new data at once- combined_chunk = self.function_args_buffer + updates_main_jsonif prev_message_type and prev_message_type != "tool_call_message":message_index += 1+ self.tool_call_name = str(self.function_name_buffer)tool_call_msg = ToolCallMessage(id=self.letta_message_id,date=datetime.now(timezone.utc),tool_call=ToolCallDelta(name=self.function_name_buffer,- arguments=combined_chunk,+ arguments=None,tool_call_id=self.function_id_buffer,),- # name=name,otid=Message.generate_otid_from_id(self.letta_message_id, message_index),)prev_message_type = tool_call_msg.message_typeyield tool_call_msg- # clear buffer- self.function_args_buffer = None- self.function_id_buffer = None++ # Record what the last function name we flushed was+ self.last_flushed_function_name = self.function_name_buffer+ if self.last_flushed_function_id is None:+ self.last_flushed_function_id = self.function_id_buffer+ # Clear the buffer+ self.function_name_buffer = None+ self.function_id_buffer = None+ # Since we're clearing the name buffer, we should store+ # any updates to the arguments inside a separate buffer++ # Add any main_json updates to the arguments buffer+ if self.function_args_buffer is None:+ self.function_args_buffer = updates_main_jsonelse:- # If there's no buffer to clear, just output a new chunk with new data- if prev_message_type and prev_message_type != "tool_call_message":- message_index += 1- tool_call_msg = ToolCallMessage(- id=self.letta_message_id,- date=datetime.now(timezone.utc),- tool_call=ToolCallDelta(- name=None,- arguments=updates_main_json,- tool_call_id=self.function_id_buffer,- ),- # name=name,- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),- )- prev_message_type = tool_call_msg.message_type- yield tool_call_msg- self.function_id_buffer = None+ self.function_args_buffer += updates_main_json++ # If there was nothing in the name buffer, we can proceed to+ # output the arguments chunk as a ToolCallMessage+ else:++ # use_assisitant_message means that we should also not release main_json raw, and instead should only release the contents of "message": "..."+ if self.use_assistant_message and (+ self.last_flushed_function_name is not None+ and self.last_flushed_function_name == self.assistant_message_tool_name+ ):+ # do an additional parse on the updates_main_json+ if self.function_args_buffer:+ updates_main_json = self.function_args_buffer + updates_main_json+ self.function_args_buffer = None++ # Pretty gross hardcoding that assumes that if we're toggling into the keywords, we have the full prefix+ match_str = '{"' + self.assistant_message_tool_kwarg + '":"'+ if updates_main_json == match_str:+ updates_main_json = None++ else:+ # Some hardcoding to strip off the trailing "}"+ if updates_main_json in ["}", '"}']:+ updates_main_json = None+ if updates_main_json and len(updates_main_json) > 0 and updates_main_json[-1:] == '"':+ updates_main_json = updates_main_json[:-1]++ if not updates_main_json:+ # early exit to turn into content mode+ continue++ # There may be a buffer from a previous chunk, for example+ # if the previous chunk had arguments but we needed to flush name+ if self.function_args_buffer:+ # In this case, we should release the buffer + new data at once+ combined_chunk = self.function_args_buffer + updates_main_json++ if prev_message_type and prev_message_type != "assistant_message":+ message_index += 1+ assistant_message = AssistantMessage(+ id=self.letta_message_id,+ date=datetime.now(timezone.utc),+ content=combined_chunk,+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),+ )+ prev_message_type = assistant_message.message_type+ yield assistant_message+ # Store the ID of the tool call so allow skipping the corresponding response+ if self.function_id_buffer:+ self.prev_assistant_message_id = self.function_id_buffer+ # clear buffer+ self.function_args_buffer = None+ self.function_id_buffer = None++ else:+ # If there's no buffer to clear, just output a new chunk with new data+ # TODO: THIS IS HORRIBLE+ # TODO: WE USE THE OLD JSON PARSER EARLIER (WHICH DOES NOTHING) AND NOW THE NEW JSON PARSER+ # TODO: THIS IS TOTALLY WRONG AND BAD, BUT SAVING FOR A LARGER REWRITE IN THE NEAR FUTURE+ parsed_args = self.optimistic_json_parser.parse(self.current_function_arguments)++ if parsed_args.get(self.assistant_message_tool_kwarg) and parsed_args.get(+ self.assistant_message_tool_kwarg+ ) != self.current_json_parse_result.get(self.assistant_message_tool_kwarg):+ new_content = parsed_args.get(self.assistant_message_tool_kwarg)+ prev_content = self.current_json_parse_result.get(self.assistant_message_tool_kwarg, "")+ # TODO: Assumes consistent state and that prev_content is subset of new_content+ diff = new_content.replace(prev_content, "", 1)+ self.current_json_parse_result = parsed_args+ if prev_message_type and prev_message_type != "assistant_message":+ message_index += 1+ assistant_message = AssistantMessage(+ id=self.letta_message_id,+ date=datetime.now(timezone.utc),+ content=diff,+ # name=name,+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),+ )+ prev_message_type = assistant_message.message_type+ yield assistant_message++ # Store the ID of the tool call so allow skipping the corresponding response+ if self.function_id_buffer:+ self.prev_assistant_message_id = self.function_id_buffer+ # clear buffers+ self.function_id_buffer = None+ else:++ # There may be a buffer from a previous chunk, for example+ # if the previous chunk had arguments but we needed to flush name+ if self.function_args_buffer:+ # In this case, we should release the buffer + new data at once+ combined_chunk = self.function_args_buffer + updates_main_json+ if prev_message_type and prev_message_type != "tool_call_message":+ message_index += 1+ tool_call_msg = ToolCallMessage(+ id=self.letta_message_id,+ date=datetime.now(timezone.utc),+ tool_call=ToolCallDelta(+ name=self.function_name_buffer,+ arguments=combined_chunk,+ tool_call_id=self.function_id_buffer,+ ),+ # name=name,+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),+ )+ prev_message_type = tool_call_msg.message_type+ yield tool_call_msg+ # clear buffer+ self.function_args_buffer = None+ self.function_id_buffer = None+ else:+ # If there's no buffer to clear, just output a new chunk with new data+ if prev_message_type and prev_message_type != "tool_call_message":+ message_index += 1+ tool_call_msg = ToolCallMessage(+ id=self.letta_message_id,+ date=datetime.now(timezone.utc),+ tool_call=ToolCallDelta(+ name=None,+ arguments=updates_main_json,+ tool_call_id=self.function_id_buffer,+ ),+ # name=name,+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),+ )+ prev_message_type = tool_call_msg.message_type+ yield tool_call_msg+ self.function_id_buffer = None+ except Exception as e:+ logger.error("Error processing stream: %s", e)+ raise+ finally:+ logger.info("OpenAIStreamingInterface: Stream processing complete.")