Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: Fix chat completions interface and move fast chat completions endpoint to /voice (#1139)

Matthew Zhou15cd8035a426
diff --git a/letta/server/rest_api/chat_completions_interface.py b/letta/server/rest_api/chat_completions_interface.py
index 6a1190a61..062d82ce6 100644
--- a/letta/server/rest_api/chat_completions_interface.py
+++ b/letta/server/rest_api/chat_completions_interface.py
@@ -225,10 +225,10 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):
combined_args = "".join(self.current_function_arguments)
parsed_args = OptimisticJSONParser().parse(combined_args)
- # TODO: Make this less brittle! This depends on `message` coming first!
- # This is a heuristic we use to know if we're done with the `message` part of `send_message`
- if len(parsed_args.keys()) > 1:
- self._found_message_tool_kwarg = True
+ if parsed_args.get(self.assistant_message_tool_kwarg) and parsed_args.get(
+ self.assistant_message_tool_kwarg
+ ) != self.current_json_parse_result.get(self.assistant_message_tool_kwarg):
+ self.current_json_parse_result = parsed_args
return ChatCompletionChunk(
id=chunk.id,
object=chunk.object,
@@ -237,31 +237,11 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):
choices=[
Choice(
index=choice.index,
- delta=ChoiceDelta(),
- finish_reason="stop",
+ delta=ChoiceDelta(content=self.current_function_arguments[-1], role=self.ASSISTANT_STR),
+ finish_reason=None,
)
],
)
- else:
- # If the parsed result is different
- # This is an edge case we need to consider. E.g. if the last streamed token is '}', we shouldn't stream that out
- if parsed_args != self.current_json_parse_result:
- self.current_json_parse_result = parsed_args
- # If we can see a "message" field, return it as partial content
- if self.assistant_message_tool_kwarg in parsed_args and parsed_args[self.assistant_message_tool_kwarg]:
- return ChatCompletionChunk(
- id=chunk.id,
- object=chunk.object,
- created=chunk.created.timestamp(),
- model=chunk.model,
- choices=[
- Choice(
- index=choice.index,
- delta=ChoiceDelta(content=self.current_function_arguments[-1], role=self.ASSISTANT_STR),
- finish_reason=None,
- )
- ],
- )
# If there's a finish reason, pass that along
if choice.finish_reason is not None:
diff --git a/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py b/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py
index 3ba586045..ecf10fd77 100644
--- a/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py
+++ b/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py
@@ -1,51 +1,19 @@
import asyncio
-import json
-import uuid
from typing import TYPE_CHECKING, List, Optional, Union
-import httpx
-import openai
from fastapi import APIRouter, Body, Depends, Header, HTTPException
from fastapi.responses import StreamingResponse
-from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDelta
from openai.types.chat.completion_create_params import CompletionCreateParams
-from starlette.concurrency import run_in_threadpool
from letta.agent import Agent
-from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, LETTA_TOOL_SET, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG
-from letta.helpers.tool_execution_helper import (
- add_pre_execution_message,
- enable_strict_mode,
- execute_external_tool,
- remove_request_heartbeat,
-)
+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
from letta.log import get_logger
-from letta.orm.enums import ToolType
from letta.schemas.message import Message, MessageCreate
-from letta.schemas.openai.chat_completion_request import (
- AssistantMessage,
- ChatCompletionRequest,
- Tool,
- ToolCall,
- ToolCallFunction,
- ToolMessage,
- UserMessage,
-)
from letta.schemas.user import User
from letta.server.rest_api.chat_completions_interface import ChatCompletionsStreamingInterface
-from letta.server.rest_api.optimistic_json_parser import OptimisticJSONParser
# TODO this belongs in a controller!
-from letta.server.rest_api.utils import (
- convert_letta_messages_to_openai,
- create_assistant_messages_from_openai_response,
- create_tool_call_messages_from_openai_response,
- create_user_message,
- get_letta_server,
- get_messages_from_completion_request,
- sse_async_generator,
-)
-from letta.settings import model_settings
+from letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request, sse_async_generator
if TYPE_CHECKING:
from letta.server.server import SyncServer
@@ -55,273 +23,6 @@ router = APIRouter(prefix="/v1", tags=["chat_completions"])
logger = get_logger(__name__)
- "/fast/chat/completions",
- response_model=None,
- operation_id="create_fast_chat_completions",
- responses={
- 200: {
- "description": "Successful response",
- "content": {
- "text/event-stream": {"description": "Server-Sent Events stream"},
- },
- }
- },
-)
-async def create_fast_chat_completions(
- completion_request: CompletionCreateParams = Body(...),
- server: "SyncServer" = Depends(get_letta_server),
- user_id: Optional[str] = Header(None, alias="user_id"),
-):
- actor = server.user_manager.get_user_or_default(user_id=user_id)
-
- agent_id = str(completion_request.get("user", None))
- if agent_id is None:
- raise HTTPException(status_code=400, detail="Must pass agent_id in the 'user' field")
-
- agent_state = server.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)
- if agent_state.llm_config.model_endpoint_type != "openai":
- raise HTTPException(status_code=400, detail="Only OpenAI models are supported by this endpoint.")
-
- # Convert Letta messages to OpenAI messages
- in_context_messages = server.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=actor)
- openai_messages = convert_letta_messages_to_openai(in_context_messages)
-
- # Also parse user input from completion_request and append
- input_message = get_messages_from_completion_request(completion_request)[-1]
- openai_messages.append(input_message)
-
- # Tools we allow this agent to call
- tools = [t for t in agent_state.tools if t.name not in LETTA_TOOL_SET and t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]
-
- # Initial request
- openai_request = ChatCompletionRequest(
- model=agent_state.llm_config.model,
- messages=openai_messages,
- # TODO: This nested thing here is so ugly, need to refactor
- tools=(
- [
- Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))
- for t in tools
- ]
- if tools
- else None
- ),
- tool_choice="auto",
- user=user_id,
- max_completion_tokens=agent_state.llm_config.max_tokens,
- temperature=agent_state.llm_config.temperature,
- stream=True,
- )
-
- # Create the OpenAI async client
- client = openai.AsyncClient(
- api_key=model_settings.openai_api_key,
- max_retries=0,
- http_client=httpx.AsyncClient(
- timeout=httpx.Timeout(connect=15.0, read=30.0, write=15.0, pool=15.0),
- follow_redirects=True,
- limits=httpx.Limits(
- max_connections=50,
- max_keepalive_connections=50,
- keepalive_expiry=120,
- ),
- ),
- )
-
- # The messages we want to persist to the Letta agent
- user_message = create_user_message(input_message=input_message, agent_id=agent_id, actor=actor)
- message_db_queue = [user_message]
-
- async def event_stream():
- """
- A function-calling loop:
- - We stream partial tokens.
- - If we detect a tool call (finish_reason="tool_calls"), we parse it,
- add two messages to the conversation:
- (a) assistant message with tool_calls referencing the same ID
- (b) a tool message referencing that ID, containing the tool result.
- - Re-invoke the OpenAI request with updated conversation, streaming again.
- - End when finish_reason="stop" or no more tool calls.
- """
-
- # We'll keep updating this conversation in a loop
- conversation = openai_messages[:]
-
- while True:
- # Make the streaming request to OpenAI
- stream = await client.chat.completions.create(**openai_request.model_dump(exclude_unset=True))
-
- content_buffer = []
- tool_call_name = None
- tool_call_args_str = ""
- tool_call_id = None
- tool_call_happened = False
- finish_reason_stop = False
- optimistic_json_parser = OptimisticJSONParser(strict=True)
- current_parsed_json_result = {}
-
- async with stream:
- async for chunk in stream:
- choice = chunk.choices[0]
- delta = choice.delta
- finish_reason = choice.finish_reason # "tool_calls", "stop", or None
-
- if delta.content:
- content_buffer.append(delta.content)
- yield f"data: {chunk.model_dump_json()}\n\n"
-
- # CASE B: Partial tool call info
- if delta.tool_calls:
- # Typically there's only one in delta.tool_calls
- tc = delta.tool_calls[0]
- if tc.function.name:
- tool_call_name = tc.function.name
- if tc.function.arguments:
- tool_call_args_str += tc.function.arguments
-
- # See if we can stream out the pre-execution message
- parsed_args = optimistic_json_parser.parse(tool_call_args_str)
- if parsed_args.get(
- PRE_EXECUTION_MESSAGE_ARG
- ) and current_parsed_json_result.get( # Ensure key exists and is not None/empty
- PRE_EXECUTION_MESSAGE_ARG
- ) != parsed_args.get(
- PRE_EXECUTION_MESSAGE_ARG
- ):
- # Only stream if there's something new to stream
- # We do this way to avoid hanging JSON at the end of the stream, e.g. '}'
- if parsed_args != current_parsed_json_result:
- current_parsed_json_result = parsed_args
- synthetic_chunk = ChatCompletionChunk(
- id=chunk.id,
- object=chunk.object,
- created=chunk.created,
- model=chunk.model,
- choices=[
- Choice(
- index=choice.index,
- delta=ChoiceDelta(content=tc.function.arguments, role="assistant"),
- finish_reason=None,
- )
- ],
- )
-
- yield f"data: {synthetic_chunk.model_dump_json()}\n\n"
-
- # We might generate a unique ID for the tool call
- if tc.id:
- tool_call_id = tc.id
-
- # Check finish_reason
- if finish_reason == "tool_calls":
- tool_call_happened = True
- break
- elif finish_reason == "stop":
- finish_reason_stop = True
- break
-
- if content_buffer:
- # We treat that partial text as an assistant message
- content = "".join(content_buffer)
- conversation.append({"role": "assistant", "content": content})
-
- # Create an assistant message here to persist later
- assistant_messages = create_assistant_messages_from_openai_response(
- response_text=content, agent_id=agent_id, model=agent_state.llm_config.model, actor=actor
- )
- message_db_queue.extend(assistant_messages)
-
- if tool_call_happened:
- # Parse the tool call arguments
- try:
- tool_args = json.loads(tool_call_args_str)
- except json.JSONDecodeError:
- tool_args = {}
-
- if not tool_call_id:
- # If no tool_call_id given by the model, generate one
- tool_call_id = f"call_{uuid.uuid4().hex[:8]}"
-
- # 1) Insert the "assistant" message with the tool_calls field
- # referencing the same tool_call_id
- assistant_tool_call_msg = AssistantMessage(
- content=None,
- tool_calls=[ToolCall(id=tool_call_id, function=ToolCallFunction(name=tool_call_name, arguments=tool_call_args_str))],
- )
-
- conversation.append(assistant_tool_call_msg.model_dump())
-
- # 2) Execute the tool
- target_tool = next((x for x in tools if x.name == tool_call_name), None)
- if not target_tool:
- # Tool not found, handle error
- yield f"data: {json.dumps({'error': 'Tool not found', 'tool': tool_call_name})}\n\n"
- break
-
- try:
- tool_result, _ = execute_external_tool(
- agent_state=agent_state,
- function_name=tool_call_name,
- function_args=tool_args,
- target_letta_tool=target_tool,
- actor=actor,
- allow_agent_state_modifications=False,
- )
- function_call_success = True
- except Exception as e:
- tool_result = f"Failed to call tool. Error: {e}"
- function_call_success = False
-
- # 3) Insert the "tool" message referencing the same tool_call_id
- tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)
-
- conversation.append(tool_message.model_dump())
-
- # 4) Add a user message prompting the tool call result summarization
- heartbeat_user_message = UserMessage(
- content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user.",
- )
- conversation.append(heartbeat_user_message.model_dump())
-
- # Now, re-invoke OpenAI with the updated conversation
- openai_request.messages = conversation
-
- # Create a tool call message and append to message_db_queue
- tool_call_messages = create_tool_call_messages_from_openai_response(
- agent_id=agent_state.id,
- model=agent_state.llm_config.model,
- function_name=tool_call_name,
- function_arguments=tool_args,
- tool_call_id=tool_call_id,
- function_call_success=function_call_success,
- function_response=tool_result,
- actor=actor,
- add_heartbeat_request_system_message=True,
- )
- message_db_queue.extend(tool_call_messages)
-
- continue # Start the while loop again
-
- if finish_reason_stop:
- break
-
- # If we reach here, no tool call, no "stop", but we've ended streaming
- # Possibly a model error or some other finish reason. We'll just end.
- break
-
- await run_in_threadpool(
- server.agent_manager.append_to_in_context_messages,
- message_db_queue,
- agent_id=agent_id,
- actor=actor,
- )
-
- yield "data: [DONE]\n\n"
-
- return StreamingResponse(event_stream(), media_type="text/event-stream")
-
-
@router.post(
"/chat/completions",
response_model=None,
diff --git a/letta/server/rest_api/routers/v1/__init__.py b/letta/server/rest_api/routers/v1/__init__.py
index 9f14ff9f8..d4f86bf81 100644
--- a/letta/server/rest_api/routers/v1/__init__.py
+++ b/letta/server/rest_api/routers/v1/__init__.py
@@ -11,6 +11,7 @@ from letta.server.rest_api.routers.v1.sources import router as sources_router
from letta.server.rest_api.routers.v1.steps import router as steps_router
from letta.server.rest_api.routers.v1.tags import router as tags_router
from letta.server.rest_api.routers.v1.tools import router as tools_router
+from letta.server.rest_api.routers.v1.voice import router as voice_router
ROUTERS = [
tools_router,
@@ -26,4 +27,5 @@ ROUTERS = [
runs_router,
steps_router,
tags_router,
+ voice_router,
]
diff --git a/letta/server/rest_api/routers/v1/voice.py b/letta/server/rest_api/routers/v1/voice.py
new file mode 100644
index 000000000..2e8c54f8e
--- /dev/null
+++ b/letta/server/rest_api/routers/v1/voice.py
@@ -0,0 +1,315 @@
+import json
+import uuid
+from typing import TYPE_CHECKING, Optional
+
+import httpx
+import openai
+from fastapi import APIRouter, Body, Depends, Header, HTTPException
+from fastapi.responses import StreamingResponse
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDelta
+from openai.types.chat.completion_create_params import CompletionCreateParams
+from starlette.concurrency import run_in_threadpool
+
+from letta.constants import LETTA_TOOL_SET, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG
+from letta.helpers.tool_execution_helper import (
+ add_pre_execution_message,
+ enable_strict_mode,
+ execute_external_tool,
+ remove_request_heartbeat,
+)
+from letta.log import get_logger
+from letta.orm.enums import ToolType
+from letta.schemas.openai.chat_completion_request import (
+ AssistantMessage,
+ ChatCompletionRequest,
+ Tool,
+ ToolCall,
+ ToolCallFunction,
+ ToolMessage,
+ UserMessage,
+)
+from letta.server.rest_api.optimistic_json_parser import OptimisticJSONParser
+from letta.server.rest_api.utils import (
+ convert_letta_messages_to_openai,
+ create_assistant_messages_from_openai_response,
+ create_tool_call_messages_from_openai_response,
+ create_user_message,
+ get_letta_server,
+ get_messages_from_completion_request,
+)
+from letta.settings import model_settings
+
+if TYPE_CHECKING:
+ from letta.server.server import SyncServer
+
+
+router = APIRouter(prefix="/voice", tags=["voice"])
+
+logger = get_logger(__name__)
+
+
+ "/chat/completions",
+ response_model=None,
+ operation_id="create_voice_chat_completions",
+ responses={
+ 200: {
+ "description": "Successful response",
+ "content": {
+ "text/event-stream": {"description": "Server-Sent Events stream"},
+ },
+ }
+ },
+)
+async def create_voice_chat_completions(
+ completion_request: CompletionCreateParams = Body(...),
+ server: "SyncServer" = Depends(get_letta_server),
+ user_id: Optional[str] = Header(None, alias="user_id"),
+):
+ actor = server.user_manager.get_user_or_default(user_id=user_id)
+
+ agent_id = str(completion_request.get("user", None))
+ if agent_id is None:
+ raise HTTPException(status_code=400, detail="Must pass agent_id in the 'user' field")
+
+ agent_state = server.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)
+ if agent_state.llm_config.model_endpoint_type != "openai":
+ raise HTTPException(status_code=400, detail="Only OpenAI models are supported by this endpoint.")
+
+ # Convert Letta messages to OpenAI messages
+ in_context_messages = server.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=actor)
+ openai_messages = convert_letta_messages_to_openai(in_context_messages)
+
+ # Also parse user input from completion_request and append
+ input_message = get_messages_from_completion_request(completion_request)[-1]
+ openai_messages.append(input_message)
+
+ # Tools we allow this agent to call
+ tools = [t for t in agent_state.tools if t.name not in LETTA_TOOL_SET and t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]
+
+ # Initial request
+ openai_request = ChatCompletionRequest(
+ model=agent_state.llm_config.model,
+ messages=openai_messages,
+ # TODO: This nested thing here is so ugly, need to refactor
+ tools=(
+ [
+ Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))
+ for t in tools
+ ]
+ if tools
+ else None
+ ),
+ tool_choice="auto",
+ user=user_id,
+ max_completion_tokens=agent_state.llm_config.max_tokens,
+ temperature=agent_state.llm_config.temperature,
+ stream=True,
+ )
+
+ # Create the OpenAI async client
+ client = openai.AsyncClient(
+ api_key=model_settings.openai_api_key,
+ max_retries=0,
+ http_client=httpx.AsyncClient(
+ timeout=httpx.Timeout(connect=15.0, read=30.0, write=15.0, pool=15.0),
+ follow_redirects=True,
+ limits=httpx.Limits(
+ max_connections=50,
+ max_keepalive_connections=50,
+ keepalive_expiry=120,
+ ),
+ ),
+ )
+
+ # The messages we want to persist to the Letta agent
+ user_message = create_user_message(input_message=input_message, agent_id=agent_id, actor=actor)
+ message_db_queue = [user_message]
+
+ async def event_stream():
+ """
+ A function-calling loop:
+ - We stream partial tokens.
+ - If we detect a tool call (finish_reason="tool_calls"), we parse it,
+ add two messages to the conversation:
+ (a) assistant message with tool_calls referencing the same ID
+ (b) a tool message referencing that ID, containing the tool result.
+ - Re-invoke the OpenAI request with updated conversation, streaming again.
+ - End when finish_reason="stop" or no more tool calls.
+ """
+
+ # We'll keep updating this conversation in a loop
+ conversation = openai_messages[:]
+
+ while True:
+ # Make the streaming request to OpenAI
+ stream = await client.chat.completions.create(**openai_request.model_dump(exclude_unset=True))
+
+ content_buffer = []
+ tool_call_name = None
+ tool_call_args_str = ""
+ tool_call_id = None
+ tool_call_happened = False
+ finish_reason_stop = False
+ optimistic_json_parser = OptimisticJSONParser(strict=True)
+ current_parsed_json_result = {}
+
+ async with stream:
+ async for chunk in stream:
+ choice = chunk.choices[0]
+ delta = choice.delta
+ finish_reason = choice.finish_reason # "tool_calls", "stop", or None
+
+ if delta.content:
+ content_buffer.append(delta.content)
+ yield f"data: {chunk.model_dump_json()}\n\n"
+
+ # CASE B: Partial tool call info
+ if delta.tool_calls:
+ # Typically there's only one in delta.tool_calls
+ tc = delta.tool_calls[0]
+ if tc.function.name:
+ tool_call_name = tc.function.name
+ if tc.function.arguments:
+ tool_call_args_str += tc.function.arguments
+
+ # See if we can stream out the pre-execution message
+ parsed_args = optimistic_json_parser.parse(tool_call_args_str)
+ if parsed_args.get(
+ PRE_EXECUTION_MESSAGE_ARG
+ ) and current_parsed_json_result.get( # Ensure key exists and is not None/empty
+ PRE_EXECUTION_MESSAGE_ARG
+ ) != parsed_args.get(
+ PRE_EXECUTION_MESSAGE_ARG
+ ):
+ # Only stream if there's something new to stream
+ # We do this way to avoid hanging JSON at the end of the stream, e.g. '}'
+ if parsed_args != current_parsed_json_result:
+ current_parsed_json_result = parsed_args
+ synthetic_chunk = ChatCompletionChunk(
+ id=chunk.id,
+ object=chunk.object,
+ created=chunk.created,
+ model=chunk.model,
+ choices=[
+ Choice(
+ index=choice.index,
+ delta=ChoiceDelta(content=tc.function.arguments, role="assistant"),
+ finish_reason=None,
+ )
+ ],
+ )
+
+ yield f"data: {synthetic_chunk.model_dump_json()}\n\n"
+
+ # We might generate a unique ID for the tool call
+ if tc.id:
+ tool_call_id = tc.id
+
+ # Check finish_reason
+ if finish_reason == "tool_calls":
+ tool_call_happened = True
+ break
+ elif finish_reason == "stop":
+ finish_reason_stop = True
+ break
+
+ if content_buffer:
+ # We treat that partial text as an assistant message
+ content = "".join(content_buffer)
+ conversation.append({"role": "assistant", "content": content})
+
+ # Create an assistant message here to persist later
+ assistant_messages = create_assistant_messages_from_openai_response(
+ response_text=content, agent_id=agent_id, model=agent_state.llm_config.model, actor=actor
+ )
+ message_db_queue.extend(assistant_messages)
+
+ if tool_call_happened:
+ # Parse the tool call arguments
+ try:
+ tool_args = json.loads(tool_call_args_str)
+ except json.JSONDecodeError:
+ tool_args = {}
+
+ if not tool_call_id:
+ # If no tool_call_id given by the model, generate one
+ tool_call_id = f"call_{uuid.uuid4().hex[:8]}"
+
+ # 1) Insert the "assistant" message with the tool_calls field
+ # referencing the same tool_call_id
+ assistant_tool_call_msg = AssistantMessage(
+ content=None,
+ tool_calls=[ToolCall(id=tool_call_id, function=ToolCallFunction(name=tool_call_name, arguments=tool_call_args_str))],
+ )
+
+ conversation.append(assistant_tool_call_msg.model_dump())
+
+ # 2) Execute the tool
+ target_tool = next((x for x in tools if x.name == tool_call_name), None)
+ if not target_tool:
+ # Tool not found, handle error
+ yield f"data: {json.dumps({'error': 'Tool not found', 'tool': tool_call_name})}\n\n"
+ break
+
+ try:
+ tool_result, _ = execute_external_tool(
+ agent_state=agent_state,
+ function_name=tool_call_name,
+ function_args=tool_args,
+ target_letta_tool=target_tool,
+ actor=actor,
+ allow_agent_state_modifications=False,
+ )
+ function_call_success = True
+ except Exception as e:
+ tool_result = f"Failed to call tool. Error: {e}"
+ function_call_success = False
+
+ # 3) Insert the "tool" message referencing the same tool_call_id
+ tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)
+
+ conversation.append(tool_message.model_dump())
+
+ # 4) Add a user message prompting the tool call result summarization
+ heartbeat_user_message = UserMessage(
+ content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user.",
+ )
+ conversation.append(heartbeat_user_message.model_dump())
+
+ # Now, re-invoke OpenAI with the updated conversation
+ openai_request.messages = conversation
+
+ # Create a tool call message and append to message_db_queue
+ tool_call_messages = create_tool_call_messages_from_openai_response(
+ agent_id=agent_state.id,
+ model=agent_state.llm_config.model,
+ function_name=tool_call_name,
+ function_arguments=tool_args,
+ tool_call_id=tool_call_id,
+ function_call_success=function_call_success,
+ function_response=tool_result,
+ actor=actor,
+ add_heartbeat_request_system_message=True,
+ )
+ message_db_queue.extend(tool_call_messages)
+
+ continue # Start the while loop again
+
+ if finish_reason_stop:
+ break
+
+ # If we reach here, no tool call, no "stop", but we've ended streaming
+ # Possibly a model error or some other finish reason. We'll just end.
+ break
+
+ await run_in_threadpool(
+ server.agent_manager.append_to_in_context_messages,
+ message_db_queue,
+ agent_id=agent_id,
+ actor=actor,
+ )
+
+ yield "data: [DONE]\n\n"
+
+ return StreamingResponse(event_stream(), media_type="text/event-stream")
diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.py
index 0dfea79e7..495a3ecd7 100644
--- a/tests/integration_test_chat_completions.py
+++ b/tests/integration_test_chat_completions.py
@@ -9,7 +9,6 @@ from openai import AsyncOpenAI
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
from letta import create_client
-from letta.client.streaming import _sse_post
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import MessageStreamStatus
from letta.schemas.llm_config import LLMConfig
@@ -153,37 +152,29 @@ def _assert_valid_chunk(chunk, idx, chunks):
# --- Test Cases --- #
[email protected]("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])
[email protected]("endpoint", ["chat/completions", "fast/chat/completions"])
-def test_chat_completions_streaming(mock_e2b_api_key_none, client, agent, message, endpoint):
- """Tests chat completion streaming via SSE."""
[email protected]("message", ["Tell me something interesting about bananas."])
[email protected]("endpoint", ["v1/voice"])
+async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):
+ """Tests chat completion streaming using the Async OpenAI client."""
request = _get_chat_request(agent.id, message)
- response = _sse_post(f"{client.base_url}/openai/{client.api_prefix}/{endpoint}", request.model_dump(exclude_none=True), client.headers)
-
- try:
- chunks = list(response)
- assert len(chunks) > 5, "Streaming response did not return enough chunks (may have failed silently)."
-
- for idx, chunk in enumerate(chunks):
- assert chunk, f"Empty chunk received at index {idx}."
- print(chunk)
- _assert_valid_chunk(chunk, idx, chunks)
- except Exception as e:
- pytest.fail(f"Streaming failed with exception: {e}")
-
- # Test that we can still follow up and send a message
- client.user_message(agent_id=agent.id, message="So what should I wear today?")
+ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}", max_retries=0)
+ stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))
+ async with stream:
+ async for chunk in stream:
+ assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"
+ assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."
@pytest.mark.asyncio
@pytest.mark.parametrize("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])
[email protected]("endpoint", ["", "fast"])
-async def test_chat_completions_streaming_async(mock_e2b_api_key_none, client, agent, message, endpoint):
[email protected]("endpoint", ["openai/v1", "v1/voice"])
+async def test_chat_completions_streaming_openai_client(mock_e2b_api_key_none, client, agent, message, endpoint):
"""Tests chat completion streaming using the Async OpenAI client."""
request = _get_chat_request(agent.id, message)
- async_client = AsyncOpenAI(base_url=f"{client.base_url}/openai/{client.api_prefix}/{endpoint}", max_retries=0)
+ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}", max_retries=0)
stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))
received_chunks = 0
@@ -193,7 +184,6 @@ async def test_chat_completions_streaming_async(mock_e2b_api_key_none, client, a
try:
async with stream:
async for chunk in stream:
- print(chunk)
assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"
assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."