Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: Fix chat completions interface and move fast chat completions endpoint to /voice (#1139)
Matthew Zhou15cd8035a426
diff --git a/letta/server/rest_api/chat_completions_interface.py b/letta/server/rest_api/chat_completions_interface.pyindex 6a1190a61..062d82ce6 100644--- a/letta/server/rest_api/chat_completions_interface.py+++ b/letta/server/rest_api/chat_completions_interface.py@@ -225,10 +225,10 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):combined_args = "".join(self.current_function_arguments)parsed_args = OptimisticJSONParser().parse(combined_args)- # TODO: Make this less brittle! This depends on `message` coming first!- # This is a heuristic we use to know if we're done with the `message` part of `send_message`- if len(parsed_args.keys()) > 1:- self._found_message_tool_kwarg = True+ if parsed_args.get(self.assistant_message_tool_kwarg) and parsed_args.get(+ self.assistant_message_tool_kwarg+ ) != self.current_json_parse_result.get(self.assistant_message_tool_kwarg):+ self.current_json_parse_result = parsed_argsreturn ChatCompletionChunk(id=chunk.id,object=chunk.object,@@ -237,31 +237,11 @@ class ChatCompletionsStreamingInterface(AgentChunkStreamingInterface):choices=[Choice(index=choice.index,- delta=ChoiceDelta(),- finish_reason="stop",+ delta=ChoiceDelta(content=self.current_function_arguments[-1], role=self.ASSISTANT_STR),+ finish_reason=None,)],)- else:- # If the parsed result is different- # This is an edge case we need to consider. E.g. if the last streamed token is '}', we shouldn't stream that out- if parsed_args != self.current_json_parse_result:- self.current_json_parse_result = parsed_args- # If we can see a "message" field, return it as partial content- if self.assistant_message_tool_kwarg in parsed_args and parsed_args[self.assistant_message_tool_kwarg]:- return ChatCompletionChunk(- id=chunk.id,- object=chunk.object,- created=chunk.created.timestamp(),- model=chunk.model,- choices=[- Choice(- index=choice.index,- delta=ChoiceDelta(content=self.current_function_arguments[-1], role=self.ASSISTANT_STR),- finish_reason=None,- )- ],- )# If there's a finish reason, pass that alongif choice.finish_reason is not None:diff --git a/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py b/letta/server/rest_api/routers/openai/chat_completions/chat_completions.pyindex 3ba586045..ecf10fd77 100644--- a/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py+++ b/letta/server/rest_api/routers/openai/chat_completions/chat_completions.py@@ -1,51 +1,19 @@import asyncio-import json-import uuidfrom typing import TYPE_CHECKING, List, Optional, Union-import httpx-import openaifrom fastapi import APIRouter, Body, Depends, Header, HTTPExceptionfrom fastapi.responses import StreamingResponse-from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDeltafrom openai.types.chat.completion_create_params import CompletionCreateParams-from starlette.concurrency import run_in_threadpoolfrom letta.agent import Agent-from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, LETTA_TOOL_SET, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG-from letta.helpers.tool_execution_helper import (- add_pre_execution_message,- enable_strict_mode,- execute_external_tool,- remove_request_heartbeat,-)+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.log import get_logger-from letta.orm.enums import ToolTypefrom letta.schemas.message import Message, MessageCreate-from letta.schemas.openai.chat_completion_request import (- AssistantMessage,- ChatCompletionRequest,- Tool,- ToolCall,- ToolCallFunction,- ToolMessage,- UserMessage,-)from letta.schemas.user import Userfrom letta.server.rest_api.chat_completions_interface import ChatCompletionsStreamingInterface-from letta.server.rest_api.optimistic_json_parser import OptimisticJSONParser# TODO this belongs in a controller!-from letta.server.rest_api.utils import (- convert_letta_messages_to_openai,- create_assistant_messages_from_openai_response,- create_tool_call_messages_from_openai_response,- create_user_message,- get_letta_server,- get_messages_from_completion_request,- sse_async_generator,-)-from letta.settings import model_settings+from letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request, sse_async_generatorif TYPE_CHECKING:from letta.server.server import SyncServer@@ -55,273 +23,6 @@ router = APIRouter(prefix="/v1", tags=["chat_completions"])logger = get_logger(__name__)- "/fast/chat/completions",- response_model=None,- operation_id="create_fast_chat_completions",- responses={- 200: {- "description": "Successful response",- "content": {- "text/event-stream": {"description": "Server-Sent Events stream"},- },- }- },-)-async def create_fast_chat_completions(- completion_request: CompletionCreateParams = Body(...),- server: "SyncServer" = Depends(get_letta_server),- user_id: Optional[str] = Header(None, alias="user_id"),-):- actor = server.user_manager.get_user_or_default(user_id=user_id)-- agent_id = str(completion_request.get("user", None))- if agent_id is None:- raise HTTPException(status_code=400, detail="Must pass agent_id in the 'user' field")-- agent_state = server.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)- if agent_state.llm_config.model_endpoint_type != "openai":- raise HTTPException(status_code=400, detail="Only OpenAI models are supported by this endpoint.")-- # Convert Letta messages to OpenAI messages- in_context_messages = server.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=actor)- openai_messages = convert_letta_messages_to_openai(in_context_messages)-- # Also parse user input from completion_request and append- input_message = get_messages_from_completion_request(completion_request)[-1]- openai_messages.append(input_message)-- # Tools we allow this agent to call- tools = [t for t in agent_state.tools if t.name not in LETTA_TOOL_SET and t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]-- # Initial request- openai_request = ChatCompletionRequest(- model=agent_state.llm_config.model,- messages=openai_messages,- # TODO: This nested thing here is so ugly, need to refactor- tools=(- [- Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))- for t in tools- ]- if tools- else None- ),- tool_choice="auto",- user=user_id,- max_completion_tokens=agent_state.llm_config.max_tokens,- temperature=agent_state.llm_config.temperature,- stream=True,- )-- # Create the OpenAI async client- client = openai.AsyncClient(- api_key=model_settings.openai_api_key,- max_retries=0,- http_client=httpx.AsyncClient(- timeout=httpx.Timeout(connect=15.0, read=30.0, write=15.0, pool=15.0),- follow_redirects=True,- limits=httpx.Limits(- max_connections=50,- max_keepalive_connections=50,- keepalive_expiry=120,- ),- ),- )-- # The messages we want to persist to the Letta agent- user_message = create_user_message(input_message=input_message, agent_id=agent_id, actor=actor)- message_db_queue = [user_message]-- async def event_stream():- """- A function-calling loop:- - We stream partial tokens.- - If we detect a tool call (finish_reason="tool_calls"), we parse it,- add two messages to the conversation:- (a) assistant message with tool_calls referencing the same ID- (b) a tool message referencing that ID, containing the tool result.- - Re-invoke the OpenAI request with updated conversation, streaming again.- - End when finish_reason="stop" or no more tool calls.- """-- # We'll keep updating this conversation in a loop- conversation = openai_messages[:]-- while True:- # Make the streaming request to OpenAI- stream = await client.chat.completions.create(**openai_request.model_dump(exclude_unset=True))-- content_buffer = []- tool_call_name = None- tool_call_args_str = ""- tool_call_id = None- tool_call_happened = False- finish_reason_stop = False- optimistic_json_parser = OptimisticJSONParser(strict=True)- current_parsed_json_result = {}-- async with stream:- async for chunk in stream:- choice = chunk.choices[0]- delta = choice.delta- finish_reason = choice.finish_reason # "tool_calls", "stop", or None-- if delta.content:- content_buffer.append(delta.content)- yield f"data: {chunk.model_dump_json()}\n\n"-- # CASE B: Partial tool call info- if delta.tool_calls:- # Typically there's only one in delta.tool_calls- tc = delta.tool_calls[0]- if tc.function.name:- tool_call_name = tc.function.name- if tc.function.arguments:- tool_call_args_str += tc.function.arguments-- # See if we can stream out the pre-execution message- parsed_args = optimistic_json_parser.parse(tool_call_args_str)- if parsed_args.get(- PRE_EXECUTION_MESSAGE_ARG- ) and current_parsed_json_result.get( # Ensure key exists and is not None/empty- PRE_EXECUTION_MESSAGE_ARG- ) != parsed_args.get(- PRE_EXECUTION_MESSAGE_ARG- ):- # Only stream if there's something new to stream- # We do this way to avoid hanging JSON at the end of the stream, e.g. '}'- if parsed_args != current_parsed_json_result:- current_parsed_json_result = parsed_args- synthetic_chunk = ChatCompletionChunk(- id=chunk.id,- object=chunk.object,- created=chunk.created,- model=chunk.model,- choices=[- Choice(- index=choice.index,- delta=ChoiceDelta(content=tc.function.arguments, role="assistant"),- finish_reason=None,- )- ],- )-- yield f"data: {synthetic_chunk.model_dump_json()}\n\n"-- # We might generate a unique ID for the tool call- if tc.id:- tool_call_id = tc.id-- # Check finish_reason- if finish_reason == "tool_calls":- tool_call_happened = True- break- elif finish_reason == "stop":- finish_reason_stop = True- break-- if content_buffer:- # We treat that partial text as an assistant message- content = "".join(content_buffer)- conversation.append({"role": "assistant", "content": content})-- # Create an assistant message here to persist later- assistant_messages = create_assistant_messages_from_openai_response(- response_text=content, agent_id=agent_id, model=agent_state.llm_config.model, actor=actor- )- message_db_queue.extend(assistant_messages)-- if tool_call_happened:- # Parse the tool call arguments- try:- tool_args = json.loads(tool_call_args_str)- except json.JSONDecodeError:- tool_args = {}-- if not tool_call_id:- # If no tool_call_id given by the model, generate one- tool_call_id = f"call_{uuid.uuid4().hex[:8]}"-- # 1) Insert the "assistant" message with the tool_calls field- # referencing the same tool_call_id- assistant_tool_call_msg = AssistantMessage(- content=None,- tool_calls=[ToolCall(id=tool_call_id, function=ToolCallFunction(name=tool_call_name, arguments=tool_call_args_str))],- )-- conversation.append(assistant_tool_call_msg.model_dump())-- # 2) Execute the tool- target_tool = next((x for x in tools if x.name == tool_call_name), None)- if not target_tool:- # Tool not found, handle error- yield f"data: {json.dumps({'error': 'Tool not found', 'tool': tool_call_name})}\n\n"- break-- try:- tool_result, _ = execute_external_tool(- agent_state=agent_state,- function_name=tool_call_name,- function_args=tool_args,- target_letta_tool=target_tool,- actor=actor,- allow_agent_state_modifications=False,- )- function_call_success = True- except Exception as e:- tool_result = f"Failed to call tool. Error: {e}"- function_call_success = False-- # 3) Insert the "tool" message referencing the same tool_call_id- tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)-- conversation.append(tool_message.model_dump())-- # 4) Add a user message prompting the tool call result summarization- heartbeat_user_message = UserMessage(- content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user.",- )- conversation.append(heartbeat_user_message.model_dump())-- # Now, re-invoke OpenAI with the updated conversation- openai_request.messages = conversation-- # Create a tool call message and append to message_db_queue- tool_call_messages = create_tool_call_messages_from_openai_response(- agent_id=agent_state.id,- model=agent_state.llm_config.model,- function_name=tool_call_name,- function_arguments=tool_args,- tool_call_id=tool_call_id,- function_call_success=function_call_success,- function_response=tool_result,- actor=actor,- add_heartbeat_request_system_message=True,- )- message_db_queue.extend(tool_call_messages)-- continue # Start the while loop again-- if finish_reason_stop:- break-- # If we reach here, no tool call, no "stop", but we've ended streaming- # Possibly a model error or some other finish reason. We'll just end.- break-- await run_in_threadpool(- server.agent_manager.append_to_in_context_messages,- message_db_queue,- agent_id=agent_id,- actor=actor,- )-- yield "data: [DONE]\n\n"-- return StreamingResponse(event_stream(), media_type="text/event-stream")--@router.post("/chat/completions",response_model=None,diff --git a/letta/server/rest_api/routers/v1/__init__.py b/letta/server/rest_api/routers/v1/__init__.pyindex 9f14ff9f8..d4f86bf81 100644--- a/letta/server/rest_api/routers/v1/__init__.py+++ b/letta/server/rest_api/routers/v1/__init__.py@@ -11,6 +11,7 @@ from letta.server.rest_api.routers.v1.sources import router as sources_routerfrom letta.server.rest_api.routers.v1.steps import router as steps_routerfrom letta.server.rest_api.routers.v1.tags import router as tags_routerfrom letta.server.rest_api.routers.v1.tools import router as tools_router+from letta.server.rest_api.routers.v1.voice import router as voice_routerROUTERS = [tools_router,@@ -26,4 +27,5 @@ ROUTERS = [runs_router,steps_router,tags_router,+ voice_router,]diff --git a/letta/server/rest_api/routers/v1/voice.py b/letta/server/rest_api/routers/v1/voice.pynew file mode 100644index 000000000..2e8c54f8e--- /dev/null+++ b/letta/server/rest_api/routers/v1/voice.py@@ -0,0 +1,315 @@+import json+import uuid+from typing import TYPE_CHECKING, Optional++import httpx+import openai+from fastapi import APIRouter, Body, Depends, Header, HTTPException+from fastapi.responses import StreamingResponse+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, Choice, ChoiceDelta+from openai.types.chat.completion_create_params import CompletionCreateParams+from starlette.concurrency import run_in_threadpool++from letta.constants import LETTA_TOOL_SET, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG+from letta.helpers.tool_execution_helper import (+ add_pre_execution_message,+ enable_strict_mode,+ execute_external_tool,+ remove_request_heartbeat,+)+from letta.log import get_logger+from letta.orm.enums import ToolType+from letta.schemas.openai.chat_completion_request import (+ AssistantMessage,+ ChatCompletionRequest,+ Tool,+ ToolCall,+ ToolCallFunction,+ ToolMessage,+ UserMessage,+)+from letta.server.rest_api.optimistic_json_parser import OptimisticJSONParser+from letta.server.rest_api.utils import (+ convert_letta_messages_to_openai,+ create_assistant_messages_from_openai_response,+ create_tool_call_messages_from_openai_response,+ create_user_message,+ get_letta_server,+ get_messages_from_completion_request,+)+from letta.settings import model_settings++if TYPE_CHECKING:+ from letta.server.server import SyncServer+++router = APIRouter(prefix="/voice", tags=["voice"])++logger = get_logger(__name__)+++ "/chat/completions",+ response_model=None,+ operation_id="create_voice_chat_completions",+ responses={+ 200: {+ "description": "Successful response",+ "content": {+ "text/event-stream": {"description": "Server-Sent Events stream"},+ },+ }+ },+)+async def create_voice_chat_completions(+ completion_request: CompletionCreateParams = Body(...),+ server: "SyncServer" = Depends(get_letta_server),+ user_id: Optional[str] = Header(None, alias="user_id"),+):+ actor = server.user_manager.get_user_or_default(user_id=user_id)++ agent_id = str(completion_request.get("user", None))+ if agent_id is None:+ raise HTTPException(status_code=400, detail="Must pass agent_id in the 'user' field")++ agent_state = server.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)+ if agent_state.llm_config.model_endpoint_type != "openai":+ raise HTTPException(status_code=400, detail="Only OpenAI models are supported by this endpoint.")++ # Convert Letta messages to OpenAI messages+ in_context_messages = server.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=actor)+ openai_messages = convert_letta_messages_to_openai(in_context_messages)++ # Also parse user input from completion_request and append+ input_message = get_messages_from_completion_request(completion_request)[-1]+ openai_messages.append(input_message)++ # Tools we allow this agent to call+ tools = [t for t in agent_state.tools if t.name not in LETTA_TOOL_SET and t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]++ # Initial request+ openai_request = ChatCompletionRequest(+ model=agent_state.llm_config.model,+ messages=openai_messages,+ # TODO: This nested thing here is so ugly, need to refactor+ tools=(+ [+ Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))+ for t in tools+ ]+ if tools+ else None+ ),+ tool_choice="auto",+ user=user_id,+ max_completion_tokens=agent_state.llm_config.max_tokens,+ temperature=agent_state.llm_config.temperature,+ stream=True,+ )++ # Create the OpenAI async client+ client = openai.AsyncClient(+ api_key=model_settings.openai_api_key,+ max_retries=0,+ http_client=httpx.AsyncClient(+ timeout=httpx.Timeout(connect=15.0, read=30.0, write=15.0, pool=15.0),+ follow_redirects=True,+ limits=httpx.Limits(+ max_connections=50,+ max_keepalive_connections=50,+ keepalive_expiry=120,+ ),+ ),+ )++ # The messages we want to persist to the Letta agent+ user_message = create_user_message(input_message=input_message, agent_id=agent_id, actor=actor)+ message_db_queue = [user_message]++ async def event_stream():+ """+ A function-calling loop:+ - We stream partial tokens.+ - If we detect a tool call (finish_reason="tool_calls"), we parse it,+ add two messages to the conversation:+ (a) assistant message with tool_calls referencing the same ID+ (b) a tool message referencing that ID, containing the tool result.+ - Re-invoke the OpenAI request with updated conversation, streaming again.+ - End when finish_reason="stop" or no more tool calls.+ """++ # We'll keep updating this conversation in a loop+ conversation = openai_messages[:]++ while True:+ # Make the streaming request to OpenAI+ stream = await client.chat.completions.create(**openai_request.model_dump(exclude_unset=True))++ content_buffer = []+ tool_call_name = None+ tool_call_args_str = ""+ tool_call_id = None+ tool_call_happened = False+ finish_reason_stop = False+ optimistic_json_parser = OptimisticJSONParser(strict=True)+ current_parsed_json_result = {}++ async with stream:+ async for chunk in stream:+ choice = chunk.choices[0]+ delta = choice.delta+ finish_reason = choice.finish_reason # "tool_calls", "stop", or None++ if delta.content:+ content_buffer.append(delta.content)+ yield f"data: {chunk.model_dump_json()}\n\n"++ # CASE B: Partial tool call info+ if delta.tool_calls:+ # Typically there's only one in delta.tool_calls+ tc = delta.tool_calls[0]+ if tc.function.name:+ tool_call_name = tc.function.name+ if tc.function.arguments:+ tool_call_args_str += tc.function.arguments++ # See if we can stream out the pre-execution message+ parsed_args = optimistic_json_parser.parse(tool_call_args_str)+ if parsed_args.get(+ PRE_EXECUTION_MESSAGE_ARG+ ) and current_parsed_json_result.get( # Ensure key exists and is not None/empty+ PRE_EXECUTION_MESSAGE_ARG+ ) != parsed_args.get(+ PRE_EXECUTION_MESSAGE_ARG+ ):+ # Only stream if there's something new to stream+ # We do this way to avoid hanging JSON at the end of the stream, e.g. '}'+ if parsed_args != current_parsed_json_result:+ current_parsed_json_result = parsed_args+ synthetic_chunk = ChatCompletionChunk(+ id=chunk.id,+ object=chunk.object,+ created=chunk.created,+ model=chunk.model,+ choices=[+ Choice(+ index=choice.index,+ delta=ChoiceDelta(content=tc.function.arguments, role="assistant"),+ finish_reason=None,+ )+ ],+ )++ yield f"data: {synthetic_chunk.model_dump_json()}\n\n"++ # We might generate a unique ID for the tool call+ if tc.id:+ tool_call_id = tc.id++ # Check finish_reason+ if finish_reason == "tool_calls":+ tool_call_happened = True+ break+ elif finish_reason == "stop":+ finish_reason_stop = True+ break++ if content_buffer:+ # We treat that partial text as an assistant message+ content = "".join(content_buffer)+ conversation.append({"role": "assistant", "content": content})++ # Create an assistant message here to persist later+ assistant_messages = create_assistant_messages_from_openai_response(+ response_text=content, agent_id=agent_id, model=agent_state.llm_config.model, actor=actor+ )+ message_db_queue.extend(assistant_messages)++ if tool_call_happened:+ # Parse the tool call arguments+ try:+ tool_args = json.loads(tool_call_args_str)+ except json.JSONDecodeError:+ tool_args = {}++ if not tool_call_id:+ # If no tool_call_id given by the model, generate one+ tool_call_id = f"call_{uuid.uuid4().hex[:8]}"++ # 1) Insert the "assistant" message with the tool_calls field+ # referencing the same tool_call_id+ assistant_tool_call_msg = AssistantMessage(+ content=None,+ tool_calls=[ToolCall(id=tool_call_id, function=ToolCallFunction(name=tool_call_name, arguments=tool_call_args_str))],+ )++ conversation.append(assistant_tool_call_msg.model_dump())++ # 2) Execute the tool+ target_tool = next((x for x in tools if x.name == tool_call_name), None)+ if not target_tool:+ # Tool not found, handle error+ yield f"data: {json.dumps({'error': 'Tool not found', 'tool': tool_call_name})}\n\n"+ break++ try:+ tool_result, _ = execute_external_tool(+ agent_state=agent_state,+ function_name=tool_call_name,+ function_args=tool_args,+ target_letta_tool=target_tool,+ actor=actor,+ allow_agent_state_modifications=False,+ )+ function_call_success = True+ except Exception as e:+ tool_result = f"Failed to call tool. Error: {e}"+ function_call_success = False++ # 3) Insert the "tool" message referencing the same tool_call_id+ tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)++ conversation.append(tool_message.model_dump())++ # 4) Add a user message prompting the tool call result summarization+ heartbeat_user_message = UserMessage(+ content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user.",+ )+ conversation.append(heartbeat_user_message.model_dump())++ # Now, re-invoke OpenAI with the updated conversation+ openai_request.messages = conversation++ # Create a tool call message and append to message_db_queue+ tool_call_messages = create_tool_call_messages_from_openai_response(+ agent_id=agent_state.id,+ model=agent_state.llm_config.model,+ function_name=tool_call_name,+ function_arguments=tool_args,+ tool_call_id=tool_call_id,+ function_call_success=function_call_success,+ function_response=tool_result,+ actor=actor,+ add_heartbeat_request_system_message=True,+ )+ message_db_queue.extend(tool_call_messages)++ continue # Start the while loop again++ if finish_reason_stop:+ break++ # If we reach here, no tool call, no "stop", but we've ended streaming+ # Possibly a model error or some other finish reason. We'll just end.+ break++ await run_in_threadpool(+ server.agent_manager.append_to_in_context_messages,+ message_db_queue,+ agent_id=agent_id,+ actor=actor,+ )++ yield "data: [DONE]\n\n"++ return StreamingResponse(event_stream(), media_type="text/event-stream")diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.pyindex 0dfea79e7..495a3ecd7 100644--- a/tests/integration_test_chat_completions.py+++ b/tests/integration_test_chat_completions.py@@ -9,7 +9,6 @@ from openai import AsyncOpenAIfrom openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta import create_client-from letta.client.streaming import _sse_postfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import MessageStreamStatusfrom letta.schemas.llm_config import LLMConfig@@ -153,37 +152,29 @@ def _assert_valid_chunk(chunk, idx, chunks):# --- Test Cases --- #[email protected]("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])[email protected]("endpoint", ["chat/completions", "fast/chat/completions"])-def test_chat_completions_streaming(mock_e2b_api_key_none, client, agent, message, endpoint):- """Tests chat completion streaming via SSE."""[email protected]("message", ["Tell me something interesting about bananas."])[email protected]("endpoint", ["v1/voice"])+async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):+ """Tests chat completion streaming using the Async OpenAI client."""request = _get_chat_request(agent.id, message)- response = _sse_post(f"{client.base_url}/openai/{client.api_prefix}/{endpoint}", request.model_dump(exclude_none=True), client.headers)-- try:- chunks = list(response)- assert len(chunks) > 5, "Streaming response did not return enough chunks (may have failed silently)."-- for idx, chunk in enumerate(chunks):- assert chunk, f"Empty chunk received at index {idx}."- print(chunk)- _assert_valid_chunk(chunk, idx, chunks)- except Exception as e:- pytest.fail(f"Streaming failed with exception: {e}")-- # Test that we can still follow up and send a message- client.user_message(agent_id=agent.id, message="So what should I wear today?")+ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}", max_retries=0)+ stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))+ async with stream:+ async for chunk in stream:+ assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"+ assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."@pytest.mark.asyncio@pytest.mark.parametrize("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])[email protected]("endpoint", ["", "fast"])-async def test_chat_completions_streaming_async(mock_e2b_api_key_none, client, agent, message, endpoint):[email protected]("endpoint", ["openai/v1", "v1/voice"])+async def test_chat_completions_streaming_openai_client(mock_e2b_api_key_none, client, agent, message, endpoint):"""Tests chat completion streaming using the Async OpenAI client."""request = _get_chat_request(agent.id, message)- async_client = AsyncOpenAI(base_url=f"{client.base_url}/openai/{client.api_prefix}/{endpoint}", max_retries=0)+ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}", max_retries=0)stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))received_chunks = 0@@ -193,7 +184,6 @@ async def test_chat_completions_streaming_async(mock_e2b_api_key_none, client, atry:async with stream:async for chunk in stream:- print(chunk)assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."