Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add batch create functionality to SqlAlchemyBase (#1332)
Matthew Zhou804e50e9dbcf
diff --git a/letta/agents/ephemeral_memory_agent.py b/letta/agents/ephemeral_memory_agent.pynew file mode 100644index 000000000..03fc64e07--- /dev/null+++ b/letta/agents/ephemeral_memory_agent.py@@ -0,0 +1,114 @@+from typing import AsyncGenerator, Dict, List++import openai++from letta.agents.base_agent import BaseAgent+from letta.helpers.tool_execution_helper import enable_strict_mode+from letta.orm.enums import ToolType+from letta.schemas.agent import AgentState+from letta.schemas.enums import MessageRole+from letta.schemas.letta_message import UserMessage+from letta.schemas.letta_message_content import TextContent+from letta.schemas.message import Message+from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, Tool+from letta.schemas.user import User+from letta.services.agent_manager import AgentManager+from letta.services.message_manager import MessageManager+++class EphemeralMemoryAgent(BaseAgent):+ """+ A stateless agent that helps with offline memory computations.++ """++ def __init__(+ self,+ agent_id: str,+ openai_client: openai.AsyncClient,+ message_manager: MessageManager,+ agent_manager: AgentManager,+ actor: User,+ ):+ super().__init__(+ agent_id=agent_id,+ openai_client=openai_client,+ message_manager=message_manager,+ agent_manager=agent_manager,+ actor=actor,+ )++ async def step(self, input_message: UserMessage) -> List[Message]:+ """+ Synchronous method that takes a user's input text and returns a summary from OpenAI.+ Returns a list of ephemeral Message objects containing both the user text and the assistant summary.+ """+ agent_state = self.agent_manager.get_agent_by_id(agent_id=self.agent_id, actor=self.actor)++ input_message = self.pre_process_input_message(input_message=input_message)+ request = self._build_openai_request([input_message], agent_state)++ chat_completion = await self.openai_client.chat.completions.create(**request.model_dump(exclude_unset=True))++ return [+ Message(+ role=MessageRole.assistant,+ content=[TextContent(text=chat_completion.choices[0].message.content.strip())],+ )+ ]++ def pre_process_input_message(self, input_message: UserMessage) -> Dict:+ input_prompt_augmented = f"""+ You are a memory recall agent whose job is to comb through a large set of messages and write relevant memories in relation to a user query.+ Your response will directly populate a "memory block" called "human" that describes the user, that will be used to answer more questions in the future.+ You should err on the side of being more verbose, and also try to *predict* the trajectory of the conversation, and pull memories or messages you think will be relevant to where the conversation is going.++ Your response should include:+ - A high level summary of the relevant events/timeline of the conversation relevant to the query+ - Direct citations of quotes from the messages you used while creating the summary++ Here is a history of the messages so far:++ {self._format_messages_llm_friendly()}++ This is the query:++ "{input_message.content}"++ Your response:+ """++ input_message.content = input_prompt_augmented+ # print(input_prompt_augmented)+ return input_message.model_dump()++ def _format_messages_llm_friendly(self):+ messages = self.message_manager.list_messages_for_agent(agent_id=self.agent_id, actor=self.actor)++ llm_friendly_messages = [f"{m.role}: {m.content[0].text}" for m in messages if m.content and isinstance(m.content[0], TextContent)]+ return "\n".join(llm_friendly_messages)++ def _build_openai_request(self, openai_messages: List[Dict], agent_state: AgentState) -> ChatCompletionRequest:+ openai_request = ChatCompletionRequest(+ model=agent_state.llm_config.model,+ messages=openai_messages,+ # tools=self._build_tool_schemas(agent_state),+ # tool_choice="auto",+ user=self.actor.id,+ max_completion_tokens=agent_state.llm_config.max_tokens,+ temperature=agent_state.llm_config.temperature,+ stream=False,+ )+ return openai_request++ def _build_tool_schemas(self, agent_state: AgentState) -> List[Tool]:+ # Only include memory tools+ tools = [t for t in agent_state.tools if t.tool_type in {ToolType.LETTA_CORE, ToolType.LETTA_MEMORY_CORE}]++ return [Tool(type="function", function=enable_strict_mode(t.json_schema)) for t in tools]++ async def step_stream(self, input_message: UserMessage) -> AsyncGenerator[str, None]:+ """+ This agent is synchronous-only. If called in an async context, raise an error.+ """+ raise NotImplementedError("EphemeralMemoryAgent does not support async step.")diff --git a/letta/agents/low_latency_agent.py b/letta/agents/voice_agent.pysimilarity index 62%rename from letta/agents/low_latency_agent.pyrename to letta/agents/voice_agent.pyindex 8fe3b7c02..006b5827f 100644--- a/letta/agents/low_latency_agent.py+++ b/letta/agents/voice_agent.py@@ -6,14 +6,10 @@ import openaifrom letta.agents.base_agent import BaseAgentfrom letta.agents.ephemeral_agent import EphemeralAgent+from letta.agents.ephemeral_memory_agent import EphemeralMemoryAgentfrom letta.constants import NON_USER_MSG_PREFIXfrom letta.helpers.datetime_helpers import get_utc_time-from letta.helpers.tool_execution_helper import (- add_pre_execution_message,- enable_strict_mode,- execute_external_tool,- remove_request_heartbeat,-)+from letta.helpers.tool_execution_helper import add_pre_execution_message, execute_external_tool, remove_request_heartbeatfrom letta.interfaces.openai_chat_completions_streaming_interface import OpenAIChatCompletionsStreamingInterfacefrom letta.log import get_loggerfrom letta.orm.enums import ToolType@@ -48,7 +44,7 @@ from letta.utils import united_difflogger = get_logger(__name__)-class LowLatencyAgent(BaseAgent):+class VoiceAgent(BaseAgent):"""A function-calling loop for streaming OpenAI responses with tool execution.This agent:@@ -65,9 +61,9 @@ class LowLatencyAgent(BaseAgent):agent_manager: AgentManager,block_manager: BlockManager,actor: User,+ message_buffer_limit: int,+ message_buffer_min: int,summarization_mode: SummarizationMode = SummarizationMode.STATIC_MESSAGE_BUFFER,- message_buffer_limit: int = 10,- message_buffer_min: int = 4,):super().__init__(agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor@@ -89,65 +85,72 @@ class LowLatencyAgent(BaseAgent):)self.message_buffer_limit = message_buffer_limitself.message_buffer_min = message_buffer_min+ self.offline_memory_agent = EphemeralMemoryAgent(+ agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor+ )async def step(self, input_message: UserMessage) -> List[Message]:raise NotImplementedError("LowLatencyAgent does not have a synchronous step implemented currently.")async def step_stream(self, input_message: UserMessage) -> AsyncGenerator[str, None]:"""- Async generator that yields partial tokens as SSE events, handles tool calls,- and streams error messages if OpenAI API failures occur.+ Main streaming loop that yields partial tokens.+ Whenever we detect a tool call, we yield from _handle_ai_response as well."""- input_message = self.pre_process_input_message(input_message=input_message)- agent_state = self.agent_manager.get_agent_by_id(agent_id=self.agent_id, actor=self.actor)+ input_message = self.pre_process_input_message(input_message)+ agent_state = self.agent_manager.get_agent_by_id(self.agent_id, actor=self.actor)in_context_messages = self.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=self.actor)letta_message_db_queue = [create_user_message(input_message=input_message, agent_id=agent_state.id, actor=self.actor)]in_memory_message_history = [input_message]+ # TODO: Define max steps herewhile True:- # Constantly pull down and integrate memory blocks- in_context_messages = self._rebuild_memory(in_context_messages=in_context_messages, agent_state=agent_state)+ step_first = True- # Convert Letta messages to OpenAI messages+ # Rebuild memory each loop+ in_context_messages = self._rebuild_memory(in_context_messages, agent_state)openai_messages = convert_letta_messages_to_openai(in_context_messages)openai_messages.extend(in_memory_message_history)+request = self._build_openai_request(openai_messages, agent_state)- # Execute the requeststream = await self.openai_client.chat.completions.create(**request.model_dump(exclude_unset=True))streaming_interface = OpenAIChatCompletionsStreamingInterface(stream_pre_execution_message=True)- async for sse in streaming_interface.process(stream):- yield sse-- # Process the AI response (buffered messages, tool execution, etc.)- continue_execution = await self._handle_ai_response(- streaming_interface, agent_state, in_memory_message_history, letta_message_db_queue+ # 1) Yield partial tokens from OpenAI+ async for sse_chunk in streaming_interface.process(stream):+ if step_first:+ step_first = False+ yield sse_chunk++ # 2) Now handle the final AI response. This might yield more text (stalling, etc.)+ should_continue = await self._handle_ai_response(+ streaming_interface,+ agent_state,+ in_memory_message_history,+ letta_message_db_queue,)- if not continue_execution:+ if not should_continue:break- # Rebuild context window+ # Rebuild context window if desiredawait self._rebuild_context_window(in_context_messages, letta_message_db_queue, agent_state)-yield "data: [DONE]\n\n"async def _handle_ai_response(self,- streaming_interface: OpenAIChatCompletionsStreamingInterface,+ streaming_interface: "OpenAIChatCompletionsStreamingInterface",agent_state: AgentState,in_memory_message_history: List[Dict[str, Any]],letta_message_db_queue: List[Any],) -> bool:"""- Handles AI response processing, including buffering messages, detecting tool calls,- executing tools, and deciding whether to continue execution.-- Returns:- bool: True if execution should continue, False if the step loop should terminate.+ Now that streaming is done, handle the final AI response.+ This might yield additional SSE tokens if we do stalling.+ At the end, set self._continue_execution accordingly."""- # Handle assistant message buffering+ # 1. If we have any leftover content from partial stream, store it as an assistant messageif streaming_interface.content_buffer:content = "".join(streaming_interface.content_buffer)in_memory_message_history.append({"role": "assistant", "content": content})@@ -160,76 +163,78 @@ class LowLatencyAgent(BaseAgent):)letta_message_db_queue.extend(assistant_msgs)- # Handle tool execution if a tool call occurred+ # 2. If a tool call was requested, handle itif streaming_interface.tool_call_happened:+ tool_call_name = streaming_interface.tool_call_name+ tool_call_args_str = streaming_interface.tool_call_args_str or "{}"try:- tool_args = json.loads(streaming_interface.tool_call_args_str)+ tool_args = json.loads(tool_call_args_str)except json.JSONDecodeError:tool_args = {}tool_call_id = streaming_interface.tool_call_id or f"call_{uuid.uuid4().hex[:8]}"-assistant_tool_call_msg = AssistantMessage(content=None,tool_calls=[ToolCall(id=tool_call_id,function=ToolCallFunction(- name=streaming_interface.tool_call_name,- arguments=streaming_interface.tool_call_args_str,+ name=tool_call_name,+ arguments=tool_call_args_str,),)],)in_memory_message_history.append(assistant_tool_call_msg.model_dump())- tool_result, function_call_success = await self._execute_tool(- tool_name=streaming_interface.tool_call_name,+ tool_result, success_flag = await self._execute_tool(+ tool_name=tool_call_name,tool_args=tool_args,agent_state=agent_state,)- tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)+ # 3. Provide function_call response back into the conversation+ tool_message = ToolMessage(+ content=json.dumps({"result": tool_result}),+ tool_call_id=tool_call_id,+ )in_memory_message_history.append(tool_message.model_dump())+ # 4. Insert heartbeat message for follow-upheartbeat_user_message = UserMessage(content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user.")in_memory_message_history.append(heartbeat_user_message.model_dump())+ # 5. Also store in DBtool_call_messages = create_tool_call_messages_from_openai_response(agent_id=agent_state.id,model=agent_state.llm_config.model,- function_name=streaming_interface.tool_call_name,+ function_name=tool_call_name,function_arguments=tool_args,tool_call_id=tool_call_id,- function_call_success=function_call_success,+ function_call_success=success_flag,function_response=tool_result,actor=self.actor,add_heartbeat_request_system_message=True,)letta_message_db_queue.extend(tool_call_messages)- # Continue execution by restarting the loop with updated context+ # Because we have new data, we want to continue the while-loop in `step_stream`return True-- # Exit the loop if finish_reason_stop or no tool call occurred- return not streaming_interface.finish_reason_stop+ else:+ # If we got here, there's no tool call. If finish_reason_stop => done+ return not streaming_interface.finish_reason_stopasync def _rebuild_context_window(self, in_context_messages: List[Message], letta_message_db_queue: List[Message], agent_state: AgentState) -> None:new_letta_messages = self.message_manager.create_many_messages(letta_message_db_queue, actor=self.actor)+ new_in_context_messages = in_context_messages + new_letta_messages- # TODO: Make this more general and configurable, less brittle- target_block = next(b for b in agent_state.memory.blocks if b.label == self.summary_block_label)- previous_summary = self.block_manager.get_block_by_id(block_id=target_block.id, actor=self.actor).value- new_in_context_messages, summary_str, updated = await self.summarizer.summarize(- in_context_messages=in_context_messages, new_letta_messages=new_letta_messages, previous_summary=previous_summary- )-- if updated:- self.block_manager.update_block(block_id=target_block.id, block_update=BlockUpdate(value=summary_str), actor=self.actor)+ if len(new_in_context_messages) > self.message_buffer_limit:+ cutoff = len(new_in_context_messages) - self.message_buffer_limit+ new_in_context_messages = [new_in_context_messages[0]] + new_in_context_messages[cutoff:]self.agent_manager.set_in_context_messages(agent_id=self.agent_id, message_ids=[m.id for m in new_in_context_messages], actor=self.actor@@ -249,8 +254,8 @@ class LowLatencyAgent(BaseAgent):memory_edit_timestamp = get_utc_time()- num_messages = self.message_manager.size(actor=actor, agent_id=agent_id)- num_archival_memories = self.passage_manager.size(actor=actor, agent_id=agent_id)+ num_messages = self.message_manager.size(actor=self.actor, agent_id=agent_state.id)+ num_archival_memories = self.passage_manager.size(actor=self.actor, agent_id=agent_state.id)new_system_message_str = compile_system_message(system_prompt=agent_state.system,@@ -296,29 +301,68 @@ class LowLatencyAgent(BaseAgent):else:tools = agent_state.tools+ # Special tool state+ recall_memory_utterance_description = (+ "A lengthier message to be uttered while your memories of the current conversation are being re-contextualized."+ "You should stall naturally and show the user you're thinking hard. The main thing is to not leave the user in silence."+ "You MUST also include punctuation at the end of this message."+ )+ recall_memory_json = Tool(+ type="function",+ function=add_pre_execution_message(+ {+ "name": "recall_memory",+ "description": "Retrieve relevant information from memory based on a given query. Use when you don't remember the answer to a question.",+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "A description of what the model is trying to recall from memory.",+ }+ },+ "required": ["query"],+ },+ },+ description=recall_memory_utterance_description,+ ),+ )+# TODO: Customize whether or not to have heartbeats, pre_exec_message, etc.- return [- Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))- for t in tools+ return [recall_memory_json] + [+ Tool(type="function", function=add_pre_execution_message(remove_request_heartbeat(t.json_schema))) for t in tools]async def _execute_tool(self, tool_name: str, tool_args: dict, agent_state: AgentState) -> Tuple[str, bool]:"""Executes a tool and returns (result, success_flag)."""- target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)- if not target_tool:- return f"Tool not found: {tool_name}", False+ # Special memory case+ if tool_name == "recall_memory":+ # TODO: Make this safe+ await self._recall_memory(tool_args["query"], agent_state)+ return f"Successfully recalled memory and populated {self.summary_block_label} block.", True+ else:+ target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)+ if not target_tool:+ return f"Tool not found: {tool_name}", False- try:- tool_result, _ = execute_external_tool(- agent_state=agent_state,- function_name=tool_name,- function_args=tool_args,- target_letta_tool=target_tool,- actor=self.actor,- allow_agent_state_modifications=False,- )- return tool_result, True- except Exception as e:- return f"Failed to call tool. Error: {e}", False+ try:+ tool_result, _ = execute_external_tool(+ agent_state=agent_state,+ function_name=tool_name,+ function_args=tool_args,+ target_letta_tool=target_tool,+ actor=self.actor,+ allow_agent_state_modifications=False,+ )+ return tool_result, True+ except Exception as e:+ return f"Failed to call tool. Error: {e}", False++ async def _recall_memory(self, query, agent_state: AgentState) -> None:+ results = await self.offline_memory_agent.step(UserMessage(content=query))+ target_block = next(b for b in agent_state.memory.blocks if b.label == self.summary_block_label)+ self.block_manager.update_block(+ block_id=target_block.id, block_update=BlockUpdate(value=results[0].content[0].text), actor=self.actor+ )diff --git a/letta/helpers/tool_execution_helper.py b/letta/helpers/tool_execution_helper.pyindex 948772eec..886cf9950 100644--- a/letta/helpers/tool_execution_helper.py+++ b/letta/helpers/tool_execution_helper.py@@ -40,7 +40,7 @@ def enable_strict_mode(tool_schema: Dict[str, Any]) -> Dict[str, Any]:return schema-def add_pre_execution_message(tool_schema: Dict[str, Any]) -> Dict[str, Any]:+def add_pre_execution_message(tool_schema: Dict[str, Any], description: Optional[str] = None) -> Dict[str, Any]:"""Adds a `pre_execution_message` parameter to a tool schema to prompt a natural, human-like message before executing the tool.Args:@@ -58,14 +58,17 @@ def add_pre_execution_message(tool_schema: Dict[str, Any]) -> Dict[str, Any]:properties = parameters.get("properties", {})required = parameters.get("required", [])- # Define the new `pre_execution_message` field with a refined description- pre_execution_message_field = {- "type": "string",- "description": (+ # Define the new `pre_execution_message` field+ if not description:+ # Default description+ description = ("A concise message to be uttered before executing this tool. ""This should sound natural, as if a person is casually announcing their next action.""You MUST also include punctuation at the end of this message."- ),+ )+ pre_execution_message_field = {+ "type": "string",+ "description": description,}# Ensure the pre-execution message is the first field in propertiesdiff --git a/letta/orm/sqlalchemy_base.py b/letta/orm/sqlalchemy_base.pyindex 3b45c6eeb..5d91f3d7a 100644--- a/letta/orm/sqlalchemy_base.py+++ b/letta/orm/sqlalchemy_base.py@@ -331,6 +331,50 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):except (DBAPIError, IntegrityError) as e:self._handle_dbapi_error(e)+ @classmethod+ @handle_db_timeout+ def batch_create(cls, items: List["SqlalchemyBase"], db_session: "Session", actor: Optional["User"] = None) -> List["SqlalchemyBase"]:+ """+ Create multiple records in a single transaction for better performance.++ Args:+ items: List of model instances to create+ db_session: SQLAlchemy session+ actor: Optional user performing the action++ Returns:+ List of created model instances+ """+ logger.debug(f"Batch creating {len(items)} {cls.__name__} items with actor={actor}")++ if not items:+ return []++ # Set created/updated by fields if actor is provided+ if actor:+ for item in items:+ item._set_created_and_updated_by_fields(actor.id)++ try:+ with db_session as session:+ session.add_all(items)+ session.flush() # Flush to generate IDs but don't commit yet++ # Collect IDs to fetch the complete objects after commit+ item_ids = [item.id for item in items]++ session.commit()++ # Re-query the objects to get them with relationships loaded+ query = select(cls).where(cls.id.in_(item_ids))+ if hasattr(cls, "created_at"):+ query = query.order_by(cls.created_at)++ return list(session.execute(query).scalars())++ except (DBAPIError, IntegrityError) as e:+ cls._handle_dbapi_error(e)+@handle_db_timeoutdef delete(self, db_session: "Session", actor: Optional["User"] = None) -> "SqlalchemyBase":logger.debug(f"Soft deleting {self.__class__.__name__} with ID: {self.id} with actor={actor}")diff --git a/letta/server/rest_api/routers/v1/voice.py b/letta/server/rest_api/routers/v1/voice.pyindex 7e3871b87..5f187c5ee 100644--- a/letta/server/rest_api/routers/v1/voice.py+++ b/letta/server/rest_api/routers/v1/voice.py@@ -6,7 +6,7 @@ from fastapi import APIRouter, Body, Depends, Headerfrom fastapi.responses import StreamingResponsefrom openai.types.chat.completion_create_params import CompletionCreateParams-from letta.agents.low_latency_agent import LowLatencyAgent+from letta.agents.voice_agent import VoiceAgentfrom letta.log import get_loggerfrom letta.schemas.openai.chat_completions import UserMessagefrom letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request@@ -16,7 +16,7 @@ if TYPE_CHECKING:from letta.server.server import SyncServer-router = APIRouter(prefix="/voice", tags=["voice"])+router = APIRouter(prefix="/voice-beta", tags=["voice"])logger = get_logger(__name__)@@ -61,14 +61,14 @@ async def create_voice_chat_completions()# Instantiate our LowLatencyAgent- agent = LowLatencyAgent(+ agent = VoiceAgent(agent_id=agent_id,openai_client=client,message_manager=server.message_manager,agent_manager=server.agent_manager,block_manager=server.block_manager,actor=actor,- message_buffer_limit=10,+ message_buffer_limit=100,message_buffer_min=4,)diff --git a/letta/services/message_manager.py b/letta/services/message_manager.pyindex 387168dc4..96b6f14cf 100644--- a/letta/services/message_manager.py+++ b/letta/services/message_manager.py@@ -63,8 +63,33 @@ class MessageManager:@enforce_typesdef create_many_messages(self, pydantic_msgs: List[PydanticMessage], actor: PydanticUser) -> List[PydanticMessage]:- """Create multiple messages."""- return [self.create_message(m, actor=actor) for m in pydantic_msgs]+ """+ Create multiple messages in a single database transaction.++ Args:+ pydantic_msgs: List of Pydantic message models to create+ actor: User performing the action++ Returns:+ List of created Pydantic message models+ """+ if not pydantic_msgs:+ return []++ # Create ORM model instances for all messages+ orm_messages = []+ for pydantic_msg in pydantic_msgs:+ # Set the organization id of the Pydantic message+ pydantic_msg.organization_id = actor.organization_id+ msg_data = pydantic_msg.model_dump(to_orm=True)+ orm_messages.append(MessageModel(**msg_data))++ # Use the batch_create method for efficient creation+ with self.session_maker() as session:+ created_messages = MessageModel.batch_create(orm_messages, session, actor=actor)++ # Convert back to Pydantic models+ return [msg.to_pydantic() for msg in created_messages]@enforce_typesdef update_message_by_letta_message(diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.pyindex 3ae24ec6c..c7e77a970 100644--- a/tests/integration_test_chat_completions.py+++ b/tests/integration_test_chat_completions.py@@ -15,7 +15,9 @@ from letta.schemas.llm_config import LLMConfigfrom letta.schemas.openai.chat_completion_request import ChatCompletionRequest, UserMessagefrom letta.schemas.tool import ToolCreatefrom letta.schemas.usage import LettaUsageStatistics+from letta.services.agent_manager import AgentManagerfrom letta.services.tool_manager import ToolManager+from letta.services.user_manager import UserManager# --- Server Management --- #@@ -111,7 +113,9 @@ def composio_gmail_get_profile_tool(default_user):def agent(client, roll_dice_tool, weather_tool, composio_gmail_get_profile_tool):"""Creates an agent and ensures cleanup after tests."""agent_state = client.create_agent(- name=f"test_compl_{str(uuid.uuid4())[5:]}", tool_ids=[roll_dice_tool.id, weather_tool.id, composio_gmail_get_profile_tool.id]+ name=f"test_compl_{str(uuid.uuid4())[5:]}",+ tool_ids=[roll_dice_tool.id, weather_tool.id, composio_gmail_get_profile_tool.id],+ include_base_tools=False,)yield agent_stateclient.delete_agent(agent_state.id)@@ -152,8 +156,8 @@ def _assert_valid_chunk(chunk, idx, chunks):@pytest.mark.asyncio[email protected]("message", ["How are you?"])[email protected]("endpoint", ["v1/voice"])[email protected]("message", ["Hi how are you today?"])[email protected]("endpoint", ["v1/voice-beta"])async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):"""Tests chat completion streaming using the Async OpenAI client."""request = _get_chat_request(message)@@ -165,9 +169,30 @@ async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):print(chunk)[email protected]("message", ["Use recall memory tool to recall what my name is."])[email protected]("endpoint", ["v1/voice-beta"])+async def test_voice_recall_memory(mock_e2b_api_key_none, client, agent, message, endpoint):+ """Tests chat completion streaming using the Async OpenAI client."""+ request = _get_chat_request(message)++ # Insert some messages about my name+ client.user_message(agent.id, "My name is Matt")++ # Wipe the in context messages+ actor = UserManager().get_default_user()+ AgentManager().set_in_context_messages(agent_id=agent.id, message_ids=[agent.message_ids[0]], actor=actor)++ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}/{agent.id}", max_retries=0)+ stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))+ async with stream:+ async for chunk in stream:+ print(chunk)++@pytest.mark.asyncio@pytest.mark.parametrize("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])[email protected]("endpoint", ["openai/v1", "v1/voice"])[email protected]("endpoint", ["openai/v1", "v1/voice-beta"])async def test_chat_completions_streaming_openai_client(mock_e2b_api_key_none, client, agent, message, endpoint):"""Tests chat completion streaming using the Async OpenAI client."""request = _get_chat_request(message)