Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Add batch create functionality to SqlAlchemyBase (#1332)

Matthew Zhou3eefff925f86
diff --git a/letta/agents/ephemeral_memory_agent.py b/letta/agents/ephemeral_memory_agent.py
new file mode 100644
index 000000000..03fc64e07
--- /dev/null
+++ b/letta/agents/ephemeral_memory_agent.py
@@ -0,0 +1,114 @@
+from typing import AsyncGenerator, Dict, List
+
+import openai
+
+from letta.agents.base_agent import BaseAgent
+from letta.helpers.tool_execution_helper import enable_strict_mode
+from letta.orm.enums import ToolType
+from letta.schemas.agent import AgentState
+from letta.schemas.enums import MessageRole
+from letta.schemas.letta_message import UserMessage
+from letta.schemas.letta_message_content import TextContent
+from letta.schemas.message import Message
+from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, Tool
+from letta.schemas.user import User
+from letta.services.agent_manager import AgentManager
+from letta.services.message_manager import MessageManager
+
+
+class EphemeralMemoryAgent(BaseAgent):
+ """
+ A stateless agent that helps with offline memory computations.
+
+ """
+
+ def __init__(
+ self,
+ agent_id: str,
+ openai_client: openai.AsyncClient,
+ message_manager: MessageManager,
+ agent_manager: AgentManager,
+ actor: User,
+ ):
+ super().__init__(
+ agent_id=agent_id,
+ openai_client=openai_client,
+ message_manager=message_manager,
+ agent_manager=agent_manager,
+ actor=actor,
+ )
+
+ async def step(self, input_message: UserMessage) -> List[Message]:
+ """
+ Synchronous method that takes a user's input text and returns a summary from OpenAI.
+ Returns a list of ephemeral Message objects containing both the user text and the assistant summary.
+ """
+ agent_state = self.agent_manager.get_agent_by_id(agent_id=self.agent_id, actor=self.actor)
+
+ input_message = self.pre_process_input_message(input_message=input_message)
+ request = self._build_openai_request([input_message], agent_state)
+
+ chat_completion = await self.openai_client.chat.completions.create(**request.model_dump(exclude_unset=True))
+
+ return [
+ Message(
+ role=MessageRole.assistant,
+ content=[TextContent(text=chat_completion.choices[0].message.content.strip())],
+ )
+ ]
+
+ def pre_process_input_message(self, input_message: UserMessage) -> Dict:
+ input_prompt_augmented = f"""
+ You are a memory recall agent whose job is to comb through a large set of messages and write relevant memories in relation to a user query.
+ Your response will directly populate a "memory block" called "human" that describes the user, that will be used to answer more questions in the future.
+ You should err on the side of being more verbose, and also try to *predict* the trajectory of the conversation, and pull memories or messages you think will be relevant to where the conversation is going.
+
+ Your response should include:
+ - A high level summary of the relevant events/timeline of the conversation relevant to the query
+ - Direct citations of quotes from the messages you used while creating the summary
+
+ Here is a history of the messages so far:
+
+ {self._format_messages_llm_friendly()}
+
+ This is the query:
+
+ "{input_message.content}"
+
+ Your response:
+ """
+
+ input_message.content = input_prompt_augmented
+ # print(input_prompt_augmented)
+ return input_message.model_dump()
+
+ def _format_messages_llm_friendly(self):
+ messages = self.message_manager.list_messages_for_agent(agent_id=self.agent_id, actor=self.actor)
+
+ llm_friendly_messages = [f"{m.role}: {m.content[0].text}" for m in messages if m.content and isinstance(m.content[0], TextContent)]
+ return "\n".join(llm_friendly_messages)
+
+ def _build_openai_request(self, openai_messages: List[Dict], agent_state: AgentState) -> ChatCompletionRequest:
+ openai_request = ChatCompletionRequest(
+ model=agent_state.llm_config.model,
+ messages=openai_messages,
+ # tools=self._build_tool_schemas(agent_state),
+ # tool_choice="auto",
+ user=self.actor.id,
+ max_completion_tokens=agent_state.llm_config.max_tokens,
+ temperature=agent_state.llm_config.temperature,
+ stream=False,
+ )
+ return openai_request
+
+ def _build_tool_schemas(self, agent_state: AgentState) -> List[Tool]:
+ # Only include memory tools
+ tools = [t for t in agent_state.tools if t.tool_type in {ToolType.LETTA_CORE, ToolType.LETTA_MEMORY_CORE}]
+
+ return [Tool(type="function", function=enable_strict_mode(t.json_schema)) for t in tools]
+
+ async def step_stream(self, input_message: UserMessage) -> AsyncGenerator[str, None]:
+ """
+ This agent is synchronous-only. If called in an async context, raise an error.
+ """
+ raise NotImplementedError("EphemeralMemoryAgent does not support async step.")
diff --git a/letta/agents/low_latency_agent.py b/letta/agents/voice_agent.py
similarity index 62%
rename from letta/agents/low_latency_agent.py
rename to letta/agents/voice_agent.py
index 8fe3b7c02..006b5827f 100644
--- a/letta/agents/low_latency_agent.py
+++ b/letta/agents/voice_agent.py
@@ -6,14 +6,10 @@ import openai
from letta.agents.base_agent import BaseAgent
from letta.agents.ephemeral_agent import EphemeralAgent
+from letta.agents.ephemeral_memory_agent import EphemeralMemoryAgent
from letta.constants import NON_USER_MSG_PREFIX
from letta.helpers.datetime_helpers import get_utc_time
-from letta.helpers.tool_execution_helper import (
- add_pre_execution_message,
- enable_strict_mode,
- execute_external_tool,
- remove_request_heartbeat,
-)
+from letta.helpers.tool_execution_helper import add_pre_execution_message, execute_external_tool, remove_request_heartbeat
from letta.interfaces.openai_chat_completions_streaming_interface import OpenAIChatCompletionsStreamingInterface
from letta.log import get_logger
from letta.orm.enums import ToolType
@@ -48,7 +44,7 @@ from letta.utils import united_diff
logger = get_logger(__name__)
-class LowLatencyAgent(BaseAgent):
+class VoiceAgent(BaseAgent):
"""
A function-calling loop for streaming OpenAI responses with tool execution.
This agent:
@@ -65,9 +61,9 @@ class LowLatencyAgent(BaseAgent):
agent_manager: AgentManager,
block_manager: BlockManager,
actor: User,
+ message_buffer_limit: int,
+ message_buffer_min: int,
summarization_mode: SummarizationMode = SummarizationMode.STATIC_MESSAGE_BUFFER,
- message_buffer_limit: int = 10,
- message_buffer_min: int = 4,
):
super().__init__(
agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor
@@ -89,65 +85,72 @@ class LowLatencyAgent(BaseAgent):
)
self.message_buffer_limit = message_buffer_limit
self.message_buffer_min = message_buffer_min
+ self.offline_memory_agent = EphemeralMemoryAgent(
+ agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor
+ )
async def step(self, input_message: UserMessage) -> List[Message]:
raise NotImplementedError("LowLatencyAgent does not have a synchronous step implemented currently.")
async def step_stream(self, input_message: UserMessage) -> AsyncGenerator[str, None]:
"""
- Async generator that yields partial tokens as SSE events, handles tool calls,
- and streams error messages if OpenAI API failures occur.
+ Main streaming loop that yields partial tokens.
+ Whenever we detect a tool call, we yield from _handle_ai_response as well.
"""
- input_message = self.pre_process_input_message(input_message=input_message)
- agent_state = self.agent_manager.get_agent_by_id(agent_id=self.agent_id, actor=self.actor)
+ input_message = self.pre_process_input_message(input_message)
+ agent_state = self.agent_manager.get_agent_by_id(self.agent_id, actor=self.actor)
in_context_messages = self.message_manager.get_messages_by_ids(message_ids=agent_state.message_ids, actor=self.actor)
letta_message_db_queue = [create_user_message(input_message=input_message, agent_id=agent_state.id, actor=self.actor)]
in_memory_message_history = [input_message]
+ # TODO: Define max steps here
while True:
- # Constantly pull down and integrate memory blocks
- in_context_messages = self._rebuild_memory(in_context_messages=in_context_messages, agent_state=agent_state)
+ step_first = True
- # Convert Letta messages to OpenAI messages
+ # Rebuild memory each loop
+ in_context_messages = self._rebuild_memory(in_context_messages, agent_state)
openai_messages = convert_letta_messages_to_openai(in_context_messages)
openai_messages.extend(in_memory_message_history)
+
request = self._build_openai_request(openai_messages, agent_state)
- # Execute the request
stream = await self.openai_client.chat.completions.create(**request.model_dump(exclude_unset=True))
streaming_interface = OpenAIChatCompletionsStreamingInterface(stream_pre_execution_message=True)
- async for sse in streaming_interface.process(stream):
- yield sse
-
- # Process the AI response (buffered messages, tool execution, etc.)
- continue_execution = await self._handle_ai_response(
- streaming_interface, agent_state, in_memory_message_history, letta_message_db_queue
+ # 1) Yield partial tokens from OpenAI
+ async for sse_chunk in streaming_interface.process(stream):
+ if step_first:
+ step_first = False
+ yield sse_chunk
+
+ # 2) Now handle the final AI response. This might yield more text (stalling, etc.)
+ should_continue = await self._handle_ai_response(
+ streaming_interface,
+ agent_state,
+ in_memory_message_history,
+ letta_message_db_queue,
)
- if not continue_execution:
+ if not should_continue:
break
- # Rebuild context window
+ # Rebuild context window if desired
await self._rebuild_context_window(in_context_messages, letta_message_db_queue, agent_state)
-
yield "data: [DONE]\n\n"
async def _handle_ai_response(
self,
- streaming_interface: OpenAIChatCompletionsStreamingInterface,
+ streaming_interface: "OpenAIChatCompletionsStreamingInterface",
agent_state: AgentState,
in_memory_message_history: List[Dict[str, Any]],
letta_message_db_queue: List[Any],
) -> bool:
"""
- Handles AI response processing, including buffering messages, detecting tool calls,
- executing tools, and deciding whether to continue execution.
-
- Returns:
- bool: True if execution should continue, False if the step loop should terminate.
+ Now that streaming is done, handle the final AI response.
+ This might yield additional SSE tokens if we do stalling.
+ At the end, set self._continue_execution accordingly.
"""
- # Handle assistant message buffering
+ # 1. If we have any leftover content from partial stream, store it as an assistant message
if streaming_interface.content_buffer:
content = "".join(streaming_interface.content_buffer)
in_memory_message_history.append({"role": "assistant", "content": content})
@@ -160,76 +163,78 @@ class LowLatencyAgent(BaseAgent):
)
letta_message_db_queue.extend(assistant_msgs)
- # Handle tool execution if a tool call occurred
+ # 2. If a tool call was requested, handle it
if streaming_interface.tool_call_happened:
+ tool_call_name = streaming_interface.tool_call_name
+ tool_call_args_str = streaming_interface.tool_call_args_str or "{}"
try:
- tool_args = json.loads(streaming_interface.tool_call_args_str)
+ tool_args = json.loads(tool_call_args_str)
except json.JSONDecodeError:
tool_args = {}
tool_call_id = streaming_interface.tool_call_id or f"call_{uuid.uuid4().hex[:8]}"
-
assistant_tool_call_msg = AssistantMessage(
content=None,
tool_calls=[
ToolCall(
id=tool_call_id,
function=ToolCallFunction(
- name=streaming_interface.tool_call_name,
- arguments=streaming_interface.tool_call_args_str,
+ name=tool_call_name,
+ arguments=tool_call_args_str,
),
)
],
)
in_memory_message_history.append(assistant_tool_call_msg.model_dump())
- tool_result, function_call_success = await self._execute_tool(
- tool_name=streaming_interface.tool_call_name,
+ tool_result, success_flag = await self._execute_tool(
+ tool_name=tool_call_name,
tool_args=tool_args,
agent_state=agent_state,
)
- tool_message = ToolMessage(content=json.dumps({"result": tool_result}), tool_call_id=tool_call_id)
+ # 3. Provide function_call response back into the conversation
+ tool_message = ToolMessage(
+ content=json.dumps({"result": tool_result}),
+ tool_call_id=tool_call_id,
+ )
in_memory_message_history.append(tool_message.model_dump())
+ # 4. Insert heartbeat message for follow-up
heartbeat_user_message = UserMessage(
content=f"{NON_USER_MSG_PREFIX} Tool finished executing. Summarize the result for the user."
)
in_memory_message_history.append(heartbeat_user_message.model_dump())
+ # 5. Also store in DB
tool_call_messages = create_tool_call_messages_from_openai_response(
agent_id=agent_state.id,
model=agent_state.llm_config.model,
- function_name=streaming_interface.tool_call_name,
+ function_name=tool_call_name,
function_arguments=tool_args,
tool_call_id=tool_call_id,
- function_call_success=function_call_success,
+ function_call_success=success_flag,
function_response=tool_result,
actor=self.actor,
add_heartbeat_request_system_message=True,
)
letta_message_db_queue.extend(tool_call_messages)
- # Continue execution by restarting the loop with updated context
+ # Because we have new data, we want to continue the while-loop in `step_stream`
return True
-
- # Exit the loop if finish_reason_stop or no tool call occurred
- return not streaming_interface.finish_reason_stop
+ else:
+ # If we got here, there's no tool call. If finish_reason_stop => done
+ return not streaming_interface.finish_reason_stop
async def _rebuild_context_window(
self, in_context_messages: List[Message], letta_message_db_queue: List[Message], agent_state: AgentState
) -> None:
new_letta_messages = self.message_manager.create_many_messages(letta_message_db_queue, actor=self.actor)
+ new_in_context_messages = in_context_messages + new_letta_messages
- # TODO: Make this more general and configurable, less brittle
- target_block = next(b for b in agent_state.memory.blocks if b.label == self.summary_block_label)
- previous_summary = self.block_manager.get_block_by_id(block_id=target_block.id, actor=self.actor).value
- new_in_context_messages, summary_str, updated = await self.summarizer.summarize(
- in_context_messages=in_context_messages, new_letta_messages=new_letta_messages, previous_summary=previous_summary
- )
-
- if updated:
- self.block_manager.update_block(block_id=target_block.id, block_update=BlockUpdate(value=summary_str), actor=self.actor)
+ if len(new_in_context_messages) > self.message_buffer_limit:
+ cutoff = len(new_in_context_messages) - self.message_buffer_limit
+ new_in_context_messages = [new_in_context_messages[0]] + new_in_context_messages[cutoff:]
self.agent_manager.set_in_context_messages(
agent_id=self.agent_id, message_ids=[m.id for m in new_in_context_messages], actor=self.actor
@@ -249,8 +254,8 @@ class LowLatencyAgent(BaseAgent):
memory_edit_timestamp = get_utc_time()
- num_messages = self.message_manager.size(actor=actor, agent_id=agent_id)
- num_archival_memories = self.passage_manager.size(actor=actor, agent_id=agent_id)
+ num_messages = self.message_manager.size(actor=self.actor, agent_id=agent_state.id)
+ num_archival_memories = self.passage_manager.size(actor=self.actor, agent_id=agent_state.id)
new_system_message_str = compile_system_message(
system_prompt=agent_state.system,
@@ -296,29 +301,68 @@ class LowLatencyAgent(BaseAgent):
else:
tools = agent_state.tools
+ # Special tool state
+ recall_memory_utterance_description = (
+ "A lengthier message to be uttered while your memories of the current conversation are being re-contextualized."
+ "You should stall naturally and show the user you're thinking hard. The main thing is to not leave the user in silence."
+ "You MUST also include punctuation at the end of this message."
+ )
+ recall_memory_json = Tool(
+ type="function",
+ function=add_pre_execution_message(
+ {
+ "name": "recall_memory",
+ "description": "Retrieve relevant information from memory based on a given query. Use when you don't remember the answer to a question.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "A description of what the model is trying to recall from memory.",
+ }
+ },
+ "required": ["query"],
+ },
+ },
+ description=recall_memory_utterance_description,
+ ),
+ )
+
# TODO: Customize whether or not to have heartbeats, pre_exec_message, etc.
- return [
- Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))
- for t in tools
+ return [recall_memory_json] + [
+ Tool(type="function", function=add_pre_execution_message(remove_request_heartbeat(t.json_schema))) for t in tools
]
async def _execute_tool(self, tool_name: str, tool_args: dict, agent_state: AgentState) -> Tuple[str, bool]:
"""
Executes a tool and returns (result, success_flag).
"""
- target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)
- if not target_tool:
- return f"Tool not found: {tool_name}", False
+ # Special memory case
+ if tool_name == "recall_memory":
+ # TODO: Make this safe
+ await self._recall_memory(tool_args["query"], agent_state)
+ return f"Successfully recalled memory and populated {self.summary_block_label} block.", True
+ else:
+ target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)
+ if not target_tool:
+ return f"Tool not found: {tool_name}", False
- try:
- tool_result, _ = execute_external_tool(
- agent_state=agent_state,
- function_name=tool_name,
- function_args=tool_args,
- target_letta_tool=target_tool,
- actor=self.actor,
- allow_agent_state_modifications=False,
- )
- return tool_result, True
- except Exception as e:
- return f"Failed to call tool. Error: {e}", False
+ try:
+ tool_result, _ = execute_external_tool(
+ agent_state=agent_state,
+ function_name=tool_name,
+ function_args=tool_args,
+ target_letta_tool=target_tool,
+ actor=self.actor,
+ allow_agent_state_modifications=False,
+ )
+ return tool_result, True
+ except Exception as e:
+ return f"Failed to call tool. Error: {e}", False
+
+ async def _recall_memory(self, query, agent_state: AgentState) -> None:
+ results = await self.offline_memory_agent.step(UserMessage(content=query))
+ target_block = next(b for b in agent_state.memory.blocks if b.label == self.summary_block_label)
+ self.block_manager.update_block(
+ block_id=target_block.id, block_update=BlockUpdate(value=results[0].content[0].text), actor=self.actor
+ )
diff --git a/letta/helpers/tool_execution_helper.py b/letta/helpers/tool_execution_helper.py
index 948772eec..886cf9950 100644
--- a/letta/helpers/tool_execution_helper.py
+++ b/letta/helpers/tool_execution_helper.py
@@ -40,7 +40,7 @@ def enable_strict_mode(tool_schema: Dict[str, Any]) -> Dict[str, Any]:
return schema
-def add_pre_execution_message(tool_schema: Dict[str, Any]) -> Dict[str, Any]:
+def add_pre_execution_message(tool_schema: Dict[str, Any], description: Optional[str] = None) -> Dict[str, Any]:
"""Adds a `pre_execution_message` parameter to a tool schema to prompt a natural, human-like message before executing the tool.
Args:
@@ -58,14 +58,17 @@ def add_pre_execution_message(tool_schema: Dict[str, Any]) -> Dict[str, Any]:
properties = parameters.get("properties", {})
required = parameters.get("required", [])
- # Define the new `pre_execution_message` field with a refined description
- pre_execution_message_field = {
- "type": "string",
- "description": (
+ # Define the new `pre_execution_message` field
+ if not description:
+ # Default description
+ description = (
"A concise message to be uttered before executing this tool. "
"This should sound natural, as if a person is casually announcing their next action."
"You MUST also include punctuation at the end of this message."
- ),
+ )
+ pre_execution_message_field = {
+ "type": "string",
+ "description": description,
}
# Ensure the pre-execution message is the first field in properties
diff --git a/letta/orm/sqlalchemy_base.py b/letta/orm/sqlalchemy_base.py
index 3b45c6eeb..5d91f3d7a 100644
--- a/letta/orm/sqlalchemy_base.py
+++ b/letta/orm/sqlalchemy_base.py
@@ -331,6 +331,50 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
except (DBAPIError, IntegrityError) as e:
self._handle_dbapi_error(e)
+ @classmethod
+ @handle_db_timeout
+ def batch_create(cls, items: List["SqlalchemyBase"], db_session: "Session", actor: Optional["User"] = None) -> List["SqlalchemyBase"]:
+ """
+ Create multiple records in a single transaction for better performance.
+
+ Args:
+ items: List of model instances to create
+ db_session: SQLAlchemy session
+ actor: Optional user performing the action
+
+ Returns:
+ List of created model instances
+ """
+ logger.debug(f"Batch creating {len(items)} {cls.__name__} items with actor={actor}")
+
+ if not items:
+ return []
+
+ # Set created/updated by fields if actor is provided
+ if actor:
+ for item in items:
+ item._set_created_and_updated_by_fields(actor.id)
+
+ try:
+ with db_session as session:
+ session.add_all(items)
+ session.flush() # Flush to generate IDs but don't commit yet
+
+ # Collect IDs to fetch the complete objects after commit
+ item_ids = [item.id for item in items]
+
+ session.commit()
+
+ # Re-query the objects to get them with relationships loaded
+ query = select(cls).where(cls.id.in_(item_ids))
+ if hasattr(cls, "created_at"):
+ query = query.order_by(cls.created_at)
+
+ return list(session.execute(query).scalars())
+
+ except (DBAPIError, IntegrityError) as e:
+ cls._handle_dbapi_error(e)
+
@handle_db_timeout
def delete(self, db_session: "Session", actor: Optional["User"] = None) -> "SqlalchemyBase":
logger.debug(f"Soft deleting {self.__class__.__name__} with ID: {self.id} with actor={actor}")
diff --git a/letta/server/rest_api/routers/v1/voice.py b/letta/server/rest_api/routers/v1/voice.py
index 7e3871b87..5f187c5ee 100644
--- a/letta/server/rest_api/routers/v1/voice.py
+++ b/letta/server/rest_api/routers/v1/voice.py
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Body, Depends, Header
from fastapi.responses import StreamingResponse
from openai.types.chat.completion_create_params import CompletionCreateParams
-from letta.agents.low_latency_agent import LowLatencyAgent
+from letta.agents.voice_agent import VoiceAgent
from letta.log import get_logger
from letta.schemas.openai.chat_completions import UserMessage
from letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
from letta.server.server import SyncServer
-router = APIRouter(prefix="/voice", tags=["voice"])
+router = APIRouter(prefix="/voice-beta", tags=["voice"])
logger = get_logger(__name__)
@@ -61,14 +61,14 @@ async def create_voice_chat_completions(
)
# Instantiate our LowLatencyAgent
- agent = LowLatencyAgent(
+ agent = VoiceAgent(
agent_id=agent_id,
openai_client=client,
message_manager=server.message_manager,
agent_manager=server.agent_manager,
block_manager=server.block_manager,
actor=actor,
- message_buffer_limit=10,
+ message_buffer_limit=100,
message_buffer_min=4,
)
diff --git a/letta/services/message_manager.py b/letta/services/message_manager.py
index 387168dc4..96b6f14cf 100644
--- a/letta/services/message_manager.py
+++ b/letta/services/message_manager.py
@@ -63,8 +63,33 @@ class MessageManager:
@enforce_types
def create_many_messages(self, pydantic_msgs: List[PydanticMessage], actor: PydanticUser) -> List[PydanticMessage]:
- """Create multiple messages."""
- return [self.create_message(m, actor=actor) for m in pydantic_msgs]
+ """
+ Create multiple messages in a single database transaction.
+
+ Args:
+ pydantic_msgs: List of Pydantic message models to create
+ actor: User performing the action
+
+ Returns:
+ List of created Pydantic message models
+ """
+ if not pydantic_msgs:
+ return []
+
+ # Create ORM model instances for all messages
+ orm_messages = []
+ for pydantic_msg in pydantic_msgs:
+ # Set the organization id of the Pydantic message
+ pydantic_msg.organization_id = actor.organization_id
+ msg_data = pydantic_msg.model_dump(to_orm=True)
+ orm_messages.append(MessageModel(**msg_data))
+
+ # Use the batch_create method for efficient creation
+ with self.session_maker() as session:
+ created_messages = MessageModel.batch_create(orm_messages, session, actor=actor)
+
+ # Convert back to Pydantic models
+ return [msg.to_pydantic() for msg in created_messages]
@enforce_types
def update_message_by_letta_message(
diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.py
index 3ae24ec6c..c7e77a970 100644
--- a/tests/integration_test_chat_completions.py
+++ b/tests/integration_test_chat_completions.py
@@ -15,7 +15,9 @@ from letta.schemas.llm_config import LLMConfig
from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, UserMessage
from letta.schemas.tool import ToolCreate
from letta.schemas.usage import LettaUsageStatistics
+from letta.services.agent_manager import AgentManager
from letta.services.tool_manager import ToolManager
+from letta.services.user_manager import UserManager
# --- Server Management --- #
@@ -111,7 +113,9 @@ def composio_gmail_get_profile_tool(default_user):
def agent(client, roll_dice_tool, weather_tool, composio_gmail_get_profile_tool):
"""Creates an agent and ensures cleanup after tests."""
agent_state = client.create_agent(
- name=f"test_compl_{str(uuid.uuid4())[5:]}", tool_ids=[roll_dice_tool.id, weather_tool.id, composio_gmail_get_profile_tool.id]
+ name=f"test_compl_{str(uuid.uuid4())[5:]}",
+ tool_ids=[roll_dice_tool.id, weather_tool.id, composio_gmail_get_profile_tool.id],
+ include_base_tools=False,
)
yield agent_state
client.delete_agent(agent_state.id)
@@ -152,8 +156,8 @@ def _assert_valid_chunk(chunk, idx, chunks):
@pytest.mark.asyncio
[email protected]("message", ["How are you?"])
[email protected]("endpoint", ["v1/voice"])
[email protected]("message", ["Hi how are you today?"])
[email protected]("endpoint", ["v1/voice-beta"])
async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):
"""Tests chat completion streaming using the Async OpenAI client."""
request = _get_chat_request(message)
@@ -165,9 +169,30 @@ async def test_latency(mock_e2b_api_key_none, client, agent, message, endpoint):
print(chunk)
[email protected]("message", ["Use recall memory tool to recall what my name is."])
[email protected]("endpoint", ["v1/voice-beta"])
+async def test_voice_recall_memory(mock_e2b_api_key_none, client, agent, message, endpoint):
+ """Tests chat completion streaming using the Async OpenAI client."""
+ request = _get_chat_request(message)
+
+ # Insert some messages about my name
+ client.user_message(agent.id, "My name is Matt")
+
+ # Wipe the in context messages
+ actor = UserManager().get_default_user()
+ AgentManager().set_in_context_messages(agent_id=agent.id, message_ids=[agent.message_ids[0]], actor=actor)
+
+ async_client = AsyncOpenAI(base_url=f"{client.base_url}/{endpoint}/{agent.id}", max_retries=0)
+ stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))
+ async with stream:
+ async for chunk in stream:
+ print(chunk)
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])
[email protected]("endpoint", ["openai/v1", "v1/voice"])
[email protected]("endpoint", ["openai/v1", "v1/voice-beta"])
async def test_chat_completions_streaming_openai_client(mock_e2b_api_key_none, client, agent, message, endpoint):
"""Tests chat completion streaming using the Async OpenAI client."""
request = _get_chat_request(message)