Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: Refresh blocks in between steps (#1335)
Matthew Zhou82f9377c8af5
diff --git a/letta/agents/voice_agent.py b/letta/agents/voice_agent.pyindex 006b5827f..f60bb6612 100644--- a/letta/agents/voice_agent.py+++ b/letta/agents/voice_agent.py@@ -5,11 +5,15 @@ from typing import Any, AsyncGenerator, Dict, List, Tupleimport openaifrom letta.agents.base_agent import BaseAgent-from letta.agents.ephemeral_agent import EphemeralAgentfrom letta.agents.ephemeral_memory_agent import EphemeralMemoryAgentfrom letta.constants import NON_USER_MSG_PREFIXfrom letta.helpers.datetime_helpers import get_utc_time-from letta.helpers.tool_execution_helper import add_pre_execution_message, execute_external_tool, remove_request_heartbeat+from letta.helpers.tool_execution_helper import (+ add_pre_execution_message,+ enable_strict_mode,+ execute_external_tool,+ remove_request_heartbeat,+)from letta.interfaces.openai_chat_completions_streaming_interface import OpenAIChatCompletionsStreamingInterfacefrom letta.log import get_loggerfrom letta.orm.enums import ToolType@@ -38,7 +42,6 @@ from letta.services.helpers.agent_manager_helper import compile_system_messagefrom letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.summarizer.enums import SummarizationMode-from letta.services.summarizer.summarizer import Summarizerfrom letta.utils import united_difflogger = get_logger(__name__)@@ -75,16 +78,16 @@ class VoiceAgent(BaseAgent):self.passage_manager = PassageManager() # TODO: pass this in# TODO: This is not guaranteed to exist!self.summary_block_label = "human"- self.summarizer = Summarizer(- mode=summarization_mode,- summarizer_agent=EphemeralAgent(- agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor- ),- message_buffer_limit=message_buffer_limit,- message_buffer_min=message_buffer_min,- )+ # self.summarizer = Summarizer(+ # mode=summarization_mode,+ # summarizer_agent=EphemeralAgent(+ # agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor+ # ),+ # message_buffer_limit=message_buffer_limit,+ # message_buffer_min=message_buffer_min,+ # )self.message_buffer_limit = message_buffer_limit- self.message_buffer_min = message_buffer_min+ # self.message_buffer_min = message_buffer_minself.offline_memory_agent = EphemeralMemoryAgent(agent_id=agent_id, openai_client=openai_client, message_manager=message_manager, agent_manager=agent_manager, actor=actor)@@ -105,8 +108,6 @@ class VoiceAgent(BaseAgent):# TODO: Define max steps herewhile True:- step_first = True-# Rebuild memory each loopin_context_messages = self._rebuild_memory(in_context_messages, agent_state)openai_messages = convert_letta_messages_to_openai(in_context_messages)@@ -119,8 +120,6 @@ class VoiceAgent(BaseAgent):# 1) Yield partial tokens from OpenAIasync for sse_chunk in streaming_interface.process(stream):- if step_first:- step_first = Falseyield sse_chunk# 2) Now handle the final AI response. This might yield more text (stalling, etc.)@@ -241,6 +240,14 @@ class VoiceAgent(BaseAgent):)def _rebuild_memory(self, in_context_messages: List[Message], agent_state: AgentState) -> List[Message]:+ # Refresh memory+ # TODO: This only happens for the summary block+ # TODO: We want to extend this refresh to be general, and stick it in agent_manager+ for i, b in enumerate(agent_state.memory.blocks):+ if b.label == self.summary_block_label:+ agent_state.memory.blocks[i] = self.block_manager.get_block_by_id(block_id=b.id, actor=self.actor)+ break+# TODO: This is a pretty brittle pattern established all over our code, need to get rid of thiscurr_system_message = in_context_messages[0]curr_memory_str = agent_state.memory.compile()@@ -309,28 +316,31 @@ class VoiceAgent(BaseAgent):)recall_memory_json = Tool(type="function",- function=add_pre_execution_message(- {- "name": "recall_memory",- "description": "Retrieve relevant information from memory based on a given query. Use when you don't remember the answer to a question.",- "parameters": {- "type": "object",- "properties": {- "query": {- "type": "string",- "description": "A description of what the model is trying to recall from memory.",- }+ function=enable_strict_mode(+ add_pre_execution_message(+ {+ "name": "recall_memory",+ "description": "Retrieve relevant information from memory based on a given query. Use when you don't remember the answer to a question.",+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "A description of what the model is trying to recall from memory.",+ }+ },+ "required": ["query"],},- "required": ["query"],},- },- description=recall_memory_utterance_description,+ description=recall_memory_utterance_description,+ )),)# TODO: Customize whether or not to have heartbeats, pre_exec_message, etc.return [recall_memory_json] + [- Tool(type="function", function=add_pre_execution_message(remove_request_heartbeat(t.json_schema))) for t in tools+ Tool(type="function", function=enable_strict_mode(add_pre_execution_message(remove_request_heartbeat(t.json_schema))))+ for t in tools]async def _execute_tool(self, tool_name: str, tool_args: dict, agent_state: AgentState) -> Tuple[str, bool]:diff --git a/letta/helpers/tool_execution_helper.py b/letta/helpers/tool_execution_helper.pyindex 886cf9950..f307d730b 100644--- a/letta/helpers/tool_execution_helper.py+++ b/letta/helpers/tool_execution_helper.py@@ -36,7 +36,6 @@ def enable_strict_mode(tool_schema: Dict[str, Any]) -> Dict[str, Any]:# Set additionalProperties to Falseparameters["additionalProperties"] = Falseschema["parameters"] = parameters-return schemadiff --git a/letta/schemas/openai/chat_completion_request.py b/letta/schemas/openai/chat_completion_request.pyindex 12486bca9..f8ad12644 100644--- a/letta/schemas/openai/chat_completion_request.py+++ b/letta/schemas/openai/chat_completion_request.py@@ -82,6 +82,7 @@ class FunctionSchema(BaseModel):name: strdescription: Optional[str] = Noneparameters: Optional[Dict[str, Any]] = None # JSON Schema for the parameters+ strict: bool = Falseclass Tool(BaseModel):diff --git a/letta/server/rest_api/routers/v1/voice.py b/letta/server/rest_api/routers/v1/voice.pyindex 5f187c5ee..5e32b60f5 100644--- a/letta/server/rest_api/routers/v1/voice.py+++ b/letta/server/rest_api/routers/v1/voice.py@@ -68,8 +68,8 @@ async def create_voice_chat_completions(agent_manager=server.agent_manager,block_manager=server.block_manager,actor=actor,- message_buffer_limit=100,- message_buffer_min=4,+ message_buffer_limit=50,+ message_buffer_min=10,)# Return the streaming generatordiff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 6bf4a5cfd..d9e3894aa 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -755,7 +755,7 @@ class AgentManager:if updated_value != agent_state.memory.get_block(label).value:# update the block if it's changedblock_id = agent_state.memory.get_block(label).id- block = self.block_manager.update_block(block_id=block_id, block_update=BlockUpdate(value=updated_value), actor=actor)+ self.block_manager.update_block(block_id=block_id, block_update=BlockUpdate(value=updated_value), actor=actor)# refresh memory from DB (using block ids)agent_state.memory = Memory(diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.pyindex c7e77a970..36c7b742d 100644--- a/tests/integration_test_chat_completions.py+++ b/tests/integration_test_chat_completions.py@@ -110,11 +110,11 @@ def composio_gmail_get_profile_tool(default_user):@pytest.fixture(scope="function")-def agent(client, roll_dice_tool, weather_tool, composio_gmail_get_profile_tool):+def agent(client, roll_dice_tool, weather_tool):"""Creates an agent and ensures cleanup after tests."""agent_state = client.create_agent(name=f"test_compl_{str(uuid.uuid4())[5:]}",- tool_ids=[roll_dice_tool.id, weather_tool.id, composio_gmail_get_profile_tool.id],+ tool_ids=[roll_dice_tool.id, weather_tool.id],include_base_tools=False,)yield agent_state