Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Integrate tool executor into VoiceAgent (#2872)
Matthew Zhou0931ab943a96
diff --git a/letta/agents/voice_agent.py b/letta/agents/voice_agent.pyindex e35e53855..e31e983fe 100644--- a/letta/agents/voice_agent.py+++ b/letta/agents/voice_agent.py@@ -9,14 +9,9 @@ import openaifrom letta.agents.base_agent import BaseAgentfrom letta.agents.exceptions import IncompatibleAgentTypefrom letta.agents.voice_sleeptime_agent import VoiceSleeptimeAgent-from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX+from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG, REQUEST_HEARTBEAT_PARAMfrom letta.helpers.datetime_helpers import get_utc_time-from letta.helpers.tool_execution_helper import (- add_pre_execution_message,- enable_strict_mode,- execute_external_tool,- remove_request_heartbeat,-)+from letta.helpers.tool_execution_helper import add_pre_execution_message, enable_strict_mode, remove_request_heartbeatfrom letta.interfaces.openai_chat_completions_streaming_interface import OpenAIChatCompletionsStreamingInterfacefrom letta.log import get_loggerfrom letta.orm.enums import ToolType@@ -47,6 +42,7 @@ from letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.summarizer.enums import SummarizationModefrom letta.services.summarizer.summarizer import Summarizer+from letta.services.tool_executor.tool_execution_manager import ToolExecutionManagerfrom letta.settings import model_settingslogger = get_logger(__name__)@@ -124,7 +120,11 @@ class VoiceAgent(BaseAgent):user_query = input_messages[0].content[0].text- agent_state = await self.agent_manager.get_agent_by_id_async(self.agent_id, actor=self.actor)+ agent_state = await self.agent_manager.get_agent_by_id_async(+ agent_id=self.agent_id,+ include_relationships=["tools", "memory", "tool_exec_environment_variables", "multi_agent_group"],+ actor=self.actor,+ )# TODO: Refactor this so it uses our in-house clients# TODO: For now, piggyback off of OpenAI client for ease@@ -332,7 +332,12 @@ class VoiceAgent(BaseAgent):def _build_tool_schemas(self, agent_state: AgentState, external_tools_only=True) -> List[Tool]:if external_tools_only:- tools = [t for t in agent_state.tools if t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]+ tools = [+ t+ for t in agent_state.tools+ if t.tool_type+ in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM, ToolType.LETTA_FILES_CORE, ToolType.LETTA_BUILTIN, ToolType.EXTERNAL_MCP}+ ]else:tools = agent_state.tools@@ -401,12 +406,10 @@ class VoiceAgent(BaseAgent):async def _execute_tool(self, user_query: str, tool_name: str, tool_args: dict, agent_state: AgentState) -> "ToolExecutionResult":"""- Executes a tool and returns (result, success_flag).+ Executes a tool and returns the ToolExecutionResult."""from letta.schemas.tool_execution_result import ToolExecutionResult- print("EXECUTING TOOL")-# Special memory caseif tool_name == "search_memory":tool_result = await self._search_memory(@@ -420,26 +423,39 @@ class VoiceAgent(BaseAgent):func_return=tool_result,status="success",)- else:- target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)- if not target_tool:- return ToolExecutionResult(- func_return=f"Tool not found: {tool_name}",- status="error",- )- try:- tool_result, _ = execute_external_tool(- agent_state=agent_state,- function_name=tool_name,- function_args=tool_args,- target_letta_tool=target_tool,- actor=self.actor,- allow_agent_state_modifications=False,- )- return ToolExecutionResult(func_return=tool_result, status="success")- except Exception as e:- return ToolExecutionResult(func_return=f"Failed to call tool. Error: {e}", status="error")+ # Find the target tool+ target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)+ if not target_tool:+ return ToolExecutionResult(+ func_return=f"Tool {tool_name} not found",+ status="error",+ )++ # Use ToolExecutionManager for modern tool execution+ sandbox_env_vars = {var.key: var.value for var in agent_state.tool_exec_environment_variables}+ tool_execution_manager = ToolExecutionManager(+ agent_state=agent_state,+ message_manager=self.message_manager,+ agent_manager=self.agent_manager,+ block_manager=self.block_manager,+ passage_manager=self.passage_manager,+ sandbox_env_vars=sandbox_env_vars,+ actor=self.actor,+ )++ # Remove request heartbeat / pre_exec_message+ tool_args.pop(PRE_EXECUTION_MESSAGE_ARG, None)+ tool_args.pop(REQUEST_HEARTBEAT_PARAM, None)++ tool_execution_result = await tool_execution_manager.execute_tool_async(+ function_name=tool_name,+ function_args=tool_args,+ tool=target_tool,+ step_id=None, # VoiceAgent doesn't use step tracking currently+ )++ return tool_execution_resultasync def _search_memory(self,diff --git a/letta/interfaces/openai_chat_completions_streaming_interface.py b/letta/interfaces/openai_chat_completions_streaming_interface.pyindex b0450674c..a58ee5545 100644--- a/letta/interfaces/openai_chat_completions_streaming_interface.py+++ b/letta/interfaces/openai_chat_completions_streaming_interface.py@@ -16,7 +16,6 @@ class OpenAIChatCompletionsStreamingInterface:"""def __init__(self, stream_pre_execution_message: bool = True):- print("CHAT COMPLETITION INTERFACE")self.optimistic_json_parser: OptimisticJSONParser = OptimisticJSONParser()self.stream_pre_execution_message: bool = stream_pre_execution_messagediff --git a/tests/integration_test_voice_agent.py b/tests/integration_test_voice_agent.pyindex 7c9e3e9e5..d25ccabe6 100644--- a/tests/integration_test_voice_agent.py+++ b/tests/integration_test_voice_agent.py@@ -23,7 +23,6 @@ from letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message, MessageCreatefrom letta.schemas.openai.chat_completion_request import ChatCompletionRequestfrom letta.schemas.openai.chat_completion_request import UserMessage as OpenAIUserMessage-from letta.schemas.tool import ToolCreatefrom letta.schemas.usage import LettaUsageStatisticsfrom letta.server.server import SyncServerfrom letta.services.agent_manager import AgentManager@@ -158,8 +157,8 @@ def server():# --- Client Setup --- #-async def roll_dice_tool(server):[email protected](scope="module")+async def roll_dice_tool(server, actor):def roll_dice():"""Rolls a 6 sided die.@@ -169,51 +168,13 @@ async def roll_dice_tool(server):"""return "Rolled a 10!"- actor = server.user_manager.get_user_or_default()tool = server.tool_manager.create_or_update_tool(create_tool_from_func(func=roll_dice), actor=actor)yield tool@pytest.fixture-async def weather_tool(server):- def get_weather(location: str) -> str:- """- Fetches the current weather for a given location.-- Parameters:- location (str): The location to get the weather for.-- Returns:- str: A formatted string describing the weather in the given location.-- Raises:- RuntimeError: If the request to fetch weather data fails.- """- import requests-- url = f"https://wttr.in/{location}?format=%C+%t"-- response = requests.get(url)- if response.status_code == 200:- weather_data = response.text- return f"The weather in {location} is {weather_data}."- else:- raise RuntimeError(f"Failed to get weather data, status code: {response.status_code}")-- actor = server.user_manager.get_user_or_default()- tool = server.tool_manager.create_or_update_tool(create_tool_from_func(func=get_weather), actor=actor)- yield tool---def composio_gmail_get_profile_tool(default_user):- tool_create = ToolCreate.from_composio(action_name="GMAIL_GET_PROFILE")- tool = ToolManager().create_or_update_composio_tool(tool_create=tool_create, actor=default_user)- yield tool---def voice_agent(server, actor):+def voice_agent(server, actor, roll_dice_tool):+ run_code_tool = server.tool_manager.get_tool_by_name("run_code", actor)main_agent = server.create_agent(request=CreateAgent(agent_type=AgentType.voice_convo_agent,@@ -231,6 +192,7 @@ def voice_agent(server, actor):model="openai/gpt-4o-mini",embedding="openai/text-embedding-ada-002",enable_sleeptime=True,+ tool_ids=[roll_dice_tool.id, run_code_tool.id],),actor=actor,)@@ -292,10 +254,16 @@ def _assert_valid_chunk(chunk, idx, chunks):@pytest.mark.asyncio(loop_scope="module")@pytest.mark.parametrize("model", ["openai/gpt-4o-mini", "anthropic/claude-3-5-sonnet-20241022"])-async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, server, server_url, group_id, actor):- request = _get_chat_request("How are you?")+ "message", ["How are you?", "Use the roll_dice tool to roll a die for me", "Use the run_code tool to calculate 2+2"]+)+async def test_model_compatibility(model, message, server, server_url, actor, roll_dice_tool):+ request = _get_chat_request(message)server.tool_manager.upsert_base_tools(actor=actor)+ # Get the run_code tool+ run_code_tool = server.tool_manager.get_tool_by_name("run_code", actor)+main_agent = server.create_agent(request=CreateAgent(agent_type=AgentType.voice_convo_agent,@@ -313,6 +281,7 @@ async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, servmodel=model,embedding="openai/text-embedding-ada-002",enable_sleeptime=True,+ tool_ids=[roll_dice_tool.id, run_code_tool.id],),actor=actor,)@@ -324,6 +293,43 @@ async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, servif chunk.choices and chunk.choices[0].delta.content:print(chunk.choices[0].delta.content)+ # Get the messages and assert based on the message type+ messages = await server.message_manager.list_messages_for_agent_async(agent_id=main_agent.id, actor=actor)++ # Find user message with our request+ user_messages = [msg for msg in messages if msg.role == MessageRole.user and message in str(msg.content)]+ assert len(user_messages) >= 1, f"Should find user message containing: {message}"++ # Find assistant messages with tool calls+ assistant_tool_messages = [msg for msg in messages if msg.role == MessageRole.assistant and msg.tool_calls]++ if "How are you?" in message:+ # Generic greeting should have no tool calls (just conversational)+ tool_call_names = [+ tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls if tc.function.name not in ["send_message"]+ ]+ assert len(tool_call_names) == 0, f"Generic greeting should not use tools, but found: {tool_call_names}"++ elif "roll_dice" in message:+ # Should have roll_dice tool call+ tool_call_names = [tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls]+ assert "roll_dice" in tool_call_names, f"Should call roll_dice tool, found calls: {tool_call_names}"++ # Should have tool response with dice result+ tool_responses = [msg for msg in messages if msg.role == MessageRole.tool and msg.name == "roll_dice"]+ assert len(tool_responses) >= 1, "Should have roll_dice tool response"+ assert "Rolled a 10!" in str(tool_responses[0].content), "Should contain dice result"++ elif "run_code" in message:+ # Should have run_code tool call+ tool_call_names = [tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls]+ assert "run_code" in tool_call_names, f"Should call run_code tool, found calls: {tool_call_names}"++ # Should have tool response with calculation result+ tool_responses = [msg for msg in messages if msg.role == MessageRole.tool and msg.name == "run_code"]+ assert len(tool_responses) >= 1, "Should have run_code tool response"+ assert "4" in str(tool_responses[0].content), "Should contain calculation result '4'"+@pytest.mark.asyncio(loop_scope="module")@pytest.mark.parametrize("message", ["Use search memory tool to recall what my name is."])