Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Integrate tool executor into VoiceAgent (#2872)

Matthew Zhou0931ab943a96
diff --git a/letta/agents/voice_agent.py b/letta/agents/voice_agent.py
index e35e53855..e31e983fe 100644
--- a/letta/agents/voice_agent.py
+++ b/letta/agents/voice_agent.py
@@ -9,14 +9,9 @@ import openai
from letta.agents.base_agent import BaseAgent
from letta.agents.exceptions import IncompatibleAgentType
from letta.agents.voice_sleeptime_agent import VoiceSleeptimeAgent
-from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX
+from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX, PRE_EXECUTION_MESSAGE_ARG, REQUEST_HEARTBEAT_PARAM
from letta.helpers.datetime_helpers import get_utc_time
-from letta.helpers.tool_execution_helper import (
- add_pre_execution_message,
- enable_strict_mode,
- execute_external_tool,
- remove_request_heartbeat,
-)
+from letta.helpers.tool_execution_helper import add_pre_execution_message, enable_strict_mode, remove_request_heartbeat
from letta.interfaces.openai_chat_completions_streaming_interface import OpenAIChatCompletionsStreamingInterface
from letta.log import get_logger
from letta.orm.enums import ToolType
@@ -47,6 +42,7 @@ from letta.services.message_manager import MessageManager
from letta.services.passage_manager import PassageManager
from letta.services.summarizer.enums import SummarizationMode
from letta.services.summarizer.summarizer import Summarizer
+from letta.services.tool_executor.tool_execution_manager import ToolExecutionManager
from letta.settings import model_settings
logger = get_logger(__name__)
@@ -124,7 +120,11 @@ class VoiceAgent(BaseAgent):
user_query = input_messages[0].content[0].text
- agent_state = await self.agent_manager.get_agent_by_id_async(self.agent_id, actor=self.actor)
+ agent_state = await self.agent_manager.get_agent_by_id_async(
+ agent_id=self.agent_id,
+ include_relationships=["tools", "memory", "tool_exec_environment_variables", "multi_agent_group"],
+ actor=self.actor,
+ )
# TODO: Refactor this so it uses our in-house clients
# TODO: For now, piggyback off of OpenAI client for ease
@@ -332,7 +332,12 @@ class VoiceAgent(BaseAgent):
def _build_tool_schemas(self, agent_state: AgentState, external_tools_only=True) -> List[Tool]:
if external_tools_only:
- tools = [t for t in agent_state.tools if t.tool_type in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM}]
+ tools = [
+ t
+ for t in agent_state.tools
+ if t.tool_type
+ in {ToolType.EXTERNAL_COMPOSIO, ToolType.CUSTOM, ToolType.LETTA_FILES_CORE, ToolType.LETTA_BUILTIN, ToolType.EXTERNAL_MCP}
+ ]
else:
tools = agent_state.tools
@@ -401,12 +406,10 @@ class VoiceAgent(BaseAgent):
async def _execute_tool(self, user_query: str, tool_name: str, tool_args: dict, agent_state: AgentState) -> "ToolExecutionResult":
"""
- Executes a tool and returns (result, success_flag).
+ Executes a tool and returns the ToolExecutionResult.
"""
from letta.schemas.tool_execution_result import ToolExecutionResult
- print("EXECUTING TOOL")
-
# Special memory case
if tool_name == "search_memory":
tool_result = await self._search_memory(
@@ -420,26 +423,39 @@ class VoiceAgent(BaseAgent):
func_return=tool_result,
status="success",
)
- else:
- target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)
- if not target_tool:
- return ToolExecutionResult(
- func_return=f"Tool not found: {tool_name}",
- status="error",
- )
- try:
- tool_result, _ = execute_external_tool(
- agent_state=agent_state,
- function_name=tool_name,
- function_args=tool_args,
- target_letta_tool=target_tool,
- actor=self.actor,
- allow_agent_state_modifications=False,
- )
- return ToolExecutionResult(func_return=tool_result, status="success")
- except Exception as e:
- return ToolExecutionResult(func_return=f"Failed to call tool. Error: {e}", status="error")
+ # Find the target tool
+ target_tool = next((x for x in agent_state.tools if x.name == tool_name), None)
+ if not target_tool:
+ return ToolExecutionResult(
+ func_return=f"Tool {tool_name} not found",
+ status="error",
+ )
+
+ # Use ToolExecutionManager for modern tool execution
+ sandbox_env_vars = {var.key: var.value for var in agent_state.tool_exec_environment_variables}
+ tool_execution_manager = ToolExecutionManager(
+ agent_state=agent_state,
+ message_manager=self.message_manager,
+ agent_manager=self.agent_manager,
+ block_manager=self.block_manager,
+ passage_manager=self.passage_manager,
+ sandbox_env_vars=sandbox_env_vars,
+ actor=self.actor,
+ )
+
+ # Remove request heartbeat / pre_exec_message
+ tool_args.pop(PRE_EXECUTION_MESSAGE_ARG, None)
+ tool_args.pop(REQUEST_HEARTBEAT_PARAM, None)
+
+ tool_execution_result = await tool_execution_manager.execute_tool_async(
+ function_name=tool_name,
+ function_args=tool_args,
+ tool=target_tool,
+ step_id=None, # VoiceAgent doesn't use step tracking currently
+ )
+
+ return tool_execution_result
async def _search_memory(
self,
diff --git a/letta/interfaces/openai_chat_completions_streaming_interface.py b/letta/interfaces/openai_chat_completions_streaming_interface.py
index b0450674c..a58ee5545 100644
--- a/letta/interfaces/openai_chat_completions_streaming_interface.py
+++ b/letta/interfaces/openai_chat_completions_streaming_interface.py
@@ -16,7 +16,6 @@ class OpenAIChatCompletionsStreamingInterface:
"""
def __init__(self, stream_pre_execution_message: bool = True):
- print("CHAT COMPLETITION INTERFACE")
self.optimistic_json_parser: OptimisticJSONParser = OptimisticJSONParser()
self.stream_pre_execution_message: bool = stream_pre_execution_message
diff --git a/tests/integration_test_voice_agent.py b/tests/integration_test_voice_agent.py
index 7c9e3e9e5..d25ccabe6 100644
--- a/tests/integration_test_voice_agent.py
+++ b/tests/integration_test_voice_agent.py
@@ -23,7 +23,6 @@ from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message, MessageCreate
from letta.schemas.openai.chat_completion_request import ChatCompletionRequest
from letta.schemas.openai.chat_completion_request import UserMessage as OpenAIUserMessage
-from letta.schemas.tool import ToolCreate
from letta.schemas.usage import LettaUsageStatistics
from letta.server.server import SyncServer
from letta.services.agent_manager import AgentManager
@@ -158,8 +157,8 @@ def server():
# --- Client Setup --- #
-async def roll_dice_tool(server):
[email protected](scope="module")
+async def roll_dice_tool(server, actor):
def roll_dice():
"""
Rolls a 6 sided die.
@@ -169,51 +168,13 @@ async def roll_dice_tool(server):
"""
return "Rolled a 10!"
- actor = server.user_manager.get_user_or_default()
tool = server.tool_manager.create_or_update_tool(create_tool_from_func(func=roll_dice), actor=actor)
yield tool
@pytest.fixture
-async def weather_tool(server):
- def get_weather(location: str) -> str:
- """
- Fetches the current weather for a given location.
-
- Parameters:
- location (str): The location to get the weather for.
-
- Returns:
- str: A formatted string describing the weather in the given location.
-
- Raises:
- RuntimeError: If the request to fetch weather data fails.
- """
- import requests
-
- url = f"https://wttr.in/{location}?format=%C+%t"
-
- response = requests.get(url)
- if response.status_code == 200:
- weather_data = response.text
- return f"The weather in {location} is {weather_data}."
- else:
- raise RuntimeError(f"Failed to get weather data, status code: {response.status_code}")
-
- actor = server.user_manager.get_user_or_default()
- tool = server.tool_manager.create_or_update_tool(create_tool_from_func(func=get_weather), actor=actor)
- yield tool
-
-
-def composio_gmail_get_profile_tool(default_user):
- tool_create = ToolCreate.from_composio(action_name="GMAIL_GET_PROFILE")
- tool = ToolManager().create_or_update_composio_tool(tool_create=tool_create, actor=default_user)
- yield tool
-
-
-def voice_agent(server, actor):
+def voice_agent(server, actor, roll_dice_tool):
+ run_code_tool = server.tool_manager.get_tool_by_name("run_code", actor)
main_agent = server.create_agent(
request=CreateAgent(
agent_type=AgentType.voice_convo_agent,
@@ -231,6 +192,7 @@ def voice_agent(server, actor):
model="openai/gpt-4o-mini",
embedding="openai/text-embedding-ada-002",
enable_sleeptime=True,
+ tool_ids=[roll_dice_tool.id, run_code_tool.id],
),
actor=actor,
)
@@ -292,10 +254,16 @@ def _assert_valid_chunk(chunk, idx, chunks):
@pytest.mark.asyncio(loop_scope="module")
@pytest.mark.parametrize("model", ["openai/gpt-4o-mini", "anthropic/claude-3-5-sonnet-20241022"])
-async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, server, server_url, group_id, actor):
- request = _get_chat_request("How are you?")
+ "message", ["How are you?", "Use the roll_dice tool to roll a die for me", "Use the run_code tool to calculate 2+2"]
+)
+async def test_model_compatibility(model, message, server, server_url, actor, roll_dice_tool):
+ request = _get_chat_request(message)
server.tool_manager.upsert_base_tools(actor=actor)
+ # Get the run_code tool
+ run_code_tool = server.tool_manager.get_tool_by_name("run_code", actor)
+
main_agent = server.create_agent(
request=CreateAgent(
agent_type=AgentType.voice_convo_agent,
@@ -313,6 +281,7 @@ async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, serv
model=model,
embedding="openai/text-embedding-ada-002",
enable_sleeptime=True,
+ tool_ids=[roll_dice_tool.id, run_code_tool.id],
),
actor=actor,
)
@@ -324,6 +293,43 @@ async def test_model_compatibility(disable_e2b_api_key, voice_agent, model, serv
if chunk.choices and chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content)
+ # Get the messages and assert based on the message type
+ messages = await server.message_manager.list_messages_for_agent_async(agent_id=main_agent.id, actor=actor)
+
+ # Find user message with our request
+ user_messages = [msg for msg in messages if msg.role == MessageRole.user and message in str(msg.content)]
+ assert len(user_messages) >= 1, f"Should find user message containing: {message}"
+
+ # Find assistant messages with tool calls
+ assistant_tool_messages = [msg for msg in messages if msg.role == MessageRole.assistant and msg.tool_calls]
+
+ if "How are you?" in message:
+ # Generic greeting should have no tool calls (just conversational)
+ tool_call_names = [
+ tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls if tc.function.name not in ["send_message"]
+ ]
+ assert len(tool_call_names) == 0, f"Generic greeting should not use tools, but found: {tool_call_names}"
+
+ elif "roll_dice" in message:
+ # Should have roll_dice tool call
+ tool_call_names = [tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls]
+ assert "roll_dice" in tool_call_names, f"Should call roll_dice tool, found calls: {tool_call_names}"
+
+ # Should have tool response with dice result
+ tool_responses = [msg for msg in messages if msg.role == MessageRole.tool and msg.name == "roll_dice"]
+ assert len(tool_responses) >= 1, "Should have roll_dice tool response"
+ assert "Rolled a 10!" in str(tool_responses[0].content), "Should contain dice result"
+
+ elif "run_code" in message:
+ # Should have run_code tool call
+ tool_call_names = [tc.function.name for msg in assistant_tool_messages for tc in msg.tool_calls]
+ assert "run_code" in tool_call_names, f"Should call run_code tool, found calls: {tool_call_names}"
+
+ # Should have tool response with calculation result
+ tool_responses = [msg for msg in messages if msg.role == MessageRole.tool and msg.name == "run_code"]
+ assert len(tool_responses) >= 1, "Should have run_code tool response"
+ assert "4" in str(tool_responses[0].content), "Should contain calculation result '4'"
+
@pytest.mark.asyncio(loop_scope="module")
@pytest.mark.parametrize("message", ["Use search memory tool to recall what my name is."])