Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Refactor mcp client and make stdio errors more manageable (#1291)
Matthew Zhou40c70b46cf26
diff --git a/letta/agent.py b/letta/agent.pyindex 11414bce7..5286f9cbf 100644--- a/letta/agent.py+++ b/letta/agent.py@@ -22,11 +22,11 @@ from letta.errors import ContextWindowExceededErrorfrom letta.functions.ast_parsers import coerce_dict_args_by_annotations, get_function_annotations_from_sourcefrom letta.functions.functions import get_function_from_modulefrom letta.functions.helpers import execute_composio_action, generate_composio_action_from_func_name+from letta.functions.mcp_client.base_client import BaseMCPClientfrom letta.helpers import ToolRulesSolverfrom letta.helpers.composio_helpers import get_composio_api_keyfrom letta.helpers.datetime_helpers import get_utc_timefrom letta.helpers.json_helpers import json_dumps, json_loads-from letta.helpers.mcp_helpers import BaseMCPClientfrom letta.interface import AgentInterfacefrom letta.llm_api.helpers import calculate_summarizer_cutoff, get_token_counts_for_messages, is_context_overflow_errorfrom letta.llm_api.llm_api_tools import creatediff --git a/letta/functions/mcp_client/__init__.py b/letta/functions/mcp_client/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/letta/functions/mcp_client/base_client.py b/letta/functions/mcp_client/base_client.pynew file mode 100644index 000000000..91d46d912--- /dev/null+++ b/letta/functions/mcp_client/base_client.py@@ -0,0 +1,61 @@+import asyncio+from typing import List, Optional, Tuple++from mcp import ClientSession, Tool++from letta.functions.mcp_client.types import BaseServerConfig+from letta.log import get_logger++logger = get_logger(__name__)+++class BaseMCPClient:+ def __init__(self):+ self.session: Optional[ClientSession] = None+ self.stdio = None+ self.write = None+ self.initialized = False+ self.loop = asyncio.new_event_loop()+ self.cleanup_funcs = []++ def connect_to_server(self, server_config: BaseServerConfig):+ asyncio.set_event_loop(self.loop)+ success = self._initialize_connection(server_config)++ if success:+ self.loop.run_until_complete(self.session.initialize())+ self.initialized = True+ else:+ raise RuntimeError(+ f"Connecting to MCP server failed. Please review your server config: {server_config.model_dump_json(indent=4)}"+ )++ def _initialize_connection(self, server_config: BaseServerConfig) -> bool:+ raise NotImplementedError("Subclasses must implement _initialize_connection")++ def list_tools(self) -> List[Tool]:+ self._check_initialized()+ response = self.loop.run_until_complete(self.session.list_tools())+ return response.tools++ def execute_tool(self, tool_name: str, tool_args: dict) -> Tuple[str, bool]:+ self._check_initialized()+ result = self.loop.run_until_complete(self.session.call_tool(tool_name, tool_args))+ return str(result.content), result.isError++ def _check_initialized(self):+ if not self.initialized:+ logger.error("MCPClient has not been initialized")+ raise RuntimeError("MCPClient has not been initialized")++ def cleanup(self):+ try:+ for cleanup_func in self.cleanup_funcs:+ cleanup_func()+ self.initialized = False+ if not self.loop.is_closed():+ self.loop.close()+ except Exception as e:+ logger.warning(e)+ finally:+ logger.info("Cleaned up MCP clients on shutdown.")diff --git a/letta/functions/mcp_client/sse_client.py b/letta/functions/mcp_client/sse_client.pynew file mode 100644index 000000000..daf31367a--- /dev/null+++ b/letta/functions/mcp_client/sse_client.py@@ -0,0 +1,21 @@+from mcp import ClientSession+from mcp.client.sse import sse_client++from letta.functions.mcp_client.base_client import BaseMCPClient+from letta.functions.mcp_client.types import SSEServerConfig++# see: https://modelcontextprotocol.io/quickstart/user+MCP_CONFIG_TOPLEVEL_KEY = "mcpServers"+++class SSEMCPClient(BaseMCPClient):+ def _initialize_connection(self, server_config: SSEServerConfig) -> bool:+ sse_cm = sse_client(url=server_config.server_url)+ sse_transport = self.loop.run_until_complete(sse_cm.__aenter__())+ self.stdio, self.write = sse_transport+ self.cleanup_funcs.append(lambda: self.loop.run_until_complete(sse_cm.__aexit__(None, None, None)))++ session_cm = ClientSession(self.stdio, self.write)+ self.session = self.loop.run_until_complete(session_cm.__aenter__())+ self.cleanup_funcs.append(lambda: self.loop.run_until_complete(session_cm.__aexit__(None, None, None)))+ return Truediff --git a/letta/functions/mcp_client/stdio_client.py b/letta/functions/mcp_client/stdio_client.pynew file mode 100644index 000000000..b79608894--- /dev/null+++ b/letta/functions/mcp_client/stdio_client.py@@ -0,0 +1,103 @@+import sys+from contextlib import asynccontextmanager++import anyio+import anyio.lowlevel+import mcp.types as types+from anyio.streams.text import TextReceiveStream+from mcp import ClientSession, StdioServerParameters+from mcp.client.stdio import get_default_environment++from letta.functions.mcp_client.base_client import BaseMCPClient+from letta.functions.mcp_client.types import StdioServerConfig+from letta.log import get_logger++logger = get_logger(__name__)+++class StdioMCPClient(BaseMCPClient):+ def _initialize_connection(self, server_config: StdioServerConfig) -> bool:+ try:+ server_params = StdioServerParameters(command=server_config.command, args=server_config.args)+ stdio_cm = forked_stdio_client(server_params)+ stdio_transport = self.loop.run_until_complete(stdio_cm.__aenter__())+ self.stdio, self.write = stdio_transport+ self.cleanup_funcs.append(lambda: self.loop.run_until_complete(stdio_cm.__aexit__(None, None, None)))++ session_cm = ClientSession(self.stdio, self.write)+ self.session = self.loop.run_until_complete(session_cm.__aenter__())+ self.cleanup_funcs.append(lambda: self.loop.run_until_complete(session_cm.__aexit__(None, None, None)))+ return True+ except Exception:+ return False+++@asynccontextmanager+async def forked_stdio_client(server: StdioServerParameters):+ """+ Client transport for stdio: this will connect to a server by spawning a+ process and communicating with it over stdin/stdout.+ """+ read_stream_writer, read_stream = anyio.create_memory_object_stream(0)+ write_stream, write_stream_reader = anyio.create_memory_object_stream(0)++ try:+ process = await anyio.open_process(+ [server.command, *server.args],+ env=server.env or get_default_environment(),+ stderr=sys.stderr, # Consider logging stderr somewhere instead of silencing it+ )+ except OSError as exc:+ raise RuntimeError(f"Failed to spawn process: {server.command} {server.args}") from exc++ async def stdout_reader():+ assert process.stdout, "Opened process is missing stdout"+ buffer = ""+ try:+ async with read_stream_writer:+ async for chunk in TextReceiveStream(+ process.stdout,+ encoding=server.encoding,+ errors=server.encoding_error_handler,+ ):+ lines = (buffer + chunk).split("\n")+ buffer = lines.pop()+ for line in lines:+ try:+ message = types.JSONRPCMessage.model_validate_json(line)+ except Exception as exc:+ await read_stream_writer.send(exc)+ continue+ await read_stream_writer.send(message)+ except anyio.ClosedResourceError:+ await anyio.lowlevel.checkpoint()++ async def stdin_writer():+ assert process.stdin, "Opened process is missing stdin"+ try:+ async with write_stream_reader:+ async for message in write_stream_reader:+ json = message.model_dump_json(by_alias=True, exclude_none=True)+ await process.stdin.send(+ (json + "\n").encode(+ encoding=server.encoding,+ errors=server.encoding_error_handler,+ )+ )+ except anyio.ClosedResourceError:+ await anyio.lowlevel.checkpoint()++ async def watch_process_exit():+ returncode = await process.wait()+ if returncode != 0:+ raise RuntimeError(f"Subprocess exited with code {returncode}. Command: {server.command} {server.args}")++ async with anyio.create_task_group() as tg, process:+ tg.start_soon(stdout_reader)+ tg.start_soon(stdin_writer)+ tg.start_soon(watch_process_exit)++ with anyio.move_on_after(0.2):+ await anyio.sleep_forever()++ yield read_stream, write_streamdiff --git a/letta/functions/mcp_client/types.py b/letta/functions/mcp_client/types.pynew file mode 100644index 000000000..2d8b7af6d--- /dev/null+++ b/letta/functions/mcp_client/types.py@@ -0,0 +1,48 @@+from enum import Enum+from typing import List, Optional++from mcp import Tool+from pydantic import BaseModel, Field+++class MCPTool(Tool):+ """A simple wrapper around MCP's tool definition (to avoid conflict with our own)"""+++class MCPServerType(str, Enum):+ SSE = "sse"+ STDIO = "stdio"+++class BaseServerConfig(BaseModel):+ server_name: str = Field(..., description="The name of the server")+ type: MCPServerType+++class SSEServerConfig(BaseServerConfig):+ type: MCPServerType = MCPServerType.SSE+ server_url: str = Field(..., description="The URL of the server (MCP SSE client will connect to this URL)")++ def to_dict(self) -> dict:+ values = {+ "transport": "sse",+ "url": self.server_url,+ }+ return values+++class StdioServerConfig(BaseServerConfig):+ type: MCPServerType = MCPServerType.STDIO+ command: str = Field(..., description="The command to run (MCP 'local' client will run this command)")+ args: List[str] = Field(..., description="The arguments to pass to the command")+ env: Optional[dict[str, str]] = Field(None, description="Environment variables to set")++ def to_dict(self) -> dict:+ values = {+ "transport": "stdio",+ "command": self.command,+ "args": self.args,+ }+ if self.env is not None:+ values["env"] = self.env+ return valuesdiff --git a/letta/functions/schema_generator.py b/letta/functions/schema_generator.pyindex 30fd9cb71..62b134aed 100644--- a/letta/functions/schema_generator.py+++ b/letta/functions/schema_generator.py@@ -6,7 +6,7 @@ from composio.client.collections import ActionParametersModelfrom docstring_parser import parsefrom pydantic import BaseModel-from letta.helpers.mcp_helpers import MCPTool+from letta.functions.mcp_client.types import MCPTooldef is_optional(annotation):diff --git a/letta/helpers/mcp_helpers.py b/letta/helpers/mcp_helpers.pydeleted file mode 100644index 450622a39..000000000--- a/letta/helpers/mcp_helpers.py+++ /dev/null@@ -1,129 +0,0 @@-import asyncio-from enum import Enum-from typing import List, Optional, Tuple--from mcp import ClientSession, StdioServerParameters, Tool-from mcp.client.sse import sse_client-from mcp.client.stdio import stdio_client-from pydantic import BaseModel, Field--from letta.log import get_logger--logger = get_logger(__name__)--# see: https://modelcontextprotocol.io/quickstart/user-MCP_CONFIG_TOPLEVEL_KEY = "mcpServers"---class MCPTool(Tool):- """A simple wrapper around MCP's tool definition (to avoid conflict with our own)"""---class MCPServerType(str, Enum):- SSE = "sse"- STDIO = "stdio"---class BaseServerConfig(BaseModel):- server_name: str = Field(..., description="The name of the server")- type: MCPServerType---class SSEServerConfig(BaseServerConfig):- type: MCPServerType = MCPServerType.SSE- server_url: str = Field(..., description="The URL of the server (MCP SSE client will connect to this URL)")-- def to_dict(self) -> dict:- values = {- "transport": "sse",- "url": self.server_url,- }- return values---class StdioServerConfig(BaseServerConfig):- type: MCPServerType = MCPServerType.STDIO- command: str = Field(..., description="The command to run (MCP 'local' client will run this command)")- args: List[str] = Field(..., description="The arguments to pass to the command")- env: Optional[dict[str, str]] = Field(None, description="Environment variables to set")-- def to_dict(self) -> dict:- values = {- "transport": "stdio",- "command": self.command,- "args": self.args,- }- if self.env is not None:- values["env"] = self.env- return values---class BaseMCPClient:- def __init__(self):- self.session: Optional[ClientSession] = None- self.stdio = None- self.write = None- self.initialized = False- self.loop = asyncio.new_event_loop()- self.cleanup_funcs = []-- def connect_to_server(self, server_config: BaseServerConfig):- asyncio.set_event_loop(self.loop)- self._initialize_connection(server_config)- self.loop.run_until_complete(self.session.initialize())- self.initialized = True-- def _initialize_connection(self, server_config: BaseServerConfig):- raise NotImplementedError("Subclasses must implement _initialize_connection")-- def list_tools(self) -> List[Tool]:- self._check_initialized()- response = self.loop.run_until_complete(self.session.list_tools())- return response.tools-- def execute_tool(self, tool_name: str, tool_args: dict) -> Tuple[str, bool]:- self._check_initialized()- result = self.loop.run_until_complete(self.session.call_tool(tool_name, tool_args))- return str(result.content), result.isError-- def _check_initialized(self):- if not self.initialized:- logger.error("MCPClient has not been initialized")- raise RuntimeError("MCPClient has not been initialized")-- def cleanup(self):- try:- for cleanup_func in self.cleanup_funcs:- cleanup_func()- self.initialized = False- if not self.loop.is_closed():- self.loop.close()- except Exception as e:- logger.warning(e)- finally:- logger.info("Cleaned up MCP clients on shutdown.")---class StdioMCPClient(BaseMCPClient):- def _initialize_connection(self, server_config: StdioServerConfig):- server_params = StdioServerParameters(command=server_config.command, args=server_config.args)- stdio_cm = stdio_client(server_params)- stdio_transport = self.loop.run_until_complete(stdio_cm.__aenter__())- self.stdio, self.write = stdio_transport- self.cleanup_funcs.append(lambda: self.loop.run_until_complete(stdio_cm.__aexit__(None, None, None)))-- session_cm = ClientSession(self.stdio, self.write)- self.session = self.loop.run_until_complete(session_cm.__aenter__())- self.cleanup_funcs.append(lambda: self.loop.run_until_complete(session_cm.__aexit__(None, None, None)))---class SSEMCPClient(BaseMCPClient):- def _initialize_connection(self, server_config: SSEServerConfig):- sse_cm = sse_client(url=server_config.server_url)- sse_transport = self.loop.run_until_complete(sse_cm.__aenter__())- self.stdio, self.write = sse_transport- self.cleanup_funcs.append(lambda: self.loop.run_until_complete(sse_cm.__aexit__(None, None, None)))-- session_cm = ClientSession(self.stdio, self.write)- self.session = self.loop.run_until_complete(session_cm.__aenter__())- self.cleanup_funcs.append(lambda: self.loop.run_until_complete(session_cm.__aexit__(None, None, None)))diff --git a/letta/schemas/tool.py b/letta/schemas/tool.pyindex b2499a6f1..55fac00c8 100644--- a/letta/schemas/tool.py+++ b/letta/schemas/tool.py@@ -17,12 +17,12 @@ from letta.functions.helpers import (generate_mcp_tool_wrapper,generate_model_from_args_json_schema,)+from letta.functions.mcp_client.types import MCPToolfrom letta.functions.schema_generator import (generate_schema_from_args_schema_v2,generate_tool_schema_for_composio,generate_tool_schema_for_mcp,)-from letta.helpers.mcp_helpers import MCPToolfrom letta.log import get_loggerfrom letta.orm.enums import ToolTypefrom letta.schemas.letta_base import LettaBasediff --git a/letta/server/rest_api/routers/v1/tools.py b/letta/server/rest_api/routers/v1/tools.pyindex 2290d281a..b4423027c 100644--- a/letta/server/rest_api/routers/v1/tools.py+++ b/letta/server/rest_api/routers/v1/tools.py@@ -12,8 +12,8 @@ from composio.exceptions import (from fastapi import APIRouter, Body, Depends, Header, HTTPExceptionfrom letta.errors import LettaToolCreateError+from letta.functions.mcp_client.types import MCPTool, SSEServerConfig, StdioServerConfigfrom letta.helpers.composio_helpers import get_composio_api_key-from letta.helpers.mcp_helpers import MCPTool, SSEServerConfig, StdioServerConfigfrom letta.log import get_loggerfrom letta.orm.errors import UniqueConstraintViolationErrorfrom letta.schemas.letta_message import ToolReturnMessagediff --git a/letta/server/server.py b/letta/server/server.pyindex 65e018327..8e9ff8dc6 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -20,18 +20,12 @@ from letta.agent import Agent, save_agentfrom letta.config import LettaConfigfrom letta.data_sources.connectors import DataConnector, load_datafrom letta.dynamic_multi_agent import DynamicMultiAgent+from letta.functions.mcp_client.base_client import BaseMCPClient+from letta.functions.mcp_client.sse_client import MCP_CONFIG_TOPLEVEL_KEY, SSEMCPClient+from letta.functions.mcp_client.stdio_client import StdioMCPClient+from letta.functions.mcp_client.types import MCPServerType, MCPTool, SSEServerConfig, StdioServerConfigfrom letta.helpers.datetime_helpers import get_utc_timefrom letta.helpers.json_helpers import json_dumps, json_loads-from letta.helpers.mcp_helpers import (- MCP_CONFIG_TOPLEVEL_KEY,- BaseMCPClient,- MCPServerType,- MCPTool,- SSEMCPClient,- SSEServerConfig,- StdioMCPClient,- StdioServerConfig,-)# TODO use custom interfacefrom letta.interface import AgentInterface # abstract@@ -343,11 +337,12 @@ class SyncServer(Server):self.mcp_clients[server_name] = StdioMCPClient()else:raise ValueError(f"Invalid MCP server config: {server_config}")+try:self.mcp_clients[server_name].connect_to_server(server_config)- except:- logger.exception(f"Failed to connect to MCP server: {server_name}")- raise+ except Exception as e:+ logger.error(e)+ self.mcp_clients.pop(server_name)# Print out the tools that are connectedfor server_name, client in self.mcp_clients.items():diff --git a/tests/test_base_functions.py b/tests/test_base_functions.pyindex 8b1336385..037eda2f4 100644--- a/tests/test_base_functions.py+++ b/tests/test_base_functions.py@@ -98,216 +98,3 @@ def test_recall(client, agent_obj):# Conversation searchresult = base_functions.conversation_search(agent_obj, "banana")assert keyword in result---# This test is nondeterministic, so we retry until we get the perfect behavior from the LLM-@retry_until_success(max_attempts=2, sleep_time_seconds=2)-def test_send_message_to_agent(client, agent_obj, other_agent_obj):- secret_word = "banana"-- # Encourage the agent to send a message to the other agent_obj with the secret string- client.send_message(- agent_id=agent_obj.agent_state.id,- role="user",- message=f"Use your tool to send a message to another agent with id {other_agent_obj.agent_state.id} to share the secret word: {secret_word}!",- )-- # Conversation search the other agent- messages = client.get_messages(other_agent_obj.agent_state.id)- # Check for the presence of system message- for m in reversed(messages):- print(f"\n\n {other_agent_obj.agent_state.id} -> {m.model_dump_json(indent=4)}")- if isinstance(m, SystemMessage):- assert secret_word in m.content- break-- # Search the sender agent for the response from another agent- in_context_messages = agent_obj.agent_manager.get_in_context_messages(agent_id=agent_obj.agent_state.id, actor=agent_obj.user)- found = False- target_snippet = f"{other_agent_obj.agent_state.id} said:"-- for m in in_context_messages:- if target_snippet in m.text:- found = True- break-- # Compute the joined string first- joined_messages = "\n".join([m.text for m in in_context_messages[1:]])- print(f"In context messages of the sender agent (without system):\n\n{joined_messages}")- if not found:- raise Exception(f"Was not able to find an instance of the target snippet: {target_snippet}")-- # Test that the agent can still receive messages fine- response = client.send_message(agent_id=agent_obj.agent_state.id, role="user", message="So what did the other agent say?")- print(response.messages)---@retry_until_success(max_attempts=2, sleep_time_seconds=2)-def test_send_message_to_agents_with_tags_simple(client):- worker_tags = ["worker", "user-456"]-- # Clean up first from possibly failed tests- prev_worker_agents = client.server.agent_manager.list_agents(client.user, tags=worker_tags, match_all_tags=True)- for agent in prev_worker_agents:- client.delete_agent(agent.id)-- secret_word = "banana"-- # Create "manager" agent- send_message_to_agents_matching_all_tags_tool_id = client.get_tool_id(name="send_message_to_agents_matching_all_tags")- manager_agent_state = client.create_agent(tool_ids=[send_message_to_agents_matching_all_tags_tool_id])- manager_agent = client.server.load_agent(agent_id=manager_agent_state.id, actor=client.user)-- # Create 3 non-matching worker agents (These should NOT get the message)- worker_agents = []- worker_tags = ["worker", "user-123"]- for _ in range(3):- worker_agent_state = client.create_agent(include_multi_agent_tools=False, tags=worker_tags)- worker_agent = client.server.load_agent(agent_id=worker_agent_state.id, actor=client.user)- worker_agents.append(worker_agent)-- # Create 3 worker agents that should get the message- worker_agents = []- worker_tags = ["worker", "user-456"]- for _ in range(3):- worker_agent_state = client.create_agent(include_multi_agent_tools=False, tags=worker_tags)- worker_agent = client.server.load_agent(agent_id=worker_agent_state.id, actor=client.user)- worker_agents.append(worker_agent)-- # Encourage the manager to send a message to the other agent_obj with the secret string- response = client.send_message(- agent_id=manager_agent.agent_state.id,- role="user",- message=f"Send a message to all agents with tags {worker_tags} informing them of the secret word: {secret_word}!",- )-- for m in response.messages:- if isinstance(m, ToolReturnMessage):- tool_response = eval(json.loads(m.tool_return)["message"])- print(f"\n\nManager agent tool response: \n{tool_response}\n\n")- assert len(tool_response) == len(worker_agents)-- # We can break after this, the ToolReturnMessage after is not related- break-- # Conversation search the worker agents- for agent in worker_agents:- messages = client.get_messages(agent.agent_state.id)- # Check for the presence of system message- for m in reversed(messages):- print(f"\n\n {agent.agent_state.id} -> {m.model_dump_json(indent=4)}")- if isinstance(m, SystemMessage):- assert secret_word in m.content- break-- # Test that the agent can still receive messages fine- response = client.send_message(agent_id=manager_agent.agent_state.id, role="user", message="So what did the other agents say?")- print("Manager agent followup message: \n\n" + "\n".join([str(m) for m in response.messages]))-- # Clean up agents- client.delete_agent(manager_agent_state.id)- for agent in worker_agents:- client.delete_agent(agent.agent_state.id)---# This test is nondeterministic, so we retry until we get the perfect behavior from the LLM-@retry_until_success(max_attempts=2, sleep_time_seconds=2)-def test_send_message_to_agents_with_tags_complex_tool_use(client, roll_dice_tool):- worker_tags = ["dice-rollers"]-- # Clean up first from possibly failed tests- prev_worker_agents = client.server.agent_manager.list_agents(client.user, tags=worker_tags, match_all_tags=True)- for agent in prev_worker_agents:- client.delete_agent(agent.id)-- # Create "manager" agent- send_message_to_agents_matching_all_tags_tool_id = client.get_tool_id(name="send_message_to_agents_matching_all_tags")- manager_agent_state = client.create_agent(tool_ids=[send_message_to_agents_matching_all_tags_tool_id])- manager_agent = client.server.load_agent(agent_id=manager_agent_state.id, actor=client.user)-- # Create 3 worker agents- worker_agents = []- worker_tags = ["dice-rollers"]- for _ in range(2):- worker_agent_state = client.create_agent(include_multi_agent_tools=False, tags=worker_tags, tool_ids=[roll_dice_tool.id])- worker_agent = client.server.load_agent(agent_id=worker_agent_state.id, actor=client.user)- worker_agents.append(worker_agent)-- # Encourage the manager to send a message to the other agent_obj with the secret string- broadcast_message = f"Send a message to all agents with tags {worker_tags} asking them to roll a dice for you!"- response = client.send_message(- agent_id=manager_agent.agent_state.id,- role="user",- message=broadcast_message,- )-- for m in response.messages:- if isinstance(m, ToolReturnMessage):- tool_response = eval(json.loads(m.tool_return)["message"])- print(f"\n\nManager agent tool response: \n{tool_response}\n\n")- assert len(tool_response) == len(worker_agents)-- # We can break after this, the ToolReturnMessage after is not related- break-- # Test that the agent can still receive messages fine- response = client.send_message(agent_id=manager_agent.agent_state.id, role="user", message="So what did the other agents say?")- print("Manager agent followup message: \n\n" + "\n".join([str(m) for m in response.messages]))-- # Clean up agents- client.delete_agent(manager_agent_state.id)- for agent in worker_agents:- client.delete_agent(agent.agent_state.id)---@retry_until_success(max_attempts=5, sleep_time_seconds=2)-def test_agents_async_simple(client):- """- Test two agents with multi-agent tools sending messages back and forth to count to 5.- The chain is started by prompting one of the agents.- """- # Cleanup from potentially failed previous runs- existing_agents = client.server.agent_manager.list_agents(client.user)- for agent in existing_agents:- client.delete_agent(agent.id)-- # Create two agents with multi-agent tools- send_message_to_agent_async_tool_id = client.get_tool_id(name="send_message_to_agent_async")- memory_a = ChatMemory(- human="Chad - I'm interested in hearing poem.",- persona="You are an AI agent that can communicate with your agent buddy using `send_message_to_agent_async`, who has some great poem ideas (so I've heard).",- )- charles_state = client.create_agent(name="charles", memory=memory_a, tool_ids=[send_message_to_agent_async_tool_id])- charles = client.server.load_agent(agent_id=charles_state.id, actor=client.user)-- memory_b = ChatMemory(- human="No human - you are to only communicate with the other AI agent.",- persona="You are an AI agent that can communicate with your agent buddy using `send_message_to_agent_async`, who is interested in great poem ideas.",- )- sarah_state = client.create_agent(name="sarah", memory=memory_b, tool_ids=[send_message_to_agent_async_tool_id])-- # Start the count chain with Agent1- initial_prompt = f"I want you to talk to the other agent with ID {sarah_state.id} using `send_message_to_agent_async`. Specifically, I want you to ask him for a poem idea, and then craft a poem for me."- client.send_message(- agent_id=charles.agent_state.id,- role="user",- message=initial_prompt,- )-- found_in_charles = wait_for_incoming_message(- client=client,- agent_id=charles_state.id,- substring="[Incoming message from agent with ID",- max_wait_seconds=10,- sleep_interval=0.5,- )- assert found_in_charles, "Charles never received the system message from Sarah (timed out)."-- found_in_sarah = wait_for_incoming_message(- client=client,- agent_id=sarah_state.id,- substring="[Incoming message from agent with ID",- max_wait_seconds=10,- sleep_interval=0.5,- )- assert found_in_sarah, "Sarah never received the system message from Charles (timed out)."