Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

chore: migrate multi agent integration test to v1 sdk [LET-6152] (#6113)

Christina Tong2a8523aa01af
diff --git a/tests/sdk_v1/integration/integration_test_multi_agent.py b/tests/sdk_v1/integration/integration_test_multi_agent.py
new file mode 100644
index 000000000..255cbb8ad
--- /dev/null
+++ b/tests/sdk_v1/integration/integration_test_multi_agent.py
@@ -0,0 +1,375 @@
+import ast
+import json
+import os
+import threading
+import time
+
+import pytest
+import requests
+from dotenv import load_dotenv
+from letta_client import Letta
+from letta_client.types import AgentState, MessageCreateParam, ToolReturnMessage
+from letta_client.types.agents import SystemMessage
+
+from tests.helpers.utils import retry_until_success
+
+
[email protected](scope="module")
+def server_url() -> str:
+ """
+ Provides the URL for the Letta server.
+ If LETTA_SERVER_URL is not set, starts the server in a background thread
+ and polls until it's accepting connections.
+ """
+
+ def _run_server() -> None:
+ load_dotenv()
+ from letta.server.rest_api.app import start_server
+
+ start_server(debug=True)
+
+ url: str = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")
+
+ if not os.getenv("LETTA_SERVER_URL"):
+ thread = threading.Thread(target=_run_server, daemon=True)
+ thread.start()
+
+ # Poll until the server is up (or timeout)
+ timeout_seconds = 30
+ deadline = time.time() + timeout_seconds
+ while time.time() < deadline:
+ try:
+ resp = requests.get(url + "/v1/health")
+ if resp.status_code < 500:
+ break
+ except requests.exceptions.RequestException:
+ pass
+ time.sleep(0.1)
+ else:
+ raise RuntimeError(f"Could not reach {url} within {timeout_seconds}s")
+
+ yield url
+
+
[email protected](scope="module")
+def client(server_url: str) -> Letta:
+ """
+ Creates and returns a synchronous Letta REST client for testing.
+ """
+ client_instance = Letta(base_url=server_url)
+ client_instance.tools.upsert_base_tools()
+ yield client_instance
+
+
[email protected](autouse=True)
+def remove_stale_agents(client):
+ stale_agents = client.agents.list(limit=300)
+ for agent in stale_agents:
+ client.agents.delete(agent_id=agent.id)
+
+
[email protected](scope="function")
+def agent_obj(client: Letta) -> AgentState:
+ """Create a test agent that we can call functions on"""
+ send_message_to_agent_tool = list(client.tools.list(name="send_message_to_agent_and_wait_for_reply"))[0]
+ agent_state_instance = client.agents.create(
+ agent_type="letta_v1_agent",
+ include_base_tools=True,
+ tool_ids=[send_message_to_agent_tool.id],
+ model="openai/gpt-4o",
+ embedding="letta/letta-free",
+ context_window_limit=32000,
+ )
+ yield agent_state_instance
+
+
[email protected](scope="function")
+def other_agent_obj(client: Letta) -> AgentState:
+ """Create another test agent that we can call functions on"""
+ agent_state_instance = client.agents.create(
+ agent_type="letta_v1_agent",
+ include_base_tools=True,
+ include_multi_agent_tools=False,
+ model="openai/gpt-4o",
+ embedding="letta/letta-free",
+ context_window_limit=32000,
+ )
+
+ yield agent_state_instance
+
+
+def roll_dice_tool(client: Letta):
+ def roll_dice():
+ """
+ Rolls a 6 sided die.
+
+ Returns:
+ str: The roll result.
+ """
+ return "Rolled a 5!"
+
+ # Use SDK method to create tool from function
+ tool = client.tools.upsert_from_function(func=roll_dice)
+
+ # Yield the created tool
+ yield tool
+
+
+@retry_until_success(max_attempts=5, sleep_time_seconds=2)
+def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent_obj: AgentState):
+ secret_word = "banana"
+
+ # Encourage the agent to send a message to the other agent_obj with the secret string
+ response = client.agents.messages.send(
+ agent_id=agent_obj.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content=f"Use your tool to send a message to another agent with id {other_agent_obj.id} to share the secret word: {secret_word}!",
+ )
+ ],
+ )
+
+ # Get messages from the other agent
+ messages_page = client.agents.messages.list(agent_id=other_agent_obj.id)
+ messages = messages_page.items
+
+ # Check for the presence of system message with secret word
+ found_secret = False
+ for m in reversed(messages):
+ print(f"\n\n {other_agent_obj.id} -> {m.model_dump_json(indent=4)}")
+ if isinstance(m, SystemMessage):
+ if secret_word in m.content:
+ found_secret = True
+ break
+
+ assert found_secret, f"Secret word '{secret_word}' not found in system messages of agent {other_agent_obj.id}"
+
+ # Search the sender agent for the response from another agent
+ in_context_messages_page = client.agents.messages.list(agent_id=agent_obj.id)
+ in_context_messages = in_context_messages_page.items
+ found = False
+ target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': ["
+
+ for m in in_context_messages:
+ # Check ToolReturnMessage for the response
+ if isinstance(m, ToolReturnMessage):
+ if target_snippet in m.tool_return:
+ found = True
+ break
+ # Handle different message content structures
+ elif hasattr(m, "content"):
+ if isinstance(m.content, list) and len(m.content) > 0:
+ content_text = m.content[0].text if hasattr(m.content[0], "text") else str(m.content[0])
+ else:
+ content_text = str(m.content)
+
+ if target_snippet in content_text:
+ found = True
+ break
+
+ if not found:
+ # Print debug info
+ joined = "\n".join(
+ [
+ str(
+ m.content[0].text
+ if hasattr(m, "content") and isinstance(m.content, list) and len(m.content) > 0 and hasattr(m.content[0], "text")
+ else m.content
+ if hasattr(m, "content")
+ else f"<{type(m).__name__}>"
+ )
+ for m in in_context_messages[1:]
+ ]
+ )
+ print(f"In context messages of the sender agent (without system):\n\n{joined}")
+ raise Exception(f"Was not able to find an instance of the target snippet: {target_snippet}")
+
+ # Test that the agent can still receive messages fine
+ response = client.agents.messages.send(
+ agent_id=agent_obj.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content="So what did the other agent say?",
+ )
+ ],
+ )
+ print(response.messages)
+
+
+@retry_until_success(max_attempts=5, sleep_time_seconds=2)
+def test_send_message_to_agents_with_tags_simple(client: Letta):
+ worker_tags_123 = ["worker", "user-123"]
+ worker_tags_456 = ["worker", "user-456"]
+
+ secret_word = "banana"
+
+ # Create "manager" agent
+ send_message_to_agents_matching_tags_tool_id = list(client.tools.list(name="send_message_to_agents_matching_tags"))[0].id
+ manager_agent_state = client.agents.create(
+ agent_type="letta_v1_agent",
+ name="manager_agent",
+ tool_ids=[send_message_to_agents_matching_tags_tool_id],
+ model="openai/gpt-4o-mini",
+ embedding="letta/letta-free",
+ )
+
+ # Create 2 non-matching worker agents (These should NOT get the message)
+ worker_agents_123 = []
+ for idx in range(2):
+ worker_agent_state = client.agents.create(
+ agent_type="letta_v1_agent",
+ name=f"not_worker_{idx}",
+ include_multi_agent_tools=False,
+ tags=worker_tags_123,
+ model="openai/gpt-4o-mini",
+ embedding="letta/letta-free",
+ )
+ worker_agents_123.append(worker_agent_state)
+
+ # Create 2 worker agents that should get the message
+ worker_agents_456 = []
+ for idx in range(2):
+ worker_agent_state = client.agents.create(
+ agent_type="letta_v1_agent",
+ name=f"worker_{idx}",
+ include_multi_agent_tools=False,
+ tags=worker_tags_456,
+ model="openai/gpt-4o-mini",
+ embedding="letta/letta-free",
+ )
+ worker_agents_456.append(worker_agent_state)
+
+ # Encourage the manager to send a message to the other agent_obj with the secret string
+ response = client.agents.messages.send(
+ agent_id=manager_agent_state.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content=f"Send a message to all agents with tags {worker_tags_456} informing them of the secret word: {secret_word}!",
+ )
+ ],
+ )
+
+ for m in response.messages:
+ if isinstance(m, ToolReturnMessage):
+ tool_response = ast.literal_eval(m.tool_return)
+ print(f"\n\nManager agent tool response: \n{tool_response}\n\n")
+ assert len(tool_response) == len(worker_agents_456)
+
+ # Verify responses from all expected worker agents
+ worker_agent_ids = {agent.id for agent in worker_agents_456}
+ returned_agent_ids = set()
+ for json_str in tool_response:
+ response_obj = json.loads(json_str)
+ assert response_obj["agent_id"] in worker_agent_ids
+ assert response_obj["response_messages"] != ["<no response>"]
+ returned_agent_ids.add(response_obj["agent_id"])
+ break
+
+ # Check messages in the worker agents that should have received the message
+ for agent_state in worker_agents_456:
+ messages_page = client.agents.messages.list(agent_state.id)
+ messages = messages_page.items
+ # Check for the presence of system message
+ found_secret = False
+ for m in reversed(messages):
+ print(f"\n\n {agent_state.id} -> {m.model_dump_json(indent=4)}")
+ if isinstance(m, SystemMessage):
+ if secret_word in m.content:
+ found_secret = True
+ break
+ assert found_secret, f"Secret word not found in messages for agent {agent_state.id}"
+
+ # Ensure it's NOT in the non matching worker agents
+ for agent_state in worker_agents_123:
+ messages_page = client.agents.messages.list(agent_state.id)
+ messages = messages_page.items
+ # Check for the presence of system message
+ for m in reversed(messages):
+ print(f"\n\n {agent_state.id} -> {m.model_dump_json(indent=4)}")
+ if isinstance(m, SystemMessage):
+ assert secret_word not in m.content, f"Secret word should not be in agent {agent_state.id}"
+
+ # Test that the agent can still receive messages fine
+ response = client.agents.messages.send(
+ agent_id=manager_agent_state.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content="So what did the other agent say?",
+ )
+ ],
+ )
+ print("Manager agent followup message: \n\n" + "\n".join([str(m) for m in response.messages]))
+
+
+@retry_until_success(max_attempts=5, sleep_time_seconds=2)
+def test_send_message_to_agents_with_tags_complex_tool_use(client: Letta, roll_dice_tool):
+ # Create "manager" agent
+ send_message_to_agents_matching_tags_tool_id = list(client.tools.list(name="send_message_to_agents_matching_tags"))[0].id
+ manager_agent_state = client.agents.create(
+ agent_type="letta_v1_agent",
+ tool_ids=[send_message_to_agents_matching_tags_tool_id],
+ model="openai/gpt-4o-mini",
+ embedding="letta/letta-free",
+ )
+
+ # Create 2 worker agents
+ worker_agents = []
+ worker_tags = ["dice-rollers"]
+ for _ in range(2):
+ worker_agent_state = client.agents.create(
+ agent_type="letta_v1_agent",
+ include_multi_agent_tools=False,
+ tags=worker_tags,
+ tool_ids=[roll_dice_tool.id],
+ model="openai/gpt-4o-mini",
+ embedding="letta/letta-free",
+ )
+ worker_agents.append(worker_agent_state)
+
+ # Encourage the manager to send a message to the other agent_obj with the secret string
+ broadcast_message = f"Send a message to all agents with tags {worker_tags} asking them to roll a dice for you!"
+ response = client.agents.messages.send(
+ agent_id=manager_agent_state.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content=broadcast_message,
+ )
+ ],
+ )
+
+ for m in response.messages:
+ if isinstance(m, ToolReturnMessage):
+ # Parse tool_return string to get list of responses
+ tool_response = ast.literal_eval(m.tool_return)
+ print(f"\n\nManager agent tool response: \n{tool_response}\n\n")
+ assert len(tool_response) == len(worker_agents)
+
+ # Verify responses from all expected worker agents
+ worker_agent_ids = {agent.id for agent in worker_agents}
+ returned_agent_ids = set()
+ all_responses = []
+ for json_str in tool_response:
+ response_obj = json.loads(json_str)
+ assert response_obj["agent_id"] in worker_agent_ids
+ assert response_obj["response_messages"] != ["<no response>"]
+ returned_agent_ids.add(response_obj["agent_id"])
+ all_responses.extend(response_obj["response_messages"])
+ break
+
+ # Test that the agent can still receive messages fine
+ response = client.agents.messages.send(
+ agent_id=manager_agent_state.id,
+ messages=[
+ MessageCreateParam(
+ role="user",
+ content="So what did the other agent say?",
+ )
+ ],
+ )
+ print("Manager agent followup message: \n\n" + "\n".join([str(m) for m in response.messages]))