Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: multi_agent flaky test (#9314)

Kian Jonesb0c40b6b1d03
diff --git a/tests/integration_test_multi_agent.py b/tests/integration_test_multi_agent.py
index f4aec50f0..92fa120c8 100644
--- a/tests/integration_test_multi_agent.py
+++ b/tests/integration_test_multi_agent.py
@@ -135,18 +135,50 @@ def roll_dice_tool(client: Letta):
def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent_obj: AgentState):
secret_word = "banana"
- # Encourage the agent to send a message to the other agent_obj with the secret string
+ # Send a message to the agent asking it to use the tool
response = client.agents.messages.create(
agent_id=agent_obj.id,
messages=[
MessageCreateParam(
role="user",
- content=f"Use your tool to send a message to another agent with id {other_agent_obj.id} to share the secret word: {secret_word}!",
+ content=f"IMPORTANT: You MUST use the send_message_to_agent_and_wait_for_reply tool RIGHT NOW to send a message to agent {other_agent_obj.id}. Include the exact secret word '{secret_word}' in your message. Call the tool immediately.",
)
],
)
- # Get messages from the other agent
+ # FIRST: Verify the sender agent actually called the tool
+ # This catches LLM non-determinism early with a clear error message
+ found_tool_call = False
+ tool_return_message = None
+ target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': ["
+
+ for m in response.messages:
+ if isinstance(m, ToolReturnMessage):
+ if target_snippet in m.tool_return:
+ found_tool_call = True
+ tool_return_message = m
+ break
+
+ if not found_tool_call:
+ # Print debug info to help diagnose the issue
+ print("\n=== DEBUG: Sender agent did not call the tool ===")
+ print(f"Response messages from agent {agent_obj.id}:")
+ for i, m in enumerate(response.messages):
+ print(f"\nMessage {i} ({type(m).__name__}):")
+ if isinstance(m, ToolReturnMessage):
+ print(f" Tool return: {m.tool_return}")
+ elif hasattr(m, "content"):
+ print(f" Content: {m.content}")
+ else:
+ print(f" {m}")
+ raise AssertionError(
+ f"Sender agent {agent_obj.id} did not call send_message_to_agent_and_wait_for_reply tool. "
+ f"This is likely LLM non-determinism. Check debug output above for what the agent did instead."
+ )
+
+ print(f"\n✓ Tool call verified: {tool_return_message.tool_return[:200]}...")
+
+ # SECOND: Now verify the message arrived at the receiver
messages_page = client.agents.messages.list(agent_id=other_agent_obj.id)
messages = messages_page.items
@@ -161,46 +193,6 @@ def test_send_message_to_agent(client: Letta, agent_obj: AgentState, other_agent
assert found_secret, f"Secret word '{secret_word}' not found in system messages of agent {other_agent_obj.id}"
- # Search the sender agent for the response from another agent
- in_context_messages_page = client.agents.messages.list(agent_id=agent_obj.id)
- in_context_messages = in_context_messages_page.items
- found = False
- target_snippet = f"'agent_id': '{other_agent_obj.id}', 'response': ["
-
- for m in in_context_messages:
- # Check ToolReturnMessage for the response
- if isinstance(m, ToolReturnMessage):
- if target_snippet in m.tool_return:
- found = True
- break
- # Handle different message content structures
- elif hasattr(m, "content"):
- if isinstance(m.content, list) and len(m.content) > 0:
- content_text = m.content[0].text if hasattr(m.content[0], "text") else str(m.content[0])
- else:
- content_text = str(m.content)
-
- if target_snippet in content_text:
- found = True
- break
-
- if not found:
- # Print debug info
- joined = "\n".join(
- [
- str(
- m.content[0].text
- if hasattr(m, "content") and isinstance(m.content, list) and len(m.content) > 0 and hasattr(m.content[0], "text")
- else m.content
- if hasattr(m, "content")
- else f"<{type(m).__name__}>"
- )
- for m in in_context_messages[1:]
- ]
- )
- print(f"In context messages of the sender agent (without system):\n\n{joined}")
- raise Exception(f"Was not able to find an instance of the target snippet: {target_snippet}")
-
# Test that the agent can still receive messages fine
response = client.agents.messages.create(
agent_id=agent_obj.id,