Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: migrate integration_test_human_in_the_loop to sdk v1 [LET-5979] (#5878)
Christina Tong255fdfecf2f7
diff --git a/tests/sdk_v1/integration/integration_test_human_in_the_loop.py b/tests/sdk_v1/integration/integration_test_human_in_the_loop.pynew file mode 100644index 000000000..2b9455157--- /dev/null+++ b/tests/sdk_v1/integration/integration_test_human_in_the_loop.py@@ -0,0 +1,1181 @@+import logging+import uuid+from typing import List++import pytest+from letta_client import APIError, Letta+from letta_client.types import AgentState, MessageCreateParam+from letta_client.types.agents import ApprovalCreateParam++logger = logging.getLogger(__name__)++# ------------------------------+# Helper Functions and Constants+# ------------------------------++USER_MESSAGE_OTID = str(uuid.uuid4())+USER_MESSAGE_CONTENT = "This is an automated test message. Call the get_secret_code_tool to get the code for text 'hello world'."+USER_MESSAGE_TEST_APPROVAL: List[MessageCreateParam] = [+ MessageCreateParam(+ role="user",+ content=USER_MESSAGE_CONTENT,+ otid=USER_MESSAGE_OTID,+ )+]+FAKE_REQUEST_ID = str(uuid.uuid4())+SECRET_CODE = str(740845635798344975)+USER_MESSAGE_FOLLOW_UP_OTID = str(uuid.uuid4())+USER_MESSAGE_FOLLOW_UP_CONTENT = "Thank you for the secret code."+USER_MESSAGE_FOLLOW_UP: List[MessageCreateParam] = [+ MessageCreateParam(+ role="user",+ content=USER_MESSAGE_FOLLOW_UP_CONTENT,+ otid=USER_MESSAGE_FOLLOW_UP_OTID,+ )+]+USER_MESSAGE_PARALLEL_TOOL_CALL_CONTENT = "This is an automated test message. Call the get_secret_code_tool 3 times in parallel for the following inputs: 'hello world', 'hello letta', 'hello test', and also call the roll_dice_tool once with a 16-sided dice."+USER_MESSAGE_PARALLEL_TOOL_CALL: List[MessageCreateParam] = [+ MessageCreateParam(+ role="user",+ content=USER_MESSAGE_PARALLEL_TOOL_CALL_CONTENT,+ otid=USER_MESSAGE_OTID,+ )+]+++def get_secret_code_tool(input_text: str) -> str:+ """+ A tool that returns the secret code based on the input. This tool requires approval before execution.+ Args:+ input_text (str): The input text to process.+ Returns:+ str: The secret code based on the input text.+ """+ return str(abs(hash(input_text)))+++def roll_dice_tool(num_sides: int) -> str:+ """+ A tool that returns a random number between 1 and num_sides.+ Args:+ num_sides (int): The number of sides on the die.+ Returns:+ str: The random number between 1 and num_sides.+ """+ import random++ return str(random.randint(1, num_sides))+++def accumulate_chunks(stream):+ messages = []+ current_message = None+ prev_message_type = None++ for chunk in stream:+ # Handle chunks that might not have message_type (like pings)+ if not hasattr(chunk, "message_type"):+ continue++ current_message_type = getattr(chunk, "message_type", None)++ if prev_message_type != current_message_type:+ # Save the previous message if it exists+ if current_message is not None:+ messages.append(current_message)+ # Start a new message+ current_message = chunk+ else:+ # Accumulate content for same message type (token streaming)+ if current_message is not None and hasattr(current_message, "content") and hasattr(chunk, "content"):+ current_message.content += chunk.content++ prev_message_type = current_message_type++ # Don't forget the last message+ if current_message is not None:+ messages.append(current_message)++ return [m for m in messages if m is not None]+++def approve_tool_call(client: Letta, agent_id: str, tool_call_id: str):+ client.agents.messages.send(+ agent_id=agent_id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ )+++# ------------------------------+# Fixtures+# ------------------------------+# Note: server_url and client fixtures are inherited from tests/sdk_v1/conftest.py++[email protected](scope="function")+def approval_tool_fixture(client: Letta):+ """+ Creates and returns a tool that requires approval for testing.+ """+ client.tools.upsert_base_tools()+ approval_tool = client.tools.upsert_from_function(+ func=get_secret_code_tool,+ default_requires_approval=True,+ )+ yield approval_tool++ client.tools.delete(tool_id=approval_tool.id)++[email protected](scope="function")+def dice_tool_fixture(client: Letta):+ client.tools.upsert_base_tools()+ dice_tool = client.tools.upsert_from_function(+ func=roll_dice_tool,+ )+ yield dice_tool++ client.tools.delete(tool_id=dice_tool.id)++[email protected](scope="function")+def agent(client: Letta, approval_tool_fixture, dice_tool_fixture) -> AgentState:+ """+ Creates and returns an agent state for testing with a pre-configured agent.+ The agent is configured with the requires_approval_tool.+ """+ agent_state = client.agents.create(+ name="approval_test_agent",+ agent_type="letta_v1_agent",+ include_base_tools=False,+ tool_ids=[approval_tool_fixture.id, dice_tool_fixture.id],+ include_base_tool_rules=False,+ tool_rules=[],+ model="anthropic/claude-sonnet-4-5-20250929",+ embedding="openai/text-embedding-3-small",+ tags=["approval_test"],+ )+ # Enable parallel tool calls for testing+ agent_state = client.agents.modify(agent_id=agent_state.id, parallel_tool_calls=True)+ yield agent_state++ client.agents.delete(agent_id=agent_state.id)+++# ------------------------------+# Error Test Cases+# ------------------------------+++def test_send_approval_without_pending_request(client, agent):+ with pytest.raises(APIError, match="No tool call is currently awaiting approval"):+ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy+ approval_request_id=FAKE_REQUEST_ID, # legacy+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": FAKE_REQUEST_ID,+ },+ ],+ ),+ ],+ )+++def test_send_user_message_with_pending_request(client, agent):+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )++ with pytest.raises(APIError, match="Please approve or deny the pending request before continuing"):+ client.agents.messages.send(+ agent_id=agent.id,+ messages=[MessageCreateParam(role="user", content="hi")],+ )++ approve_tool_call(client, agent.id, response.messages[2].tool_call.tool_call_id)+++def test_send_approval_message_with_incorrect_request_id(client, agent):+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )++ with pytest.raises(APIError, match="Invalid tool call IDs"):+ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy+ approval_request_id=FAKE_REQUEST_ID, # legacy+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": FAKE_REQUEST_ID,+ },+ ],+ ),+ ],+ )++ approve_tool_call(client, agent.id, response.messages[2].tool_call.tool_call_id)+++# ------------------------------+# Request Test Cases+# ------------------------------+++def test_invoke_approval_request(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )++ messages = response.messages++ assert messages is not None+ assert len(messages) == 3+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "approval_request_message"+ assert messages[2].tool_call is not None+ assert messages[2].tool_call.name == "get_secret_code_tool"+ assert messages[2].tool_calls is not None+ assert len(messages[2].tool_calls) == 1+ assert messages[2].tool_calls[0].name == "get_secret_code_tool"++ # v3/v1 path: approval request tool args must not include request_heartbeat+ import json as _json++ _args = _json.loads(messages[2].tool_call.arguments)+ assert "request_heartbeat" not in _args++ client.agents.retrieve(agent_id=agent.id)++ approve_tool_call(client, agent.id, response.messages[2].tool_call.tool_call_id)+++def test_invoke_approval_request_stream(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 5+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "approval_request_message"+ assert messages[2].tool_call is not None+ assert messages[2].tool_call.name == "get_secret_code_tool"+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"++ client.agents.retrieve(agent_id=agent.id)++ approve_tool_call(client, agent.id, messages[2].tool_call.tool_call_id)+++def test_invoke_tool_after_turning_off_requires_approval(+ client: Letta,+ agent: AgentState,+ approval_tool_fixture,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ stream_tokens=True,+ )+ messages = accumulate_chunks(response)++ client.agents.tools.update_approval(+ agent_id=agent.id,+ tool_name=approval_tool_fixture.name,+ body_requires_approval=False,+ )++ response = client.agents.messages.stream(agent_id=agent.id, messages=USER_MESSAGE_TEST_APPROVAL, stream_tokens=True)++ messages = accumulate_chunks(response)++ assert messages is not None+ assert 6 <= len(messages) <= 9+ idx = 0++ assert messages[idx].message_type == "reasoning_message"+ idx += 1++ try:+ assert messages[idx].message_type == "assistant_message"+ idx += 1+ except:+ pass++ assert messages[idx].message_type == "tool_call_message"+ idx += 1+ assert messages[idx].message_type == "tool_return_message"+ idx += 1++ assert messages[idx].message_type == "reasoning_message"+ idx += 1+ try:+ assert messages[idx].message_type == "assistant_message"+ idx += 1+ except:+ assert messages[idx].message_type == "tool_call_message"+ idx += 1+ assert messages[idx].message_type == "tool_return_message"+ idx += 1+++# ------------------------------+# Approve Test Cases+# ------------------------------+++def test_approve_tool_call_request(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 3 or len(messages) == 5 or len(messages) == 6+ assert messages[0].message_type == "tool_return_message"+ assert messages[0].tool_call_id == tool_call_id+ assert messages[0].status == "success"+ if len(messages) == 4:+ assert messages[1].message_type == "stop_reason"+ assert messages[2].message_type == "usage_statistics"+ elif len(messages) == 5:+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"+ elif len(messages) == 6:+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "tool_call_message"+ assert messages[3].message_type == "tool_return_message"+ assert messages[4].message_type == "stop_reason"+ assert messages[5].message_type == "usage_statistics"+++def test_approve_cursor_fetch(+ client: Letta,+ agent: AgentState,+) -> None:+ last_message_cursor = client.agents.messages.list(agent_id=agent.id, limit=1).items[0].id+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ last_message_id = response.messages[0].id+ tool_call_id = response.messages[2].tool_call.tool_call_id++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ messages = messages_page.items+ assert len(messages) == 4+ assert messages[0].message_type == "user_message"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "approval_request_message"+ # Ensure no request_heartbeat on approval request+ import json as _json++ _args = _json.loads(messages[3].tool_call.arguments)+ assert "request_heartbeat" not in _args++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ )++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_id)+ messages = messages_page.items+ assert len(messages) == 2 or len(messages) == 4+ assert messages[0].message_type == "approval_response_message"+ assert messages[0].approval_request_id == tool_call_id+ assert messages[0].approve is True+ assert messages[0].approvals[0].approve is True+ assert messages[0].approvals[0].tool_call_id == tool_call_id+ assert messages[1].message_type == "tool_return_message"+ assert messages[1].status == "success"+ if len(messages) == 4:+ assert messages[2].message_type == "reasoning_message"+ assert messages[3].message_type == "assistant_message"+++def test_approve_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e+++def test_approve_and_follow_up(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ )++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4 or len(messages) == 5+ if len(messages) == 4:+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"+ elif len(messages) == 5:+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "tool_call_message"+ assert messages[2].message_type == "tool_return_message"+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"+++# ------------------------------+# Deny Test Cases+# ------------------------------+++def test_deny_tool_call_request(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": f"You don't need to call the tool, the secret code is {SECRET_CODE}",+ },+ ],+ ),+ ],+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 5+ assert messages[0].message_type == "tool_return_message"+ assert messages[0].tool_call_id == tool_call_id+ assert messages[0].status == "error"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert SECRET_CODE in messages[2].content+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"+++def test_deny_cursor_fetch(+ client: Letta,+ agent: AgentState,+) -> None:+ last_message_cursor = client.agents.messages.list(agent_id=agent.id, limit=1).items[0].id+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ last_message_id = response.messages[0].id+ tool_call_id = response.messages[2].tool_call.tool_call_id++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ messages = messages_page.items+ assert len(messages) == 4+ assert messages[0].message_type == "user_message"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "approval_request_message"+ assert messages[3].tool_call.tool_call_id == tool_call_id+ # Ensure no request_heartbeat on approval request+ # import json as _json++ # _args = _json.loads(messages[2].tool_call.arguments)+ # assert "request_heartbeat" not in _args++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": f"You don't need to call the tool, the secret code is {SECRET_CODE}",+ },+ ],+ ),+ ],+ )++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_id)+ messages = messages_page.items+ assert len(messages) == 4+ assert messages[0].message_type == "approval_response_message"+ assert messages[0].approvals[0].approve == False+ assert messages[0].approvals[0].tool_call_id == tool_call_id+ assert messages[0].approvals[0].reason == f"You don't need to call the tool, the secret code is {SECRET_CODE}"+ assert messages[1].message_type == "tool_return_message"+ assert messages[1].status == "error"+ assert messages[2].message_type == "reasoning_message"+ assert messages[3].message_type == "assistant_message"+++def test_deny_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason="Cancelled by user. Instead of responding, wait for next user input before replying.", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": "Cancelled by user. Instead of responding, wait for next user input before replying.",+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e+++def test_deny_and_follow_up(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": f"You don't need to call the tool, the secret code is {SECRET_CODE}",+ },+ ],+ ),+ ],+ )++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"+++# --------------------------------+# Client-Side Execution Test Cases+# --------------------------------+++def test_client_side_tool_call_request(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 5+ assert messages[0].message_type == "tool_return_message"+ assert messages[0].tool_call_id == tool_call_id+ assert messages[0].status == "success"+ assert messages[0].tool_return == SECRET_CODE+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert SECRET_CODE in messages[2].content+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"+++def test_client_side_tool_call_cursor_fetch(+ client: Letta,+ agent: AgentState,+) -> None:+ last_message_cursor = client.agents.messages.list(agent_id=agent.id, limit=1).items[0].id+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ last_message_id = response.messages[0].id+ tool_call_id = response.messages[2].tool_call.tool_call_id++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ messages = messages_page.items+ assert len(messages) == 4+ assert messages[0].message_type == "user_message"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "approval_request_message"+ assert messages[3].tool_call.tool_call_id == tool_call_id+ # Ensure no request_heartbeat on approval request+ # import json as _json++ # _args = _json.loads(messages[2].tool_call.arguments)+ # assert "request_heartbeat" not in _args++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_id)+ messages = messages_page.items+ assert len(messages) == 4+ assert messages[0].message_type == "approval_response_message"+ assert messages[0].approvals[0].type == "tool"+ assert messages[0].approvals[0].tool_call_id == tool_call_id+ assert messages[0].approvals[0].tool_return == SECRET_CODE+ assert messages[0].approvals[0].status == "success"+ assert messages[1].message_type == "tool_return_message"+ assert messages[1].status == "success"+ assert messages[1].tool_call_id == tool_call_id+ assert messages[1].tool_return == SECRET_CODE+ assert messages[2].message_type == "reasoning_message"+ assert messages[3].message_type == "assistant_message"+++def test_client_side_tool_call_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason="Cancelled by user. Instead of responding, wait for next user input before replying.", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e+++def test_client_side_tool_call_and_follow_up(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"+++def test_parallel_tool_calling(+ client: Letta,+ agent: AgentState,+) -> None:+ # Parallel tool calling only works for Anthropic models+ retrieved_agent = client.agents.retrieve(agent_id=agent.id)+ model = None+ if hasattr(retrieved_agent, "llm_config") and retrieved_agent.llm_config and hasattr(retrieved_agent.llm_config, "model"):+ model = retrieved_agent.llm_config.model+ elif hasattr(retrieved_agent, "model") and retrieved_agent.model:+ model = retrieved_agent.model++ if not model or not model.startswith("anthropic/"):+ pytest.skip("Parallel tool calling test only applies to Anthropic models.")++ last_message_cursor = client.agents.messages.list(agent_id=agent.id, limit=1).items[0].id+ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=USER_MESSAGE_PARALLEL_TOOL_CALL,+ )++ messages = response.messages++ assert messages is not None+ assert len(messages) == 3 or len(messages) == 4+ assert messages[0].message_type == "reasoning_message"++ # Handle cases where assistant_message might be missing+ idx = 1+ if len(messages) == 4:+ # If 4 messages, expect assistant_message, tool_call_message, approval_request_message+ assert messages[1].message_type == "assistant_message"+ idx = 2+ else:+ # If 3 messages, might skip assistant_message and go straight to tool_call_message+ pass++ # Find the tool_call_message and approval_request_message dynamically+ # roll_dice_tool doesn't require approval, so it might be executed immediately+ # and appear as tool_return_message instead of tool_call_message+ tool_call_msg_idx = None+ approval_request_msg_idx = None+ tool_return_msg_idx = None+ dice_tool_call_id = None++ for i, msg in enumerate(messages):+ if msg.message_type == "tool_call_message":+ tool_call_msg_idx = i+ elif msg.message_type == "approval_request_message":+ approval_request_msg_idx = i+ elif msg.message_type == "tool_return_message":+ tool_return_msg_idx = i++ assert approval_request_msg_idx is not None, f"Expected approval_request_message. Message types: {[m.message_type for m in messages]}"++ # Try to find roll_dice_tool - it could be in tool_call_message or already executed in tool_return_message+ if tool_call_msg_idx is not None:+ # Check if tool_call_message has roll_dice_tool+ tool_calls = messages[tool_call_msg_idx].tool_calls+ for tool_call in tool_calls:+ if tool_call.name == "roll_dice_tool":+ assert "6" in tool_call.arguments+ dice_tool_call_id = tool_call.tool_call_id+ break++ # If we didn't find it in tool_call_message, check tool_return_message+ if dice_tool_call_id is None and tool_return_msg_idx is not None:+ tool_return_msg = messages[tool_return_msg_idx]+ if hasattr(tool_return_msg, "tool_call_id") and tool_return_msg.tool_call_id:+ dice_tool_call_id = tool_return_msg.tool_call_id++ # If still not found, check if roll_dice_tool is in approval_request_message's tool_calls+ if dice_tool_call_id is None and approval_request_msg_idx is not None:+ approval_msg = messages[approval_request_msg_idx]+ if hasattr(approval_msg, "tool_calls") and approval_msg.tool_calls:+ for tool_call in approval_msg.tool_calls:+ if tool_call.name == "roll_dice_tool":+ assert "6" in tool_call.arguments+ dice_tool_call_id = tool_call.tool_call_id+ break++ # Get approval_request_message tool calls+ approval_msg = messages[approval_request_msg_idx]+ assert approval_msg.tool_call is not None+ assert approval_msg.tool_call.name == "get_secret_code_tool"++ # Find the 3 get_secret_code_tool calls (might also have roll_dice_tool if combined)+ get_secret_code_calls = []+ for tool_call in approval_msg.tool_calls:+ if tool_call.name == "get_secret_code_tool":+ get_secret_code_calls.append(tool_call)+ elif tool_call.name == "roll_dice_tool" and dice_tool_call_id is None:+ # Found roll_dice_tool in approval_request_message+ assert "6" in tool_call.arguments+ dice_tool_call_id = tool_call.tool_call_id++ assert len(get_secret_code_calls) == 3, (+ f"Expected 3 get_secret_code_tool calls, found {len(get_secret_code_calls)}. All tool calls: {[tc.name for tc in approval_msg.tool_calls]}"+ )++ assert "hello world" in get_secret_code_calls[0].arguments+ approve_tool_call_id = get_secret_code_calls[0].tool_call_id+ assert "hello letta" in get_secret_code_calls[1].arguments+ deny_tool_call_id = get_secret_code_calls[1].tool_call_id+ assert "hello test" in get_secret_code_calls[2].arguments+ client_side_tool_call_id = get_secret_code_calls[2].tool_call_id++ # If we still don't have dice_tool_call_id, get it from DB messages+ if dice_tool_call_id is None:+ db_messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ db_messages = db_messages_page.items+ # Look for tool_call_message or tool_return_message with roll_dice_tool+ for db_msg in db_messages:+ if db_msg.message_type == "tool_call_message" and hasattr(db_msg, "tool_calls") and db_msg.tool_calls:+ for tool_call in db_msg.tool_calls:+ if tool_call.name == "roll_dice_tool":+ dice_tool_call_id = tool_call.tool_call_id+ break+ if dice_tool_call_id:+ break+ elif db_msg.message_type == "tool_return_message" and hasattr(db_msg, "tool_call_id") and db_msg.tool_call_id:+ # Check if this might be roll_dice_tool by checking nearby messages+ if dice_tool_call_id is None and len(db_messages) > 2:+ potential_id = db_msg.tool_call_id+ dice_tool_call_id = potential_id+ break++ # Ensure we have dice_tool_call_id before proceeding+ assert dice_tool_call_id is not None, (+ f"Could not find roll_dice_tool call_id. Message types in initial response: {[m.message_type for m in messages]}"+ )++ # ensure context is not bricked+ client.agents.retrieve(agent_id=agent.id)++ response = client.agents.messages.send(+ agent_id=agent.id,+ messages=[+ ApprovalCreateParam(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": approve_tool_call_id,+ },+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": deny_tool_call_id,+ },+ {+ "type": "tool",+ "tool_call_id": client_side_tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ messages = response.messages++ assert messages is not None+ assert len(messages) == 1 or len(messages) == 3 or len(messages) == 4+ assert messages[0].message_type == "tool_return_message"+ assert len(messages[0].tool_returns) == 4+ for tool_return in messages[0].tool_returns:+ if tool_return.tool_call_id == approve_tool_call_id:+ assert tool_return.status == "success"+ elif tool_return.tool_call_id == deny_tool_call_id:+ assert tool_return.status == "error"+ elif tool_return.tool_call_id == client_side_tool_call_id:+ assert tool_return.status == "success"+ assert tool_return.tool_return == SECRET_CODE+ else:+ assert tool_return.tool_call_id == dice_tool_call_id+ assert tool_return.status == "success"+ if len(messages) == 3:+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ elif len(messages) == 4:+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "tool_call_message"+ assert messages[3].message_type == "tool_return_message"++ # ensure context is not bricked+ client.agents.retrieve(agent_id=agent.id)++ messages_page = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ messages = messages_page.items+ assert len(messages) > 6+ assert messages[0].message_type == "user_message"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "tool_call_message"+ assert messages[4].message_type == "approval_request_message"+ assert messages[5].message_type == "approval_response_message"+ assert messages[6].message_type == "tool_return_message"++ response = client.agents.messages.stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"