Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: amend hitl tests for new agent loop (#5176)
cthomasa7fa6bb33f45
diff --git a/tests/integration_test_human_in_the_loop.py b/tests/integration_test_human_in_the_loop.pyindex 7648badd3..7a50685b8 100644--- a/tests/integration_test_human_in_the_loop.py+++ b/tests/integration_test_human_in_the_loop.py@@ -11,6 +11,7 @@ from dotenv import load_dotenvfrom letta_client import AgentState, ApprovalCreate, Letta, MessageCreate, Toolfrom letta_client.core.api_error import ApiError+from letta.adapters.simple_llm_stream_adapter import SimpleLLMStreamAdapterfrom letta.interfaces.anthropic_streaming_interface import AnthropicStreamingInterfacefrom letta.log import get_loggerfrom letta.schemas.enums import AgentType@@ -22,7 +23,7 @@ logger = get_logger(__name__)# ------------------------------USER_MESSAGE_OTID = str(uuid.uuid4())-USER_MESSAGE_CONTENT = "This is an automated test message. Call the get_secret_code_tool to get the code for text 'hello world'. Make sure to set request_heartbeat to True."+USER_MESSAGE_CONTENT = "This is an automated test message. Call the get_secret_code_tool to get the code for text 'hello world'."USER_MESSAGE_TEST_APPROVAL: List[MessageCreate] = [MessageCreate(role="user",@@ -137,12 +138,11 @@ def agent(client: Letta, approval_tool_fixture) -> AgentState:Creates and returns an agent state for testing with a pre-configured agent.The agent is configured with the requires_approval_tool."""- send_message_tool = client.tools.list(name="send_message")[0]agent_state = client.agents.create(name="approval_test_agent",agent_type=AgentType.letta_v1_agent,include_base_tools=False,- tool_ids=[send_message_tool.id, approval_tool_fixture.id],+ tool_ids=[approval_tool_fixture.id],model="anthropic/claude-sonnet-4-20250514",embedding="openai/text-embedding-3-small",tags=["approval_test"],@@ -214,10 +214,10 @@ def test_send_message_with_requires_approval_tool(assert messages[1].message_type == "assistant_message"assert messages[2].message_type == "approval_request_message"# v3/v1 path: approval request tool args must not include request_heartbeat- import json as _json+ # import json as _json- _args = _json.loads(messages[2].tool_call.arguments)- assert "request_heartbeat" not in _args+ # _args = _json.loads(messages[2].tool_call.arguments)+ # assert "request_heartbeat" not in _argsassert messages[3].message_type == "stop_reason"assert messages[4].message_type == "usage_statistics"@@ -254,14 +254,18 @@ def test_send_message_after_turning_off_requires_approval(messages = accumulate_chunks(response)assert messages is not None- assert len(messages) == 6 or len(messages) == 8+ assert len(messages) == 6 or len(messages) == 8 or len(messages) == 9assert messages[0].message_type == "reasoning_message"assert messages[1].message_type == "assistant_message"assert messages[2].message_type == "tool_call_message"assert messages[3].message_type == "tool_return_message"- if len(messages) > 6:+ if len(messages) == 8:assert messages[4].message_type == "reasoning_message"assert messages[5].message_type == "assistant_message"+ elif len(messages) == 9:+ assert messages[4].message_type == "reasoning_message"+ assert messages[5].message_type == "tool_call_message"+ assert messages[6].message_type == "tool_return_message"# ------------------------------@@ -280,10 +284,10 @@ def test_approve_tool_call_request(approval_request_id = response.messages[0].idtool_call_id = response.messages[2].tool_call.tool_call_id# Ensure no request_heartbeat on approval request- import json as _json+ # import json as _json- _args = _json.loads(response.messages[0].tool_call.arguments)- assert "request_heartbeat" not in _args+ # _args = _json.loads(response.messages[0].tool_call.arguments)+ # assert "request_heartbeat" not in _argsresponse = client.agents.messages.create_stream(agent_id=agent.id,@@ -419,8 +423,8 @@ def test_approve_and_follow_up_with_error()approval_request_id = response.messages[0].id- # Mock the streaming interface to return no tool call on the follow up request heartbeat message- with patch.object(AnthropicStreamingInterface, "get_tool_call_object", return_value=None):+ # Mock the streaming adapter to return llm invocation failure on the follow up turn+ with patch.object(SimpleLLMStreamAdapter, "invoke_llm", side_effect=ValueError("TEST: Mocked error")):response = client.agents.messages.create_stream(agent_id=agent.id,messages=[@@ -437,7 +441,7 @@ def test_approve_and_follow_up_with_error(assert messages is not Nonestop_reason_message = [m for m in messages if m.message_type == "stop_reason"][0]assert stop_reason_message- assert stop_reason_message.stop_reason == "no_tool_call"+ assert stop_reason_message.stop_reason == "invalid_llm_response"# Ensure that agent is not brickedresponse = client.agents.messages.create_stream(@@ -448,11 +452,13 @@ def test_approve_and_follow_up_with_error(messages = accumulate_chunks(response)assert messages is not None- assert len(messages) == 4+ assert len(messages) == 4 or len(messages) == 5assert messages[0].message_type == "reasoning_message"- assert messages[1].message_type == "assistant_message"- assert messages[2].message_type == "stop_reason"- assert messages[3].message_type == "usage_statistics"+ if len(messages) == 4:+ assert messages[1].message_type == "assistant_message"+ else:+ assert messages[1].message_type == "tool_call_message"+ assert messages[2].message_type == "tool_return_message"# ------------------------------@@ -469,7 +475,7 @@ def test_deny_tool_call_request(messages=USER_MESSAGE_TEST_APPROVAL,)approval_request_id = response.messages[0].id- tool_call_id = response.messages[1].tool_call.tool_call_id+ tool_call_id = response.messages[2].tool_call.tool_call_idresponse = client.agents.messages.create_stream(agent_id=agent.id,@@ -508,16 +514,17 @@ def test_deny_cursor_fetch(approval_request_id = response.messages[0].idmessages = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)- assert len(messages) == 3+ assert len(messages) == 4assert messages[0].message_type == "user_message"assert messages[1].message_type == "reasoning_message"- assert messages[2].message_type == "approval_request_message"- assert messages[2].id == approval_request_id+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "approval_request_message"+ assert messages[3].id == approval_request_id# Ensure no request_heartbeat on approval request- import json as _json+ # import json as _json- _args = _json.loads(messages[2].tool_call.arguments)- assert "request_heartbeat" not in _args+ # _args = _json.loads(messages[2].tool_call.arguments)+ # assert "request_heartbeat" not in _argslast_message_cursor = approval_request_idclient.agents.messages.create(@@ -532,13 +539,12 @@ def test_deny_cursor_fetch()messages = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)- assert len(messages) == 5+ assert len(messages) == 4assert messages[0].message_type == "approval_response_message"assert messages[1].message_type == "tool_return_message"assert messages[1].status == "error"- assert messages[2].message_type == "user_message" # heartbeat- assert messages[3].message_type == "reasoning_message"- assert messages[4].message_type == "assistant_message"+ assert messages[2].message_type == "reasoning_message"+ assert messages[3].message_type == "assistant_message"def test_deny_and_follow_up(@@ -588,8 +594,8 @@ def test_deny_and_follow_up_with_error()approval_request_id = response.messages[0].id- # Mock the streaming interface to return no tool call on the follow up request heartbeat message- with patch.object(AnthropicStreamingInterface, "get_tool_call_object", return_value=None):+ # Mock the streaming adapter to return llm invocation failure on the follow up turn+ with patch.object(SimpleLLMStreamAdapter, "invoke_llm", side_effect=ValueError("TEST: Mocked error")):response = client.agents.messages.create_stream(agent_id=agent.id,messages=[@@ -607,7 +613,7 @@ def test_deny_and_follow_up_with_error(assert messages is not Nonestop_reason_message = [m for m in messages if m.message_type == "stop_reason"][0]assert stop_reason_message- assert stop_reason_message.stop_reason == "no_tool_call"+ assert stop_reason_message.stop_reason == "invalid_llm_response"# Ensure that agent is not brickedresponse = client.agents.messages.create_stream(