Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add comprehensive testing for client side tool calling (#5539)
cthomas5a475fd1a537
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex fa0931292..d73f26f4f 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -27,7 +27,7 @@ from letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.enums import MessageRole-from letta.schemas.letta_message import LettaMessage, MessageType+from letta.schemas.letta_message import ApprovalReturn, LettaMessage, MessageTypefrom letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContentfrom letta.schemas.letta_response import LettaResponsefrom letta.schemas.letta_stop_reason import LettaStopReason, StopReasonType@@ -318,19 +318,19 @@ class LettaAgentV3(LettaAgentV2):pending_tool_calls = {backfill_tool_call_id if a.tool_call_id.startswith("message-") else a.tool_call_id: afor a in approval_response.approvals- if a.type == "approval" and a.approve+ if isinstance(a, ApprovalReturn) and a.approve}tool_calls = [t for t in approval_request.tool_calls if t.id in pending_tool_calls]# Get tool calls that were denied- denies = {d.tool_call_id: d for d in approval_response.approvals if d.type == "approval" and not d.approve}+ denies = {d.tool_call_id: d for d in approval_response.approvals if isinstance(d, ApprovalReturn) and not d.approve}tool_call_denials = [ToolCallDenial(**t.model_dump(), reason=denies.get(t.id).reason) for t in approval_request.tool_calls if t.id in denies]# Get tool calls that were executed client sideif approval_response.approvals:- tool_returns = [r for r in approval_response.approvals if r.type == "tool"]+ tool_returns = [r for r in approval_response.approvals if isinstance(r, ToolReturn)]step_id = approval_request.step_idstep_metrics = await self.step_manager.get_step_metrics_async(step_id=step_id, actor=self.actor)diff --git a/tests/integration_test_human_in_the_loop.py b/tests/integration_test_human_in_the_loop.pyindex 7bf580599..5d905a62f 100644--- a/tests/integration_test_human_in_the_loop.py+++ b/tests/integration_test_human_in_the_loop.py@@ -220,7 +220,7 @@ def test_send_approval_message_with_incorrect_request_id(client, agent):# -------------------------------def test_send_message_with_requires_approval_tool(+def test_invoke_approval_request(client: Letta,agent: AgentState,) -> None:@@ -246,7 +246,45 @@ def test_send_message_with_requires_approval_tool(assert messages[4].message_type == "usage_statistics"-def test_send_message_after_turning_off_requires_approval(+def test_invoke_approval_request_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.context.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e+++def test_invoke_tool_after_turning_off_requires_approval(client: Letta,agent: AgentState,approval_tool_fixture: Tool,@@ -434,6 +472,44 @@ def test_approve_cursor_fetch(assert messages[3].message_type == "assistant_message"+def test_approve_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=False, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ approvals=[+ {+ "type": "approval",+ "approve": True,+ "tool_call_id": tool_call_id,+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.context.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e++def test_approve_and_follow_up(client: Letta,agent: AgentState,@@ -643,6 +719,46 @@ def test_deny_cursor_fetch(assert messages[3].message_type == "assistant_message"+def test_deny_with_context_check(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason="Cancelled by user. Instead of responding, wait for next user input before replying.", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": "Cancelled by user. Instead of responding, wait for next user input before replying.",+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ try:+ client.agents.context.retrieve(agent_id=agent.id)+ except Exception as e:+ if len(messages) > 4:+ raise ValueError("Model did not respond with only reasoning content, please rerun test to repro edge case.")+ raise e++def test_deny_and_follow_up(client: Letta,agent: AgentState,@@ -688,7 +804,170 @@ def test_deny_and_follow_up(assert messages[3].message_type == "usage_statistics"-def test_deny_with_context_check(+def test_deny_and_follow_up_with_error(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ # Mock the streaming adapter to return llm invocation failure on the follow up turn+ with patch.object(SimpleLLMStreamAdapter, "invoke_llm", side_effect=ValueError("TEST: Mocked error")):+ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "approval",+ "approve": False,+ "tool_call_id": tool_call_id,+ "reason": f"You don't need to call the tool, the secret code is {SECRET_CODE}",+ },+ ],+ ),+ ],+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ stop_reason_message = [m for m in messages if m.message_type == "stop_reason"][0]+ assert stop_reason_message+ assert stop_reason_message.stop_reason == "invalid_llm_response"++ # Ensure that agent is not bricked+ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"+++# --------------------------------+# Client-Side Execution Test Cases+# --------------------------------+++def test_client_side_tool_call_request(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 5+ assert messages[0].message_type == "tool_return_message"+ assert messages[0].tool_call_id == tool_call_id+ assert messages[0].status == "success"+ assert messages[0].tool_return == SECRET_CODE+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert SECRET_CODE in messages[2].content+ assert messages[3].message_type == "stop_reason"+ assert messages[4].message_type == "usage_statistics"+++def test_client_side_tool_call_cursor_fetch(+ client: Letta,+ agent: AgentState,+) -> None:+ last_message_cursor = client.agents.messages.list(agent_id=agent.id, limit=1)[0].id+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ last_message_id = response.messages[0].id+ tool_call_id = response.messages[2].tool_call.tool_call_id++ messages = client.agents.messages.list(agent_id=agent.id, after=last_message_cursor)+ assert len(messages) == 4+ assert messages[0].message_type == "user_message"+ assert messages[1].message_type == "reasoning_message"+ assert messages[2].message_type == "assistant_message"+ assert messages[3].message_type == "approval_request_message"+ assert messages[3].tool_call.tool_call_id == tool_call_id+ # Ensure no request_heartbeat on approval request+ # import json as _json++ # _args = _json.loads(messages[2].tool_call.arguments)+ # assert "request_heartbeat" not in _args++ client.agents.messages.create(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ messages = client.agents.messages.list(agent_id=agent.id, after=last_message_id)+ assert len(messages) == 4+ assert messages[0].message_type == "approval_response_message"+ assert messages[0].approvals[0]["type"] == "tool"+ assert messages[0].approvals[0]["tool_call_id"] == tool_call_id+ assert messages[0].approvals[0]["tool_return"] == SECRET_CODE+ assert messages[0].approvals[0]["status"] == "success"+ assert messages[1].message_type == "tool_return_message"+ assert messages[1].status == "success"+ assert messages[1].tool_call_id == tool_call_id+ assert messages[1].tool_return == SECRET_CODE+ assert messages[2].message_type == "reasoning_message"+ assert messages[3].message_type == "assistant_message"+++def test_client_side_tool_call_with_context_check(client: Letta,agent: AgentState,) -> None:@@ -707,10 +986,10 @@ def test_deny_with_context_check(reason="Cancelled by user. Instead of responding, wait for next user input before replying.", # legacyapprovals=[{- "type": "approval",- "approve": False,+ "type": "tool","tool_call_id": tool_call_id,- "reason": "Cancelled by user. Instead of responding, wait for next user input before replying.",+ "tool_return": SECRET_CODE,+ "status": "success",},],),@@ -728,7 +1007,52 @@ def test_deny_with_context_check(raise e-def test_deny_and_follow_up_with_error(+def test_client_side_tool_call_and_follow_up(+ client: Letta,+ agent: AgentState,+) -> None:+ response = client.agents.messages.create(+ agent_id=agent.id,+ messages=USER_MESSAGE_TEST_APPROVAL,+ )+ tool_call_id = response.messages[2].tool_call.tool_call_id++ client.agents.messages.create(+ agent_id=agent.id,+ messages=[+ ApprovalCreate(+ approve=True, # legacy (passing incorrect value to ensure it is overridden)+ approval_request_id=FAKE_REQUEST_ID, # legacy (passing incorrect value to ensure it is overridden)+ reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacy+ approvals=[+ {+ "type": "tool",+ "tool_call_id": tool_call_id,+ "tool_return": SECRET_CODE,+ "status": "success",+ },+ ],+ ),+ ],+ )++ response = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=USER_MESSAGE_FOLLOW_UP,+ stream_tokens=True,+ )++ messages = accumulate_chunks(response)++ assert messages is not None+ assert len(messages) == 4+ assert messages[0].message_type == "reasoning_message"+ assert messages[1].message_type == "assistant_message"+ assert messages[2].message_type == "stop_reason"+ assert messages[3].message_type == "usage_statistics"+++def test_client_side_tool_call_and_follow_up_with_error(client: Letta,agent: AgentState,) -> None:@@ -749,10 +1073,10 @@ def test_deny_and_follow_up_with_error(reason=f"You don't need to call the tool, the secret code is {SECRET_CODE}", # legacyapprovals=[{- "type": "approval",- "approve": False,+ "type": "tool","tool_call_id": tool_call_id,- "reason": f"You don't need to call the tool, the secret code is {SECRET_CODE}",+ "tool_return": SECRET_CODE,+ "status": "success",},],),