Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: fix cancellation issues without making too many changes to `message_ids` persistence (#6442)
Sarah Woodersf417e536385c
diff --git a/letta/agents/helpers.py b/letta/agents/helpers.pyindex 5cddbb89b..796a47711 100644--- a/letta/agents/helpers.py+++ b/letta/agents/helpers.py@@ -162,7 +162,8 @@ async def _prepare_in_context_messages_no_persist_async(new_in_context_messages.extend(follow_up_messages)else:# User is trying to send a regular message- if current_in_context_messages and current_in_context_messages[-1].role == "approval":+ # if current_in_context_messages and current_in_context_messages[-1].role == "approval":+ if current_in_context_messages and current_in_context_messages[-1].is_approval_request():raise PendingApprovalError(pending_request_id=current_in_context_messages[-1].id)# Create a new user message from the input but dont store it yetdiff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 3af7fad0e..ce6915400 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -44,6 +44,7 @@ from letta.server.rest_api.utils import (create_approval_request_message_from_llm_response,create_letta_messages_from_llm_response,create_parallel_tool_messages_from_llm_response,+ create_tool_returns_for_denials,)from letta.services.helpers.tool_parser_helper import runtime_override_tool_json_schemafrom letta.services.summarizer.summarizer_all import summarize_all@@ -701,6 +702,14 @@ class LettaAgentV3(LettaAgentV2):finally:self.logger.debug("Running cleanup for agent loop run: %s", run_id)self.logger.info("Running final update. Step Progression: %s", step_progression)++ # update message ids+ message_ids = [m.id for m in messages]+ await self.agent_manager.update_message_ids_async(+ agent_id=self.agent_state.id,+ message_ids=message_ids,+ actor=self.actor,+ )try:if step_progression == StepProgression.FINISHED:if not self.should_continue:@@ -932,19 +941,15 @@ class LettaAgentV3(LettaAgentV2):# 4. Handle denial casesif tool_call_denials:+ # Convert ToolCallDenial objects to ToolReturn objects using shared helper+ # Group denials by reason to potentially batch them, but for now process individuallyfor tool_call_denial in tool_call_denials:- tool_call_id = tool_call_denial.id or f"call_{uuid.uuid4().hex[:8]}"- packaged_function_response = package_function_response(- was_success=False,- response_string=f"Error: request to call tool denied. User reason: {tool_call_denial.reason}",+ denial_returns = create_tool_returns_for_denials(+ tool_calls=[tool_call_denial],+ denial_reason=tool_call_denial.reason,timezone=agent_state.timezone,)- tool_return = ToolReturn(- tool_call_id=tool_call_id,- func_response=packaged_function_response,- status="error",- )- result_tool_returns.append(tool_return)+ result_tool_returns.extend(denial_returns)# 5. Unified tool execution path (works for both single and multiple tools)diff --git a/letta/constants.py b/letta/constants.pyindex 4b3549a5b..72ec844e8 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -195,6 +195,8 @@ PRE_EXECUTION_MESSAGE_ARG = "pre_exec_msg"REQUEST_HEARTBEAT_PARAM = "request_heartbeat"REQUEST_HEARTBEAT_DESCRIPTION = "Request an immediate heartbeat after function execution. You MUST set this value to `True` if you want to send a follow-up message or run a follow-up tool call (chain multiple tools together). If set to `False` (the default), then the chain of execution will end immediately after this function call."+# Automated tool call denials+TOOL_CALL_DENIAL_ON_CANCEL = "The user cancelled the request, so the tool call was denied."# Structured output modelsSTRUCTURED_OUTPUT_MODELS = {"gpt-4o", "gpt-4o-mini"}diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 41ae70224..7e8330c98 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -1659,6 +1659,7 @@ async def cancel_message(Note to cancel active runs associated with an agent, redis is required."""+ # TODO: WHY DOES THIS CANCEL A LIST OF RUNS?actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)if not settings.track_agent_run:raise HTTPException(status_code=400, detail="Agent run tracking is disabled")@@ -1685,12 +1686,12 @@ async def cancel_message(if run.metadata.get("lettuce"):lettuce_client = await LettuceClient.create()await lettuce_client.cancel(run_id)- success = await server.run_manager.update_run_by_id_async(- run_id=run_id,- update=RunUpdate(status=RunStatus.cancelled),- actor=actor,- )- results[run_id] = "cancelled" if success else "failed"+ try:+ run = await server.run_manager.cancel_run(actor=actor, agent_id=agent_id, run_id=run_id)+ except Exception as e:+ results[run_id] = "failed"+ continue+ results[run_id] = "cancelled"return resultsdiff --git a/letta/server/rest_api/utils.py b/letta/server/rest_api/utils.pyindex 8b2125d0d..810847312 100644--- a/letta/server/rest_api/utils.py+++ b/letta/server/rest_api/utils.py@@ -213,6 +213,80 @@ def create_approval_response_message_from_input(]+def create_tool_returns_for_denials(+ tool_calls: List[OpenAIToolCall],+ denial_reason: str,+ timezone: str,+) -> List[ToolReturn]:+ """+ Create ToolReturn objects with error status for denied tool calls.++ This is used when tool calls are denied either by:+ - User explicitly denying approval+ - Run cancellation (automated denial)++ Args:+ tool_calls: List of tool calls that were denied+ denial_reason: Reason for denial (e.g., user reason or cancellation message)+ timezone: Agent timezone for timestamp formatting++ Returns:+ List of ToolReturn objects with error status+ """+ tool_returns = []+ for tool_call in tool_calls:+ tool_call_id = tool_call.id or f"call_{uuid.uuid4().hex[:8]}"+ packaged_function_response = package_function_response(+ was_success=False,+ response_string=f"Error: request to call tool denied. User reason: {denial_reason}",+ timezone=timezone,+ )+ tool_return = ToolReturn(+ tool_call_id=tool_call_id,+ func_response=packaged_function_response,+ status="error",+ )+ tool_returns.append(tool_return)+ return tool_returns+++def create_tool_message_from_returns(+ agent_id: str,+ model: str,+ tool_returns: List[ToolReturn],+ run_id: Optional[str] = None,+ step_id: Optional[str] = None,+) -> Message:+ """+ Create a tool message with error returns for denied/failed tool calls.++ This creates a properly formatted tool message that can be added to the+ conversation history to reflect tool call denials or failures.++ Args:+ agent_id: ID of the agent+ model: Model identifier+ tool_returns: List of ToolReturn objects (typically with error status)+ run_id: Optional run ID+ step_id: Optional step ID++ Returns:+ Message with role="tool" containing the tool returns+ """+ return Message(+ role=MessageRole.tool,+ content=[TextContent(text=tr.func_response) for tr in tool_returns],+ agent_id=agent_id,+ model=model,+ tool_calls=[],+ tool_call_id=tool_returns[0].tool_call_id if tool_returns else None,+ tool_returns=tool_returns,+ run_id=run_id,+ step_id=step_id,+ created_at=get_utc_time(),+ )++def create_approval_request_message_from_llm_response(agent_id: str,model: str,diff --git a/letta/services/run_manager.py b/letta/services/run_manager.pyindex b890de50b..52f6ad2ab 100644--- a/letta/services/run_manager.py+++ b/letta/services/run_manager.py@@ -1,9 +1,11 @@from datetime import datetime+from multiprocessing import Valuefrom pickletools import pyunicodefrom typing import List, Literal, Optionalfrom httpx import AsyncClient+from letta.errors import LettaInvalidArgumentErrorfrom letta.helpers.datetime_helpers import get_utc_timefrom letta.log import get_loggerfrom letta.orm.agent import Agent as AgentModel@@ -314,7 +316,7 @@ class RunManager:needs_callback = Falsecallback_url = Nonenot_completed_before = not bool(run.completed_at)- is_terminal_update = update.status in {RunStatus.completed, RunStatus.failed}+ is_terminal_update = update.status in {RunStatus.completed, RunStatus.failed, RunStatus.cancelled}if is_terminal_update and not_completed_before and run.callback_url:needs_callback = Truecallback_url = run.callback_url@@ -558,3 +560,129 @@ class RunManager:actor=actor, run_id=run_id, limit=limit, before=before, after=after, order="asc" if ascending else "desc")return steps++ @enforce_types+ async def cancel_run(self, actor: PydanticUser, agent_id: Optional[str] = None, run_id: Optional[str] = None) -> None:+ """Cancel a run."""++ # make sure run_id and agent_id are not both None+ if not run_id:+ # get the last agent run+ if not agent_id:+ raise ValueError("Agent ID is required to cancel a run by ID")+ logger.warning("Cannot find run associated with agent to cancel in redis, fetching from db.")+ run_ids = await self.list_runs(+ actor=actor,+ ascending=False,+ agent_id=agent_id,+ )+ run_ids = [run.id for run in run_ids]+ else:+ # get the agent+ run = await self.get_run_by_id(run_id=run_id, actor=actor)+ if not run:+ raise NoResultFound(f"Run with id {run_id} not found")+ agent_id = run.agent_id++ logger.debug(f"Cancelling run {run_id} for agent {agent_id}")++ # check if run can be cancelled (cannot cancel a completed, failed, or cancelled run)+ if run.stop_reason and run.stop_reason not in [StopReasonType.requires_approval]:+ logger.error(f"Run {run_id} cannot be cancelled because it is already terminated with stop reason: {run.stop_reason.value}")+ raise LettaInvalidArgumentError(+ f"Run {run_id} cannot be cancelled because it is already terminated with stop reason: {run.stop_reason.value}"+ )++ # Check if agent is waiting for approval by examining the last message+ agent_state = await self.agent_manager.get_agent_by_id_async(agent_id=agent_id, actor=actor)+ current_in_context_messages = await self.message_manager.get_messages_by_ids_async(message_ids=agent_state.message_ids, actor=actor)+ was_pending_approval = current_in_context_messages and current_in_context_messages[-1].is_approval_request()++ # cancel the run+ # NOTE: this should update the agent's last stop reason to cancelled+ run = await self.update_run_by_id_async(+ run_id=run_id, update=RunUpdate(status=RunStatus.cancelled, stop_reason=StopReasonType.cancelled), actor=actor+ )++ # cleanup the agent's state+ # if was pending approval, we need to cleanup the approval state+ if was_pending_approval:+ logger.debug(f"Agent was waiting for approval, adding denial messages for run {run_id}")+ approval_request_message = current_in_context_messages[-1]++ # Ensure the approval request has tool calls to deny+ if approval_request_message.tool_calls:+ from letta.constants import TOOL_CALL_DENIAL_ON_CANCEL+ from letta.schemas.letta_message import ApprovalReturn+ from letta.schemas.message import ApprovalCreate+ from letta.server.rest_api.utils import (+ create_approval_response_message_from_input,+ create_tool_message_from_returns,+ create_tool_returns_for_denials,+ )++ # Create denials for ALL pending tool calls+ denials = [+ ApprovalReturn(+ tool_call_id=tool_call.id,+ approve=False,+ reason=TOOL_CALL_DENIAL_ON_CANCEL,+ )+ for tool_call in approval_request_message.tool_calls+ ]++ # Create an ApprovalCreate input with the denials+ approval_input = ApprovalCreate(+ approvals=denials,+ approval_request_id=approval_request_message.id,+ )++ # Use the standard function to create properly formatted approval response messages+ approval_response_messages = create_approval_response_message_from_input(+ agent_state=agent_state,+ input_message=approval_input,+ run_id=run_id,+ )++ # Create tool returns for ALL denied tool calls using shared helper+ # This handles all pending tool calls at once since they all have the same denial reason+ tool_returns = create_tool_returns_for_denials(+ tool_calls=approval_request_message.tool_calls, # ALL pending tool calls+ denial_reason=TOOL_CALL_DENIAL_ON_CANCEL,+ timezone=agent_state.timezone,+ )++ # Create tool message with all denial returns using shared helper+ tool_message = create_tool_message_from_returns(+ agent_id=agent_state.id,+ model=agent_state.llm_config.model,+ tool_returns=tool_returns,+ run_id=run_id,+ )++ # Combine approval response and tool messages+ new_messages = approval_response_messages + [tool_message]++ # Insert the approval response and tool messages into the database+ persisted_messages = await self.message_manager.create_many_messages_async(+ pydantic_msgs=new_messages,+ actor=actor,+ run_id=run_id,+ )+ logger.debug(f"Persisted {len(persisted_messages)} messages (approval + tool returns)")++ # Update the agent's message_ids to include the new messages (approval + tool message)+ agent_state.message_ids = agent_state.message_ids + [m.id for m in persisted_messages]+ await self.agent_manager.update_message_ids_async(agent_id=agent_state.id, message_ids=agent_state.message_ids, actor=actor)++ logger.debug(+ f"Inserted approval response with {len(denials)} denials and tool return message for cancelled run {run_id}. "+ f"Approval request message ID: {approval_request_message.id}"+ )+ else:+ logger.warning(+ f"Last message is an approval request but has no tool_calls. "+ f"Message ID: {approval_request_message.id}, Run ID: {run_id}"+ )++ return rundiff --git a/tests/integration_test_cancellation.py b/tests/integration_test_cancellation.pyindex 45042897b..c6791612b 100644--- a/tests/integration_test_cancellation.py+++ b/tests/integration_test_cancellation.py@@ -173,7 +173,7 @@ async def test_background_streaming_cancellation() -> None:agent_state = await client.agents.update(agent_id=agent_state.id, llm_config=llm_config)- delay = 5 if llm_config.model == "gpt-5" else 1.5+ delay = 1.5_cancellation_task = asyncio.create_task(cancel_run_after_delay(client, agent_state.id, delay=delay))response = await client.agents.messages.stream(diff --git a/tests/managers/test_cancellation.py b/tests/managers/test_cancellation.pynew file mode 100644index 000000000..554b700f0--- /dev/null+++ b/tests/managers/test_cancellation.py@@ -0,0 +1,1455 @@+"""+Tests for agent cancellation at different points in the execution loop.++These tests use mocking and deterministic control to test cancellation at specific+points in the agent execution flow, covering all the issues documented in CANCELLATION_ISSUES.md.+"""++import asyncio+from typing import AsyncGenerator+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from letta.agents.agent_loop import AgentLoop+from letta.constants import TOOL_CALL_DENIAL_ON_CANCEL+from letta.schemas.agent import CreateAgent+from letta.schemas.embedding_config import EmbeddingConfig+from letta.schemas.enums import MessageRole, RunStatus+from letta.schemas.letta_request import LettaStreamingRequest+from letta.schemas.llm_config import LLMConfig+from letta.schemas.message import MessageCreate+from letta.schemas.model import ModelSettings+from letta.schemas.run import Run as PydanticRun, RunUpdate+from letta.server.server import SyncServer+from letta.services.streaming_service import StreamingService+++async def test_agent_with_tool(server: SyncServer, default_user, print_tool):+ """Create a test agent with letta_v1_agent type (uses LettaAgentV3)."""+ agent_state = await server.agent_manager.create_agent_async(+ agent_create=CreateAgent(+ name="test_cancellation_agent",+ agent_type="letta_v1_agent",+ memory_blocks=[],+ llm_config=LLMConfig.default_config("gpt-4o-mini"),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ tool_ids=[print_tool.id],+ include_base_tools=False,+ ),+ actor=default_user,+ )+ yield agent_state+++async def test_run(server: SyncServer, default_user, test_agent_with_tool):+ """Create a test run for cancellation testing."""+ run = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=test_agent_with_tool.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )+ yield run+++class TestMessageStateDesyncIssues:+ """+ Test Issue #2: Message State Desync Issues+ Tests that message state stays consistent between client and server during cancellation.+ """++ @pytest.mark.asyncio+ async def test_message_state_consistency_after_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test that message state is consistent after cancellation.++ Verifies:+ - response_messages list matches persisted messages+ - response_messages_for_metadata list matches persisted messages+ - agent.message_ids includes all persisted messages+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Call print_tool with 'test'")]++ # Cancel after first step+ call_count = [0]++ async def mock_check_cancellation(run_id):+ call_count[0] += 1+ if call_count[0] > 1:+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=run_id,+ )+ return True+ return False++ agent_loop._check_run_cancellation = mock_check_cancellation++ # Execute step+ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Get messages from database+ db_messages = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run.id,+ limit=1000,+ )++ # Verify response_messages count matches result messages+ assert len(agent_loop.response_messages) == len(result.messages), (+ f"response_messages ({len(agent_loop.response_messages)}) should match result.messages ({len(result.messages)})"+ )++ # Verify persisted message count is reasonable+ assert len(db_messages) > 0, "Should have persisted messages from completed step"++ # CRITICAL CHECK: Verify agent state after cancellation+ agent_after_cancel = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )++ # Verify last_stop_reason is set to cancelled+ assert agent_after_cancel.last_stop_reason == "cancelled", (+ f"Agent's last_stop_reason should be 'cancelled', got '{agent_after_cancel.last_stop_reason}'"+ )++ agent_message_ids = set(agent_after_cancel.message_ids or [])+ db_message_ids = {m.id for m in db_messages}++ # Check for desync: every message in DB must be in agent.message_ids+ messages_in_db_not_in_agent = db_message_ids - agent_message_ids++ assert len(messages_in_db_not_in_agent) == 0, (+ f"MESSAGE DESYNC: {len(messages_in_db_not_in_agent)} messages in DB but not in agent.message_ids\n"+ f"Missing message IDs: {messages_in_db_not_in_agent}\n"+ f"This indicates message_ids was not updated after cancellation."+ )++ @pytest.mark.asyncio+ async def test_agent_can_continue_after_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test that agent can continue execution after a cancelled run.++ Verifies:+ - Agent state is not corrupted after cancellation+ - Subsequent runs complete successfully+ - Message IDs are properly updated+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # First run: cancel it+ input_messages_1 = [MessageCreate(role=MessageRole.user, content="First message")]++ # Cancel immediately+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ result_1 = await agent_loop.step(+ input_messages=input_messages_1,+ max_steps=5,+ run_id=test_run.id,+ )++ assert result_1.stop_reason.stop_reason == "cancelled"++ # Get agent state after cancellation+ agent_after_cancel = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )++ # Verify last_stop_reason is set to cancelled+ assert agent_after_cancel.last_stop_reason == "cancelled", (+ f"Agent's last_stop_reason should be 'cancelled', got '{agent_after_cancel.last_stop_reason}'"+ )++ message_ids_after_cancel = len(agent_after_cancel.message_ids or [])++ # Second run: complete it successfully+ test_run_2 = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=test_agent_with_tool.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )++ # Reload agent loop with fresh state+ agent_loop_2 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ input_messages_2 = [MessageCreate(role=MessageRole.user, content="Second message")]++ result_2 = await agent_loop_2.step(+ input_messages=input_messages_2,+ max_steps=5,+ run_id=test_run_2.id,+ )++ # Verify second run completed successfully+ assert result_2.stop_reason.stop_reason != "cancelled", f"Second run should complete, got {result_2.stop_reason.stop_reason}"++ # Get agent state after completion+ agent_after_complete = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )+ message_ids_after_complete = len(agent_after_complete.message_ids or [])++ # Verify message count increased+ assert message_ids_after_complete >= message_ids_after_cancel, (+ f"Message IDs should increase or stay same: "+ f"after_cancel={message_ids_after_cancel}, after_complete={message_ids_after_complete}"+ )++ # CRITICAL CHECK: Verify agent.message_ids consistency with DB for BOTH runs+ # Check first run (cancelled)+ db_messages_run1 = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run.id,+ limit=1000,+ )++ # Check second run (completed)+ db_messages_run2 = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run_2.id,+ limit=1000,+ )++ agent_message_ids = set(agent_after_complete.message_ids or [])+ all_db_message_ids = {m.id for m in db_messages_run1} | {m.id for m in db_messages_run2}++ # Check for desync: every message in DB must be in agent.message_ids+ messages_in_db_not_in_agent = all_db_message_ids - agent_message_ids++ assert len(messages_in_db_not_in_agent) == 0, (+ f"MESSAGE DESYNC: {len(messages_in_db_not_in_agent)} messages in DB but not in agent.message_ids\n"+ f"Missing message IDs: {messages_in_db_not_in_agent}\n"+ f"Run 1 (cancelled) had {len(db_messages_run1)} messages\n"+ f"Run 2 (completed) had {len(db_messages_run2)} messages\n"+ f"Agent has {len(agent_message_ids)} message_ids total\n"+ f"This indicates message_ids was not updated properly after cancellation or continuation."+ )++ @pytest.mark.asyncio+ async def test_approval_request_message_ids_desync_with_background_token_streaming(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ bash_tool,+ ):+ """+ Test for the specific desync bug with BACKGROUND + TOKEN STREAMING.++ This is the EXACT scenario where the bug occurs in production:+ - background=True (background streaming)+ - stream_tokens=True (token streaming)+ - Agent calls HITL tool requiring approval+ - Run is cancelled during approval++ Bug Scenario:+ 1. Agent calls HITL tool requiring approval+ 2. Approval request message is persisted to DB+ 3. Run is cancelled while processing in background with token streaming+ 4. Approval request message ID is NOT in agent.message_ids+ 5. Result: "Desync detected - cursor last: X, in-context last: Y"+ """+ # Add bash_tool to agent (requires approval)+ await server.agent_manager.attach_tool_async(+ agent_id=test_agent_with_tool.id,+ tool_id=bash_tool.id,+ actor=default_user,+ )++ # Get initial message count+ agent_before = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )+ initial_message_ids = set(agent_before.message_ids or [])+ print(f"\nInitial message_ids count: {len(initial_message_ids)}")++ # Create streaming service+ streaming_service = StreamingService(server)++ # Create request with BACKGROUND + TOKEN STREAMING (the key conditions!)+ request = LettaStreamingRequest(+ messages=[MessageCreate(role=MessageRole.user, content="Please run the bash_tool with operation 'test'")],+ max_steps=5,+ stream_tokens=True, # TOKEN STREAMING - KEY CONDITION+ background=True, # BACKGROUND STREAMING - KEY CONDITION+ )++ print("\n🔥 Starting agent with BACKGROUND + TOKEN STREAMING...")+ print(f" stream_tokens={request.stream_tokens}")+ print(f" background={request.background}")++ # Start the background streaming agent+ run, stream_response = await streaming_service.create_agent_stream(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ request=request,+ run_type="test_desync",+ )++ assert run is not None, "Run should be created for background streaming"+ print(f"\n✅ Run created: {run.id}")+ print(f" Status: {run.status}")++ # Cancel almost immediately - we want to interrupt DURING processing, not after+ # The bug happens when cancellation interrupts the approval flow mid-execution+ print("\n⏳ Starting background task, will cancel quickly to catch mid-execution...")+ await asyncio.sleep(0.3) # Just enough time for LLM to start, but not complete++ # NOW CANCEL THE RUN WHILE IT'S STILL PROCESSING - This is where the bug happens!+ print("\n❌ CANCELLING RUN while in background + token streaming mode (MID-EXECUTION)...")+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=run.id,+ )++ # Give cancellation time to propagate and background task to react+ print("⏳ Waiting for cancellation to propagate through background task...")+ await asyncio.sleep(2) # Let the background task detect cancellation and clean up++ # Check run status after cancellation+ run_status = await server.run_manager.get_run_by_id(run.id, actor=default_user)+ print(f"\n📊 Run status after cancel: {run_status.status}")+ print(f" Stop reason: {run_status.stop_reason}")++ # Get messages from DB AFTER cancellation+ db_messages_after_cancel = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=run.id,+ limit=1000,+ )+ print(f"\n📨 Messages in DB after cancel: {len(db_messages_after_cancel)}")+ for msg in db_messages_after_cancel:+ print(f" - {msg.id}: role={msg.role}")++ # Get agent state AFTER cancellation+ agent_after_cancel = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )++ # Verify last_stop_reason is set to cancelled+ print(f"\n🔍 Agent last_stop_reason: {agent_after_cancel.last_stop_reason}")+ assert agent_after_cancel.last_stop_reason == "cancelled", (+ f"Agent's last_stop_reason should be 'cancelled', got '{agent_after_cancel.last_stop_reason}'"+ )++ agent_message_ids = set(agent_after_cancel.message_ids or [])+ new_message_ids = agent_message_ids - initial_message_ids+ print(f"\n📝 Agent message_ids after cancel: {len(agent_message_ids)}")+ print(f" New message_ids in this run: {len(new_message_ids)}")++ db_message_ids = {m.id for m in db_messages_after_cancel}++ # CRITICAL CHECK: Every message in DB must be in agent.message_ids+ messages_in_db_not_in_agent = db_message_ids - agent_message_ids++ if messages_in_db_not_in_agent:+ # THIS IS THE DESYNC BUG!+ print("\n❌ DESYNC BUG DETECTED!")+ print(f"🐛 Found {len(messages_in_db_not_in_agent)} messages in DB but NOT in agent.message_ids")+ print(" This bug occurs specifically with: background=True + stream_tokens=True")++ missing_messages = [m for m in db_messages_after_cancel if m.id in messages_in_db_not_in_agent]++ print("\n🔍 Missing messages details:")+ for m in missing_messages:+ print(f" - ID: {m.id}")+ print(f" Role: {m.role}")+ print(f" Created: {m.created_at}")+ if hasattr(m, "content"):+ content_preview = str(m.content)[:100] if m.content else "None"+ print(f" Content: {content_preview}...")++ # Get the last message IDs for the exact error message format+ cursor_last = list(db_message_ids)[-1] if db_message_ids else None+ in_context_last = list(agent_message_ids)[-1] if agent_message_ids else None++ print("\n💥 This causes the EXACT error reported:")+ print(f" 'Desync detected - cursor last: {cursor_last},")+ print(f" in-context last: {in_context_last}'")++ assert False, (+ f"🐛 DESYNC DETECTED IN BACKGROUND + TOKEN STREAMING MODE\n\n"+ f"Found {len(messages_in_db_not_in_agent)} messages in DB but not in agent.message_ids\n\n"+ f"This reproduces the reported bug:\n"+ f" 'Desync detected - cursor last: {cursor_last},\n"+ f" in-context last: {in_context_last}'\n\n"+ f"Missing message IDs: {messages_in_db_not_in_agent}\n\n"+ f"Root cause: With background=True + stream_tokens=True, approval request messages\n"+ f"are persisted to DB but NOT added to agent.message_ids when cancellation occurs\n"+ f"during HITL approval flow.\n\n"+ f"Fix location: Check approval flow in letta_agent_v3.py:442-486 and background\n"+ f"streaming wrapper in streaming_service.py:138-146"+ )++ # Also check reverse: agent.message_ids shouldn't have messages not in DB+ messages_in_agent_not_in_db = agent_message_ids - db_message_ids+ messages_in_agent_not_in_db = messages_in_agent_not_in_db - initial_message_ids++ if messages_in_agent_not_in_db:+ print("\n❌ REVERSE DESYNC DETECTED!")+ print(f"Found {len(messages_in_agent_not_in_db)} message IDs in agent.message_ids but NOT in DB")++ assert False, (+ f"REVERSE DESYNC: {len(messages_in_agent_not_in_db)} messages in agent.message_ids but not in DB\n"+ f"Message IDs: {messages_in_agent_not_in_db}"+ )++ # If we get here, message IDs are consistent!+ print("\n✅ No desync detected - message IDs are consistent between DB and agent state")+ print(f" DB message count: {len(db_message_ids)}")+ print(f" Agent message_ids count: {len(agent_message_ids)}")+ print("\n Either the bug is fixed, or we need to adjust test timing/conditions.")+++class TestStreamingCancellation:+ """+ Test cancellation during different streaming modes.+ """++ @pytest.mark.asyncio+ async def test_token_streaming_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test cancellation during token streaming mode.++ This tests Issue #3: Cancellation During LLM Streaming (token mode).++ Verifies:+ - Cancellation can be detected during token streaming+ - Partial messages are handled correctly+ - Stop reason is set to 'cancelled'+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Hello")]++ # Cancel after first chunk+ cancel_triggered = [False]++ async def mock_check_cancellation(run_id):+ if cancel_triggered[0]:+ return True+ return False++ agent_loop._check_run_cancellation = mock_check_cancellation++ # Mock streaming+ async def cancel_during_stream():+ """Generator that simulates streaming and cancels mid-stream."""+ chunks_yielded = 0+ stream = agent_loop.stream(+ input_messages=input_messages,+ max_steps=5,+ stream_tokens=True,+ run_id=test_run.id,+ )++ async for chunk in stream:+ chunks_yielded += 1+ yield chunk++ # Cancel after a few chunks+ if chunks_yielded == 2 and not cancel_triggered[0]:+ cancel_triggered[0] = True+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Consume the stream+ chunks = []+ try:+ async for chunk in cancel_during_stream():+ chunks.append(chunk)+ except Exception as e:+ # May raise exception on cancellation+ pass++ # Verify we got some chunks before cancellation+ assert len(chunks) > 0, "Should receive at least some chunks before cancellation"++ @pytest.mark.asyncio+ async def test_step_streaming_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test cancellation during step streaming mode (not token streaming).++ Verifies:+ - Cancellation detected between steps+ - Completed steps are streamed fully+ - Partial step is not streamed+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Call print_tool with 'message'")]++ # Cancel after first step+ call_count = [0]++ async def mock_check_cancellation(run_id):+ call_count[0] += 1+ if call_count[0] > 1:+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=run_id,+ )+ return True+ return False++ agent_loop._check_run_cancellation = mock_check_cancellation++ # Stream with step streaming (not token streaming)+ chunks = []+ stream = agent_loop.stream(+ input_messages=input_messages,+ max_steps=5,+ stream_tokens=False, # Step streaming+ run_id=test_run.id,+ )++ async for chunk in stream:+ chunks.append(chunk)++ # Verify we got chunks from the first step+ assert len(chunks) > 0, "Should receive chunks from first step before cancellation"++ # Verify cancellation was detected+ assert agent_loop.stop_reason.stop_reason == "cancelled"+++class TestToolExecutionCancellation:+ """+ Test cancellation during tool execution.+ This tests Issue #2C: Token streaming tool return desync.+ """++ @pytest.mark.asyncio+ async def test_cancellation_during_tool_execution(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ print_tool,+ ):+ """+ Test cancellation while tool is executing.++ Verifies:+ - Tool execution completes or is interrupted cleanly+ - Tool return messages are consistent+ - Database state matches client state+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Call print_tool with 'test message'")]++ # Mock the tool execution to detect cancellation+ tool_execution_started = [False]+ tool_execution_completed = [False]++ original_execute = agent_loop._execute_tool++ async def mock_execute_tool(target_tool, tool_args, agent_state, agent_step_span, step_id):+ tool_execution_started[0] = True++ # Cancel during tool execution+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Call original (tool execution should complete)+ result = await original_execute(target_tool, tool_args, agent_state, agent_step_span, step_id)++ tool_execution_completed[0] = True+ return result++ agent_loop._execute_tool = mock_execute_tool++ # Execute step+ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Verify tool execution started+ assert tool_execution_started[0], "Tool execution should have started"++ # Verify cancellation was eventually detected+ # (may be after tool completes, at next step boundary)+ assert result.stop_reason.stop_reason == "cancelled"++ # If tool completed, verify its messages are persisted+ if tool_execution_completed[0]:+ db_messages = await server.message_manager.list_messages(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )+ run_messages = [m for m in db_messages if m.run_id == test_run.id]+ tool_returns = [m for m in run_messages if m.role == "tool"]++ # If tool executed, should have a tool return message+ assert len(tool_returns) > 0, "Should have persisted tool return message"+++class TestResourceCleanupAfterCancellation:+ """+ Test Issue #6: Resource Cleanup Issues+ Tests that resources are properly cleaned up after cancellation.+ """++ @pytest.mark.asyncio+ async def test_stop_reason_set_correctly_on_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test that stop_reason is set to 'cancelled' not 'end_turn' or other.++ This tests Issue #6: Resource Cleanup Issues.+ The finally block should set stop_reason to 'cancelled' when appropriate.++ Verifies:+ - stop_reason is 'cancelled' when run is cancelled+ - stop_reason is not 'end_turn' or 'completed' for cancelled runs+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # Cancel before execution+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ input_messages = [MessageCreate(role=MessageRole.user, content="Hello")]++ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Verify stop reason is cancelled, not end_turn+ assert result.stop_reason.stop_reason == "cancelled", f"Stop reason should be 'cancelled', got '{result.stop_reason.stop_reason}'"++ # Verify run status in database+ run = await server.run_manager.get_run_by_id(run_id=test_run.id, actor=default_user)+ assert run.status == RunStatus.cancelled, f"Run status should be cancelled, got {run.status}"++ @pytest.mark.asyncio+ async def test_response_messages_cleared_after_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test that internal message buffers are properly managed after cancellation.++ Verifies:+ - response_messages list is in expected state after cancellation+ - No memory leaks from accumulated messages+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # Execute and cancel+ call_count = [0]++ async def mock_check_cancellation(run_id):+ call_count[0] += 1+ if call_count[0] > 1:+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=run_id,+ )+ return True+ return False++ agent_loop._check_run_cancellation = mock_check_cancellation++ input_messages = [MessageCreate(role=MessageRole.user, content="Call print_tool with 'test'")]++ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Verify response_messages is not empty (contains messages from completed step)+ # or is properly cleared depending on implementation+ response_msg_count = len(agent_loop.response_messages)++ # The exact behavior may vary, but we're checking that the state is reasonable+ assert response_msg_count >= 0, "response_messages should be in valid state"++ # Verify no excessive accumulation+ assert response_msg_count < 100, "response_messages should not have excessive accumulation"+++class TestApprovalFlowCancellation:+ """+ Test Issue #5: Approval Flow + Cancellation+ Tests edge cases with HITL tool approvals and cancellation.+ """++ @pytest.mark.asyncio+ async def test_cancellation_while_waiting_for_approval(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ bash_tool,+ ):+ """+ Test cancellation while agent is waiting for tool approval.++ This tests the scenario where:+ 1. Agent calls a tool requiring approval+ 2. Run is cancelled while waiting for approval+ 3. Agent should detect cancellation and not process approval++ Verifies:+ - Run status is cancelled+ - Agent does not process approval after cancellation+ - No tool execution happens+ """+ # Add bash_tool which requires approval to agent+ await server.agent_manager.attach_tool_async(+ agent_id=test_agent_with_tool.id,+ tool_id=bash_tool.id,+ actor=default_user,+ )++ # Reload agent with new tool+ test_agent_with_tool = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ )++ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Call bash_tool with operation 'test'")]++ # Execute step - should stop at approval request+ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Verify we got approval request+ assert result.stop_reason.stop_reason == "requires_approval", f"Should stop for approval, got {result.stop_reason.stop_reason}"++ # Now cancel the run while "waiting for approval"+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Reload agent loop with fresh state+ agent_loop_2 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ # Try to continue - should detect cancellation+ result_2 = await agent_loop_2.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Hello")], # No new input, just continuing+ max_steps=5,+ run_id=test_run.id,+ )++ # Should detect cancellation+ assert result_2.stop_reason.stop_reason == "cancelled", f"Should detect cancellation, got {result_2.stop_reason.stop_reason}"++ @pytest.mark.asyncio+ async def test_agent_state_after_cancelled_approval(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ bash_tool,+ ):+ """+ Test that agent state is consistent after approval request is cancelled.++ This addresses the issue where agents say they are "awaiting approval"+ even though the run is cancelled.++ Verifies:+ - Agent can continue after cancelled approval+ - No phantom "awaiting approval" state+ - Messages reflect actual state+ """+ # Add bash_tool which requires approval+ await server.agent_manager.attach_tool_async(+ agent_id=test_agent_with_tool.id,+ tool_id=bash_tool.id,+ actor=default_user,+ )++ # Reload agent with new tool+ test_agent_with_tool = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ )++ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # First run: trigger approval request then cancel+ input_messages_1 = [MessageCreate(role=MessageRole.user, content="Call bash_tool with operation 'test'")]++ result_1 = await agent_loop.step(+ input_messages=input_messages_1,+ max_steps=5,+ run_id=test_run.id,+ )++ assert result_1.stop_reason.stop_reason == "requires_approval"++ # Cancel the run+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Get messages to check for "awaiting approval" state+ messages_after_cancel = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run.id,+ limit=1000,+ )++ # Check for approval request messages+ approval_messages = [m for m in messages_after_cancel if m.role == "approval_request"]++ # Second run: try to execute normally (should work, not stuck in approval)+ test_run_2 = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=test_agent_with_tool.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )++ agent_loop_2 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ # Call a different tool that doesn't require approval+ input_messages_2 = [MessageCreate(role=MessageRole.user, content="Call print_tool with message 'hello'")]++ result_2 = await agent_loop_2.step(+ input_messages=input_messages_2,+ max_steps=5,+ run_id=test_run_2.id,+ )++ # Should complete normally, not be stuck in approval state+ assert result_2.stop_reason.stop_reason != "requires_approval", "Agent should not be stuck in approval state from cancelled run"++ @pytest.mark.asyncio+ async def test_approval_state_persisted_correctly_after_cancel(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ bash_tool,+ ):+ """+ Test that approval state is correctly persisted/cleaned after cancellation.++ This addresses the specific issue mentioned:+ "agents say they are awaiting approval despite the run not being shown as pending approval"++ Verifies:+ - Run status matches actual state+ - No phantom "pending approval" status+ - Messages accurately reflect cancellation+ """+ # Add bash_tool+ await server.agent_manager.attach_tool_async(+ agent_id=test_agent_with_tool.id,+ tool_id=bash_tool.id,+ actor=default_user,+ )++ test_agent_with_tool = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ )++ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # Trigger approval+ result = await agent_loop.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Call bash_tool with 'test'")],+ max_steps=5,+ run_id=test_run.id,+ )++ assert result.stop_reason.stop_reason == "requires_approval"++ # Cancel the run+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Verify run status is cancelled, NOT pending_approval+ run_after_cancel = await server.run_manager.get_run_by_id(run_id=test_run.id, actor=default_user)+ assert run_after_cancel.status == RunStatus.cancelled, f"Run status should be cancelled, got {run_after_cancel.status}"++ # Agent should be able to start fresh run+ test_run_3 = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=test_agent_with_tool.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )++ agent_loop_3 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ # Should be able to make normal call+ result_3 = await agent_loop_3.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Call print_tool with 'test'")],+ max_steps=5,+ run_id=test_run_3.id,+ )++ # Should complete normally+ assert result_3.stop_reason.stop_reason != "requires_approval", "New run should not be stuck in approval state"++ @pytest.mark.asyncio+ async def test_approval_request_message_ids_desync(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ bash_tool,+ ):+ """+ Test for the specific desync bug reported:+ "Desync detected - cursor last: message-X, in-context last: message-Y"++ Bug Scenario:+ 1. Agent calls HITL tool requiring approval+ 2. Approval request message is persisted to DB+ 3. Run is cancelled+ 4. Approval request message ID is NOT in agent.message_ids+ 5. Result: cursor desync between DB and agent state++ This is the root cause of the reported error:+ "Desync detected - cursor last: message-c07fa1ec..., in-context last: message-a2615dc3..."++ The bug happens because:+ - Database contains the approval_request message+ - Agent's message_ids list does NOT contain the approval_request message ID+ - Causes cursor/pagination to fail++ Verifies:+ - If approval request is in DB, it must be in agent.message_ids+ - Cancellation doesn't cause partial message persistence+ - Cursor consistency between DB and agent state+ """+ # Add bash_tool which requires approval+ await server.agent_manager.attach_tool_async(+ agent_id=test_agent_with_tool.id,+ tool_id=bash_tool.id,+ actor=default_user,+ )++ # Get initial message count+ agent_before = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )+ initial_message_ids = set(agent_before.message_ids or [])++ # Reload agent with new tool+ test_agent_with_tool = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ )++ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # Trigger approval request+ result = await agent_loop.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Call bash_tool with 'test'")],+ max_steps=5,+ run_id=test_run.id,+ )++ assert result.stop_reason.stop_reason == "requires_approval", f"Expected requires_approval, got {result.stop_reason.stop_reason}"++ # Get all messages from database for this run+ db_messages = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run.id,+ limit=1000,+ )++ # Cancel the run+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Get agent state after cancellation+ agent_after_cancel = await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ )++ agent_message_ids = set(agent_after_cancel.message_ids or [])++ # Get all messages from database again+ db_messages_after = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=test_agent_with_tool.id,+ run_id=test_run.id,+ limit=1000,+ )++ db_message_ids = {m.id for m in db_messages_after}++ # CRITICAL CHECK: Every message in DB must be in agent.message_ids+ messages_in_db_not_in_agent = db_message_ids - agent_message_ids++ if messages_in_db_not_in_agent:+ # THIS IS THE DESYNC BUG!+ missing_messages = [m for m in db_messages_after if m.id in messages_in_db_not_in_agent]+ missing_details = [f"ID: {m.id}, Role: {m.role}, Created: {m.created_at}" for m in missing_messages]++ # Get the cursor values that would cause the error+ cursor_last = list(db_message_ids)[-1] if db_message_ids else None+ in_context_last = list(agent_message_ids)[-1] if agent_message_ids else None++ assert False, (+ f"DESYNC DETECTED: {len(messages_in_db_not_in_agent)} messages in DB but not in agent.message_ids\n\n"+ f"This is the reported bug:\n"+ f" 'Desync detected - cursor last: {cursor_last}, in-context last: {in_context_last}'\n\n"+ f"Missing messages:\n" + "\n".join(missing_details) + "\n\n"+ f"Agent message_ids count: {len(agent_message_ids)}\n"+ f"DB messages count: {len(db_message_ids)}\n\n"+ f"Root cause: Approval request message was persisted to DB but not added to agent.message_ids\n"+ f"when cancellation occurred during HITL approval flow."+ )++ # Also check the inverse: agent.message_ids shouldn't have messages not in DB+ messages_in_agent_not_in_db = agent_message_ids - db_message_ids+ messages_in_agent_not_in_db = messages_in_agent_not_in_db - initial_message_ids++ if messages_in_agent_not_in_db:+ assert False, (+ f"REVERSE DESYNC: {len(messages_in_agent_not_in_db)} messages in agent.message_ids but not in DB\n"+ f"Message IDs: {messages_in_agent_not_in_db}"+ )++ @pytest.mark.asyncio+ async def test_parallel_tool_calling_cancellation_with_denials(+ self,+ server: SyncServer,+ default_user,+ bash_tool,+ ):+ """+ Test that parallel tool calls receive proper denial messages on cancellation.++ This tests the scenario where:+ 1. Agent has parallel tool calling enabled+ 2. Agent calls a tool 3 times in parallel (requiring approval)+ 3. Run is cancelled while waiting for approval+ 4. All 3 tool calls receive denial messages with TOOL_CALL_DENIAL_ON_CANCEL+ 5. Agent can still be messaged again (creating a new run)++ Verifies:+ - All parallel tool calls get proper denial messages+ - Denial messages contain TOOL_CALL_DENIAL_ON_CANCEL reason+ - Agent state is not corrupted+ - New runs can be created after cancellation+ """+ # Create agent with parallel tool calling enabled+ config = LLMConfig.default_config("gpt-4o-mini")+ config.parallel_tool_calls = True+ agent_state = await server.agent_manager.create_agent_async(+ agent_create=CreateAgent(+ name="test_parallel_tool_calling_agent",+ agent_type="letta_v1_agent",+ memory_blocks=[],+ llm_config=LLMConfig.default_config("gpt-4o-mini"),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ tool_ids=[bash_tool.id],+ include_base_tools=False,+ ),+ actor=default_user,+ )++ # Create a run+ test_run = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=agent_state.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )++ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=agent_state, actor=default_user)++ # Prompt the agent to call bash_tool 3 times+ # The agent should make parallel tool calls since parallel_tool_calls is enabled+ input_messages = [+ MessageCreate(+ role=MessageRole.user,+ content="Please call bash_tool three times with operations: 'ls', 'pwd', and 'echo test'",+ )+ ]++ # Execute step - should stop at approval request with multiple tool calls+ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Verify we got approval request+ assert result.stop_reason.stop_reason == "requires_approval", f"Should stop for approval, got {result.stop_reason.stop_reason}"++ # Get the approval request message to see how many tool calls were made+ db_messages_before_cancel = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=agent_state.id,+ run_id=test_run.id,+ limit=1000,+ )++ # should not be possible to message the agent (Pending approval)+ from letta.errors import PendingApprovalError++ with pytest.raises(PendingApprovalError):+ test_run2 = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=agent_state.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )+ await agent_loop.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Hello, how are you?")],+ max_steps=5,+ run_id=test_run2.id,+ )++ from letta.schemas.letta_message import ApprovalRequestMessage++ approval_request_messages = [m for m in result.messages if isinstance(m, ApprovalRequestMessage)]+ assert len(approval_request_messages) > 0, "Should have at least one approval request message"++ # Get the last approval request message (should have the tool calls)+ approval_request = approval_request_messages[-1]+ tool_calls = approval_request.tool_calls if hasattr(approval_request, "tool_calls") else []+ num_tool_calls = len(tool_calls)++ print(f"\nFound {num_tool_calls} tool calls in approval request")++ # The agent might not always make exactly 3 parallel calls depending on the LLM,+ # but we should have at least 1 tool call. For the test to be meaningful,+ # we want multiple tool calls, but we'll verify whatever the LLM decides+ assert num_tool_calls >= 1, f"Should have at least 1 tool call, got {num_tool_calls}"++ # Now cancel the run while "waiting for approval"+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Get messages after cancellation+ db_messages_after_cancel = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=agent_state.id,+ run_id=test_run.id,+ limit=1000,+ )++ # Find tool return messages (these should be the denial messages)+ tool_return_messages = [m for m in db_messages_after_cancel if m.role == "tool"]++ print(f"Found {len(tool_return_messages)} tool return messages after cancellation")++ # Verify we got denial messages for all tool calls+ assert len(tool_return_messages) == num_tool_calls, (+ f"Should have {num_tool_calls} tool return messages (one per tool call), got {len(tool_return_messages)}"+ )++ # Verify each tool return message contains the denial reason+ for tool_return_msg in tool_return_messages:+ # Check if message has tool_returns (new format) or tool_return (old format)+ print("TOOL RETURN MESSAGE:\n\n", tool_return_msg)+ if hasattr(tool_return_msg, "tool_returns") and tool_return_msg.tool_returns:+ # New format: list of tool returns+ for tool_return in tool_return_msg.tool_returns:+ assert TOOL_CALL_DENIAL_ON_CANCEL in tool_return.func_response, (+ f"Tool return should contain denial message, got: {tool_return.tool_return}"+ )+ elif hasattr(tool_return_msg, "tool_return"):+ # Old format: single tool_return field+ assert TOOL_CALL_DENIAL_ON_CANCEL in tool_return_msg.content, (+ f"Tool return should contain denial message, got: {tool_return_msg.tool_return}"+ )+ elif hasattr(tool_return_msg, "content"):+ # Check content field+ content_str = str(tool_return_msg.content)+ assert TOOL_CALL_DENIAL_ON_CANCEL in content_str, f"Tool return content should contain denial message, got: {content_str}"++ # Verify run status is cancelled+ run_after_cancel = await server.run_manager.get_run_by_id(run_id=test_run.id, actor=default_user)+ assert run_after_cancel.status == RunStatus.cancelled, f"Run status should be cancelled, got {run_after_cancel.status}"++ # Verify agent can be messaged again (create a new run)+ test_run_2 = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=agent_state.id,+ status=RunStatus.created,+ ),+ actor=default_user,+ )++ # Reload agent loop with fresh state+ agent_loop_2 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=agent_state.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ # Send a simple message that doesn't require approval+ input_messages_2 = [MessageCreate(role=MessageRole.user, content="Hello, how are you?")]++ result_2 = await agent_loop_2.step(+ input_messages=input_messages_2,+ max_steps=5,+ run_id=test_run_2.id,+ )++ # Verify second run completed successfully (not cancelled, not stuck in approval)+ assert result_2.stop_reason.stop_reason != "cancelled", (+ f"Second run should not be cancelled, got {result_2.stop_reason.stop_reason}"+ )+ assert result_2.stop_reason.stop_reason != "requires_approval", (+ f"Second run should not require approval for simple message, got {result_2.stop_reason.stop_reason}"+ )++ # Verify the second run has messages+ db_messages_run2 = await server.message_manager.list_messages(+ actor=default_user,+ agent_id=agent_state.id,+ run_id=test_run_2.id,+ limit=1000,+ )+ assert len(db_messages_run2) > 0, "Second run should have messages"+++class TestEdgeCases:+ """+ Test edge cases and boundary conditions for cancellation.+ """++ @pytest.mark.asyncio+ async def test_cancellation_with_max_steps_reached(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test interaction between max_steps and cancellation.++ Verifies:+ - If both max_steps and cancellation occur, correct stop_reason is set+ - Cancellation takes precedence over max_steps+ """+ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ # Cancel after second step, but max_steps=2+ call_count = [0]++ async def mock_check_cancellation(run_id):+ call_count[0] += 1+ if call_count[0] >= 2:+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=run_id,+ )+ return True+ return False++ agent_loop._check_run_cancellation = mock_check_cancellation++ input_messages = [MessageCreate(role=MessageRole.user, content="Call print_tool with 'test'")]++ result = await agent_loop.step(+ input_messages=input_messages,+ max_steps=2, # Will hit max_steps around the same time as cancellation+ run_id=test_run.id,+ )++ # Stop reason could be either cancelled or max_steps depending on timing+ # Both are acceptable in this edge case+ assert result.stop_reason.stop_reason in ["cancelled", "max_steps"], (+ f"Stop reason should be cancelled or max_steps, got {result.stop_reason.stop_reason}"+ )++ @pytest.mark.asyncio+ async def test_double_cancellation(+ self,+ server: SyncServer,+ default_user,+ test_agent_with_tool,+ test_run,+ ):+ """+ Test that cancelling an already-cancelled run is handled gracefully.++ Verifies:+ - No errors when checking already-cancelled run+ - State remains consistent+ """+ # Cancel the run+ await server.run_manager.cancel_run(+ actor=default_user,+ run_id=test_run.id,+ )++ # Load agent loop+ agent_loop = AgentLoop.load(agent_state=test_agent_with_tool, actor=default_user)++ input_messages = [MessageCreate(role=MessageRole.user, content="Hello")]++ # First execution - should detect cancellation+ result_1 = await agent_loop.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ assert result_1.stop_reason.stop_reason == "cancelled"++ # Try to execute again with same cancelled run - should handle gracefully+ agent_loop_2 = AgentLoop.load(+ agent_state=await server.agent_manager.get_agent_by_id_async(+ agent_id=test_agent_with_tool.id,+ actor=default_user,+ include_relationships=["memory", "tools", "sources"],+ ),+ actor=default_user,+ )++ result_2 = await agent_loop_2.step(+ input_messages=input_messages,+ max_steps=5,+ run_id=test_run.id,+ )++ # Should still detect as cancelled+ assert result_2.stop_reason.stop_reason == "cancelled"