Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: fix cancellation bugs and add testing (#5353)
cthomas128afeb587be
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 2d1cf841b..873645bd4 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -308,6 +308,7 @@ class LettaAgentV3(LettaAgentV2):else:# Check for job cancellation at the start of each stepif run_id and await self._check_run_cancellation(run_id):+ self.should_continue = Falseself.stop_reason = LettaStopReason(stop_reason=StopReasonType.cancelled.value)self.logger.info(f"Agent execution cancelled for run {run_id}")returndiff --git a/letta/server/rest_api/redis_stream_manager.py b/letta/server/rest_api/redis_stream_manager.pyindex 3d98205f9..b361466db 100644--- a/letta/server/rest_api/redis_stream_manager.py+++ b/letta/server/rest_api/redis_stream_manager.py@@ -213,6 +213,7 @@ async def create_background_stream_processor(run_manager: Optional run manager for updating run statusactor: Optional actor for run status updates"""+ stop_reason = Noneif writer is None:writer = RedisSSEStreamWriter(redis_client)await writer.start()@@ -232,6 +233,15 @@ async def create_background_stream_processor(if is_done:break+ try:+ # sorry for this+ maybe_json_chunk = chunk.split("data: ")[1]+ maybe_stop_reason = json.loads(maybe_json_chunk) if maybe_json_chunk and maybe_json_chunk[0] == "{" else None+ if maybe_stop_reason and maybe_stop_reason.get("message_type") == "stop_reason":+ stop_reason = maybe_stop_reason.get("stop_reason")+ except:+ pass+except Exception as e:logger.error(f"Error processing stream for run {run_id}: {e}")# Write error chunk@@ -251,9 +261,14 @@ async def create_background_stream_processor(if should_stop_writer:await writer.stop()if run_manager and actor:+ if stop_reason == "cancelled":+ run_status = RunStatus.cancelled+ else:+ run_status = RunStatus.completed+await run_manager.update_run_by_id_async(run_id=run_id,- update=RunUpdate(status=RunStatus.completed, stop_reason=StopReasonType.end_turn.value),+ update=RunUpdate(status=run_status, stop_reason=stop_reason or StopReasonType.end_turn.value),actor=actor,)diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 9dd2ca166..314207b5a 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -1364,9 +1364,14 @@ async def send_message_streaming(runs_manager = RunManager()from letta.schemas.enums import RunStatus+ if agent_loop.stop_reason.stop_reason.value == "cancelled":+ run_status = RunStatus.cancelled+ else:+ run_status = RunStatus.completed+await runs_manager.update_run_by_id_async(run_id=run.id,- update=RunUpdate(status=RunStatus.completed, stop_reason=agent_loop.stop_reason.stop_reason.value),+ update=RunUpdate(status=run_status, stop_reason=agent_loop.stop_reason.stop_reason.value),actor=actor,)@@ -1620,9 +1625,14 @@ async def _process_message_background(runs_manager = RunManager()from letta.schemas.enums import RunStatus+ if result.stop_reason.stop_reason == "cancelled":+ run_status = RunStatus.cancelled+ else:+ run_status = RunStatus.completed+await runs_manager.update_run_by_id_async(run_id=run_id,- update=RunUpdate(status=RunStatus.completed, stop_reason=result.stop_reason.stop_reason),+ update=RunUpdate(status=run_status, stop_reason=result.stop_reason.stop_reason),actor=actor,)diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.pyindex 1ea9d9aca..1eefbc31d 100644--- a/tests/integration_test_send_message_v2.py+++ b/tests/integration_test_send_message_v2.py@@ -1,4 +1,6 @@+import asyncioimport base64+import itertoolsimport jsonimport osimport threading@@ -164,6 +166,7 @@ def assert_tool_call_response(llm_config: LLMConfig,streaming: bool = False,from_db: bool = False,+ with_cancellation: bool = False,) -> None:"""Asserts that the messages list follows the expected sequence:@@ -175,10 +178,11 @@ def assert_tool_call_response(msg for msg in messages if not (isinstance(msg, LettaPing) or (hasattr(msg, "message_type") and msg.message_type == "ping"))]- expected_message_count_min, expected_message_count_max = get_expected_message_count_range(- llm_config, tool_call=True, streaming=streaming, from_db=from_db- )- assert expected_message_count_min <= len(messages) <= expected_message_count_max+ if not with_cancellation:+ expected_message_count_min, expected_message_count_max = get_expected_message_count_range(+ llm_config, tool_call=True, streaming=streaming, from_db=from_db+ )+ assert expected_message_count_min <= len(messages) <= expected_message_count_max# User message if loaded from dbindex = 0@@ -217,28 +221,30 @@ def assert_tool_call_response(assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)index += 1- # Reasoning message if reasoning enabled- otid_suffix = 0- try:- if is_reasoner_model(llm_config):- assert isinstance(messages[index], ReasoningMessage)- assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)- index += 1- otid_suffix += 1- except:- # Reasoning is non-deterministic, so don't throw if missing- pass-- # Assistant message- assert isinstance(messages[index], AssistantMessage)- assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)- index += 1- otid_suffix += 1+ # Messages from second agent step if request has not been cancelled+ if not with_cancellation:+ # Reasoning message if reasoning enabled+ otid_suffix = 0+ try:+ if is_reasoner_model(llm_config):+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)+ index += 1+ otid_suffix += 1+ except:+ # Reasoning is non-deterministic, so don't throw if missing+ pass++ # Assistant message+ assert isinstance(messages[index], AssistantMessage)+ assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)+ index += 1+ otid_suffix += 1# Stop reason and usage statistics if streamingif streaming:assert isinstance(messages[index], LettaStopReason)- assert messages[index].stop_reason == "end_turn"+ assert messages[index].stop_reason == ("cancelled" if with_cancellation else "end_turn")index += 1assert isinstance(messages[index], LettaUsageStatistics)assert messages[index].prompt_tokens > 0@@ -280,12 +286,20 @@ async def accumulate_chunks(chunks: List[Any], verify_token_streaming: bool = Fareturn [m for m in messages if m is not None]+async def cancel_run_after_delay(client: AsyncLetta, agent_id: str):+ await asyncio.sleep(0.5)+ await client.agents.messages.cancel(agent_id=agent_id)++async def wait_for_run_completion(client: AsyncLetta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:start = time.time()while True:run = await client.runs.retrieve(run_id)if run.status == "completed":return run+ if run.status == "cancelled":+ time.sleep(5)+ return runif run.status == "failed":raise RuntimeError(f"Run {run_id} did not complete: status = {run.status}")if time.time() - start > timeout:@@ -501,7 +515,20 @@ async def test_greeting(TESTED_LLM_CONFIGS,ids=[c.model for c in TESTED_LLM_CONFIGS],)[email protected]("send_type", ["step", "stream_steps", "stream_tokens", "stream_tokens_background", "async"])+ ["send_type", "cancellation"],+ list(+ itertools.product(+ ["step", "stream_steps", "stream_tokens", "stream_tokens_background", "async"], ["with_cancellation", "no_cancellation"]+ )+ ),+ ids=[+ f"{s}-{c}"+ for s, c in itertools.product(+ ["step", "stream_steps", "stream_tokens", "stream_tokens_background", "async"], ["with_cancellation", "no_cancellation"]+ )+ ],+)@pytest.mark.asyncio(loop_scope="function")async def test_tool_call(disable_e2b_api_key: Any,@@ -509,10 +536,14 @@ async def test_tool_call(agent_state: AgentState,llm_config: LLMConfig,send_type: str,+ cancellation: str,) -> None:last_message = await client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = await client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ if cancellation == "with_cancellation":+ _cancellation_task = asyncio.create_task(cancel_run_after_delay(client, agent_state.id))+if send_type == "step":response = await client.agents.messages.create(agent_id=agent_state.id,@@ -539,16 +570,22 @@ async def test_tool_call(messages = await accumulate_chunks(response)run_id = messages[0].run_id- assert_tool_call_response(messages, streaming=("stream" in send_type), llm_config=llm_config)+ assert_tool_call_response(+ messages, streaming=("stream" in send_type), llm_config=llm_config, with_cancellation=(cancellation == "with_cancellation")+ )if "background" in send_type:response = client.runs.stream(run_id=run_id, starting_after=0)messages = await accumulate_chunks(response)- assert_tool_call_response(messages, streaming=("stream" in send_type), llm_config=llm_config)+ assert_tool_call_response(+ messages, streaming=("stream" in send_type), llm_config=llm_config, with_cancellation=(cancellation == "with_cancellation")+ )messages_from_db = await client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)- assert_tool_call_response(messages_from_db, from_db=True, llm_config=llm_config)+ assert_tool_call_response(+ messages_from_db, from_db=True, llm_config=llm_config, with_cancellation=(cancellation == "with_cancellation")+ )assert run_id is not Nonerun = await client.runs.retrieve(run_id=run_id)- assert run.status == JobStatus.completed+ assert run.status == ("cancelled" if cancellation == "with_cancellation" else "completed")