Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: various hardening to prevent stale state on background mode runs (#6072)
Charles Packer2e721ddc629e
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 84a0f0130..0e142818d 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -434,6 +434,13 @@ class LettaAgentV3(LettaAgentV2):self.logger.info("switching to unconstrained mode (allowing non-tool responses)")self._require_tool_call = require_tool_call+ # Always refresh messages at the start of each step to pick up external inputs+ # (e.g., approval responses submitted by the client while this stream is running)+ try:+ messages = await self._refresh_messages(messages)+ except Exception as e:+ self.logger.warning(f"Failed to refresh messages at step start: {e}")+approval_request, approval_response = _maybe_get_approval_messages(messages)tool_call_denials, tool_returns = [], []if approval_request and approval_response:@@ -494,7 +501,6 @@ class LettaAgentV3(LettaAgentV2):step_id=step_id, run_id=run_id)- messages = await self._refresh_messages(messages)force_tool_call = valid_tools[0]["name"] if len(valid_tools) == 1 and self._require_tool_call else Nonefor llm_request_attempt in range(summarizer_settings.max_summarizer_retries + 1):try:diff --git a/letta/server/rest_api/redis_stream_manager.py b/letta/server/rest_api/redis_stream_manager.pyindex c3d1e501e..5728e19a6 100644--- a/letta/server/rest_api/redis_stream_manager.py+++ b/letta/server/rest_api/redis_stream_manager.py@@ -301,11 +301,19 @@ async def create_background_stream_processor(if should_stop_writer:await writer.stop()- # Update run status if not already set (e.g., by exception handler)- if run_manager and actor and stop_reason:+ # Derive a final stop_reason if one wasn't observed explicitly+ final_stop_reason = stop_reason+ if final_stop_reason is None:+ if saw_error:+ final_stop_reason = StopReasonType.error.value+ elif saw_done:+ # Treat DONE without an explicit stop_reason as an error to avoid masking failures+ final_stop_reason = StopReasonType.error.value++ # Update run status to reflect terminal outcome+ if run_manager and actor and final_stop_reason:# Map stop_reason to run status- # Error states -> failed- if stop_reason in [+ if final_stop_reason in [StopReasonType.error.value,StopReasonType.llm_api_error.value,StopReasonType.invalid_tool_call.value,@@ -313,11 +321,9 @@ async def create_background_stream_processor(StopReasonType.no_tool_call.value,]:run_status = RunStatus.failed- # Cancelled state- elif stop_reason == StopReasonType.cancelled.value:+ elif final_stop_reason == StopReasonType.cancelled.value:run_status = RunStatus.cancelled- # Success states -> completed- elif stop_reason in [+ elif final_stop_reason in [StopReasonType.end_turn.value,StopReasonType.max_steps.value,StopReasonType.tool_rule.value,@@ -325,17 +331,29 @@ async def create_background_stream_processor(]:run_status = RunStatus.completedelse:- # Unknown stop_reason - default to completed but log warning- logger.warning(f"Unknown stop_reason '{stop_reason}' for run {run_id}, defaulting to completed")+ logger.warning(f"Unknown stop_reason '{final_stop_reason}' for run {run_id}, defaulting to completed")run_status = RunStatus.completed- # Only update if we saw a clean terminal (don't overwrite failed status set in except block)- if not saw_error or run_status != RunStatus.completed:- await run_manager.update_run_by_id_async(- run_id=run_id,- update=RunUpdate(status=run_status, stop_reason=stop_reason),- actor=actor,- )+ await run_manager.update_run_by_id_async(+ run_id=run_id,+ update=RunUpdate(status=run_status, stop_reason=final_stop_reason),+ actor=actor,+ )++ # Belt-and-suspenders: always append a terminal [DONE] chunk to ensure clients terminate+ # Even if a previous chunk set `complete`, an extra [DONE] is harmless and ensures SDKs that+ # rely on explicit [DONE] will exit.+ logger.warning(+ "[Stream Finalizer] Appending forced [DONE] for run=%s (saw_error=%s, saw_done=%s, final_stop_reason=%s)",+ run_id,+ saw_error,+ saw_done,+ final_stop_reason,+ )+ try:+ await writer.mark_complete(run_id)+ except Exception as e:+ logger.warning(f"Failed to append terminal [DONE] for run {run_id}: {e}")async def redis_sse_stream_generator(diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex b7b90cea1..dd9240d63 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -326,20 +326,31 @@ class StreamingService:# Stream completed - check if we got a terminal eventif not saw_done and not saw_error:- # Stream ended without terminal - synthesize one- logger.warning(+ # Stream ended without terminal - treat as error to avoid hanging clients+ logger.error(f"Stream for run {run_id} ended without terminal event. "- f"Agent stop_reason: {agent_loop.stop_reason}. Synthesizing [DONE]."+ f"Agent stop_reason: {agent_loop.stop_reason}. Emitting error + [DONE].")+ error_chunk = {+ "error": {+ "type": "stream_incomplete",+ "message": "Stream ended unexpectedly without a terminal event.",+ "detail": None,+ }+ }+ yield f"event: error\ndata: {json.dumps(error_chunk)}\n\n"yield "data: [DONE]\n\n"+ saw_error = Truesaw_done = True-- # set run status after successful completion- if agent_loop.stop_reason.stop_reason.value == "cancelled":- run_status = RunStatus.cancelled+ run_status = RunStatus.failed+ stop_reason = StopReasonType.errorelse:- run_status = RunStatus.completed- stop_reason = agent_loop.stop_reason.stop_reason.value+ # set run status after successful completion+ if agent_loop.stop_reason and agent_loop.stop_reason.stop_reason.value == "cancelled":+ run_status = RunStatus.cancelled+ else:+ run_status = RunStatus.completed+ stop_reason = agent_loop.stop_reason.stop_reason.value if agent_loop.stop_reason else StopReasonType.end_turn.valueexcept LLMTimeoutError as e:run_status = RunStatus.failed