Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: various hardening to prevent stale state on background mode runs (#6072)

Charles Packer2e721ddc629e
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.py
index 84a0f0130..0e142818d 100644
--- a/letta/agents/letta_agent_v3.py
+++ b/letta/agents/letta_agent_v3.py
@@ -434,6 +434,13 @@ class LettaAgentV3(LettaAgentV2):
self.logger.info("switching to unconstrained mode (allowing non-tool responses)")
self._require_tool_call = require_tool_call
+ # Always refresh messages at the start of each step to pick up external inputs
+ # (e.g., approval responses submitted by the client while this stream is running)
+ try:
+ messages = await self._refresh_messages(messages)
+ except Exception as e:
+ self.logger.warning(f"Failed to refresh messages at step start: {e}")
+
approval_request, approval_response = _maybe_get_approval_messages(messages)
tool_call_denials, tool_returns = [], []
if approval_request and approval_response:
@@ -494,7 +501,6 @@ class LettaAgentV3(LettaAgentV2):
step_id=step_id, run_id=run_id
)
- messages = await self._refresh_messages(messages)
force_tool_call = valid_tools[0]["name"] if len(valid_tools) == 1 and self._require_tool_call else None
for llm_request_attempt in range(summarizer_settings.max_summarizer_retries + 1):
try:
diff --git a/letta/server/rest_api/redis_stream_manager.py b/letta/server/rest_api/redis_stream_manager.py
index c3d1e501e..5728e19a6 100644
--- a/letta/server/rest_api/redis_stream_manager.py
+++ b/letta/server/rest_api/redis_stream_manager.py
@@ -301,11 +301,19 @@ async def create_background_stream_processor(
if should_stop_writer:
await writer.stop()
- # Update run status if not already set (e.g., by exception handler)
- if run_manager and actor and stop_reason:
+ # Derive a final stop_reason if one wasn't observed explicitly
+ final_stop_reason = stop_reason
+ if final_stop_reason is None:
+ if saw_error:
+ final_stop_reason = StopReasonType.error.value
+ elif saw_done:
+ # Treat DONE without an explicit stop_reason as an error to avoid masking failures
+ final_stop_reason = StopReasonType.error.value
+
+ # Update run status to reflect terminal outcome
+ if run_manager and actor and final_stop_reason:
# Map stop_reason to run status
- # Error states -> failed
- if stop_reason in [
+ if final_stop_reason in [
StopReasonType.error.value,
StopReasonType.llm_api_error.value,
StopReasonType.invalid_tool_call.value,
@@ -313,11 +321,9 @@ async def create_background_stream_processor(
StopReasonType.no_tool_call.value,
]:
run_status = RunStatus.failed
- # Cancelled state
- elif stop_reason == StopReasonType.cancelled.value:
+ elif final_stop_reason == StopReasonType.cancelled.value:
run_status = RunStatus.cancelled
- # Success states -> completed
- elif stop_reason in [
+ elif final_stop_reason in [
StopReasonType.end_turn.value,
StopReasonType.max_steps.value,
StopReasonType.tool_rule.value,
@@ -325,17 +331,29 @@ async def create_background_stream_processor(
]:
run_status = RunStatus.completed
else:
- # Unknown stop_reason - default to completed but log warning
- logger.warning(f"Unknown stop_reason '{stop_reason}' for run {run_id}, defaulting to completed")
+ logger.warning(f"Unknown stop_reason '{final_stop_reason}' for run {run_id}, defaulting to completed")
run_status = RunStatus.completed
- # Only update if we saw a clean terminal (don't overwrite failed status set in except block)
- if not saw_error or run_status != RunStatus.completed:
- await run_manager.update_run_by_id_async(
- run_id=run_id,
- update=RunUpdate(status=run_status, stop_reason=stop_reason),
- actor=actor,
- )
+ await run_manager.update_run_by_id_async(
+ run_id=run_id,
+ update=RunUpdate(status=run_status, stop_reason=final_stop_reason),
+ actor=actor,
+ )
+
+ # Belt-and-suspenders: always append a terminal [DONE] chunk to ensure clients terminate
+ # Even if a previous chunk set `complete`, an extra [DONE] is harmless and ensures SDKs that
+ # rely on explicit [DONE] will exit.
+ logger.warning(
+ "[Stream Finalizer] Appending forced [DONE] for run=%s (saw_error=%s, saw_done=%s, final_stop_reason=%s)",
+ run_id,
+ saw_error,
+ saw_done,
+ final_stop_reason,
+ )
+ try:
+ await writer.mark_complete(run_id)
+ except Exception as e:
+ logger.warning(f"Failed to append terminal [DONE] for run {run_id}: {e}")
async def redis_sse_stream_generator(
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index b7b90cea1..dd9240d63 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -326,20 +326,31 @@ class StreamingService:
# Stream completed - check if we got a terminal event
if not saw_done and not saw_error:
- # Stream ended without terminal - synthesize one
- logger.warning(
+ # Stream ended without terminal - treat as error to avoid hanging clients
+ logger.error(
f"Stream for run {run_id} ended without terminal event. "
- f"Agent stop_reason: {agent_loop.stop_reason}. Synthesizing [DONE]."
+ f"Agent stop_reason: {agent_loop.stop_reason}. Emitting error + [DONE]."
)
+ error_chunk = {
+ "error": {
+ "type": "stream_incomplete",
+ "message": "Stream ended unexpectedly without a terminal event.",
+ "detail": None,
+ }
+ }
+ yield f"event: error\ndata: {json.dumps(error_chunk)}\n\n"
yield "data: [DONE]\n\n"
+ saw_error = True
saw_done = True
-
- # set run status after successful completion
- if agent_loop.stop_reason.stop_reason.value == "cancelled":
- run_status = RunStatus.cancelled
+ run_status = RunStatus.failed
+ stop_reason = StopReasonType.error
else:
- run_status = RunStatus.completed
- stop_reason = agent_loop.stop_reason.stop_reason.value
+ # set run status after successful completion
+ if agent_loop.stop_reason and agent_loop.stop_reason.stop_reason.value == "cancelled":
+ run_status = RunStatus.cancelled
+ else:
+ run_status = RunStatus.completed
+ stop_reason = agent_loop.stop_reason.stop_reason.value if agent_loop.stop_reason else StopReasonType.end_turn.value
except LLMTimeoutError as e:
run_status = RunStatus.failed