Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: handle ChatGPT keepalive and harden stream terminal state (#9783)

jnjpng6c70f44d5ff5
diff --git a/letta/llm_api/chatgpt_oauth_client.py b/letta/llm_api/chatgpt_oauth_client.py
index e5c8f4c07..fd2a6364d 100644
--- a/letta/llm_api/chatgpt_oauth_client.py
+++ b/letta/llm_api/chatgpt_oauth_client.py
@@ -1018,6 +1018,11 @@ class ChatGPTOAuthClient(LLMClientBase):
part=part,
)
+ # Keepalive events - ignore but log at info for traceability
+ elif event_type == "keepalive":
+ logger.info("ChatGPT SSE keepalive event received")
+ return None
+
# Unhandled event types
logger.warning(f"Unhandled ChatGPT SSE event type: {event_type}")
return None
diff --git a/letta/server/rest_api/redis_stream_manager.py b/letta/server/rest_api/redis_stream_manager.py
index 6ae4086dd..6e93c1d64 100644
--- a/letta/server/rest_api/redis_stream_manager.py
+++ b/letta/server/rest_api/redis_stream_manager.py
@@ -320,6 +320,60 @@ async def create_background_stream_processor(
# Write [DONE] marker to properly close the stream for clients reading from Redis
await writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)
saw_done = True
+ except asyncio.CancelledError:
+ # Task-level cancellation can happen for different reasons:
+ # - runtime/pod shutdown
+ # - parent task cancellation
+ # - rare cancellation races
+ # Distinguish explicit user run cancellation from infrastructure/task interruption.
+ logger.warning(f"Background stream processor for run {run_id} was cancelled")
+
+ run_was_explicitly_cancelled = False
+ if run_manager and actor:
+ try:
+ run = await run_manager.get_run_by_id(run_id=run_id, actor=actor)
+ run_was_explicitly_cancelled = run.status == RunStatus.cancelled
+ except Exception as lookup_err:
+ logger.warning(f"Failed to inspect run status during cancellation for run {run_id}: {lookup_err}")
+
+ if run_was_explicitly_cancelled:
+ stop_reason = StopReasonType.cancelled.value
+ logger.info(f"Run {run_id} was already cancelled; writing terminal cancelled marker")
+ try:
+ await writer.write_chunk(
+ run_id=run_id,
+ data=f"data: {LettaStopReason(stop_reason=StopReasonType.cancelled).model_dump_json()}\n\n",
+ is_complete=False,
+ )
+ await writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)
+ except Exception as write_err:
+ logger.warning(f"Failed to write terminal cancelled marker for run {run_id}: {write_err}")
+ saw_done = True
+ else:
+ stop_reason = StopReasonType.error.value
+ error_message = LettaErrorMessage(
+ run_id=run_id,
+ error_type="stream_task_cancelled",
+ message="Background stream processing was interrupted before completion. Please retry.",
+ detail=None,
+ )
+ error_metadata = {"error": error_message.model_dump()}
+ try:
+ await writer.write_chunk(
+ run_id=run_id,
+ data=f"data: {LettaStopReason(stop_reason=StopReasonType.error).model_dump_json()}\n\n",
+ is_complete=False,
+ )
+ await writer.write_chunk(
+ run_id=run_id,
+ data=f"event: error\ndata: {error_message.model_dump_json()}\n\n",
+ is_complete=False,
+ )
+ await writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)
+ except Exception as write_err:
+ logger.warning(f"Failed to write terminal error marker after cancellation for run {run_id}: {write_err}")
+ saw_error = True
+ saw_done = True
except Exception as e:
logger.error(f"Error processing stream for run {run_id}: {e}")
# Write error chunk
diff --git a/letta/server/rest_api/streaming_response.py b/letta/server/rest_api/streaming_response.py
index 9b7e5738c..21568d0dd 100644
--- a/letta/server/rest_api/streaming_response.py
+++ b/letta/server/rest_api/streaming_response.py
@@ -5,6 +5,7 @@
import asyncio
import json
import re
+import time
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from typing import Dict, Optional
@@ -51,6 +52,7 @@ async def add_keepalive_to_stream(
stream_generator: AsyncIterator[str | bytes],
run_id: str,
keepalive_interval: float = 30.0,
+ max_stream_silence: float = 1800.0,
) -> AsyncIterator[str | bytes]:
"""
Adds periodic keepalive messages to a stream to prevent connection timeouts.
@@ -59,9 +61,14 @@ async def add_keepalive_to_stream(
whether data is flowing. This ensures connections stay alive during long
operations like tool execution.
+ If no non-keepalive data is received from the wrapped stream for
+ ``max_stream_silence`` seconds, the wrapper terminates the stream to avoid
+ keeping connections alive indefinitely on ping-only hangs.
+
Args:
stream_generator: The original stream generator to wrap
keepalive_interval: Seconds between keepalive messages (default: 30)
+ max_stream_silence: Maximum seconds with no stream data before terminating (default: 1800)
Yields:
Original stream chunks interspersed with keepalive messages
@@ -71,6 +78,7 @@ async def add_keepalive_to_stream(
queue = asyncio.Queue(maxsize=1)
stream_exhausted = False
last_seq_id = None
+ last_data_time = time.monotonic()
async def stream_reader():
"""Read from the original stream and put items in the queue."""
@@ -95,6 +103,8 @@ async def add_keepalive_to_stream(
# Stream finished
break
elif msg_type == "data":
+ last_data_time = time.monotonic()
+
# Track seq_id from chunks for ping messages
if isinstance(data, str):
seq_id_match = re.search(r'"seq_id":(\d+)', data) # Look for "seq_id":<number> pattern in the SSE chunk
@@ -106,6 +116,15 @@ async def add_keepalive_to_stream(
except asyncio.TimeoutError:
# No data received within keepalive interval
if not stream_exhausted:
+ silence = time.monotonic() - last_data_time
+ if silence > max_stream_silence:
+ logger.warning(
+ f"Stream for run {run_id} has emitted only keepalives for {silence:.0f}s "
+ f"(limit {max_stream_silence}s), terminating keepalive wrapper"
+ )
+ reader_task.cancel()
+ break
+
# Send keepalive ping with the last seq_id to allow clients to track progress
yield f"data: {LettaPing(id=f'ping-{uuid4()}', date=datetime.now(timezone.utc), run_id=run_id, seq_id=last_seq_id).model_dump_json()}\n\n"
else:
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index 5d5030b72..7d51a6b43 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -497,10 +497,13 @@ class StreamingService:
# Run was explicitly cancelled - this is not an error
# The cancellation has already been handled by cancellation_aware_stream_wrapper
logger.info(f"Run {run_id} was cancelled, exiting stream gracefully")
- # Send [DONE] to properly close the stream
- yield "data: [DONE]\n\n"
+ # Mark as terminal BEFORE yielding [DONE]. Some consumers stop immediately
+ # after receiving [DONE], so code after yield may never run.
+ saw_done = True
# Don't update run status in finally - cancellation is already recorded
run_status = None # Signal to finally block to skip update
+ # Send [DONE] to properly close the stream
+ yield "data: [DONE]\n\n"
except Exception as e:
run_status = RunStatus.failed
stop_reason = LettaStopReason(stop_reason=StopReasonType.error)
@@ -521,6 +524,20 @@ class StreamingService:
# Capture for Sentry but don't re-raise to allow stream to complete gracefully
capture_sentry_exception(e)
finally:
+ # If run_status was never set and the stream ended without [DONE],
+ # mark as failed so the run doesn't stay "running" forever.
+ if run_id and self.runs_manager and run_status is None and not saw_done:
+ logger.warning(f"Run {run_id} stream ended without setting run_status or emitting [DONE]. Marking as failed.")
+ run_status = RunStatus.failed
+ stop_reason = LettaStopReason(stop_reason=StopReasonType.error)
+ error_data = {
+ "error": {
+ "run_id": run_id,
+ "error_type": "stream_incomplete",
+ "message": "Stream ended unexpectedly without a terminal event.",
+ }
+ }
+
# always update run status, whether success or failure
if run_id and self.runs_manager and run_status:
# Extract stop_reason enum value from LettaStopReason object