Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: handle ChatGPT keepalive and harden stream terminal state (#9783)
jnjpng6c70f44d5ff5
diff --git a/letta/llm_api/chatgpt_oauth_client.py b/letta/llm_api/chatgpt_oauth_client.pyindex e5c8f4c07..fd2a6364d 100644--- a/letta/llm_api/chatgpt_oauth_client.py+++ b/letta/llm_api/chatgpt_oauth_client.py@@ -1018,6 +1018,11 @@ class ChatGPTOAuthClient(LLMClientBase):part=part,)+ # Keepalive events - ignore but log at info for traceability+ elif event_type == "keepalive":+ logger.info("ChatGPT SSE keepalive event received")+ return None+# Unhandled event typeslogger.warning(f"Unhandled ChatGPT SSE event type: {event_type}")return Nonediff --git a/letta/server/rest_api/redis_stream_manager.py b/letta/server/rest_api/redis_stream_manager.pyindex 6ae4086dd..6e93c1d64 100644--- a/letta/server/rest_api/redis_stream_manager.py+++ b/letta/server/rest_api/redis_stream_manager.py@@ -320,6 +320,60 @@ async def create_background_stream_processor(# Write [DONE] marker to properly close the stream for clients reading from Redisawait writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)saw_done = True+ except asyncio.CancelledError:+ # Task-level cancellation can happen for different reasons:+ # - runtime/pod shutdown+ # - parent task cancellation+ # - rare cancellation races+ # Distinguish explicit user run cancellation from infrastructure/task interruption.+ logger.warning(f"Background stream processor for run {run_id} was cancelled")++ run_was_explicitly_cancelled = False+ if run_manager and actor:+ try:+ run = await run_manager.get_run_by_id(run_id=run_id, actor=actor)+ run_was_explicitly_cancelled = run.status == RunStatus.cancelled+ except Exception as lookup_err:+ logger.warning(f"Failed to inspect run status during cancellation for run {run_id}: {lookup_err}")++ if run_was_explicitly_cancelled:+ stop_reason = StopReasonType.cancelled.value+ logger.info(f"Run {run_id} was already cancelled; writing terminal cancelled marker")+ try:+ await writer.write_chunk(+ run_id=run_id,+ data=f"data: {LettaStopReason(stop_reason=StopReasonType.cancelled).model_dump_json()}\n\n",+ is_complete=False,+ )+ await writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)+ except Exception as write_err:+ logger.warning(f"Failed to write terminal cancelled marker for run {run_id}: {write_err}")+ saw_done = True+ else:+ stop_reason = StopReasonType.error.value+ error_message = LettaErrorMessage(+ run_id=run_id,+ error_type="stream_task_cancelled",+ message="Background stream processing was interrupted before completion. Please retry.",+ detail=None,+ )+ error_metadata = {"error": error_message.model_dump()}+ try:+ await writer.write_chunk(+ run_id=run_id,+ data=f"data: {LettaStopReason(stop_reason=StopReasonType.error).model_dump_json()}\n\n",+ is_complete=False,+ )+ await writer.write_chunk(+ run_id=run_id,+ data=f"event: error\ndata: {error_message.model_dump_json()}\n\n",+ is_complete=False,+ )+ await writer.write_chunk(run_id=run_id, data="data: [DONE]\n\n", is_complete=True)+ except Exception as write_err:+ logger.warning(f"Failed to write terminal error marker after cancellation for run {run_id}: {write_err}")+ saw_error = True+ saw_done = Trueexcept Exception as e:logger.error(f"Error processing stream for run {run_id}: {e}")# Write error chunkdiff --git a/letta/server/rest_api/streaming_response.py b/letta/server/rest_api/streaming_response.pyindex 9b7e5738c..21568d0dd 100644--- a/letta/server/rest_api/streaming_response.py+++ b/letta/server/rest_api/streaming_response.py@@ -5,6 +5,7 @@import asyncioimport jsonimport re+import timefrom collections.abc import AsyncIteratorfrom datetime import datetime, timezonefrom typing import Dict, Optional@@ -51,6 +52,7 @@ async def add_keepalive_to_stream(stream_generator: AsyncIterator[str | bytes],run_id: str,keepalive_interval: float = 30.0,+ max_stream_silence: float = 1800.0,) -> AsyncIterator[str | bytes]:"""Adds periodic keepalive messages to a stream to prevent connection timeouts.@@ -59,9 +61,14 @@ async def add_keepalive_to_stream(whether data is flowing. This ensures connections stay alive during longoperations like tool execution.+ If no non-keepalive data is received from the wrapped stream for+ ``max_stream_silence`` seconds, the wrapper terminates the stream to avoid+ keeping connections alive indefinitely on ping-only hangs.+Args:stream_generator: The original stream generator to wrapkeepalive_interval: Seconds between keepalive messages (default: 30)+ max_stream_silence: Maximum seconds with no stream data before terminating (default: 1800)Yields:Original stream chunks interspersed with keepalive messages@@ -71,6 +78,7 @@ async def add_keepalive_to_stream(queue = asyncio.Queue(maxsize=1)stream_exhausted = Falselast_seq_id = None+ last_data_time = time.monotonic()async def stream_reader():"""Read from the original stream and put items in the queue."""@@ -95,6 +103,8 @@ async def add_keepalive_to_stream(# Stream finishedbreakelif msg_type == "data":+ last_data_time = time.monotonic()+# Track seq_id from chunks for ping messagesif isinstance(data, str):seq_id_match = re.search(r'"seq_id":(\d+)', data) # Look for "seq_id":<number> pattern in the SSE chunk@@ -106,6 +116,15 @@ async def add_keepalive_to_stream(except asyncio.TimeoutError:# No data received within keepalive intervalif not stream_exhausted:+ silence = time.monotonic() - last_data_time+ if silence > max_stream_silence:+ logger.warning(+ f"Stream for run {run_id} has emitted only keepalives for {silence:.0f}s "+ f"(limit {max_stream_silence}s), terminating keepalive wrapper"+ )+ reader_task.cancel()+ break+# Send keepalive ping with the last seq_id to allow clients to track progressyield f"data: {LettaPing(id=f'ping-{uuid4()}', date=datetime.now(timezone.utc), run_id=run_id, seq_id=last_seq_id).model_dump_json()}\n\n"else:diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex 5d5030b72..7d51a6b43 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -497,10 +497,13 @@ class StreamingService:# Run was explicitly cancelled - this is not an error# The cancellation has already been handled by cancellation_aware_stream_wrapperlogger.info(f"Run {run_id} was cancelled, exiting stream gracefully")- # Send [DONE] to properly close the stream- yield "data: [DONE]\n\n"+ # Mark as terminal BEFORE yielding [DONE]. Some consumers stop immediately+ # after receiving [DONE], so code after yield may never run.+ saw_done = True# Don't update run status in finally - cancellation is already recordedrun_status = None # Signal to finally block to skip update+ # Send [DONE] to properly close the stream+ yield "data: [DONE]\n\n"except Exception as e:run_status = RunStatus.failedstop_reason = LettaStopReason(stop_reason=StopReasonType.error)@@ -521,6 +524,20 @@ class StreamingService:# Capture for Sentry but don't re-raise to allow stream to complete gracefullycapture_sentry_exception(e)finally:+ # If run_status was never set and the stream ended without [DONE],+ # mark as failed so the run doesn't stay "running" forever.+ if run_id and self.runs_manager and run_status is None and not saw_done:+ logger.warning(f"Run {run_id} stream ended without setting run_status or emitting [DONE]. Marking as failed.")+ run_status = RunStatus.failed+ stop_reason = LettaStopReason(stop_reason=StopReasonType.error)+ error_data = {+ "error": {+ "run_id": run_id,+ "error_type": "stream_incomplete",+ "message": "Stream ended unexpectedly without a terminal event.",+ }+ }+# always update run status, whether success or failureif run_id and self.runs_manager and run_status:# Extract stop_reason enum value from LettaStopReason object