Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix(core): emit initial run ping before first token (#9888)
Charles Packer26cc4749d0ac
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex acd50023f..3ebf0baeb 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -1,6 +1,7 @@import asyncioimport jsonimport time+from datetime import datetime, timezonefrom typing import AsyncIterator, Optional, Unionfrom uuid import uuid4@@ -30,7 +31,7 @@ from letta.otel.metric_registry import MetricRegistryfrom letta.schemas.agent import AgentStatefrom letta.schemas.enums import AgentType, MessageStreamStatus, RunStatusfrom letta.schemas.job import LettaRequestConfig-from letta.schemas.letta_message import AssistantMessage, LettaErrorMessage, MessageType+from letta.schemas.letta_message import AssistantMessage, LettaErrorMessage, LettaPing, MessageTypefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.letta_request import ClientToolSchema, LettaStreamingRequestfrom letta.schemas.letta_response import LettaResponse@@ -57,6 +58,42 @@ from letta.utils import safe_create_tasklogger = get_logger(__name__)+async def prepend_initial_run_ping(+ stream_generator: AsyncIterator[str | bytes],+ run_id: str,+) -> AsyncIterator[str | bytes]:+ """+ Emit an immediate run_id-bearing ping before the first stream chunk.++ Device/listener mode currently waits for the first chunk that exposes run_id+ before it can attach the run. Prepending a ping lets clients bind earlier+ without changing the streaming schema surface.+ """+ yield (+ "data: "+ + LettaPing(+ id=f"ping-{uuid4()}",+ date=datetime.now(timezone.utc),+ run_id=run_id,+ ).model_dump_json()+ + "\n\n"+ )++ try:+ async for chunk in stream_generator:+ yield chunk+ except RunCancelledException as e:+ # Forward cancellation to the inner generator so its handler fires+ # (sets saw_done, run_status=None, emits [DONE]) before the finally block.+ # Without this, aclose() sends GeneratorExit which skips the handler and+ # causes the finally block to mark the run as "failed" instead of "cancelled".+ try:+ await stream_generator.athrow(e)+ except (StopAsyncIteration, RunCancelledException):+ pass+ raise++class StreamingService:"""Service for managing agent streaming responses.@@ -198,6 +235,9 @@ class StreamingService:openai_responses_websocket=openai_responses_websocket,)+ if request.include_pings and run:+ raw_stream = prepend_initial_run_ping(raw_stream, run.id)+# handle background streaming if requestedif request.background and settings.track_agent_run:if isinstance(redis_client, NoopAsyncRedisClient):diff --git a/tests/integration_test_cancellation.py b/tests/integration_test_cancellation.pyindex 6cc7a0bc7..56849d589 100644--- a/tests/integration_test_cancellation.py+++ b/tests/integration_test_cancellation.py@@ -69,6 +69,9 @@ async def accumulate_chunks(chunks: Any) -> List[Any]:prev_message_type = Noneasync for chunk in chunks:current_message_type = chunk.message_type+ # Skip keepalive/initial pings — not content messages+ if current_message_type == "ping":+ continueif prev_message_type != current_message_type:messages.append(current_message)current_message = chunkdiff --git a/tests/integration_test_human_in_the_loop.py b/tests/integration_test_human_in_the_loop.pyindex 3b3d225cf..9ec7f783e 100644--- a/tests/integration_test_human_in_the_loop.py+++ b/tests/integration_test_human_in_the_loop.py@@ -81,6 +81,10 @@ def accumulate_chunks(stream):current_message_type = getattr(chunk, "message_type", None)+ # Skip keepalive/initial pings — not content messages+ if current_message_type == "ping":+ continue+if prev_message_type != current_message_type:# Save the previous message if it existsif current_message is not None:diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex 488a9d8cc..f54de77e6 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -954,6 +954,9 @@ def accumulate_chunks(chunks: List[Any], verify_token_streaming: bool = False) -# Handle message objectsfor chunk in chunks:current_message_type = chunk.message_type+ # Skip keepalive/initial pings — not content messages+ if current_message_type == "ping":+ continueif prev_message_type != current_message_type:messages.append(current_message)if (diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.pyindex fb19126d7..8f08d8c52 100644--- a/tests/integration_test_send_message_v2.py+++ b/tests/integration_test_send_message_v2.py@@ -297,6 +297,9 @@ async def accumulate_chunks(chunks, verify_token_streaming: bool = False) -> Listry:data = json.loads(line[6:]) # Remove 'data: ' prefixif "message_type" in data:+ # Skip keepalive/initial pings — not content messages+ if data.get("message_type") == "ping":+ continue# Create proper message type objectsmessage_type = data.get("message_type")if message_type == "assistant_message":@@ -352,6 +355,10 @@ async def accumulate_chunks(chunks, verify_token_streaming: bool = False) -> Lisasync for chunk in chunks:current_message_type = chunk.message_type+ # Skip keepalive/initial pings — not content messages+ if current_message_type == "ping":+ continue+if prev_message_type != current_message_type:if current_message is not None:messages.append(current_message)