Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix(core): emit initial run ping before first token (#9888)

Charles Packer26cc4749d0ac
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index acd50023f..3ebf0baeb 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -1,6 +1,7 @@
import asyncio
import json
import time
+from datetime import datetime, timezone
from typing import AsyncIterator, Optional, Union
from uuid import uuid4
@@ -30,7 +31,7 @@ from letta.otel.metric_registry import MetricRegistry
from letta.schemas.agent import AgentState
from letta.schemas.enums import AgentType, MessageStreamStatus, RunStatus
from letta.schemas.job import LettaRequestConfig
-from letta.schemas.letta_message import AssistantMessage, LettaErrorMessage, MessageType
+from letta.schemas.letta_message import AssistantMessage, LettaErrorMessage, LettaPing, MessageType
from letta.schemas.letta_message_content import TextContent
from letta.schemas.letta_request import ClientToolSchema, LettaStreamingRequest
from letta.schemas.letta_response import LettaResponse
@@ -57,6 +58,42 @@ from letta.utils import safe_create_task
logger = get_logger(__name__)
+async def prepend_initial_run_ping(
+ stream_generator: AsyncIterator[str | bytes],
+ run_id: str,
+) -> AsyncIterator[str | bytes]:
+ """
+ Emit an immediate run_id-bearing ping before the first stream chunk.
+
+ Device/listener mode currently waits for the first chunk that exposes run_id
+ before it can attach the run. Prepending a ping lets clients bind earlier
+ without changing the streaming schema surface.
+ """
+ yield (
+ "data: "
+ + LettaPing(
+ id=f"ping-{uuid4()}",
+ date=datetime.now(timezone.utc),
+ run_id=run_id,
+ ).model_dump_json()
+ + "\n\n"
+ )
+
+ try:
+ async for chunk in stream_generator:
+ yield chunk
+ except RunCancelledException as e:
+ # Forward cancellation to the inner generator so its handler fires
+ # (sets saw_done, run_status=None, emits [DONE]) before the finally block.
+ # Without this, aclose() sends GeneratorExit which skips the handler and
+ # causes the finally block to mark the run as "failed" instead of "cancelled".
+ try:
+ await stream_generator.athrow(e)
+ except (StopAsyncIteration, RunCancelledException):
+ pass
+ raise
+
+
class StreamingService:
"""
Service for managing agent streaming responses.
@@ -198,6 +235,9 @@ class StreamingService:
openai_responses_websocket=openai_responses_websocket,
)
+ if request.include_pings and run:
+ raw_stream = prepend_initial_run_ping(raw_stream, run.id)
+
# handle background streaming if requested
if request.background and settings.track_agent_run:
if isinstance(redis_client, NoopAsyncRedisClient):
diff --git a/tests/integration_test_cancellation.py b/tests/integration_test_cancellation.py
index 6cc7a0bc7..56849d589 100644
--- a/tests/integration_test_cancellation.py
+++ b/tests/integration_test_cancellation.py
@@ -69,6 +69,9 @@ async def accumulate_chunks(chunks: Any) -> List[Any]:
prev_message_type = None
async for chunk in chunks:
current_message_type = chunk.message_type
+ # Skip keepalive/initial pings — not content messages
+ if current_message_type == "ping":
+ continue
if prev_message_type != current_message_type:
messages.append(current_message)
current_message = chunk
diff --git a/tests/integration_test_human_in_the_loop.py b/tests/integration_test_human_in_the_loop.py
index 3b3d225cf..9ec7f783e 100644
--- a/tests/integration_test_human_in_the_loop.py
+++ b/tests/integration_test_human_in_the_loop.py
@@ -81,6 +81,10 @@ def accumulate_chunks(stream):
current_message_type = getattr(chunk, "message_type", None)
+ # Skip keepalive/initial pings — not content messages
+ if current_message_type == "ping":
+ continue
+
if prev_message_type != current_message_type:
# Save the previous message if it exists
if current_message is not None:
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index 488a9d8cc..f54de77e6 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -954,6 +954,9 @@ def accumulate_chunks(chunks: List[Any], verify_token_streaming: bool = False) -
# Handle message objects
for chunk in chunks:
current_message_type = chunk.message_type
+ # Skip keepalive/initial pings — not content messages
+ if current_message_type == "ping":
+ continue
if prev_message_type != current_message_type:
messages.append(current_message)
if (
diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.py
index fb19126d7..8f08d8c52 100644
--- a/tests/integration_test_send_message_v2.py
+++ b/tests/integration_test_send_message_v2.py
@@ -297,6 +297,9 @@ async def accumulate_chunks(chunks, verify_token_streaming: bool = False) -> Lis
try:
data = json.loads(line[6:]) # Remove 'data: ' prefix
if "message_type" in data:
+ # Skip keepalive/initial pings — not content messages
+ if data.get("message_type") == "ping":
+ continue
# Create proper message type objects
message_type = data.get("message_type")
if message_type == "assistant_message":
@@ -352,6 +355,10 @@ async def accumulate_chunks(chunks, verify_token_streaming: bool = False) -> Lis
async for chunk in chunks:
current_message_type = chunk.message_type
+ # Skip keepalive/initial pings — not content messages
+ if current_message_type == "ping":
+ continue
+
if prev_message_type != current_message_type:
if current_message is not None:
messages.append(current_message)