Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat(core): add SSE lifecycle metrics (LET-7825) (#9814)

Kian Jones5f970dedc3f3
diff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.py
index a1971f926..40845d51e 100644
--- a/letta/otel/metric_registry.py
+++ b/letta/otel/metric_registry.py
@@ -230,6 +230,45 @@ class MetricRegistry:
),
)
+ # (includes route_class)
+ @property
+ def sse_active_sessions_counter(self) -> UpDownCounter:
+ return self._get_or_create_metric(
+ "sse_active_sessions",
+ partial(
+ self._meter.create_up_down_counter,
+ name="sse_active_sessions",
+ description="Number of active SSE streaming sessions.",
+ unit="1",
+ ),
+ )
+
+ # (includes reason, route_class)
+ @property
+ def sse_disconnect_counter(self) -> Counter:
+ return self._get_or_create_metric(
+ "sse_disconnect_total",
+ partial(
+ self._meter.create_counter,
+ name="sse_disconnect_total",
+ description="Total number of non-clean SSE stream terminations.",
+ unit="1",
+ ),
+ )
+
+ # (includes route_class)
+ @property
+ def sse_duration_ms_histogram(self) -> Histogram:
+ return self._get_or_create_metric(
+ "sse_duration_ms",
+ partial(
+ self._meter.create_histogram,
+ name="sse_duration_ms",
+ description="Lifetime duration of SSE stream sessions.",
+ unit="ms",
+ ),
+ )
+
# Database connection pool metrics
# (includes engine_name, pool_mode)
@property
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index b461528cc..aba80f3da 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -1,3 +1,4 @@
+import asyncio
import json
import time
from typing import AsyncIterator, Optional, Union
@@ -247,6 +248,9 @@ class StreamingService:
if request.include_pings and settings.enable_keepalive:
stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval, run_id=run.id)
+ # Track SSE lifecycle metrics on the final stream returned to clients.
+ stream = self._create_sse_lifecycle_stream(stream, route_class=route_class)
+
result = StreamingResponseWithStatusCode(
stream,
media_type="text/event-stream",
@@ -591,6 +595,95 @@ class StreamingService:
]
return base_compatible or google_letta_v1
+ @staticmethod
+ def _map_sse_error_type_to_disconnect_reason(error_type: Optional[str]) -> str:
+ """Map stream error types to SSE disconnect reason taxonomy."""
+ if error_type == "llm_timeout":
+ return "timeout"
+ if error_type in {
+ "llm_error",
+ "llm_rate_limit",
+ "llm_authentication",
+ "llm_empty_response",
+ "internal_error",
+ "stream_incomplete",
+ }:
+ return "upstream_error"
+ return "unknown"
+
+ @staticmethod
+ def _extract_sse_error_type(chunk: str) -> Optional[str]:
+ """Extract Letta stream error_type from an SSE error chunk."""
+ if not ("\nevent: error" in chunk or chunk.startswith("event: error")):
+ return None
+
+ for line in chunk.splitlines():
+ if not line.startswith("data: "):
+ continue
+ try:
+ payload = json.loads(line[len("data: ") :])
+ except json.JSONDecodeError:
+ continue
+
+ if isinstance(payload, dict):
+ error_type = payload.get("error_type")
+ if isinstance(error_type, str):
+ return error_type
+ return None
+
+ def _create_sse_lifecycle_stream(self, stream_generator: AsyncIterator, route_class: str) -> AsyncIterator:
+ """Wrap a stream generator with SSE lifecycle metrics instrumentation."""
+
+ async def instrumented_stream():
+ start_ns = get_utc_timestamp_ns()
+ attrs = {"route_class": route_class}
+ saw_done = False
+ saw_error = False
+ error_type = None
+ disconnect_reason = None
+
+ MetricRegistry().sse_active_sessions_counter.add(1, attributes=attrs)
+
+ try:
+ async for chunk in stream_generator:
+ if isinstance(chunk, str):
+ if "\ndata: [DONE]" in chunk or chunk.startswith("data: [DONE]"):
+ saw_done = True
+ if "\nevent: error" in chunk or chunk.startswith("event: error"):
+ saw_error = True
+ parsed_error_type = self._extract_sse_error_type(chunk)
+ if parsed_error_type:
+ error_type = parsed_error_type
+
+ yield chunk
+ except asyncio.CancelledError:
+ disconnect_reason = "client_cancel"
+ raise
+ except (BrokenPipeError, ConnectionError, ConnectionResetError):
+ disconnect_reason = "network_error"
+ raise
+ except TimeoutError:
+ disconnect_reason = "timeout"
+ raise
+ finally:
+ MetricRegistry().sse_active_sessions_counter.add(-1, attributes=attrs)
+
+ duration_ms = (get_utc_timestamp_ns() - start_ns) / 1_000_000
+ MetricRegistry().sse_duration_ms_histogram.record(duration_ms, attributes=attrs)
+
+ if disconnect_reason is None and saw_error:
+ disconnect_reason = self._map_sse_error_type_to_disconnect_reason(error_type)
+ if disconnect_reason is None and not saw_done:
+ disconnect_reason = "unknown"
+
+ if disconnect_reason:
+ MetricRegistry().sse_disconnect_counter.add(
+ 1,
+ attributes={"reason": disconnect_reason, "route_class": route_class},
+ )
+
+ return instrumented_stream()
+
async def _create_run(
self, agent_id: str, request: LettaStreamingRequest, run_type: str, actor: User, conversation_id: Optional[str] = None
) -> PydanticRun: