Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(core): add SSE lifecycle metrics (LET-7825) (#9814)
Kian Jones5f970dedc3f3
diff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.pyindex a1971f926..40845d51e 100644--- a/letta/otel/metric_registry.py+++ b/letta/otel/metric_registry.py@@ -230,6 +230,45 @@ class MetricRegistry:),)+ # (includes route_class)+ @property+ def sse_active_sessions_counter(self) -> UpDownCounter:+ return self._get_or_create_metric(+ "sse_active_sessions",+ partial(+ self._meter.create_up_down_counter,+ name="sse_active_sessions",+ description="Number of active SSE streaming sessions.",+ unit="1",+ ),+ )++ # (includes reason, route_class)+ @property+ def sse_disconnect_counter(self) -> Counter:+ return self._get_or_create_metric(+ "sse_disconnect_total",+ partial(+ self._meter.create_counter,+ name="sse_disconnect_total",+ description="Total number of non-clean SSE stream terminations.",+ unit="1",+ ),+ )++ # (includes route_class)+ @property+ def sse_duration_ms_histogram(self) -> Histogram:+ return self._get_or_create_metric(+ "sse_duration_ms",+ partial(+ self._meter.create_histogram,+ name="sse_duration_ms",+ description="Lifetime duration of SSE stream sessions.",+ unit="ms",+ ),+ )+# Database connection pool metrics# (includes engine_name, pool_mode)@propertydiff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex b461528cc..aba80f3da 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -1,3 +1,4 @@+import asyncioimport jsonimport timefrom typing import AsyncIterator, Optional, Union@@ -247,6 +248,9 @@ class StreamingService:if request.include_pings and settings.enable_keepalive:stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval, run_id=run.id)+ # Track SSE lifecycle metrics on the final stream returned to clients.+ stream = self._create_sse_lifecycle_stream(stream, route_class=route_class)+result = StreamingResponseWithStatusCode(stream,media_type="text/event-stream",@@ -591,6 +595,95 @@ class StreamingService:]return base_compatible or google_letta_v1+ @staticmethod+ def _map_sse_error_type_to_disconnect_reason(error_type: Optional[str]) -> str:+ """Map stream error types to SSE disconnect reason taxonomy."""+ if error_type == "llm_timeout":+ return "timeout"+ if error_type in {+ "llm_error",+ "llm_rate_limit",+ "llm_authentication",+ "llm_empty_response",+ "internal_error",+ "stream_incomplete",+ }:+ return "upstream_error"+ return "unknown"++ @staticmethod+ def _extract_sse_error_type(chunk: str) -> Optional[str]:+ """Extract Letta stream error_type from an SSE error chunk."""+ if not ("\nevent: error" in chunk or chunk.startswith("event: error")):+ return None++ for line in chunk.splitlines():+ if not line.startswith("data: "):+ continue+ try:+ payload = json.loads(line[len("data: ") :])+ except json.JSONDecodeError:+ continue++ if isinstance(payload, dict):+ error_type = payload.get("error_type")+ if isinstance(error_type, str):+ return error_type+ return None++ def _create_sse_lifecycle_stream(self, stream_generator: AsyncIterator, route_class: str) -> AsyncIterator:+ """Wrap a stream generator with SSE lifecycle metrics instrumentation."""++ async def instrumented_stream():+ start_ns = get_utc_timestamp_ns()+ attrs = {"route_class": route_class}+ saw_done = False+ saw_error = False+ error_type = None+ disconnect_reason = None++ MetricRegistry().sse_active_sessions_counter.add(1, attributes=attrs)++ try:+ async for chunk in stream_generator:+ if isinstance(chunk, str):+ if "\ndata: [DONE]" in chunk or chunk.startswith("data: [DONE]"):+ saw_done = True+ if "\nevent: error" in chunk or chunk.startswith("event: error"):+ saw_error = True+ parsed_error_type = self._extract_sse_error_type(chunk)+ if parsed_error_type:+ error_type = parsed_error_type++ yield chunk+ except asyncio.CancelledError:+ disconnect_reason = "client_cancel"+ raise+ except (BrokenPipeError, ConnectionError, ConnectionResetError):+ disconnect_reason = "network_error"+ raise+ except TimeoutError:+ disconnect_reason = "timeout"+ raise+ finally:+ MetricRegistry().sse_active_sessions_counter.add(-1, attributes=attrs)++ duration_ms = (get_utc_timestamp_ns() - start_ns) / 1_000_000+ MetricRegistry().sse_duration_ms_histogram.record(duration_ms, attributes=attrs)++ if disconnect_reason is None and saw_error:+ disconnect_reason = self._map_sse_error_type_to_disconnect_reason(error_type)+ if disconnect_reason is None and not saw_done:+ disconnect_reason = "unknown"++ if disconnect_reason:+ MetricRegistry().sse_disconnect_counter.add(+ 1,+ attributes={"reason": disconnect_reason, "route_class": route_class},+ )++ return instrumented_stream()+async def _create_run(self, agent_id: str, request: LettaStreamingRequest, run_type: str, actor: User, conversation_id: Optional[str] = None) -> PydanticRun: