Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(core): add read-only request pressure metrics for M1 (LET-7823) (#9810)
Kian Jonesa819c79d42d7
diff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.pyindex add2e0ec3..3dcf0c6a3 100644--- a/letta/otel/metric_registry.py+++ b/letta/otel/metric_registry.py@@ -2,7 +2,7 @@ from dataclasses import dataclass, fieldfrom functools import partialfrom opentelemetry import metrics-from opentelemetry.metrics import Counter, Histogram+from opentelemetry.metrics import Counter, Histogram, UpDownCounterfrom opentelemetry.metrics._internal import Gaugefrom letta.helpers.singleton import singleton@@ -28,7 +28,7 @@ class MetricRegistry:agent_id -1:N -> tool_name"""- Instrument = Counter | Histogram | Gauge+ Instrument = Counter | Histogram | Gauge | UpDownCounter_metrics: dict[str, Instrument] = field(default_factory=dict, init=False)_meter: metrics.Meter = field(init=False)@@ -54,6 +54,54 @@ class MetricRegistry:),)+ @property+ def in_flight_foreground_counter(self) -> UpDownCounter:+ return self._get_or_create_metric(+ "in_flight_foreground",+ partial(+ self._meter.create_up_down_counter,+ name="in_flight_foreground",+ description="Number of active foreground request streams.",+ unit="1",+ ),+ )++ @property+ def in_flight_background_counter(self) -> UpDownCounter:+ return self._get_or_create_metric(+ "in_flight_background",+ partial(+ self._meter.create_up_down_counter,+ name="in_flight_background",+ description="Number of active background stream processing tasks.",+ unit="1",+ ),+ )++ @property+ def request_admission_wait_ms_histogram(self) -> Histogram:+ return self._get_or_create_metric(+ "request_admission_wait_ms",+ partial(+ self._meter.create_histogram,+ name="request_admission_wait_ms",+ description="Time spent waiting for request admission control.",+ unit="ms",+ ),+ )++ @property+ def request_timeout_counter(self) -> Counter:+ return self._get_or_create_metric(+ "request_timeout_total",+ partial(+ self._meter.create_counter,+ name="request_timeout_total",+ description="Total number of timed out requests.",+ unit="1",+ ),+ )+# (includes tool_name, tool_execution_success, & step_id on failure)@propertydef tool_execution_counter(self) -> Counter:diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex 7d51a6b43..b461528cc 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -139,6 +139,7 @@ class StreamingService:agent = agent.model_copy(update={"llm_config": override_llm_config})model_compatible_token_streaming = self._is_token_streaming_compatible(agent)+ route_class = "background" if request.background else "foreground"# Determine lock key: use conversation_id if provided, else agent_id if should_locklock_key = conversation_id if conversation_id else (agent_id if should_lock else None)@@ -147,10 +148,18 @@ class StreamingService:# This prevents concurrent message processing for the same conversation/agent# Skip locking if Redis is not available (graceful degradation)if lock_key and not isinstance(redis_client, NoopAsyncRedisClient):- await redis_client.acquire_conversation_lock(- conversation_id=lock_key,- token=str(uuid4()),- )+ admission_wait_start_ns = get_utc_timestamp_ns()+ try:+ await redis_client.acquire_conversation_lock(+ conversation_id=lock_key,+ token=str(uuid4()),+ )+ finally:+ admission_wait_ms = (get_utc_timestamp_ns() - admission_wait_start_ns) / 1_000_000+ MetricRegistry().request_admission_wait_ms_histogram.record(+ admission_wait_ms,+ attributes={"route_class": route_class},+ )# create run if tracking is enabledrun = None@@ -181,6 +190,8 @@ class StreamingService:include_compaction_messages=request.include_compaction_messages,auto_mode_enabled=auto_mode_enabled,billing_context=billing_context,+ route_class=route_class,+ is_background=request.background,)# handle background streaming if requested@@ -347,6 +358,8 @@ class StreamingService:include_compaction_messages: bool = False,auto_mode_enabled: bool = False,billing_context: BillingContext | None = None,+ route_class: str = "foreground",+ is_background: bool = False,) -> AsyncIterator:"""Create a stream with unified error handling.@@ -362,6 +375,12 @@ class StreamingService:error_data = Nonesaw_done = Falsesaw_error = False+ in_flight_attrs = {"route_class": route_class}+ in_flight_counter = (+ MetricRegistry().in_flight_background_counter if is_background else MetricRegistry().in_flight_foreground_counter+ )++ in_flight_counter.add(1, attributes=in_flight_attrs)try:stream = agent_loop.stream(@@ -419,6 +438,7 @@ class StreamingService:stop_reason = agent_loop.stop_reason if agent_loop.stop_reason else LettaStopReason(stop_reason=StopReasonType.end_turn)except LLMTimeoutError as e:+ MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)run_status = RunStatus.failedstop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error)error_message = LettaErrorMessage(@@ -549,6 +569,8 @@ class StreamingService:actor=actor,)+ in_flight_counter.add(-1, attributes=in_flight_attrs)+return error_aware_stream()def _is_token_streaming_compatible(self, agent: AgentState) -> bool: