Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat(core): add read-only request pressure metrics for M1 (LET-7823) (#9810)

Kian Jonesa819c79d42d7
diff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.py
index add2e0ec3..3dcf0c6a3 100644
--- a/letta/otel/metric_registry.py
+++ b/letta/otel/metric_registry.py
@@ -2,7 +2,7 @@ from dataclasses import dataclass, field
from functools import partial
from opentelemetry import metrics
-from opentelemetry.metrics import Counter, Histogram
+from opentelemetry.metrics import Counter, Histogram, UpDownCounter
from opentelemetry.metrics._internal import Gauge
from letta.helpers.singleton import singleton
@@ -28,7 +28,7 @@ class MetricRegistry:
agent_id -1:N -> tool_name
"""
- Instrument = Counter | Histogram | Gauge
+ Instrument = Counter | Histogram | Gauge | UpDownCounter
_metrics: dict[str, Instrument] = field(default_factory=dict, init=False)
_meter: metrics.Meter = field(init=False)
@@ -54,6 +54,54 @@ class MetricRegistry:
),
)
+ @property
+ def in_flight_foreground_counter(self) -> UpDownCounter:
+ return self._get_or_create_metric(
+ "in_flight_foreground",
+ partial(
+ self._meter.create_up_down_counter,
+ name="in_flight_foreground",
+ description="Number of active foreground request streams.",
+ unit="1",
+ ),
+ )
+
+ @property
+ def in_flight_background_counter(self) -> UpDownCounter:
+ return self._get_or_create_metric(
+ "in_flight_background",
+ partial(
+ self._meter.create_up_down_counter,
+ name="in_flight_background",
+ description="Number of active background stream processing tasks.",
+ unit="1",
+ ),
+ )
+
+ @property
+ def request_admission_wait_ms_histogram(self) -> Histogram:
+ return self._get_or_create_metric(
+ "request_admission_wait_ms",
+ partial(
+ self._meter.create_histogram,
+ name="request_admission_wait_ms",
+ description="Time spent waiting for request admission control.",
+ unit="ms",
+ ),
+ )
+
+ @property
+ def request_timeout_counter(self) -> Counter:
+ return self._get_or_create_metric(
+ "request_timeout_total",
+ partial(
+ self._meter.create_counter,
+ name="request_timeout_total",
+ description="Total number of timed out requests.",
+ unit="1",
+ ),
+ )
+
# (includes tool_name, tool_execution_success, & step_id on failure)
@property
def tool_execution_counter(self) -> Counter:
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index 7d51a6b43..b461528cc 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -139,6 +139,7 @@ class StreamingService:
agent = agent.model_copy(update={"llm_config": override_llm_config})
model_compatible_token_streaming = self._is_token_streaming_compatible(agent)
+ route_class = "background" if request.background else "foreground"
# Determine lock key: use conversation_id if provided, else agent_id if should_lock
lock_key = conversation_id if conversation_id else (agent_id if should_lock else None)
@@ -147,10 +148,18 @@ class StreamingService:
# This prevents concurrent message processing for the same conversation/agent
# Skip locking if Redis is not available (graceful degradation)
if lock_key and not isinstance(redis_client, NoopAsyncRedisClient):
- await redis_client.acquire_conversation_lock(
- conversation_id=lock_key,
- token=str(uuid4()),
- )
+ admission_wait_start_ns = get_utc_timestamp_ns()
+ try:
+ await redis_client.acquire_conversation_lock(
+ conversation_id=lock_key,
+ token=str(uuid4()),
+ )
+ finally:
+ admission_wait_ms = (get_utc_timestamp_ns() - admission_wait_start_ns) / 1_000_000
+ MetricRegistry().request_admission_wait_ms_histogram.record(
+ admission_wait_ms,
+ attributes={"route_class": route_class},
+ )
# create run if tracking is enabled
run = None
@@ -181,6 +190,8 @@ class StreamingService:
include_compaction_messages=request.include_compaction_messages,
auto_mode_enabled=auto_mode_enabled,
billing_context=billing_context,
+ route_class=route_class,
+ is_background=request.background,
)
# handle background streaming if requested
@@ -347,6 +358,8 @@ class StreamingService:
include_compaction_messages: bool = False,
auto_mode_enabled: bool = False,
billing_context: BillingContext | None = None,
+ route_class: str = "foreground",
+ is_background: bool = False,
) -> AsyncIterator:
"""
Create a stream with unified error handling.
@@ -362,6 +375,12 @@ class StreamingService:
error_data = None
saw_done = False
saw_error = False
+ in_flight_attrs = {"route_class": route_class}
+ in_flight_counter = (
+ MetricRegistry().in_flight_background_counter if is_background else MetricRegistry().in_flight_foreground_counter
+ )
+
+ in_flight_counter.add(1, attributes=in_flight_attrs)
try:
stream = agent_loop.stream(
@@ -419,6 +438,7 @@ class StreamingService:
stop_reason = agent_loop.stop_reason if agent_loop.stop_reason else LettaStopReason(stop_reason=StopReasonType.end_turn)
except LLMTimeoutError as e:
+ MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)
run_status = RunStatus.failed
stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error)
error_message = LettaErrorMessage(
@@ -549,6 +569,8 @@ class StreamingService:
actor=actor,
)
+ in_flight_counter.add(-1, attributes=in_flight_attrs)
+
return error_aware_stream()
def _is_token_streaming_compatible(self, agent: AgentState) -> bool: