Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: gate readiness on fg tasks, bg tasks, and request admission latency (#9915)

Kian Jones1a52003e3917
diff --git a/letta/monitoring/event_loop_watchdog.py b/letta/monitoring/event_loop_watchdog.py
index 61c5845cc..4e3fccf2e 100644
--- a/letta/monitoring/event_loop_watchdog.py
+++ b/letta/monitoring/event_loop_watchdog.py
@@ -212,7 +212,7 @@ class EventLoopWatchdog:
if current_lag_ms < rs.event_loop_lag_threshold_ms:
return
- from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state
+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded
if get_readiness_state() not in ("ready", "degraded"):
return # Don't override draining/warming/manual_disable
@@ -224,7 +224,7 @@ class EventLoopWatchdog:
elapsed = now - self._degraded_since
if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":
- set_readiness_state("degraded", source=f"event_loop_lag:{current_lag_ms:.0f}ms")
+ mark_degraded(f"event_loop_lag:{current_lag_ms:.0f}ms")
except Exception:
pass # Readiness gating must never crash the watchdog
@@ -235,7 +235,7 @@ class EventLoopWatchdog:
if not rs.enforcement_enabled or not rs.event_loop_lag_gating_enabled:
return
- from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state
+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state
self._degraded_since = None # Reset overload timer
@@ -249,7 +249,7 @@ class EventLoopWatchdog:
elapsed = now - self._recovery_since
if elapsed >= rs.recovery_stabilization_seconds:
- set_readiness_state("ready", source="event_loop_lag_recovered")
+ clear_degraded("event_loop_lag")
self._recovery_since = None
except Exception:
pass # Readiness gating must never crash the watchdog
diff --git a/letta/monitoring/load_gate.py b/letta/monitoring/load_gate.py
new file mode 100644
index 000000000..86ce17a18
--- /dev/null
+++ b/letta/monitoring/load_gate.py
@@ -0,0 +1,244 @@
+"""
+Per-pod readiness gating based on in-flight load and request admission wait.
+
+Three independent gates, each with its own stabilization window:
+ - fg_in_flight: foreground (interactive) runs in progress on this pod
+ - bg_in_flight: background runs in progress on this pod
+ - admission_wait: time spent waiting to acquire the conversation lock (per-request)
+
+All gates share the degraded_stabilization_seconds / recovery_stabilization_seconds
+settings from ReadinessSettings. A pod only recovers to "ready" once ALL active
+degradation sources have cleared (see readiness_state.mark_degraded / clear_degraded).
+"""
+
+import threading
+import time
+from typing import Optional
+
+from letta.log import get_logger
+
+logger = get_logger(__name__)
+
+
+def _get_readiness_settings():
+ from letta.settings import readiness_settings
+
+ return readiness_settings
+
+
+class LoadGate:
+ """Thread-safe per-pod readiness gate for in-flight counts and admission wait."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+
+ # In-flight counters tracked independently of OTel (OTel gives no read-back).
+ self._fg_count: int = 0
+ self._bg_count: int = 0
+
+ # Per-gate stabilization timers.
+ self._fg_degraded_since: Optional[float] = None
+ self._fg_recovery_since: Optional[float] = None
+
+ self._bg_degraded_since: Optional[float] = None
+ self._bg_recovery_since: Optional[float] = None
+
+ self._admission_degraded_since: Optional[float] = None
+ self._admission_recovery_since: Optional[float] = None
+
+ # ------------------------------------------------------------------
+ # In-flight tracking — called from streaming_service on start/end
+ # ------------------------------------------------------------------
+
+ def on_fg_start(self) -> None:
+ with self._lock:
+ self._fg_count += 1
+ count = self._fg_count
+ self._check_fg(count)
+
+ def on_fg_end(self) -> None:
+ with self._lock:
+ self._fg_count = max(0, self._fg_count - 1)
+ count = self._fg_count
+ self._check_fg(count)
+
+ def on_bg_start(self) -> None:
+ with self._lock:
+ self._bg_count += 1
+ count = self._bg_count
+ self._check_bg(count)
+
+ def on_bg_end(self) -> None:
+ with self._lock:
+ self._bg_count = max(0, self._bg_count - 1)
+ count = self._bg_count
+ self._check_bg(count)
+
+ def on_admission_wait(self, wait_ms: float) -> None:
+ """Evaluate admission wait after each lock acquisition."""
+ try:
+ rs = _get_readiness_settings()
+ if not rs.enforcement_enabled or not rs.admission_wait_gating_enabled:
+ return
+
+ from letta.monitoring.readiness_state import get_readiness_state
+
+ if get_readiness_state() not in ("ready", "degraded"):
+ return
+
+ now = time.monotonic()
+ if wait_ms >= rs.admission_wait_threshold_ms:
+ self._maybe_degrade_admission(now, wait_ms)
+ else:
+ self._maybe_recover_admission(now)
+ except Exception:
+ pass # Never let gating crash the request path
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _check_fg(self, count: int) -> None:
+ try:
+ rs = _get_readiness_settings()
+ if not rs.enforcement_enabled or not rs.fg_in_flight_gating_enabled:
+ return
+
+ from letta.monitoring.readiness_state import get_readiness_state
+
+ if get_readiness_state() not in ("ready", "degraded"):
+ return
+
+ now = time.monotonic()
+ if count > rs.fg_in_flight_threshold:
+ self._maybe_degrade_fg(now, count)
+ else:
+ self._maybe_recover_fg(now)
+ except Exception:
+ pass
+
+ def _check_bg(self, count: int) -> None:
+ try:
+ rs = _get_readiness_settings()
+ if not rs.enforcement_enabled or not rs.bg_in_flight_gating_enabled:
+ return
+
+ from letta.monitoring.readiness_state import get_readiness_state
+
+ if get_readiness_state() not in ("ready", "degraded"):
+ return
+
+ now = time.monotonic()
+ if count > rs.bg_in_flight_threshold:
+ self._maybe_degrade_bg(now, count)
+ else:
+ self._maybe_recover_bg(now)
+ except Exception:
+ pass
+
+ def _maybe_degrade_fg(self, now: float, count: int) -> None:
+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded
+
+ rs = _get_readiness_settings()
+ if self._fg_degraded_since is None:
+ self._fg_degraded_since = now
+ self._fg_recovery_since = None
+ return
+
+ elapsed = now - self._fg_degraded_since
+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":
+ mark_degraded(f"fg_in_flight:{count}")
+
+ def _maybe_recover_fg(self, now: float) -> None:
+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state
+
+ rs = _get_readiness_settings()
+ self._fg_degraded_since = None
+
+ if get_readiness_state() != "degraded":
+ self._fg_recovery_since = None
+ return
+
+ if self._fg_recovery_since is None:
+ self._fg_recovery_since = now
+ return
+
+ if now - self._fg_recovery_since >= rs.recovery_stabilization_seconds:
+ clear_degraded("fg_in_flight")
+ self._fg_recovery_since = None
+
+ def _maybe_degrade_bg(self, now: float, count: int) -> None:
+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded
+
+ rs = _get_readiness_settings()
+ if self._bg_degraded_since is None:
+ self._bg_degraded_since = now
+ self._bg_recovery_since = None
+ return
+
+ elapsed = now - self._bg_degraded_since
+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":
+ mark_degraded(f"bg_in_flight:{count}")
+
+ def _maybe_recover_bg(self, now: float) -> None:
+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state
+
+ rs = _get_readiness_settings()
+ self._bg_degraded_since = None
+
+ if get_readiness_state() != "degraded":
+ self._bg_recovery_since = None
+ return
+
+ if self._bg_recovery_since is None:
+ self._bg_recovery_since = now
+ return
+
+ if now - self._bg_recovery_since >= rs.recovery_stabilization_seconds:
+ clear_degraded("bg_in_flight")
+ self._bg_recovery_since = None
+
+ def _maybe_degrade_admission(self, now: float, wait_ms: float) -> None:
+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded
+
+ rs = _get_readiness_settings()
+ if self._admission_degraded_since is None:
+ self._admission_degraded_since = now
+ self._admission_recovery_since = None
+ return
+
+ elapsed = now - self._admission_degraded_since
+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":
+ mark_degraded(f"admission_wait:{wait_ms:.0f}ms")
+
+ def _maybe_recover_admission(self, now: float) -> None:
+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state
+
+ rs = _get_readiness_settings()
+ self._admission_degraded_since = None
+
+ if get_readiness_state() != "degraded":
+ self._admission_recovery_since = None
+ return
+
+ if self._admission_recovery_since is None:
+ self._admission_recovery_since = now
+ return
+
+ if now - self._admission_recovery_since >= rs.recovery_stabilization_seconds:
+ clear_degraded("admission_wait")
+ self._admission_recovery_since = None
+
+
+# Singleton — one gate per process.
+_load_gate: Optional[LoadGate] = None
+_gate_init_lock = threading.Lock()
+
+
+def get_load_gate() -> LoadGate:
+ global _load_gate
+ if _load_gate is None:
+ with _gate_init_lock:
+ if _load_gate is None:
+ _load_gate = LoadGate()
+ return _load_gate
diff --git a/letta/monitoring/readiness_state.py b/letta/monitoring/readiness_state.py
index e32457dba..a8de30798 100644
--- a/letta/monitoring/readiness_state.py
+++ b/letta/monitoring/readiness_state.py
@@ -17,6 +17,10 @@ _VALID_REASONS = {
_state_lock = threading.Lock()
_current_reason = "warming"
+# Tracks which gate sources are currently active. Recovery to "ready" only happens
+# when all sources have cleared — prevents one gate recovering while another is still firing.
+_active_degradation_sources: set = set()
+
def initialize_readiness_state(reason: str = "warming", source: str = "startup") -> str:
"""Initialize readiness telemetry state without changing probe behavior."""
@@ -55,3 +59,27 @@ def set_readiness_state(reason: str, source: str = "unknown") -> str:
def get_readiness_state() -> str:
with _state_lock:
return _current_reason
+
+
+def mark_degraded(source: str) -> None:
+ """Register a degradation source and transition to degraded state.
+
+ Multiple gates can independently call mark_degraded. The pod stays degraded
+ until all sources have called clear_degraded.
+ """
+ with _state_lock:
+ _active_degradation_sources.add(source)
+
+ set_readiness_state("degraded", source=source)
+
+
+def clear_degraded(source: str) -> None:
+ """Deregister a degradation source. Recovers to ready only when all sources clear."""
+ with _state_lock:
+ _active_degradation_sources.discard(source)
+ remaining = set(_active_degradation_sources)
+
+ if not remaining:
+ set_readiness_state("ready", source=f"{source}_recovered")
+ else:
+ logger.debug(f"Readiness gate cleared source={source}, still degraded by: {remaining}")
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index 3ebf0baeb..a7cf6cb47 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -199,6 +199,9 @@ class StreamingService:
admission_wait_ms,
attributes={"route_class": route_class},
)
+ from letta.monitoring.load_gate import get_load_gate
+
+ get_load_gate().on_admission_wait(admission_wait_ms)
# create run if tracking is enabled
run = None
@@ -430,6 +433,13 @@ class StreamingService:
)
in_flight_counter.add(1, attributes=in_flight_attrs)
+ from letta.monitoring.load_gate import get_load_gate
+
+ _load_gate = get_load_gate()
+ if is_background:
+ _load_gate.on_bg_start()
+ else:
+ _load_gate.on_fg_start()
try:
stream = agent_loop.stream(
@@ -491,6 +501,7 @@ class StreamingService:
MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)
MetricRegistry().provider_timeout_counter.add(1, attributes={"provider": provider_name})
from letta.settings import readiness_settings
+
if readiness_settings.enforcement_enabled:
set_readiness_state(reason="degraded_dependency", source="provider_timeout")
run_status = RunStatus.failed
@@ -624,6 +635,10 @@ class StreamingService:
)
in_flight_counter.add(-1, attributes=in_flight_attrs)
+ if is_background:
+ _load_gate.on_bg_end()
+ else:
+ _load_gate.on_fg_end()
return error_aware_stream()
diff --git a/letta/settings.py b/letta/settings.py
index a7b449a27..5391d1172 100644
--- a/letta/settings.py
+++ b/letta/settings.py
@@ -609,19 +609,31 @@ class ReadinessSettings(BaseSettings):
# When True, /v1/health/ returns 503 during draining state (graceful shutdown signal to k8s).
drain_returns_503: bool = Field(default=True, description="Return HTTP 503 when readiness state is 'draining'.")
- # When True, event loop lag above threshold triggers degraded_dependency state.
+ # When True, event loop lag above threshold triggers degraded state.
event_loop_lag_gating_enabled: bool = Field(default=False, description="Gate readiness on event loop lag threshold.")
event_loop_lag_threshold_ms: float = Field(
default=5000.0, ge=100.0, description="Event loop lag threshold (ms) for degraded transition."
)
+ # When True, fg in-flight count above threshold triggers degraded state.
+ fg_in_flight_gating_enabled: bool = Field(default=False, description="Gate readiness on foreground in-flight count.")
+ fg_in_flight_threshold: int = Field(default=10, ge=1, description="Foreground in-flight count threshold per pod.")
+
+ # When True, bg in-flight count above threshold triggers degraded state.
+ bg_in_flight_gating_enabled: bool = Field(default=False, description="Gate readiness on background in-flight count.")
+ bg_in_flight_threshold: int = Field(default=15, ge=1, description="Background in-flight count threshold per pod.")
+
+ # When True, request admission wait above threshold triggers degraded state.
+ admission_wait_gating_enabled: bool = Field(default=False, description="Gate readiness on request admission wait.")
+ admission_wait_threshold_ms: float = Field(default=300.0, ge=10.0, description="Admission wait threshold (ms) for degraded transition.")
+
# Stabilization window prevents flapping: state must be consistently bad for this many seconds before degrading.
degraded_stabilization_seconds: float = Field(
- default=30.0, ge=0.0, description="Seconds of sustained overload before transitioning to degraded_dependency."
+ default=30.0, ge=0.0, description="Seconds of sustained overload before transitioning to degraded."
)
# Recovery: state must be healthy for this long before returning to ready.
recovery_stabilization_seconds: float = Field(
- default=15.0, ge=0.0, description="Seconds of sustained health before recovering from degraded_dependency."
+ default=15.0, ge=0.0, description="Seconds of sustained health before recovering from degraded."
)