Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: gate readiness on fg tasks, bg tasks, and request admission latency (#9915)
Kian Jones1a52003e3917
diff --git a/letta/monitoring/event_loop_watchdog.py b/letta/monitoring/event_loop_watchdog.pyindex 61c5845cc..4e3fccf2e 100644--- a/letta/monitoring/event_loop_watchdog.py+++ b/letta/monitoring/event_loop_watchdog.py@@ -212,7 +212,7 @@ class EventLoopWatchdog:if current_lag_ms < rs.event_loop_lag_threshold_ms:return- from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state+ from letta.monitoring.readiness_state import get_readiness_state, mark_degradedif get_readiness_state() not in ("ready", "degraded"):return # Don't override draining/warming/manual_disable@@ -224,7 +224,7 @@ class EventLoopWatchdog:elapsed = now - self._degraded_sinceif elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":- set_readiness_state("degraded", source=f"event_loop_lag:{current_lag_ms:.0f}ms")+ mark_degraded(f"event_loop_lag:{current_lag_ms:.0f}ms")except Exception:pass # Readiness gating must never crash the watchdog@@ -235,7 +235,7 @@ class EventLoopWatchdog:if not rs.enforcement_enabled or not rs.event_loop_lag_gating_enabled:return- from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_stateself._degraded_since = None # Reset overload timer@@ -249,7 +249,7 @@ class EventLoopWatchdog:elapsed = now - self._recovery_sinceif elapsed >= rs.recovery_stabilization_seconds:- set_readiness_state("ready", source="event_loop_lag_recovered")+ clear_degraded("event_loop_lag")self._recovery_since = Noneexcept Exception:pass # Readiness gating must never crash the watchdogdiff --git a/letta/monitoring/load_gate.py b/letta/monitoring/load_gate.pynew file mode 100644index 000000000..86ce17a18--- /dev/null+++ b/letta/monitoring/load_gate.py@@ -0,0 +1,244 @@+"""+Per-pod readiness gating based on in-flight load and request admission wait.++Three independent gates, each with its own stabilization window:+ - fg_in_flight: foreground (interactive) runs in progress on this pod+ - bg_in_flight: background runs in progress on this pod+ - admission_wait: time spent waiting to acquire the conversation lock (per-request)++All gates share the degraded_stabilization_seconds / recovery_stabilization_seconds+settings from ReadinessSettings. A pod only recovers to "ready" once ALL active+degradation sources have cleared (see readiness_state.mark_degraded / clear_degraded).+"""++import threading+import time+from typing import Optional++from letta.log import get_logger++logger = get_logger(__name__)+++def _get_readiness_settings():+ from letta.settings import readiness_settings++ return readiness_settings+++class LoadGate:+ """Thread-safe per-pod readiness gate for in-flight counts and admission wait."""++ def __init__(self):+ self._lock = threading.Lock()++ # In-flight counters tracked independently of OTel (OTel gives no read-back).+ self._fg_count: int = 0+ self._bg_count: int = 0++ # Per-gate stabilization timers.+ self._fg_degraded_since: Optional[float] = None+ self._fg_recovery_since: Optional[float] = None++ self._bg_degraded_since: Optional[float] = None+ self._bg_recovery_since: Optional[float] = None++ self._admission_degraded_since: Optional[float] = None+ self._admission_recovery_since: Optional[float] = None++ # ------------------------------------------------------------------+ # In-flight tracking — called from streaming_service on start/end+ # ------------------------------------------------------------------++ def on_fg_start(self) -> None:+ with self._lock:+ self._fg_count += 1+ count = self._fg_count+ self._check_fg(count)++ def on_fg_end(self) -> None:+ with self._lock:+ self._fg_count = max(0, self._fg_count - 1)+ count = self._fg_count+ self._check_fg(count)++ def on_bg_start(self) -> None:+ with self._lock:+ self._bg_count += 1+ count = self._bg_count+ self._check_bg(count)++ def on_bg_end(self) -> None:+ with self._lock:+ self._bg_count = max(0, self._bg_count - 1)+ count = self._bg_count+ self._check_bg(count)++ def on_admission_wait(self, wait_ms: float) -> None:+ """Evaluate admission wait after each lock acquisition."""+ try:+ rs = _get_readiness_settings()+ if not rs.enforcement_enabled or not rs.admission_wait_gating_enabled:+ return++ from letta.monitoring.readiness_state import get_readiness_state++ if get_readiness_state() not in ("ready", "degraded"):+ return++ now = time.monotonic()+ if wait_ms >= rs.admission_wait_threshold_ms:+ self._maybe_degrade_admission(now, wait_ms)+ else:+ self._maybe_recover_admission(now)+ except Exception:+ pass # Never let gating crash the request path++ # ------------------------------------------------------------------+ # Internal helpers+ # ------------------------------------------------------------------++ def _check_fg(self, count: int) -> None:+ try:+ rs = _get_readiness_settings()+ if not rs.enforcement_enabled or not rs.fg_in_flight_gating_enabled:+ return++ from letta.monitoring.readiness_state import get_readiness_state++ if get_readiness_state() not in ("ready", "degraded"):+ return++ now = time.monotonic()+ if count > rs.fg_in_flight_threshold:+ self._maybe_degrade_fg(now, count)+ else:+ self._maybe_recover_fg(now)+ except Exception:+ pass++ def _check_bg(self, count: int) -> None:+ try:+ rs = _get_readiness_settings()+ if not rs.enforcement_enabled or not rs.bg_in_flight_gating_enabled:+ return++ from letta.monitoring.readiness_state import get_readiness_state++ if get_readiness_state() not in ("ready", "degraded"):+ return++ now = time.monotonic()+ if count > rs.bg_in_flight_threshold:+ self._maybe_degrade_bg(now, count)+ else:+ self._maybe_recover_bg(now)+ except Exception:+ pass++ def _maybe_degrade_fg(self, now: float, count: int) -> None:+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded++ rs = _get_readiness_settings()+ if self._fg_degraded_since is None:+ self._fg_degraded_since = now+ self._fg_recovery_since = None+ return++ elapsed = now - self._fg_degraded_since+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":+ mark_degraded(f"fg_in_flight:{count}")++ def _maybe_recover_fg(self, now: float) -> None:+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state++ rs = _get_readiness_settings()+ self._fg_degraded_since = None++ if get_readiness_state() != "degraded":+ self._fg_recovery_since = None+ return++ if self._fg_recovery_since is None:+ self._fg_recovery_since = now+ return++ if now - self._fg_recovery_since >= rs.recovery_stabilization_seconds:+ clear_degraded("fg_in_flight")+ self._fg_recovery_since = None++ def _maybe_degrade_bg(self, now: float, count: int) -> None:+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded++ rs = _get_readiness_settings()+ if self._bg_degraded_since is None:+ self._bg_degraded_since = now+ self._bg_recovery_since = None+ return++ elapsed = now - self._bg_degraded_since+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":+ mark_degraded(f"bg_in_flight:{count}")++ def _maybe_recover_bg(self, now: float) -> None:+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state++ rs = _get_readiness_settings()+ self._bg_degraded_since = None++ if get_readiness_state() != "degraded":+ self._bg_recovery_since = None+ return++ if self._bg_recovery_since is None:+ self._bg_recovery_since = now+ return++ if now - self._bg_recovery_since >= rs.recovery_stabilization_seconds:+ clear_degraded("bg_in_flight")+ self._bg_recovery_since = None++ def _maybe_degrade_admission(self, now: float, wait_ms: float) -> None:+ from letta.monitoring.readiness_state import get_readiness_state, mark_degraded++ rs = _get_readiness_settings()+ if self._admission_degraded_since is None:+ self._admission_degraded_since = now+ self._admission_recovery_since = None+ return++ elapsed = now - self._admission_degraded_since+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":+ mark_degraded(f"admission_wait:{wait_ms:.0f}ms")++ def _maybe_recover_admission(self, now: float) -> None:+ from letta.monitoring.readiness_state import clear_degraded, get_readiness_state++ rs = _get_readiness_settings()+ self._admission_degraded_since = None++ if get_readiness_state() != "degraded":+ self._admission_recovery_since = None+ return++ if self._admission_recovery_since is None:+ self._admission_recovery_since = now+ return++ if now - self._admission_recovery_since >= rs.recovery_stabilization_seconds:+ clear_degraded("admission_wait")+ self._admission_recovery_since = None+++# Singleton — one gate per process.+_load_gate: Optional[LoadGate] = None+_gate_init_lock = threading.Lock()+++def get_load_gate() -> LoadGate:+ global _load_gate+ if _load_gate is None:+ with _gate_init_lock:+ if _load_gate is None:+ _load_gate = LoadGate()+ return _load_gatediff --git a/letta/monitoring/readiness_state.py b/letta/monitoring/readiness_state.pyindex e32457dba..a8de30798 100644--- a/letta/monitoring/readiness_state.py+++ b/letta/monitoring/readiness_state.py@@ -17,6 +17,10 @@ _VALID_REASONS = {_state_lock = threading.Lock()_current_reason = "warming"+# Tracks which gate sources are currently active. Recovery to "ready" only happens+# when all sources have cleared — prevents one gate recovering while another is still firing.+_active_degradation_sources: set = set()+def initialize_readiness_state(reason: str = "warming", source: str = "startup") -> str:"""Initialize readiness telemetry state without changing probe behavior."""@@ -55,3 +59,27 @@ def set_readiness_state(reason: str, source: str = "unknown") -> str:def get_readiness_state() -> str:with _state_lock:return _current_reason+++def mark_degraded(source: str) -> None:+ """Register a degradation source and transition to degraded state.++ Multiple gates can independently call mark_degraded. The pod stays degraded+ until all sources have called clear_degraded.+ """+ with _state_lock:+ _active_degradation_sources.add(source)++ set_readiness_state("degraded", source=source)+++def clear_degraded(source: str) -> None:+ """Deregister a degradation source. Recovers to ready only when all sources clear."""+ with _state_lock:+ _active_degradation_sources.discard(source)+ remaining = set(_active_degradation_sources)++ if not remaining:+ set_readiness_state("ready", source=f"{source}_recovered")+ else:+ logger.debug(f"Readiness gate cleared source={source}, still degraded by: {remaining}")diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex 3ebf0baeb..a7cf6cb47 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -199,6 +199,9 @@ class StreamingService:admission_wait_ms,attributes={"route_class": route_class},)+ from letta.monitoring.load_gate import get_load_gate++ get_load_gate().on_admission_wait(admission_wait_ms)# create run if tracking is enabledrun = None@@ -430,6 +433,13 @@ class StreamingService:)in_flight_counter.add(1, attributes=in_flight_attrs)+ from letta.monitoring.load_gate import get_load_gate++ _load_gate = get_load_gate()+ if is_background:+ _load_gate.on_bg_start()+ else:+ _load_gate.on_fg_start()try:stream = agent_loop.stream(@@ -491,6 +501,7 @@ class StreamingService:MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)MetricRegistry().provider_timeout_counter.add(1, attributes={"provider": provider_name})from letta.settings import readiness_settings+if readiness_settings.enforcement_enabled:set_readiness_state(reason="degraded_dependency", source="provider_timeout")run_status = RunStatus.failed@@ -624,6 +635,10 @@ class StreamingService:)in_flight_counter.add(-1, attributes=in_flight_attrs)+ if is_background:+ _load_gate.on_bg_end()+ else:+ _load_gate.on_fg_end()return error_aware_stream()diff --git a/letta/settings.py b/letta/settings.pyindex a7b449a27..5391d1172 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -609,19 +609,31 @@ class ReadinessSettings(BaseSettings):# When True, /v1/health/ returns 503 during draining state (graceful shutdown signal to k8s).drain_returns_503: bool = Field(default=True, description="Return HTTP 503 when readiness state is 'draining'.")- # When True, event loop lag above threshold triggers degraded_dependency state.+ # When True, event loop lag above threshold triggers degraded state.event_loop_lag_gating_enabled: bool = Field(default=False, description="Gate readiness on event loop lag threshold.")event_loop_lag_threshold_ms: float = Field(default=5000.0, ge=100.0, description="Event loop lag threshold (ms) for degraded transition.")+ # When True, fg in-flight count above threshold triggers degraded state.+ fg_in_flight_gating_enabled: bool = Field(default=False, description="Gate readiness on foreground in-flight count.")+ fg_in_flight_threshold: int = Field(default=10, ge=1, description="Foreground in-flight count threshold per pod.")++ # When True, bg in-flight count above threshold triggers degraded state.+ bg_in_flight_gating_enabled: bool = Field(default=False, description="Gate readiness on background in-flight count.")+ bg_in_flight_threshold: int = Field(default=15, ge=1, description="Background in-flight count threshold per pod.")++ # When True, request admission wait above threshold triggers degraded state.+ admission_wait_gating_enabled: bool = Field(default=False, description="Gate readiness on request admission wait.")+ admission_wait_threshold_ms: float = Field(default=300.0, ge=10.0, description="Admission wait threshold (ms) for degraded transition.")+# Stabilization window prevents flapping: state must be consistently bad for this many seconds before degrading.degraded_stabilization_seconds: float = Field(- default=30.0, ge=0.0, description="Seconds of sustained overload before transitioning to degraded_dependency."+ default=30.0, ge=0.0, description="Seconds of sustained overload before transitioning to degraded.")# Recovery: state must be healthy for this long before returning to ready.recovery_stabilization_seconds: float = Field(- default=15.0, ge=0.0, description="Seconds of sustained health before recovering from degraded_dependency."+ default=15.0, ge=0.0, description="Seconds of sustained health before recovering from degraded.")