Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(readiness): M2 signal-to-state integration (event loop lag + provider/redis timeout gating) (#9829)
Kian Jones34ef36fff2cf
diff --git a/letta/data_sources/redis_client.py b/letta/data_sources/redis_client.pyindex ca921deb8..6ff6c2694 100644--- a/letta/data_sources/redis_client.py+++ b/letta/data_sources/redis_client.py@@ -135,7 +135,9 @@ class AsyncRedisClient:return await func(self, *args, **kwargs)except TimeoutError as e:MetricRegistry().redis_timeout_counter.add(1, attributes={"operation": func.__name__})- set_readiness_state(reason="degraded_dependency", source=f"redis_timeout:{func.__name__}")+ from letta.settings import readiness_settings+ if readiness_settings.enforcement_enabled:+ set_readiness_state(reason="degraded_dependency", source=f"redis_timeout:{func.__name__}")last_error = eif attempt < max_attempts - 1:await asyncio.sleep(delay * (2**attempt))diff --git a/letta/monitoring/event_loop_watchdog.py b/letta/monitoring/event_loop_watchdog.pyindex 3a4e8a315..a1dfd1c1c 100644--- a/letta/monitoring/event_loop_watchdog.py+++ b/letta/monitoring/event_loop_watchdog.py@@ -16,6 +16,13 @@ from letta.otel.metric_registry import MetricRegistrylogger = get_logger(__name__)+# Lazy import to avoid circular deps at module load time.+def _get_readiness_settings():+ from letta.settings import readiness_settings++ return readiness_settings++class EventLoopWatchdog:"""Minimal watchdog that monitors event loop health from a separate thread.@@ -39,6 +46,8 @@ class EventLoopWatchdog:self._monitoring = Falseself._last_dump_time = 0.0 # Cooldown between task dumpsself._saturation_start: Optional[float] = None # Track when saturation began+ self._degraded_since: Optional[float] = None # Track when we entered sustained overload+ self._recovery_since: Optional[float] = None # Track when we started recoveringdef start(self, loop: asyncio.AbstractEventLoop):"""Start the watchdog thread."""@@ -158,6 +167,9 @@ class EventLoopWatchdog:self._dump_asyncio_tasks() # Dump async tasksself._dump_state() # Dump thread stacksself._last_dump_time = now++ # Readiness gating: transition to degraded after sustained overload+ self._maybe_degrade_readiness(now, current_lag_ms=current_lag * 1000)else:# Reset saturation tracking when recoveredif self._saturation_start is not None:@@ -165,6 +177,9 @@ class EventLoopWatchdog:logger.info(f"Event loop saturation ended after {duration:.1f}s")self._saturation_start = None+ # Readiness gating: attempt recovery when lag is healthy+ self._maybe_recover_readiness(now)+if time_since_heartbeat > self.timeout_threshold:consecutive_hangs += 1logger.error(@@ -186,6 +201,57 @@ class EventLoopWatchdog:except Exception as e:logger.error(f"Watchdog error: {e}")+ def _maybe_degrade_readiness(self, now: float, current_lag_ms: float) -> None:+ """Transition to degraded when event loop lag exceeds threshold for sustained period."""+ try:+ rs = _get_readiness_settings()+ if not rs.enforcement_enabled or not rs.event_loop_lag_gating_enabled:+ return+ if current_lag_ms < rs.event_loop_lag_threshold_ms:+ return++ from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state++ if get_readiness_state() not in ("ready", "degraded"):+ return # Don't override draining/warming/manual_disable++ if self._degraded_since is None:+ self._degraded_since = now+ self._recovery_since = None+ return++ elapsed = now - self._degraded_since+ if elapsed >= rs.degraded_stabilization_seconds and get_readiness_state() == "ready":+ set_readiness_state("degraded", source=f"event_loop_lag:{current_lag_ms:.0f}ms")+ except Exception:+ pass # Readiness gating must never crash the watchdog++ def _maybe_recover_readiness(self, now: float) -> None:+ """Recover from event-loop-lag-induced degraded state after sustained healthy period."""+ try:+ rs = _get_readiness_settings()+ if not rs.enforcement_enabled or not rs.event_loop_lag_gating_enabled:+ return++ from letta.monitoring.readiness_state import get_readiness_state, set_readiness_state++ self._degraded_since = None # Reset overload timer++ if get_readiness_state() != "degraded":+ self._recovery_since = None+ return++ if self._recovery_since is None:+ self._recovery_since = now+ return++ elapsed = now - self._recovery_since+ if elapsed >= rs.recovery_stabilization_seconds:+ set_readiness_state("ready", source="event_loop_lag_recovered")+ self._recovery_since = None+ except Exception:+ pass # Readiness gating must never crash the watchdog+@staticmethoddef _get_executor_backlog(loop: asyncio.AbstractEventLoop) -> int:"""Best-effort backlog size for the loop's default executor work queue."""diff --git a/letta/monitoring/readiness_state.py b/letta/monitoring/readiness_state.pyindex 773984038..e32457dba 100644--- a/letta/monitoring/readiness_state.py+++ b/letta/monitoring/readiness_state.py@@ -9,6 +9,7 @@ _VALID_REASONS = {"ready","warming","draining",+ "degraded","degraded_dependency","manual_disable",}diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex 18ecbdc8b..dcc54372d 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -443,7 +443,9 @@ class StreamingService:except LLMTimeoutError as e:MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)MetricRegistry().provider_timeout_counter.add(1, attributes={"provider": provider_name})- set_readiness_state(reason="degraded_dependency", source="provider_timeout")+ from letta.settings import readiness_settings+ if readiness_settings.enforcement_enabled:+ set_readiness_state(reason="degraded_dependency", source="provider_timeout")run_status = RunStatus.failedstop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error)error_message = LettaErrorMessage(