Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(core): add readiness state telemetry stub (LET-7827) (#9816)
Kian Jones75bb40138ec6
diff --git a/letta/data_sources/redis_client.py b/letta/data_sources/redis_client.pyindex 28b2574d9..ca921deb8 100644--- a/letta/data_sources/redis_client.py+++ b/letta/data_sources/redis_client.py@@ -13,6 +13,7 @@ from letta.constants import ()from letta.errors import ConversationBusyError, MemoryRepoBusyErrorfrom letta.log import get_logger+from letta.monitoring.readiness_state import set_readiness_statefrom letta.otel.metric_registry import MetricRegistryfrom letta.settings import settings@@ -134,6 +135,7 @@ class AsyncRedisClient:return await func(self, *args, **kwargs)except TimeoutError as e:MetricRegistry().redis_timeout_counter.add(1, attributes={"operation": func.__name__})+ set_readiness_state(reason="degraded_dependency", source=f"redis_timeout:{func.__name__}")last_error = eif attempt < max_attempts - 1:await asyncio.sleep(delay * (2**attempt))diff --git a/letta/monitoring/readiness_state.py b/letta/monitoring/readiness_state.pynew file mode 100644index 000000000..773984038--- /dev/null+++ b/letta/monitoring/readiness_state.py@@ -0,0 +1,56 @@+import threading++from letta.log import get_logger+from letta.otel.metric_registry import MetricRegistry++logger = get_logger(__name__)++_VALID_REASONS = {+ "ready",+ "warming",+ "draining",+ "degraded_dependency",+ "manual_disable",+}++_state_lock = threading.Lock()+_current_reason = "warming"+++def initialize_readiness_state(reason: str = "warming", source: str = "startup") -> str:+ """Initialize readiness telemetry state without changing probe behavior."""+ if reason not in _VALID_REASONS:+ logger.warning(f"Invalid readiness state '{reason}', falling back to warming")+ reason = "warming"++ global _current_reason+ with _state_lock:+ _current_reason = reason++ MetricRegistry().readiness_state_gauge.set(1, attributes={"reason": reason})+ logger.info(f"Readiness telemetry initialized: state={reason}, source={source}")+ return reason+++def set_readiness_state(reason: str, source: str = "unknown") -> str:+ """Transition readiness telemetry state and emit metric/log signals."""+ if reason not in _VALID_REASONS:+ logger.warning(f"Ignoring unknown readiness state transition '{reason}' from source={source}")+ return get_readiness_state()++ global _current_reason+ with _state_lock:+ previous_reason = _current_reason+ if previous_reason == reason:+ return reason+ _current_reason = reason++ MetricRegistry().readiness_state_gauge.set(0, attributes={"reason": previous_reason})+ MetricRegistry().readiness_state_gauge.set(1, attributes={"reason": reason})+ logger.info(f"Readiness telemetry transition: {previous_reason} -> {reason} (source={source})")+ return reason+++def get_readiness_state() -> str:+ with _state_lock:+ return _current_reasondiff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.pyindex ee7aec711..44d825663 100644--- a/letta/otel/metric_registry.py+++ b/letta/otel/metric_registry.py@@ -243,6 +243,19 @@ class MetricRegistry:),)+ # (includes reason)+ @property+ def readiness_state_gauge(self) -> Gauge:+ return self._get_or_create_metric(+ "readiness_state",+ partial(+ self._meter.create_gauge,+ name="readiness_state",+ description="Current readiness telemetry state encoded as one-hot reason labels.",+ unit="1",+ ),+ )+# (includes reason, route_class)@propertydef sse_disconnect_counter(self) -> Counter:diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex 2dcf121a6..840bf79ec 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -178,6 +178,9 @@ async def lifespan(app_: FastAPI):FastAPI lifespan context manager with setup before the app starts pre-yield and on shutdown after the yield."""worker_id = os.getpid()+ from letta.monitoring.readiness_state import initialize_readiness_state, set_readiness_state++ initialize_readiness_state(reason="warming", source="lifespan_startup")# Initialize event loop watchdogtry:@@ -249,10 +252,14 @@ async def lifespan(app_: FastAPI):logger.info(f"[Worker {worker_id}] Scheduler initialization completed")except Exception as e:logger.error(f"[Worker {worker_id}] Scheduler initialization failed: {e}", exc_info=True)+ set_readiness_state(reason="degraded_dependency", source="scheduler_init_failure")++ set_readiness_state(reason="ready", source="lifespan_startup_complete")logger.info(f"[Worker {worker_id}] Lifespan startup completed")yield# Cleanup on shutdown+ set_readiness_state(reason="draining", source="lifespan_shutdown")logger.info(f"[Worker {worker_id}] Starting lifespan shutdown")# Stop watchdog thread (important for clean test/worker shutdown)diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex ce37c815d..be925a2d5 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -24,6 +24,7 @@ from letta.errors import ()from letta.helpers.datetime_helpers import get_utc_timestamp_nsfrom letta.log import get_logger+from letta.monitoring.readiness_state import set_readiness_statefrom letta.otel.context import get_ctx_attributesfrom letta.otel.metric_registry import MetricRegistryfrom letta.schemas.agent import AgentState@@ -446,6 +447,7 @@ class StreamingService:except LLMTimeoutError as e:MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)MetricRegistry().provider_timeout_counter.add(1, attributes={"provider": provider_name})+ set_readiness_state(reason="degraded_dependency", source="provider_timeout")run_status = RunStatus.failedstop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error)error_message = LettaErrorMessage(