Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat(core): add readiness state telemetry stub (LET-7827) (#9816)

Kian Jones75bb40138ec6
diff --git a/letta/data_sources/redis_client.py b/letta/data_sources/redis_client.py
index 28b2574d9..ca921deb8 100644
--- a/letta/data_sources/redis_client.py
+++ b/letta/data_sources/redis_client.py
@@ -13,6 +13,7 @@ from letta.constants import (
)
from letta.errors import ConversationBusyError, MemoryRepoBusyError
from letta.log import get_logger
+from letta.monitoring.readiness_state import set_readiness_state
from letta.otel.metric_registry import MetricRegistry
from letta.settings import settings
@@ -134,6 +135,7 @@ class AsyncRedisClient:
return await func(self, *args, **kwargs)
except TimeoutError as e:
MetricRegistry().redis_timeout_counter.add(1, attributes={"operation": func.__name__})
+ set_readiness_state(reason="degraded_dependency", source=f"redis_timeout:{func.__name__}")
last_error = e
if attempt < max_attempts - 1:
await asyncio.sleep(delay * (2**attempt))
diff --git a/letta/monitoring/readiness_state.py b/letta/monitoring/readiness_state.py
new file mode 100644
index 000000000..773984038
--- /dev/null
+++ b/letta/monitoring/readiness_state.py
@@ -0,0 +1,56 @@
+import threading
+
+from letta.log import get_logger
+from letta.otel.metric_registry import MetricRegistry
+
+logger = get_logger(__name__)
+
+_VALID_REASONS = {
+ "ready",
+ "warming",
+ "draining",
+ "degraded_dependency",
+ "manual_disable",
+}
+
+_state_lock = threading.Lock()
+_current_reason = "warming"
+
+
+def initialize_readiness_state(reason: str = "warming", source: str = "startup") -> str:
+ """Initialize readiness telemetry state without changing probe behavior."""
+ if reason not in _VALID_REASONS:
+ logger.warning(f"Invalid readiness state '{reason}', falling back to warming")
+ reason = "warming"
+
+ global _current_reason
+ with _state_lock:
+ _current_reason = reason
+
+ MetricRegistry().readiness_state_gauge.set(1, attributes={"reason": reason})
+ logger.info(f"Readiness telemetry initialized: state={reason}, source={source}")
+ return reason
+
+
+def set_readiness_state(reason: str, source: str = "unknown") -> str:
+ """Transition readiness telemetry state and emit metric/log signals."""
+ if reason not in _VALID_REASONS:
+ logger.warning(f"Ignoring unknown readiness state transition '{reason}' from source={source}")
+ return get_readiness_state()
+
+ global _current_reason
+ with _state_lock:
+ previous_reason = _current_reason
+ if previous_reason == reason:
+ return reason
+ _current_reason = reason
+
+ MetricRegistry().readiness_state_gauge.set(0, attributes={"reason": previous_reason})
+ MetricRegistry().readiness_state_gauge.set(1, attributes={"reason": reason})
+ logger.info(f"Readiness telemetry transition: {previous_reason} -> {reason} (source={source})")
+ return reason
+
+
+def get_readiness_state() -> str:
+ with _state_lock:
+ return _current_reason
diff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.py
index ee7aec711..44d825663 100644
--- a/letta/otel/metric_registry.py
+++ b/letta/otel/metric_registry.py
@@ -243,6 +243,19 @@ class MetricRegistry:
),
)
+ # (includes reason)
+ @property
+ def readiness_state_gauge(self) -> Gauge:
+ return self._get_or_create_metric(
+ "readiness_state",
+ partial(
+ self._meter.create_gauge,
+ name="readiness_state",
+ description="Current readiness telemetry state encoded as one-hot reason labels.",
+ unit="1",
+ ),
+ )
+
# (includes reason, route_class)
@property
def sse_disconnect_counter(self) -> Counter:
diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.py
index 2dcf121a6..840bf79ec 100644
--- a/letta/server/rest_api/app.py
+++ b/letta/server/rest_api/app.py
@@ -178,6 +178,9 @@ async def lifespan(app_: FastAPI):
FastAPI lifespan context manager with setup before the app starts pre-yield and on shutdown after the yield.
"""
worker_id = os.getpid()
+ from letta.monitoring.readiness_state import initialize_readiness_state, set_readiness_state
+
+ initialize_readiness_state(reason="warming", source="lifespan_startup")
# Initialize event loop watchdog
try:
@@ -249,10 +252,14 @@ async def lifespan(app_: FastAPI):
logger.info(f"[Worker {worker_id}] Scheduler initialization completed")
except Exception as e:
logger.error(f"[Worker {worker_id}] Scheduler initialization failed: {e}", exc_info=True)
+ set_readiness_state(reason="degraded_dependency", source="scheduler_init_failure")
+
+ set_readiness_state(reason="ready", source="lifespan_startup_complete")
logger.info(f"[Worker {worker_id}] Lifespan startup completed")
yield
# Cleanup on shutdown
+ set_readiness_state(reason="draining", source="lifespan_shutdown")
logger.info(f"[Worker {worker_id}] Starting lifespan shutdown")
# Stop watchdog thread (important for clean test/worker shutdown)
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index ce37c815d..be925a2d5 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -24,6 +24,7 @@ from letta.errors import (
)
from letta.helpers.datetime_helpers import get_utc_timestamp_ns
from letta.log import get_logger
+from letta.monitoring.readiness_state import set_readiness_state
from letta.otel.context import get_ctx_attributes
from letta.otel.metric_registry import MetricRegistry
from letta.schemas.agent import AgentState
@@ -446,6 +447,7 @@ class StreamingService:
except LLMTimeoutError as e:
MetricRegistry().request_timeout_counter.add(1, attributes=in_flight_attrs)
MetricRegistry().provider_timeout_counter.add(1, attributes={"provider": provider_name})
+ set_readiness_state(reason="degraded_dependency", source="provider_timeout")
run_status = RunStatus.failed
stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error)
error_message = LettaErrorMessage(