:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Lifecycle manager: signal trapping, component registration, graceful shutdown, readiness/liveness probes.
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::{Arc, OnceLock};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
const DEFAULT_PRESTOP_PATH: &str = "/tmp/shutdown";
const DEFAULT_OBSERVABILITY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(1);
use tokio::sync::{mpsc, oneshot};
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use crate::error::LifecycleError;
use crate::handle::{ComponentEvent, Handle, HandleInner, HEALTH_STARTING, HEALTH_UNHEALTHY};
use crate::liveness::{LivenessComponentRef, LivenessHandler};
use crate::metrics;
use crate::readiness::ReadinessHandler;
use crate::signals;
/// What the manager does when the pre-stop file appears.
/// (see tests `prestop_file_starts_shutdown_by_default`, `prestop_drain_serves_until_max`)
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum PrestopAction {
/// Start graceful shutdown, the same as SIGTERM.
#[default]
Shutdown,
/// Fail readiness and keep serving, so K8s removes the pod from the Service
/// endpoints while components still answer. Shutdown starts on SIGTERM or any
/// other trigger, and at the latest `max` after the drain starts, so a lost
/// SIGTERM cannot keep the pod in the drain until SIGKILL.
Drain { max: Duration },
}
/// Builder for [`Manager`]. Start with [`Manager::builder("name")`](Manager::builder),
/// chain `.with_*()` calls, then call [`.build()`](ManagerBuilder::build).
#[derive(Debug)]
pub struct ManagerBuilder {
name: String,
global_shutdown_timeout: Duration,
trap_signals: bool,
prestop: Option<PrestopAction>,
prestop_path: PathBuf,
health_poll_interval: Duration,
external_shutdown_token: Option<CancellationToken>,
}
impl ManagerBuilder {
/// Override the hard ceiling on total shutdown duration. If all components
/// haven't finished within this window, monitor returns
/// [`LifecycleError::ShutdownTimeout`]. Default: 60s.
pub fn with_global_shutdown_timeout(mut self, d: Duration) -> Self {
self.global_shutdown_timeout = d;
self
}
/// Install SIGINT/SIGTERM handlers. Default: true. Set false in tests.
pub fn with_trap_signals(mut self, enabled: bool) -> Self {
self.trap_signals = enabled;
self
}
/// Poll for pre-stop shutdown file (K8s pre-stop hook pattern). Default: true.
pub fn with_prestop_check(mut self, enabled: bool) -> Self {
self.prestop = enabled.then_some(self.prestop.unwrap_or_default());
self
}
/// What to do when the pre-stop file appears. Default: [`PrestopAction::Shutdown`].
/// Also turns the pre-stop check on.
/// Use [`PrestopAction::Drain`] for a server whose clients keep connections to
/// the pod after K8s removes it from the Service endpoints.
pub fn with_prestop_action(mut self, action: PrestopAction) -> Self {
self.prestop = Some(action);
self
}
/// Override the pre-stop file path. Default: `/tmp/shutdown`.
pub fn with_prestop_path(mut self, path: impl Into<PathBuf>) -> Self {
self.prestop_path = path.into();
self
}
/// Override how often the health monitor polls component heartbeats.
/// The health monitor is automatically active when any component is
/// registered with `with_liveness_deadline`. Default: 5s.
/// Lower values detect stalls faster but use more CPU.
pub fn with_health_poll_interval(mut self, d: Duration) -> Self {
self.health_poll_interval = d;
self
}
/// Use an external shutdown token. The caller controls when shutdown begins by calling
/// `token.cancel()`. Use in tests for deterministic shutdown control.
pub fn with_shutdown_token(mut self, token: CancellationToken) -> Self {
self.external_shutdown_token = Some(token);
self
}
/// Consume the builder and produce a [`Manager`].
pub fn build(self) -> Manager {
let shutdown_token = self.external_shutdown_token.unwrap_or_default();
Manager {
name: self.name,
global_shutdown_timeout: self.global_shutdown_timeout,
trap_signals: self.trap_signals,
prestop: self.prestop,
prestop_path: self.prestop_path,
health_poll_interval: self.health_poll_interval,
drain_token: shutdown_token.child_token(),
shutdown_token,
observability_shutdown_token: CancellationToken::new(),
phase_tokens: HashMap::new(),
event_tx_slot: Arc::new(OnceLock::new()),
components: HashMap::new(),
observability_components: HashMap::new(),
liveness_components: Vec::new(),
}
}
}
/// Per-component options set at registration time.
#[derive(Clone, Debug)]
pub struct ComponentOptions {
graceful_shutdown: Option<Duration>,
liveness_deadline: Option<Duration>,
stall_threshold: u32,
shutdown_phase: u8,
pub(crate) is_observability: bool,
pub(crate) is_advisory: bool,
config_errors: Vec<String>,
}
impl ComponentOptions {
/// No graceful shutdown timeout, no liveness deadline, stall threshold 1,
/// shutdown phase 0.
pub fn new() -> Self {
Self {
graceful_shutdown: None,
liveness_deadline: None,
stall_threshold: 1,
shutdown_phase: 0,
is_observability: false,
is_advisory: false,
config_errors: Vec::new(),
}
}
/// Mark this component as an observability handle (e.g. metrics server).
/// Observability handles are shut down *after* all standard components finish,
/// ensuring metrics flow during the entire graceful shutdown process.
/// Observability handles do not participate in active health monitoring;
/// combining `is_observability(true)` with `with_liveness_deadline` will panic.
/// (see test `observability_handle_shuts_down_after_standard_handles`)
pub fn is_observability(mut self, enabled: bool) -> Self {
self.is_observability = enabled;
self
}
/// Mark this component as advisory. Advisory handles participate in health
/// monitoring (gauge updates, `is_healthy()` queries) but stalls do NOT
/// trigger global shutdown. The monitor does not wait for advisory handles
/// during shutdown. Requires `with_liveness_deadline`. Cannot combine with
/// `is_observability`.
/// (see test `advisory_handle_stall_does_not_trigger_shutdown`)
pub fn is_advisory(mut self, enabled: bool) -> Self {
self.is_advisory = enabled;
self
}
/// Max time this component gets for cleanup after shutdown begins. If exceeded,
/// the manager marks the component as timed out and moves on.
/// (see test `component_timeout_then_late_drop_preserves_timeout`)
pub fn with_graceful_shutdown<D>(mut self, d: D) -> Self
where
D: TryInto<Duration>,
D::Error: std::fmt::Debug,
{
match d.try_into() {
Ok(dur) => self.graceful_shutdown = Some(dur),
Err(e) => self
.config_errors
.push(format!("invalid graceful_shutdown duration: {e:?}")),
}
self
}
/// Liveness heartbeat deadline. The component must call [`Handle::report_healthy`](crate::Handle::report_healthy)
/// within this interval or the health monitor considers it stalled. After
/// `stall_threshold` consecutive stalled checks, the manager triggers global shutdown.
/// (see test `stall_triggers_shutdown`)
pub fn with_liveness_deadline<D>(mut self, d: D) -> Self
where
D: TryInto<Duration>,
D::Error: std::fmt::Debug,
{
match d.try_into() {
Ok(dur) => self.liveness_deadline = Some(dur),
Err(e) => self
.config_errors
.push(format!("invalid liveness_deadline duration: {e:?}")),
}
self
}
/// Number of consecutive stalled health checks before the manager triggers
/// global shutdown. Default: 1 (immediate). Set higher for tolerance of
/// transient hiccups.
/// (see test `stall_threshold_allows_recovery`)
pub fn with_stall_threshold(mut self, n: u32) -> Self {
self.stall_threshold = n.max(1);
self
}
/// Order this component within graceful shutdown. Phases drain in
/// ascending order: a component in phase N+1 is not signalled — its
/// shutdown token stays uncancelled — until every phase ≤ N component
/// has finished (completed, died, or timed out its graceful window).
/// Components within one phase drain concurrently. Default: 0, which is
/// signalled the moment shutdown begins, exactly as before this option
/// existed.
///
/// Use this when one component's cleanup depends on another still
/// serving: e.g. a coordination drain (phase 0) that must complete
/// requests through the gRPC server and producer (phase 1) before they
/// stop. The component's graceful window is measured from when its
/// phase begins, not from the start of shutdown; the global shutdown
/// timeout bounds the whole sequence. Observability components remain a
/// final implicit stage after every standard phase.
/// (see test `later_phase_not_signalled_until_earlier_phase_completes`)
pub fn with_shutdown_phase(mut self, phase: u8) -> Self {
self.shutdown_phase = phase;
self
}
}
impl Default for ComponentOptions {
fn default() -> Self {
Self::new()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ComponentStatus {
Running,
ShuttingDown,
Completed,
TimedOut,
Died,
}
struct ComponentState {
graceful_shutdown: Option<Duration>,
status: ComponentStatus,
shutdown_phase: u8,
}
/// Lifecycle manager: registers components, runs monitor on a dedicated OS thread, provides readiness/liveness handlers and shutdown signal.
pub struct Manager {
name: String,
global_shutdown_timeout: Duration,
trap_signals: bool,
prestop: Option<PrestopAction>,
prestop_path: PathBuf,
health_poll_interval: Duration,
/// Child of `shutdown_token`, so readiness, which reads only this token, fails
/// both when the app drains and when it shuts down.
drain_token: CancellationToken,
shutdown_token: CancellationToken,
observability_shutdown_token: CancellationToken,
/// Tokens for shutdown phases above 0, cancelled in ascending phase
/// order once every earlier phase's components have finished. Phase 0
/// components share the root `shutdown_token` so the default behavior
/// (everything signalled at shutdown start, no monitor hop) is
/// unchanged. Independent tokens, not children of the root — a child
/// would cancel with its parent and defeat the ordering.
phase_tokens: HashMap<u8, CancellationToken>,
event_tx_slot: Arc<OnceLock<mpsc::Sender<ComponentEvent>>>,
components: HashMap<String, ComponentState>,
observability_components: HashMap<String, ComponentState>,
liveness_components: Vec<LivenessComponentRef>,
}
impl Manager {
/// Start building a manager with the given service name. The name is emitted
/// as the `service_name` label on all lifecycle metrics — use your K8s service
/// name or logical app name for dashboard filtering.
pub fn builder(name: impl Into<String>) -> ManagerBuilder {
ManagerBuilder {
name: name.into(),
global_shutdown_timeout: Duration::from_secs(60),
trap_signals: true,
prestop: Some(PrestopAction::default()),
prestop_path: PathBuf::from(DEFAULT_PRESTOP_PATH),
health_poll_interval: Duration::from_secs(5),
external_shutdown_token: None,
}
}
/// Register a component and return a [`Handle`]. The handle's drop guard (or
/// [`process_scope`](crate::Handle::process_scope) guard) notifies the manager when
/// the component exits. Register all components before calling [`monitor`](Manager::monitor)
/// or [`monitor_background`](Manager::monitor_background).
pub fn register(&mut self, tag: &str, options: ComponentOptions) -> Handle {
assert!(
options.config_errors.is_empty(),
"component '{}': {}",
tag,
options.config_errors.join("; ")
);
assert!(
!self.components.contains_key(tag)
&& !self.observability_components.contains_key(tag)
&& !self.liveness_components.iter().any(|c| c.tag == tag),
"component '{}' registered more than once",
tag
);
assert!(
!(options.is_advisory && options.is_observability),
"component '{}': advisory handles cannot be observability handles",
tag
);
assert!(
!(options.is_advisory && options.liveness_deadline.is_none()),
"component '{}': advisory handles require with_liveness_deadline",
tag
);
assert!(
!(options.is_advisory && options.graceful_shutdown.is_some()),
"component '{}': advisory handles are not waited on during shutdown; \
with_graceful_shutdown has no effect",
tag
);
assert!(
!(options.is_observability && options.liveness_deadline.is_some()),
"component '{}': observability handles cannot use liveness_deadline",
tag
);
assert!(
!(options.is_observability && options.shutdown_phase != 0),
"component '{}': observability handles are already the final shutdown stage; \
with_shutdown_phase has no effect",
tag
);
assert!(
!(options.is_advisory && options.shutdown_phase != 0),
"component '{}': advisory handles are not waited on during shutdown; \
with_shutdown_phase has no effect",
tag
);
let healthy_until_ms = Arc::new(AtomicI64::new(HEALTH_STARTING));
let health_flag = Arc::new(std::sync::atomic::AtomicBool::new(true));
let tag_owned = tag.to_string();
let is_obs = options.is_observability;
if let Some(deadline) = options.liveness_deadline {
let labels = [
("service_name".to_string(), self.name.clone()),
("component".to_string(), tag_owned.clone()),
];
let health_gauge = ::metrics::gauge!("lifecycle_component_healthy", &labels);
self.liveness_components.push(LivenessComponentRef {
tag: tag_owned.clone(),
healthy_until_ms: healthy_until_ms.clone(),
stall_threshold: options.stall_threshold,
health_gauge,
advisory: options.is_advisory,
health_flag: health_flag.clone(),
});
debug!(
component = %tag_owned,
advisory = options.is_advisory,
graceful_shutdown_secs = options.graceful_shutdown.map(|d| d.as_secs_f64()),
liveness_deadline_secs = deadline.as_secs_f64(),
stall_threshold = options.stall_threshold,
"Lifecycle: component registered"
);
} else {
debug!(
component = %tag_owned,
observability = is_obs,
graceful_shutdown_secs = options.graceful_shutdown.map(|d| d.as_secs_f64()),
"Lifecycle: component registered"
);
}
if !options.is_advisory {
let target_map = if is_obs {
&mut self.observability_components
} else {
&mut self.components
};
target_map.insert(
tag_owned.clone(),
ComponentState {
graceful_shutdown: options.graceful_shutdown,
status: ComponentStatus::Running,
shutdown_phase: options.shutdown_phase,
},
);
}
let shutdown_token = if is_obs {
self.observability_shutdown_token.clone()
} else if options.shutdown_phase == 0 {
self.shutdown_token.clone()
} else {
self.phase_tokens
.entry(options.shutdown_phase)
.or_default()
.clone()
};
let inner = Arc::new(HandleInner {
tag: tag_owned,
shutdown_token,
event_tx: self.event_tx_slot.clone(),
healthy_until_ms,
liveness_deadline: options.liveness_deadline,
health_flag,
completed: std::sync::atomic::AtomicBool::new(false),
process_scope_signalled: std::sync::atomic::AtomicBool::new(false),
});
Handle { inner }
}
/// Readiness probe handler (`/_readiness`). Returns 200 while running, 503 once the app
/// drains or shuts down. K8s uses this to stop routing traffic to the pod.
/// (see test `readiness_200_until_shutdown_then_503`)
pub fn readiness_handler(&self) -> ReadinessHandler {
ReadinessHandler::new(self.drain_token.clone())
}
/// Liveness probe handler (`/_liveness`). **Intentionally always returns 200** — the
/// lifecycle library handles health monitoring internally and triggers coordinated
/// graceful shutdown on stall, rather than delegating to K8s liveness probes.
pub fn liveness_handler(&self) -> LivenessHandler {
LivenessHandler::new()
}
fn spawn_monitor_thread(self) -> oneshot::Receiver<Result<(), LifecycleError>> {
let (tx, rx) = oneshot::channel();
let total_components = self.components.len() + self.observability_components.len();
let channel_size = total_components * 2 + 2;
let (event_tx, event_rx) = mpsc::channel(channel_size);
self.event_tx_slot
.set(event_tx)
.expect("lifecycle monitor already started");
thread::Builder::new()
.name("lifecycle-monitor".into())
.spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("failed to build lifecycle runtime");
let result = rt.block_on(self.run_monitor_loop(event_rx));
drop(tx.send(result));
})
.expect("failed to spawn lifecycle monitor thread");
rx
}
/// Spawn the monitor on a dedicated OS thread and await completion; returns when all components finish or time out.
pub async fn monitor(self) -> Result<(), LifecycleError> {
let rx = self.spawn_monitor_thread();
rx.await.map_err(|_| LifecycleError::MonitorPanicked)?
}
/// Same as [`monitor`](Manager::monitor) but returns immediately with a guard; await the guard after the HTTP server exits.
pub fn monitor_background(self) -> MonitorGuard {
MonitorGuard {
rx: self.spawn_monitor_thread(),
}
}
/// Look up which component map a tag belongs to and return a mutable reference.
fn get_component_mut(&mut self, tag: &str) -> Option<&mut ComponentState> {
self.components
.get_mut(tag)
.or_else(|| self.observability_components.get_mut(tag))
}
fn is_observability_tag(&self, tag: &str) -> bool {
self.observability_components.contains_key(tag)
}
/// Advisory components are deliberately absent from `components` and
/// `observability_components`, so `liveness_components` is the only record
/// of them. Registration requires a liveness deadline for advisory handles,
/// which guarantees every one of them is present here.
fn is_advisory_tag(&self, tag: &str) -> bool {
self.liveness_components
.iter()
.any(|c| c.tag == tag && c.advisory)
}
async fn run_monitor_loop(
mut self,
mut event_rx: mpsc::Receiver<ComponentEvent>,
) -> Result<(), LifecycleError> {
let _span = tracing::info_span!("lifecycle", app = %self.name).entered();
let name = self.name.clone();
let trap_signals = self.trap_signals;
let health_poll_interval = self.health_poll_interval;
let global_timeout = self.global_shutdown_timeout;
let shutdown_token = self.shutdown_token.clone();
let has_liveness_components = !self.liveness_components.is_empty();
debug!(
global_shutdown_timeout_secs = global_timeout.as_secs_f64(),
"Lifecycle: global shutdown timeout configured"
);
if trap_signals {
let token = shutdown_token.clone();
tokio::spawn(async move {
signals::wait_for_shutdown_signal().await;
token.cancel();
});
}
if let Some(prestop_action) = self.prestop {
let token = shutdown_token.clone();
let drain_token = self.drain_token.clone();
let prestop_path = self.prestop_path.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(1));
loop {
tokio::select! {
_ = interval.tick() => {}
_ = token.cancelled() => return,
}
if prestop_path.exists() {
break;
}
}
match prestop_action {
PrestopAction::Shutdown => {
info!(
trigger_reason = "prestop",
path = %prestop_path.display(),
"Lifecycle: shutdown initiated, prestop file detected"
);
token.cancel();
}
PrestopAction::Drain { max } => {
info!(
path = %prestop_path.display(),
"Lifecycle: drain started, prestop file detected"
);
drain_token.cancel();
tokio::select! {
_ = tokio::time::sleep(max) => {
info!(
trigger_reason = "prestop",
"Lifecycle: shutdown initiated, drain limit reached"
);
token.cancel();
}
_ = token.cancelled() => {}
}
}
}
});
}
if has_liveness_components {
let liveness_for_health = self.liveness_components.clone();
let health_token = shutdown_token.clone();
let health_event_tx = self
.event_tx_slot
.get()
.expect("event channel not initialized")
.clone();
tokio::spawn(async move {
let mut stall_counts: Vec<u32> = vec![0; liveness_for_health.len()];
let mut interval = tokio::time::interval(health_poll_interval);
loop {
tokio::select! {
_ = interval.tick() => {
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as i64;
for (i, comp) in liveness_for_health.iter().enumerate() {
let until = comp.healthy_until_ms.load(Ordering::Relaxed);
let (healthy, status) = if until == HEALTH_UNHEALTHY {
(false, "unhealthy")
} else if until == HEALTH_STARTING {
continue;
} else if until > now_ms {
(true, "healthy")
} else {
(false, "stalled")
};
comp.health_gauge.set(if healthy { 1.0 } else { 0.0 });
if healthy {
stall_counts[i] = 0;
comp.health_flag.store(true, Ordering::Relaxed);
} else {
stall_counts[i] += 1;
if stall_counts[i] >= comp.stall_threshold {
comp.health_flag.store(false, Ordering::Relaxed);
if !comp.advisory {
warn!(
component = %comp.tag,
status,
stall_count = stall_counts[i],
stall_threshold = comp.stall_threshold,
"Lifecycle: health stall threshold reached"
);
drop(health_event_tx.try_send(ComponentEvent::Failure {
tag: comp.tag.clone(),
reason: format!("health check {status} (stall count {}/{})", stall_counts[i], comp.stall_threshold),
}));
return;
}
}
}
}
}
_ = health_token.cancelled() => break,
}
}
});
}
let mut first_failure: Option<LifecycleError> = None;
// --- Stage 1: Normal operation ---
// Events from both standard and observability components trigger shutdown.
loop {
tokio::select! {
biased;
Some(event) = event_rx.recv() => {
match event {
ComponentEvent::Failure { tag, reason } => {
metrics::emit_shutdown_initiated(&name, &tag, "failure");
warn!(trigger_component = %tag, trigger_reason = "failure",
"Lifecycle: shutdown initiated: {reason:#}");
shutdown_token.cancel();
if let Some(s) = self.get_component_mut(&tag) {
s.status = ComponentStatus::Died;
}
first_failure = first_failure.or(Some(LifecycleError::ComponentFailure { tag, reason }));
break;
}
ComponentEvent::Died { tag } => {
// Advisory components are best-effort by contract: their
// death is informational, so it must not take the process
// with it. Health stalls are already exempt in the poll
// task; without this the exemption is trivially bypassed
// by dropping the handle, which is what a failed
// construction or a panicking task does.
if self.is_advisory_tag(&tag) {
info!(component = %tag,
"Lifecycle: advisory component died, continuing");
continue;
}
metrics::emit_shutdown_initiated(&name, &tag, "died");
warn!(trigger_component = %tag, trigger_reason = "died",
"Lifecycle: shutdown initiated, component died unexpectedly");
shutdown_token.cancel();
if let Some(s) = self.get_component_mut(&tag) {
s.status = ComponentStatus::Died;
}
first_failure = first_failure.or(Some(LifecycleError::ComponentDied { tag }));
break;
}
ComponentEvent::ShutdownRequested { tag } => {
metrics::emit_shutdown_initiated(&name, &tag, "requested");
info!(trigger_component = %tag, trigger_reason = "requested",
"Lifecycle: shutdown requested by component");
shutdown_token.cancel();
break;
}
ComponentEvent::WorkCompleted { tag } => {
if let Some(s) = self.get_component_mut(&tag) {
s.status = ComponentStatus::Completed;
}
if self.all_finished() {
return self.finalize(Instant::now(), first_failure);
}
}
}
}
_ = shutdown_token.cancelled() => {
metrics::emit_shutdown_initiated(&name, "system", "signal");
info!(trigger_reason = "signal", "Lifecycle: shutdown initiated");
break;
}
}
}
// --- Stage 2: Standard component drain, in shutdown-phase order ---
let shutdown_clock = Instant::now();
let global_deadline = shutdown_clock + global_timeout;
// Standard components drain in ascending shutdown-phase order: each
// phase's components are signalled and waited for together, and the
// next phase's token is only cancelled once the current phase has
// fully finished. Phase 0 shares the root token, so it is already
// signalled by the time this loop starts.
let mut phases: Vec<u8> = self.components.values().map(|s| s.shutdown_phase).collect();
phases.sort_unstable();
phases.dedup();
for phase in phases {
if phase != 0 {
info!(
shutdown_phase = phase,
"Lifecycle: advancing to shutdown phase"
);
if let Some(token) = self.phase_tokens.get(&phase) {
token.cancel();
}
}
let phase_clock = Instant::now();
for s in self
.components
.values_mut()
.filter(|s| s.shutdown_phase == phase)
{
if s.status == ComponentStatus::Running {
s.status = ComponentStatus::ShuttingDown;
}
}
if !self.phase_finished(phase) {
self.drain_standard_phase(
phase,
phase_clock,
shutdown_clock,
global_deadline,
global_timeout,
&mut event_rx,
&name,
&mut first_failure,
)
.await?;
}
}
// --- Stage 3: Observability component drain ---
if self.observability_components.is_empty() {
return self.finalize_all(first_failure);
}
info!("Lifecycle: standard components finished, draining observability components");
self.observability_shutdown_token.cancel();
for s in self.observability_components.values_mut() {
if s.status == ComponentStatus::Running {
s.status = ComponentStatus::ShuttingDown;
}
}
if !self.all_observability_finished() {
let obs_clock = Instant::now();
let mut obs_deadlines: Vec<(String, Instant)> = self
.observability_components
.iter()
.map(|(tag, s)| {
let timeout = s
.graceful_shutdown
.unwrap_or(DEFAULT_OBSERVABILITY_SHUTDOWN_TIMEOUT);
(tag.clone(), obs_clock + timeout)
})
.collect();
obs_deadlines.sort_by(|a, b| a.1.cmp(&b.1));
loop {
let now = Instant::now();
if now >= global_deadline {
return self.emit_global_timeout(&name, global_timeout);
}
let next_obs_timeout = obs_deadlines
.iter()
.find(|(tag, _)| {
self.observability_components
.get(tag)
.map(|s| s.status == ComponentStatus::ShuttingDown)
== Some(true)
})
.map(|(_, t)| *t);
let wait_duration = [
next_obs_timeout.map(|t| t.saturating_duration_since(now)),
Some(global_deadline.saturating_duration_since(now)),
]
.into_iter()
.flatten()
.min()
.unwrap_or(Duration::from_secs(1));
tokio::select! {
biased;
Some(event) = event_rx.recv() => {
match event {
ComponentEvent::WorkCompleted { tag } => {
if let Some(s) = self.observability_components.get_mut(&tag) {
if s.status == ComponentStatus::ShuttingDown {
s.status = ComponentStatus::Completed;
info!(component = %tag,
result = "completed",
"Lifecycle: observability component completed shutdown");
} else {
debug!(component = %tag, status = ?s.status,
"Lifecycle: late WorkCompleted for already-finished observability component");
}
}
}
ComponentEvent::Died { tag } => {
if let Some(s) = self.observability_components.get_mut(&tag) {
s.status = ComponentStatus::Died;
warn!(component = %tag,
result = "died",
"Lifecycle: observability component died during shutdown");
}
}
ComponentEvent::Failure { tag, reason } => {
if let Some(s) = self.observability_components.get_mut(&tag) {
s.status = ComponentStatus::Died;
warn!(component = %tag,
result = "died",
"Lifecycle: observability component failed during shutdown: {reason}");
}
first_failure = first_failure.or(Some(LifecycleError::ComponentFailure { tag, reason }));
}
ComponentEvent::ShutdownRequested { .. } => {}
}
if self.all_observability_finished() {
break;
}
}
_ = tokio::time::sleep(wait_duration) => {
let now = Instant::now();
for (tag, deadline) in &obs_deadlines {
if now >= *deadline {
if let Some(s) = self.observability_components.get_mut(tag) {
if s.status == ComponentStatus::ShuttingDown {
s.status = ComponentStatus::TimedOut;
warn!(component = %tag,
duration_secs = s.graceful_shutdown
.unwrap_or(DEFAULT_OBSERVABILITY_SHUTDOWN_TIMEOUT)
.as_secs_f64(),
result = "timeout",
"Lifecycle: observability component timed out during shutdown");
}
}
}
}
if self.all_observability_finished() {
break;
}
}
}
}
}
self.finalize_all(first_failure)
}
/// Wait until every component in `phase` finishes: completes, dies, or
/// times out its graceful window (measured from `phase_clock`, when the
/// phase was signalled). Events for components outside the phase — an
/// early exit or death of a later-phase component, or an observability
/// handle — still update that component's state so its own phase finds
/// it already finished. Errors only when the global deadline passes.
#[allow(clippy::too_many_arguments)]
async fn drain_standard_phase(
&mut self,
phase: u8,
phase_clock: Instant,
shutdown_clock: Instant,
global_deadline: Instant,
global_timeout: Duration,
event_rx: &mut mpsc::Receiver<ComponentEvent>,
name: &str,
first_failure: &mut Option<LifecycleError>,
) -> Result<(), LifecycleError> {
let mut component_deadlines: Vec<(String, Instant)> = self
.components
.iter()
.filter(|(_, s)| s.shutdown_phase == phase)
.filter_map(|(tag, s)| s.graceful_shutdown.map(|d| (tag.clone(), phase_clock + d)))
.collect();
component_deadlines.sort_by(|a, b| a.1.cmp(&b.1));
loop {
let now = Instant::now();
if now >= global_deadline {
return self.emit_global_timeout(name, global_timeout);
}
let next_component_timeout = component_deadlines
.iter()
.find(|(tag, _)| {
self.components
.get(tag)
.map(|s| s.status == ComponentStatus::ShuttingDown)
== Some(true)
})
.map(|(_, t)| *t);
let wait_duration = [
next_component_timeout.map(|t| t.saturating_duration_since(now)),
Some(global_deadline.saturating_duration_since(now)),
]
.into_iter()
.flatten()
.min()
.unwrap_or(Duration::from_secs(1));
tokio::select! {
biased;
Some(event) = event_rx.recv() => {
match event {
ComponentEvent::WorkCompleted { tag } => {
if let Some(s) = self.get_component_mut(&tag) {
// Running components can complete here too: a
// later-phase component may finish its work
// before its phase is ever signalled.
if matches!(
s.status,
ComponentStatus::ShuttingDown | ComponentStatus::Running
) {
s.status = ComponentStatus::Completed;
if !self.is_observability_tag(&tag) {
let elapsed = shutdown_clock.elapsed();
metrics::emit_component_shutdown_duration(name, &tag, "completed", elapsed.as_secs_f64());
metrics::emit_component_shutdown_result(name, &tag, "completed");
info!(component = %tag,
duration_secs = elapsed.as_secs_f64(),
result = "completed",
"Lifecycle: component completed shutdown");
}
} else {
debug!(component = %tag, status = ?s.status,
"Lifecycle: late WorkCompleted for already-finished component");
}
}
}
ComponentEvent::Died { tag } => {
if let Some(s) = self.get_component_mut(&tag) {
s.status = ComponentStatus::Died;
if !self.is_observability_tag(&tag) {
let elapsed = shutdown_clock.elapsed();
metrics::emit_component_shutdown_duration(name, &tag, "died", elapsed.as_secs_f64());
metrics::emit_component_shutdown_result(name, &tag, "died");
warn!(component = %tag,
duration_secs = elapsed.as_secs_f64(),
result = "died",
"Lifecycle: component died during shutdown");
}
}
}
ComponentEvent::Failure { tag, reason } => {
if let Some(s) = self.get_component_mut(&tag) {
s.status = ComponentStatus::Died;
if !self.is_observability_tag(&tag) {
let elapsed = shutdown_clock.elapsed();
metrics::emit_component_shutdown_duration(name, &tag, "died", elapsed.as_secs_f64());
metrics::emit_component_shutdown_result(name, &tag, "died");
warn!(component = %tag,
duration_secs = elapsed.as_secs_f64(),
result = "died",
"Lifecycle: component failed during shutdown");
}
}
*first_failure = first_failure.take().or(Some(LifecycleError::ComponentFailure { tag, reason }));
}
ComponentEvent::ShutdownRequested { .. } => {}
}
if self.phase_finished(phase) {
return Ok(());
}
}
_ = tokio::time::sleep(wait_duration) => {
let now = Instant::now();
for (tag, deadline) in &component_deadlines {
if now >= *deadline {
if let Some(s) = self.components.get_mut(tag) {
if s.status == ComponentStatus::ShuttingDown {
s.status = ComponentStatus::TimedOut;
if let Some(d) = s.graceful_shutdown {
metrics::emit_component_shutdown_duration(name, tag, "timeout", d.as_secs_f64());
}
metrics::emit_component_shutdown_result(name, tag, "timeout");
warn!(component = %tag,
duration_secs = s.graceful_shutdown.unwrap_or_default().as_secs_f64(),
result = "timeout",
"Lifecycle: component timed out during graceful shutdown");
}
}
}
}
if self.phase_finished(phase) {
return Ok(());
}
}
}
}
}
fn phase_finished(&self, phase: u8) -> bool {
self.components
.values()
.filter(|s| s.shutdown_phase == phase)
.all(|s| {
matches!(
s.status,
ComponentStatus::Completed | ComponentStatus::Died | ComponentStatus::TimedOut
)
})
}
fn all_standard_finished(&self) -> bool {
self.components.values().all(|s| {
matches!(
s.status,
ComponentStatus::Completed | ComponentStatus::Died | ComponentStatus::TimedOut
)
})
}
fn all_observability_finished(&self) -> bool {
self.observability_components.values().all(|s| {
matches!(
s.status,
ComponentStatus::Completed | ComponentStatus::Died | ComponentStatus::TimedOut
)
})
}
fn all_finished(&self) -> bool {
self.all_standard_finished() && self.all_observability_finished()
}
fn remaining_tags(&self) -> Vec<String> {
self.components
.iter()
.chain(self.observability_components.iter())
.filter(|(_, s)| {
!matches!(
s.status,
ComponentStatus::Completed | ComponentStatus::TimedOut | ComponentStatus::Died
)
})
.map(|(t, _)| t.clone())
.collect()
}
fn emit_global_timeout(
&self,
service_name: &str,
global_timeout: Duration,
) -> Result<(), LifecycleError> {
let remaining = self.remaining_tags();
for tag in &remaining {
if !self.is_observability_tag(tag) {
metrics::emit_component_shutdown_result(service_name, tag, "timeout");
}
}
warn!(
total_duration_secs = global_timeout.as_secs_f64(),
remaining = ?remaining,
"Lifecycle: global shutdown timeout reached"
);
Err(LifecycleError::ShutdownTimeout {
elapsed: global_timeout,
remaining,
})
}
fn finalize_all(&self, first_failure: Option<LifecycleError>) -> Result<(), LifecycleError> {
self.finalize(Instant::now(), first_failure)
}
fn finalize(
&self,
shutdown_clock: Instant,
first_failure: Option<LifecycleError>,
) -> Result<(), LifecycleError> {
let total = shutdown_clock.elapsed();
let clean = self
.components
.values()
.chain(self.observability_components.values())
.all(|s| s.status == ComponentStatus::Completed);
metrics::emit_shutdown_completed(&self.name, clean);
if clean {
info!(
clean = true,
total_duration_secs = total.as_secs_f64(),
"Lifecycle: shutdown complete"
);
} else {
warn!(
clean = false,
total_duration_secs = total.as_secs_f64(),
"Lifecycle: shutdown complete with failures"
);
}
first_failure.map_or(Ok(()), Err)
}
}
/// Guard returned by [`Manager::monitor_background`]; await to get the monitor result.
pub struct MonitorGuard {
rx: oneshot::Receiver<Result<(), LifecycleError>>,
}
impl MonitorGuard {
/// Await the monitor thread's completion; returns the same `Result` as [`Manager::monitor`](Manager::monitor).
pub async fn wait(self) -> Result<(), LifecycleError> {
self.rx.await.map_err(|_| LifecycleError::MonitorPanicked)?
}
}